diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 9fa9d788ff..e20d023bdc 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -1,22 +1,6 @@ -ARG DEBIAN_IMAGE=debian:bookworm-slim ARG RUST_IMAGE=rust:1.80-slim-bookworm ARG PYTHON_IMAGE=python:3.11.4-slim-bookworm -FROM ${DEBIAN_IMAGE} as downloader - -ARG TARGETPLATFORM - -SHELL ["/bin/bash", "-c"] - -RUN apt update -y -RUN apt install -y unzip curl - -RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.46.3/deno-x86_64-unknown-linux-gnu.zip -o deno.zip || true -RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.46.3/deno-aarch64-unknown-linux-gnu.zip -o deno.zip || true - - -RUN unzip deno.zip && rm deno.zip - FROM ${RUST_IMAGE} as builder @@ -31,7 +15,7 @@ ENV SQLX_OFFLINE=true RUN mkdir -p /frontend/build RUN apt-get update \ - && apt-get install -y ca-certificates tzdata libpq5 cmake\ + && apt-get install -y ca-certificates tzdata libpq5 cmake unzip\ make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \ libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \ libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev libnl-route-3-dev \ @@ -43,6 +27,9 @@ RUN wget https://golang.org/dl/go1.21.5.linux-amd64.tar.gz && tar -C /usr/local ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go +# Install UV +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv + ENV TZ=Etc/UTC ENV PYTHON_VERSION 3.11.4 @@ -53,13 +40,14 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER RUN /usr/local/bin/python3 -m pip install pip-tools -COPY --from=oven/bun:1.1.27 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.31 /usr/local/bin/bun /usr/bin/bun +ARG TARGETPLATFORM -RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.41.0/deno-x86_64-unknown-linux-gnu.zip -o deno.zip || true -RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.41.0/deno-aarch64-unknown-linux-gnu.zip -o deno.zip || true +RUN curl -Lsf https://github.com/denoland/deno/releases/download/v2.0.2/deno-x86_64-unknown-linux-gnu.zip -o deno.zip +# RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v2.0.0/deno-aarch64-unknown-linux-gnu.zip -o deno.zip || true -COPY --from=downloader --chmod=755 /deno /usr/bin/deno +RUN unzip deno.zip && rm deno.zip && mv deno /usr/bin/deno RUN apt-get update \ && apt-get install -y postgresql-client --allow-unauthenticated diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 9963ecd4d1..cbcf40ebe6 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -41,8 +41,12 @@ jobs: - name: cargo test timeout-minutes: 15 run: + /usr/bin/deno --version && + /usr/bin/bun -v && + go version && + /usr/local/bin/python3 --version && mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill - DISABLE_EMBEDDING=true RUST_LOG=info cargo test --features enterprise - --all -- --nocapture + DISABLE_EMBEDDING=true RUST_LOG=info cargo test --features + enterprise,deno_core --all -- --nocapture diff --git a/.github/workflows/build-publish-rh-image.yml b/.github/workflows/build-publish-rh-image.yml new file mode 100644 index 0000000000..b957ac5148 --- /dev/null +++ b/.github/workflows/build-publish-rh-image.yml @@ -0,0 +1,128 @@ +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +name: Build and publish windmill for RHEL9 +on: + workflow_dispatch + +permissions: write-all + +jobs: + build_ee: + runs-on: ubicloud + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Read EE repo commit hash + run: | + echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV" + + - uses: actions/checkout@v4 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ env.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 0 + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + - uses: depot/setup-action@v1 + + - name: Docker meta + id: meta-ee-public + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-rhel9 + flavor: | + latest=false + tags: | + type=sha + + - name: Login to registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Substitute EE code + run: | + ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + + - name: Copy RHEL9 Dockerfile + run: | + cp ./docker/RHEL9/Dockerfile ./Dockerfile + + - name: Build and push publicly ee amd64 + uses: depot/build-push-action@v1 + with: + context: . + platforms: linux/amd64 + push: true + build-args: | + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core + secrets: | + rh_username=${{ secrets.RH_USERNAME }} + rh_password=${{ secrets.RH_PASSWORD }} + tags: | + ${{ steps.meta-ee-public.outputs.tags }}-amd64 + labels: | + ${{ steps.meta-ee-public.outputs.labels }}-amd64 + org.opencontainers.image.licenses=Windmill-Enterprise-License + + - name: Build and push publicly ee arm64 + uses: depot/build-push-action@v1 + with: + context: . + platforms: linux/arm64 + push: true + build-args: | + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core + secrets: | + rh_username=${{ secrets.RH_USERNAME }} + rh_password=${{ secrets.RH_PASSWORD }} + tags: | + ${{ steps.meta-ee-public.outputs.tags }}-arm64 + labels: | + ${{ steps.meta-ee-public.outputs.labels }}-arm64 + org.opencontainers.image.licenses=Windmill-Enterprise-License + + - uses: shrink/actions-docker-extract@v3 + id: extract-ee-amd64 + with: + image: ${{ steps.meta-ee-public.outputs.tags}}-amd64 + path: "/windmill/target/release/windmill" + + - uses: shrink/actions-docker-extract@v3 + id: extract-ee-arm64 + with: + image: ${{ steps.meta-ee-public.outputs.tags}}-arm64 + path: "/windmill/target/release/windmill" + + - name: Rename binary with corresponding architecture + run: | + mv "${{ steps.extract-ee-amd64.outputs.destination }}/windmill" "${{ steps.extract-ee-amd64.outputs.destination }}/windmill-ee-amd64-rhel9" + mv "${{ steps.extract-ee-arm64.outputs.destination }}/windmill" "${{ steps.extract-ee-arm64.outputs.destination }}/windmill-ee-arm64-rhel9" + + - uses: actions/upload-artifact@v4 + with: + name: RHEL9-amd64 build + path: ${{ steps.extract-ee-amd64.outputs.destination }}/windmill-ee-amd64-rhel9 + + - uses: actions/upload-artifact@v4 + with: + name: RHEL9-arm64 build + path: ${{ steps.extract-ee-arm64.outputs.destination }}/windmill-ee-arm64-rhel9 + + # - name: Attach binary to release + # uses: softprops/action-gh-release@v2 + # if: startsWith(github.ref, 'refs/tags/') + # with: + # files: | + # ${{ steps.extract-ee-arm64.outputs.destination }}/windmill-ee-arm64-rhel9 + # ${{ steps.extract-ee-amd64.outputs.destination }}/windmill-ee-amd64-rhel9 diff --git a/.github/workflows/build-staging-image.yml b/.github/workflows/build-staging-image.yml index 744ae83aae..724fde4e59 100644 --- a/.github/workflows/build-staging-image.yml +++ b/.github/workflows/build-staging-image.yml @@ -62,7 +62,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | diff --git a/.github/workflows/build_windows_worker.yml b/.github/workflows/build_windows_worker.yml new file mode 100644 index 0000000000..01c38dd2b1 --- /dev/null +++ b/.github/workflows/build_windows_worker.yml @@ -0,0 +1,60 @@ +name: Build and Publish Windows Worker + +on: + push: + tags: + - "v*" + +env: + CARGO_INCREMENTAL: 0 + SQLX_OFFLINE: true + DISABLE_EMBEDDING: true + RUST_LOG: info + +jobs: + cargo_build_windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - name: Read EE repo commit hash + shell: pwsh + run: | + $ee_repo_ref = Get-Content .\backend\ee-repo-ref.txt + echo "ee_repo_ref=$ee_repo_ref" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: Checkout windmill-ee-private repository + uses: actions/checkout@v4 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ env.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 0 + + - name: Substitute EE code + shell: bash + run: | + ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + + - name: Cargo build windows + timeout-minutes: 90 + run: | + vcpkg.exe install openssl-windows:x64-windows + vcpkg.exe install openssl:x64-windows-static + vcpkg.exe integrate install + $env:VCPKGRS_DYNAMIC=1 + $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" + mkdir frontend/build && cd backend + New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core + + - name: Rename binary with corresponding architecture + run: | + Rename-Item -Path ".\backend\target\release\windmill.exe" -NewName "windmill-ee.exe" + + - name: Attach binary to release + uses: softprops/action-gh-release@v2 + with: + files: | + ./backend/target/release/windmill-ee.exe diff --git a/.github/workflows/docker-image-rpi4.yml b/.github/workflows/docker-image-rpi4.yml index 9fc20c3a87..71a796f679 100644 --- a/.github/workflows/docker-image-rpi4.yml +++ b/.github/workflows/docker-image-rpi4.yml @@ -67,7 +67,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect + features=embedding,parquet,openidconnect,deno_core tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev ${{ steps.meta-public.outputs.tags }} diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 8cac56c35b..f1f3cae9f8 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1,8 +1,10 @@ env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.event_name != 'pull_request' && github.repository || + IMAGE_NAME: + ${{ github.event_name != 'pull_request' && github.repository || 'windmill-labs/windmill-test' }} - DEV_SHA: ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}', + DEV_SHA: + ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}', github.event.number) }} name: Build windmill:main @@ -24,6 +26,7 @@ permissions: write-all jobs: build: runs-on: ubicloud + if: (github.event_name != 'issue_comment') || (contains(github.event.comment.body, '/buildimage_all') || contains(github.event.comment.body, '/buildimage_base')) steps: - uses: actions/checkout@v4 with: @@ -75,7 +78,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,jemalloc + features=embedding,parquet,openidconnect,jemalloc,deno_core tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} ${{ steps.meta-public.outputs.tags }} @@ -85,6 +88,7 @@ jobs: build_ee: runs-on: ubicloud + if: (github.event_name != 'issue_comment') || (contains(github.event.comment.body, '/buildimage_ee') || contains(github.event.comment.body, '/buildimage_nsjail')) || contains(github.event.comment.body, '/buildimage_all') steps: - uses: actions/checkout@v4 with: @@ -136,7 +140,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} ${{ steps.meta-ee-public.outputs.tags }} @@ -146,7 +150,7 @@ jobs: build_ee_312: runs-on: ubicloud - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} steps: - uses: actions/checkout@v4 with: @@ -198,7 +202,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core PYTHON_IMAGE=python:3.12.2-slim-bookworm tags: | ${{ steps.meta-ee-public-py312.outputs.tags }} @@ -242,7 +246,7 @@ jobs: attach_amd64_binary_to_release: needs: [build, build_ee] runs-on: ubicloud - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} env: ARCH: amd64 steps: @@ -335,10 +339,10 @@ jobs: with: fetch-depth: 0 - name: Prepare test run - if: ${{ ! startsWith(github.ref, 'refs/tags/') }} + if: ${{ ! startsWith(github.ref, 'refs/tags/v') }} run: cd integration_tests && ./build.sh - name: Test run - if: ${{ ! startsWith(github.ref, 'refs/tags/') }} + if: ${{ ! startsWith(github.ref, 'refs/tags/v') }} timeout-minutes: 15 env: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} @@ -354,7 +358,7 @@ jobs: tag_latest: runs-on: ubicloud needs: [run_integration_test, build] - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) steps: - uses: actions/checkout@v4 with: @@ -373,7 +377,7 @@ jobs: tag_latest_ee: runs-on: ubicloud needs: [run_integration_test, build_ee] - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) steps: - uses: actions/checkout@v4 with: @@ -392,7 +396,7 @@ jobs: verify_ee_image_vulnerabilities: runs-on: ubicloud needs: [tag_latest_ee] - # if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -434,7 +438,7 @@ jobs: build_ee_nsjail: needs: [build_ee] runs-on: ubicloud - if: github.event_name != 'pull_request' + if: (github.event_name != 'issue_comment') || (github.event_name != 'pull_request') || (contains(github.event.comment.body, '/buildimage_nsjail') || contains(github.event.comment.body, '/buildimage_all')) steps: - uses: actions/checkout@v4 with: @@ -457,6 +461,8 @@ jobs: type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=sha,enable=true,priority=100,prefix=,suffix=,format=short + type=ref,event=branch + type=ref,event=pr - name: Login to registry uses: docker/login-action@v3 @@ -474,7 +480,6 @@ jobs: file: "./docker/DockerfileNsjail" tags: | ${{ steps.meta-ee-public.outputs.tags }} - ghcr.io/windmill-labs/windmill-ee-nsjail:main labels: | ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License @@ -568,7 +573,7 @@ jobs: bucket-region: us-east-1 build_ee_cuda: - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build_ee] runs-on: ubicloud steps: @@ -587,8 +592,6 @@ jobs: with: images: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-cuda - flavor: | - latest=false tags: | type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} @@ -614,7 +617,7 @@ jobs: org.opencontainers.image.licenses=Windmill-Enterprise-License build_slim: - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build] runs-on: ubicloud steps: @@ -633,8 +636,6 @@ jobs: with: images: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-slim - flavor: | - latest=false tags: | type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} @@ -659,7 +660,7 @@ jobs: ${{ steps.meta-ee-public.outputs.labels }} build_ee_slim: - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build_ee] runs-on: ubicloud steps: @@ -678,8 +679,6 @@ jobs: with: images: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-slim - flavor: | - latest=false tags: | type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} @@ -705,7 +704,7 @@ jobs: org.opencontainers.image.licenses=Windmill-Enterprise-License build_full: - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build] runs-on: ubicloud steps: @@ -724,8 +723,6 @@ jobs: with: images: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-full - flavor: | - latest=false tags: | type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} @@ -750,7 +747,7 @@ jobs: ${{ steps.meta-public.outputs.labels }} build_ee_full: - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build_ee] runs-on: ubicloud steps: @@ -769,8 +766,6 @@ jobs: with: images: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-full - flavor: | - latest=false tags: | type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} diff --git a/CHANGELOG.md b/CHANGELOG.md index 848d9a6f10..111225c3a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,383 @@ # Changelog +## [1.416.2](https://github.com/windmill-labs/windmill/compare/v1.416.1...v1.416.2) (2024-11-02) + + +### Bug Fixes + +* apply NO_PROXY and HTTP_PROXY, HTTPS_PROXY more consistently ([567d621](https://github.com/windmill-labs/windmill/commit/567d6216d2631a90fbe59ec6142c38b3b352eea7)) + +## [1.416.1](https://github.com/windmill-labs/windmill/compare/v1.416.0...v1.416.1) (2024-11-01) + + +### Bug Fixes + +* **prometheus:** fix incorrect worker_busy set to 1 ([53f9136](https://github.com/windmill-labs/windmill/commit/53f9136658b9fc1795793d82408c1f1f04adcf06)) + +## [1.416.0](https://github.com/windmill-labs/windmill/compare/v1.415.2...v1.416.0) (2024-11-01) + + +### Features + +* private hub user accessible url setting ([#4617](https://github.com/windmill-labs/windmill/issues/4617)) ([79edf89](https://github.com/windmill-labs/windmill/commit/79edf89bd17827d5f1d946739385327b6c0520bf)) + + +### Bug Fixes + +* **frontend:** improve tag selector for workspace script drawer ([66f6985](https://github.com/windmill-labs/windmill/commit/66f69859ad2de51aef5a133df4ab4397d0f61ccf)) + +## [1.415.2](https://github.com/windmill-labs/windmill/compare/v1.415.1...v1.415.2) (2024-11-01) + + +### Bug Fixes + +* **s3:** align s3 handler additional creds providers ([984c6dd](https://github.com/windmill-labs/windmill/commit/984c6dd10c63097eb195883c4d8a9681ab1b49e0)) + +## [1.415.1](https://github.com/windmill-labs/windmill/compare/v1.415.0...v1.415.1) (2024-10-31) + + +### Bug Fixes + +* **cli:** improve --instance handling wmill instance push ([cb005a1](https://github.com/windmill-labs/windmill/commit/cb005a15baef4272bc58c7e80a43e44723556d31)) + +## [1.415.0](https://github.com/windmill-labs/windmill/compare/v1.414.2...v1.415.0) (2024-10-31) + + +### Features + +* **cli:** opts.instance as instace name and prefix ([#4609](https://github.com/windmill-labs/windmill/issues/4609)) ([a07f57e](https://github.com/windmill-labs/windmill/commit/a07f57e698107056d045d8d5c2458e04c809fcc8)) + + +### Bug Fixes + +* improve express oauth setup ([ba4aed5](https://github.com/windmill-labs/windmill/commit/ba4aed5bf51c65204332cfc158d0ffd9c7095ec7)) +* improve user resource input ([8c7f53b](https://github.com/windmill-labs/windmill/commit/8c7f53b2ebe0990cd93879258d004ac89dc8b24c)) + +## [1.414.2](https://github.com/windmill-labs/windmill/compare/v1.414.1...v1.414.2) (2024-10-29) + + +### Bug Fixes + +* **cli:** improve instance sync for CI/CD + --folder-per-instance ([212579a](https://github.com/windmill-labs/windmill/commit/212579a514d070355fe0d9e0215593bacfa05e1f)) + +## [1.414.1](https://github.com/windmill-labs/windmill/compare/v1.414.0...v1.414.1) (2024-10-29) + + +### Bug Fixes + +* **apps:** enable text selection on aggrid tables by default ([b0b9180](https://github.com/windmill-labs/windmill/commit/b0b9180fb907c92b95a48ff286eb1dae59bb4981)) +* **apps:** public apps can take full height ([703db7d](https://github.com/windmill-labs/windmill/commit/703db7d4412795b4323a2eefcc39ae3cf43bc748)) +* **bun:** handle bun lockfile created with windows ([#4602](https://github.com/windmill-labs/windmill/issues/4602)) ([dcf5e2f](https://github.com/windmill-labs/windmill/commit/dcf5e2f03f977e241f6785530dad61a70c5bdd79)) +* **frontend:** make script and schema scrollable on script detail page ([6e222b3](https://github.com/windmill-labs/windmill/commit/6e222b3b1a419e5543fdf35420a7413699688b41)) +* **frontend:** new approval steps default to timeout 1800 ([b86de62](https://github.com/windmill-labs/windmill/commit/b86de6280e03e014e8ddf85b2b5f8fd030d0467a)) + +## [1.414.0](https://github.com/windmill-labs/windmill/compare/v1.413.2...v1.414.0) (2024-10-29) + +* Issue with previous release, re-releasing + +## [1.413.2](https://github.com/windmill-labs/windmill/compare/v1.413.1...v1.413.2) (2024-10-29) + + +### Bug Fixes + +* **backend:** in flows, workspace scripts should use their set tags instead of the default one ([5b7c6d7](https://github.com/windmill-labs/windmill/commit/5b7c6d7d62dcfd09fec374e781bdf5c5bafe4a9d)) +* **cli:** fix wmill instance pull --instance ([3c62f5e](https://github.com/windmill-labs/windmill/commit/3c62f5ea83d1da8bd3705468969d56e2fe680751)) +* **frontend:** fix script and flow renaming ([d743e00](https://github.com/windmill-labs/windmill/commit/d743e0056353a4fca445a7089e3afc1fd4e8c219)) + +## [1.413.1](https://github.com/windmill-labs/windmill/compare/v1.413.0...v1.413.1) (2024-10-28) + + +### Bug Fixes + +* **cli:** fix wmill instance push --base-url and --instance ([8298710](https://github.com/windmill-labs/windmill/commit/82987105a6fd6ec272c170fb094453a0267143be)) + +## [1.413.0](https://github.com/windmill-labs/windmill/compare/v1.412.0...v1.413.0) (2024-10-28) + + +### Features + +* autoscaling v0 ([#4593](https://github.com/windmill-labs/windmill/issues/4593)) ([fe7d044](https://github.com/windmill-labs/windmill/commit/fe7d044a66e8ec223a337cb704d3e58942dd1502)) + + +### Bug Fixes + +* add run immediately popover to run again ([e54d253](https://github.com/windmill-labs/windmill/commit/e54d25368541dc6109a0f99022a120d28455f9bd)) +* **docs:** smtp setup documentation link ([#4590](https://github.com/windmill-labs/windmill/issues/4590)) ([bac3205](https://github.com/windmill-labs/windmill/commit/bac32057259d893140d649c9dfec2ca75e395ad4)) + +## [1.412.0](https://github.com/windmill-labs/windmill/compare/v1.411.1...v1.412.0) (2024-10-25) + + +### Features + +* add Spotify oauth provider ([#4581](https://github.com/windmill-labs/windmill/issues/4581)) ([a46aa64](https://github.com/windmill-labs/windmill/commit/a46aa644b096e71e75b59507224ed92f7d2f99ba)) + + +### Bug Fixes + +* **app builder:** date input default value improvements ([9f43d5d](https://github.com/windmill-labs/windmill/commit/9f43d5dcd92ddcd0c0baeaa5271779520af652f4)) +* **bash:** correctly propagate sigterm for cancelled bash scripts ([134cfdb](https://github.com/windmill-labs/windmill/commit/134cfdb30eb8d29c2ecb1b78b8fabae2b6e10700)) +* do not update created_at of scripts on lockfile generation ([d1a28eb](https://github.com/windmill-labs/windmill/commit/d1a28eb7cac5f465e2b07f870a507b0cc5cc722a)) +* initialize empty smtp settings correctly ([84e0524](https://github.com/windmill-labs/windmill/commit/84e05249505c3d2f7eb7a4917f6bca503df81017)) + +## [1.411.1](https://github.com/windmill-labs/windmill/compare/v1.411.0...v1.411.1) (2024-10-22) + + +### Bug Fixes + +* update bun to 1.1.32 ([#4568](https://github.com/windmill-labs/windmill/issues/4568)) ([0586446](https://github.com/windmill-labs/windmill/commit/058644667129f0d79ec147aacdda449142ae0ab9)) + +## [1.411.0](https://github.com/windmill-labs/windmill/compare/v1.410.3...v1.411.0) (2024-10-21) + + +### Features + +* **cli:** encrypt sensitive instance settings ([#4561](https://github.com/windmill-labs/windmill/issues/4561)) ([b8a6a11](https://github.com/windmill-labs/windmill/commit/b8a6a116354b10f5977e54edb365d6711e160538)) + + +### Bug Fixes + +* Do not ignore file resources with json file ext ([#4562](https://github.com/windmill-labs/windmill/issues/4562)) ([2079b2e](https://github.com/windmill-labs/windmill/commit/2079b2e7e19aa2fe327f2ae66d1b5eba988b9b0a)) +* update bun to 1.1.31 and deno to 2.0.2 ([0d90396](https://github.com/windmill-labs/windmill/commit/0d9039641b3348e75599937188c35dd89a000584)) + +## [1.410.3](https://github.com/windmill-labs/windmill/compare/v1.410.2...v1.410.3) (2024-10-20) + + +### Bug Fixes + +* **go-client:** reduce runtime dependencies by bumping oai-codeen to v2.4.1 ([87f5c07](https://github.com/windmill-labs/windmill/commit/87f5c078dd63cd3e0ba3e719c2da7b83f1458e2c)) + +## [1.410.1](https://github.com/windmill-labs/windmill/compare/v1.410.0...v1.410.1) (2024-10-19) + + +### Bug Fixes + +* **cli:** improve wmill init behavior ([26a40d1](https://github.com/windmill-labs/windmill/commit/26a40d19441aa816ee711ce30f3435dddd3542a7)) +* **frontend:** improve display of error handlers ([a92a2fd](https://github.com/windmill-labs/windmill/commit/a92a2fd6fd67c11c04554a90b8ae6d7a0dd9067c)) + +## [1.410.1](https://github.com/windmill-labs/windmill/compare/v1.410.0...v1.410.1) (2024-10-19) + + +### Bug Fixes + +* **cli:** improve wmill init behavior ([26a40d1](https://github.com/windmill-labs/windmill/commit/26a40d19441aa816ee711ce30f3435dddd3542a7)) + +## [1.410.0](https://github.com/windmill-labs/windmill/compare/v1.409.4...v1.410.0) (2024-10-18) + + +### Features + +* **typescript-bun:** support relative imports without the .ts extension ([248fdc2](https://github.com/windmill-labs/windmill/commit/248fdc24a61aca946902b20b9d8187101a9b7bfa)) +* websocket triggers ([#4505](https://github.com/windmill-labs/windmill/issues/4505)) ([8807e99](https://github.com/windmill-labs/windmill/commit/8807e99f06caf3e26c526eefdc83dbc2f7aa93ee)) + + +### Bug Fixes + +* cache js static assets by default ([08595c6](https://github.com/windmill-labs/windmill/commit/08595c6f14a89df1559974e8835e63881a1a9601)) +* **frontend:** add back script lockfile to script details pae ([549b11d](https://github.com/windmill-labs/windmill/commit/549b11dcfb39a60acd6d3899fce51b31149f8594)) +* improve cancelling of jobs on public apps for anonymous users ([d7cf5ea](https://github.com/windmill-labs/windmill/commit/d7cf5ea37db313e78865708bc22e87a1feea9e0d)) + +## [1.409.4](https://github.com/windmill-labs/windmill/compare/v1.409.3...v1.409.4) (2024-10-17) + + +### Bug Fixes + +* fix flow viewer renderer outside of flow details and editor context ([59a1e67](https://github.com/windmill-labs/windmill/commit/59a1e67465cd21ded5539cb5b829bc0d4e7169ed)) + +## [1.409.3](https://github.com/windmill-labs/windmill/compare/v1.409.2...v1.409.3) (2024-10-17) + + +### Bug Fixes + +* do not delete primary schedule of script/flow on redeploy even if schedule wasn't loaded ([c3c2fe4](https://github.com/windmill-labs/windmill/commit/c3c2fe462c52f48795a55025414b16f8d3d98fe0)) +* **nsjail:** improve memory reading when using nsjail ([b7ad19b](https://github.com/windmill-labs/windmill/commit/b7ad19bb75bd885a254ebac778349c7d88af8326)) + +## [1.409.2](https://github.com/windmill-labs/windmill/compare/v1.409.1...v1.409.2) (2024-10-16) + + +### Bug Fixes + +* add extra args support for exception to bun scripts ([1466da3](https://github.com/windmill-labs/windmill/commit/1466da3999add0238b9c42ca13df52194f082fc0)) +* fix script persistence in url + add support for extra error args in python ([3174024](https://github.com/windmill-labs/windmill/commit/3174024d8e6ecbe9f8c9e1ea055d611f652c0057)) + +## [1.409.1](https://github.com/windmill-labs/windmill/compare/v1.409.0...v1.409.1) (2024-10-16) + + +### Bug Fixes + +* **apidocs:** fix generated openapi files ([d24e153](https://github.com/windmill-labs/windmill/commit/d24e1530655d27ffda9bb4c19471dc1431124cc2)) +* **git-sync:** propagate update of folders with git sync ([6abb346](https://github.com/windmill-labs/windmill/commit/6abb346013da4a907a860713a8a67642985b8025)) + +## [1.409.0](https://github.com/windmill-labs/windmill/compare/v1.408.1...v1.409.0) (2024-10-16) + + +### Features + +* **frontend:** unify all triggers UX and simplify flow settings ([#4259](https://github.com/windmill-labs/windmill/issues/4259)) ([91a3d06](https://github.com/windmill-labs/windmill/commit/91a3d065298cce7a882464fa0cd31d8f1ae9dda2)) +* Scroll to element in virtual list when clicking on graph point ([#4532](https://github.com/windmill-labs/windmill/issues/4532)) ([7126ba1](https://github.com/windmill-labs/windmill/commit/7126ba12c7eb52d2cfbe8d83311b5592a5707bce)) +* **sso:** adding the ability to define a custom display name for sso ([#4529](https://github.com/windmill-labs/windmill/issues/4529)) ([99c5b3e](https://github.com/windmill-labs/windmill/commit/99c5b3ecdacb1158c2cba5b891c4c3b8b70c3b6a)) + + +### Bug Fixes + +* Add indexer backup lock to fit the deployment model ([#4531](https://github.com/windmill-labs/windmill/issues/4531)) ([411bce7](https://github.com/windmill-labs/windmill/commit/411bce7e13aabfa53db80d55261ea4511f6d1ae9)) +* **app:** accept connecting to non yet existing state output for convenience ([9eb1ecc](https://github.com/windmill-labs/windmill/commit/9eb1ecc9f3017e2f4284a827a2bcd21f36b1b8ac)) +* **app:** improve absolute url handling in download button and downloadFile ([dcdbf1a](https://github.com/windmill-labs/windmill/commit/dcdbf1afb4d5a18e00b9bbb1eb0bef129ea5667f)) +* **app:** make s3 uploads persistent across tabs change ([c3b536b](https://github.com/windmill-labs/windmill/commit/c3b536b1b8069898131768867a187b186b21e537)) +* canceled jobs button reporting 0 jobs cancelled ([#4534](https://github.com/windmill-labs/windmill/issues/4534)) ([e736572](https://github.com/windmill-labs/windmill/commit/e736572db10929ae5e123c4f6cef73b8e90fc29b)) +* **python-client:** improve get_job_status for running jobs ([a8c4ea2](https://github.com/windmill-labs/windmill/commit/a8c4ea2334d2535fa7d5d43f58d65565afe8f3e5)) +* **ui:** dark mode support for queue metrics based critical alert ([#4535](https://github.com/windmill-labs/windmill/issues/4535)) ([f38b3d1](https://github.com/windmill-labs/windmill/commit/f38b3d14e8092ae58817511aea91a4e77725ead6)) + +## [1.408.1](https://github.com/windmill-labs/windmill/compare/v1.408.0...v1.408.1) (2024-10-12) + + +### Bug Fixes + +* fix deno cache --allow-import on deno 2 ([42fe31f](https://github.com/windmill-labs/windmill/commit/42fe31f804c9e6643cd90167494e45270831e013)) + +## [1.408.0](https://github.com/windmill-labs/windmill/compare/v1.407.2...v1.408.0) (2024-10-12) + + +### Features + +* **app builder:** file download helper ([#4511](https://github.com/windmill-labs/windmill/issues/4511)) ([f82f091](https://github.com/windmill-labs/windmill/commit/f82f09129096cfff975370d8bb7b6d832a2b8f9f)) + + +### Bug Fixes + +* **cli:** handle case where 'toString' is a schema field ([568cc66](https://github.com/windmill-labs/windmill/commit/568cc66932fb0470f5e89de7b02d94dba4050638)) +* **frontend:** s3 file uploader works on public apps too ([982dde2](https://github.com/windmill-labs/windmill/commit/982dde2b9dfe6d9eda300c683af729d97a03cb4d)) +* **frontend:** set unused schema property fields to null ([be11240](https://github.com/windmill-labs/windmill/commit/be112408e7c4601314726e9517c37daaeaa1bf09)) +* improve workflow as code row-lock on db to handle more concurrency ([d2c4d3f](https://github.com/windmill-labs/windmill/commit/d2c4d3fa207379cb0b8ac180f6ccc759045580e8)) + +## [1.407.2](https://github.com/windmill-labs/windmill/compare/v1.407.1...v1.407.2) (2024-10-10) + + +### Bug Fixes + +* improve default properties of new nodes of flows (suspend, branchone, branchall) ([d9bdc5a](https://github.com/windmill-labs/windmill/commit/d9bdc5a5b08dd4d0381304656af097315398c9d4)) + +## [1.407.1](https://github.com/windmill-labs/windmill/compare/v1.407.0...v1.407.1) (2024-10-10) + + +### Bug Fixes + +* improve handling of empty lock files on deno 2.0 ([7ca5bf2](https://github.com/windmill-labs/windmill/commit/7ca5bf2faeff44a7543b1afa9369c140fcb71dfc)) + +## [1.407.0](https://github.com/windmill-labs/windmill/compare/v1.406.0...v1.407.0) (2024-10-10) + + +### Features + +* upgrade to deno 2 ([26b11a0](https://github.com/windmill-labs/windmill/commit/26b11a00150acbe101abe4bb542f24379da0cc56)) + + +### Bug Fixes + +* update internal deno runtime to latest (deno 2.0) ([c3a5736](https://github.com/windmill-labs/windmill/commit/c3a57366419882ea2de1938bea592c795b1a1d03)) + +## [1.406.0](https://github.com/windmill-labs/windmill/compare/v1.405.5...v1.406.0) (2024-10-09) + + +### Features + +* **frontend:** components can be moved inside containers by holding ctrl/cmd ([111bfc6](https://github.com/windmill-labs/windmill/commit/111bfc6a659037ae7029e8f557256e2fffcf979b)) +* **monitoring:** Critical Alerts for Jobs Waiting in Queue [enterprise] ([#4491](https://github.com/windmill-labs/windmill/issues/4491)) ([d90d6c2](https://github.com/windmill-labs/windmill/commit/d90d6c2b896c5f99e00681656f376b180901f272)) + + +### Bug Fixes + +* **cli:** instance sync push does not require sync pull ([257f097](https://github.com/windmill-labs/windmill/commit/257f0971f86938da71b879f32d93473976eaa920)) +* remove monaco-editor for app preview code path for faster app loads ([7b05033](https://github.com/windmill-labs/windmill/commit/7b0503332d1bdd7f5999a5ef99150f8c9f6f18be)) + +## [1.405.5](https://github.com/windmill-labs/windmill/compare/v1.405.4...v1.405.5) (2024-10-04) + + +### Bug Fixes + +* windows.exe build with github workflow doesn't have openssl.dll bundled in ([#4489](https://github.com/windmill-labs/windmill/issues/4489)) ([284cb40](https://github.com/windmill-labs/windmill/commit/284cb4069c97efe59b5caf3effb68c8b30e02b73)) + +## [1.405.4](https://github.com/windmill-labs/windmill/compare/v1.405.3...v1.405.4) (2024-10-04) + + +### Bug Fixes + +* **frontend:** correctly initialize step inputs on new inline script ([289ad51](https://github.com/windmill-labs/windmill/commit/289ad51374f0344582372572fc521f3b2bf12b33)) + +## [1.405.3](https://github.com/windmill-labs/windmill/compare/v1.405.2...v1.405.3) (2024-10-04) + + +### Bug Fixes + +* fix id save on apps ([b034b07](https://github.com/windmill-labs/windmill/commit/b034b070c075a0fab74678bec1fd8b829d55b204)) + +## [1.405.2](https://github.com/windmill-labs/windmill/compare/v1.405.1...v1.405.2) (2024-10-03) + + +### Bug Fixes + +* **cli:** fix opts.yes for instance sync ([26659ce](https://github.com/windmill-labs/windmill/commit/26659ce37d2887d5b98dbdbdbba27bab85d4fe3f)) +* fix uv path ([19c62ba](https://github.com/windmill-labs/windmill/commit/19c62ba195b1df85c38c748dab7d9f137696a5c3)) + +## [1.405.1](https://github.com/windmill-labs/windmill/compare/v1.405.0...v1.405.1) (2024-10-03) + + +### Bug Fixes + +* flow picker of flows + precache hub scripts as bundles ([c84e6fd](https://github.com/windmill-labs/windmill/commit/c84e6fd05de2bea426cae61fa25db0323b8770f5)) + +## [1.405.0](https://github.com/windmill-labs/windmill/compare/v1.404.1...v1.405.0) (2024-10-03) + + +### Features + +* Replace `pip-compile` with `uv` ([#4460](https://github.com/windmill-labs/windmill/issues/4460)) ([b54c9ee](https://github.com/windmill-labs/windmill/commit/b54c9ee657cc88fabe694cae39dc0d3c1918fcbb)) +* **worker:** support workers to run natively on windows ([#4446](https://github.com/windmill-labs/windmill/issues/4446)) ([f5c4727](https://github.com/windmill-labs/windmill/commit/f5c472727465dd95f5378bc08ee9bbb983f4d259)) + + +### Bug Fixes + +* **cli:** fix set client of instance when passing token and base url ([794c4cd](https://github.com/windmill-labs/windmill/commit/794c4cde3cd47042472dccdf4b60a012014dd26d)) + +## [1.404.1](https://github.com/windmill-labs/windmill/compare/v1.404.0...v1.404.1) (2024-10-03) + + +### Bug Fixes + +* flow picker of flows ([92f61f0](https://github.com/windmill-labs/windmill/commit/92f61f07ed6d354407d26843e3a270b95bae90bc)) + +## [1.404.0](https://github.com/windmill-labs/windmill/compare/v1.403.1...v1.404.0) (2024-10-03) + + +### Features + +* **frontend:** add quick access menu in flow editor ([#4415](https://github.com/windmill-labs/windmill/issues/4415)) ([45ccd45](https://github.com/windmill-labs/windmill/commit/45ccd45e306c66931880a9b8fd48bfe684c774ac)) + + +### Bug Fixes + +* **cli:** improve schedule path handling on windows ([9ac3b6b](https://github.com/windmill-labs/windmill/commit/9ac3b6b1d5d64d7467dd80506f8a8d772c4630bd)) +* fix id editor for app ([8e58e43](https://github.com/windmill-labs/windmill/commit/8e58e4320a31d71c40a5ed352416a4c2dd3adb26)) +* **frontend:** disable runnable field on route editor from detail panel ([#4469](https://github.com/windmill-labs/windmill/issues/4469)) ([3134f79](https://github.com/windmill-labs/windmill/commit/3134f79ced80aab86912643ab7a60dcf909ab104)) + +## [1.403.1](https://github.com/windmill-labs/windmill/compare/v1.403.0...v1.403.1) (2024-10-01) + + +### Bug Fixes + +* fix new instance db setup ([73ab8e1](https://github.com/windmill-labs/windmill/commit/73ab8e1653d6e0c0c69fa7dcd96583f25d13ef86)) + +## [1.403.0](https://github.com/windmill-labs/windmill/compare/v1.402.3...v1.403.0) (2024-10-01) + + +### Features + +* flow step skipping ([#4461](https://github.com/windmill-labs/windmill/issues/4461)) ([0df169e](https://github.com/windmill-labs/windmill/commit/0df169e3f996ed54b91569b13cce15d7d019a213)) + + +### Bug Fixes + +* skip one migration to avoid using md5 for azure support ([630ae5d](https://github.com/windmill-labs/windmill/commit/630ae5d425cd9957d674befd2df96e2befec52a3)) + ## [1.402.3](https://github.com/windmill-labs/windmill/compare/v1.402.2...v1.402.3) (2024-09-30) diff --git a/Dockerfile b/Dockerfile index 3bc5088aed..8683aa9738 100644 --- a/Dockerfile +++ b/Dockerfile @@ -158,8 +158,11 @@ RUN set -eux; \ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go +# Install UV +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv + RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - -RUN apt-get -y update && apt-get install -y curl nodejs awscli && apt-get clean \ +RUN apt-get -y update && apt-get install -y curl procps nodejs awscli && apt-get clean \ && rm -rf /var/lib/apt/lists/* # go build is slower the first time it is ran, so we prewarm it in the build @@ -172,9 +175,9 @@ RUN /usr/local/bin/python3 -m pip install pip-tools COPY --from=builder /frontend/build /static_frontend COPY --from=builder /windmill/target/release/windmill ${APP}/windmill -COPY --from=denoland/deno:1.46.3 --chmod=755 /usr/bin/deno /usr/bin/deno +COPY --from=denoland/deno:2.0.2 --chmod=755 /usr/bin/deno /usr/bin/deno -COPY --from=oven/bun:1.1.27 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.32 /usr/local/bin/bun /usr/bin/bun COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer diff --git a/backend/.sqlx/query-02424907504848e983bfa89eec343061932dc5b4b17cf13d5cf8d833aedbe6d5.json b/backend/.sqlx/query-02424907504848e983bfa89eec343061932dc5b4b17cf13d5cf8d833aedbe6d5.json new file mode 100644 index 0000000000..c8b5e3086f --- /dev/null +++ b/backend/.sqlx/query-02424907504848e983bfa89eec343061932dc5b4b17cf13d5cf8d833aedbe6d5.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE websocket_trigger SET server_id = $1, last_server_ping = now() WHERE enabled IS TRUE AND workspace_id = $2 AND path = $3 AND (server_id IS NULL OR last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "02424907504848e983bfa89eec343061932dc5b4b17cf13d5cf8d833aedbe6d5" +} diff --git a/backend/.sqlx/query-02b516dac764662194db1bc33e365c01f40bae70af3683f1f09748f6020f0d49.json b/backend/.sqlx/query-02b516dac764662194db1bc33e365c01f40bae70af3683f1f09748f6020f0d49.json deleted file mode 100644 index 99f3606bcd..0000000000 --- a/backend/.sqlx/query-02b516dac764662194db1bc33e365c01f40bae70af3683f1f09748f6020f0d49.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT tag, count(*) as count FROM queue WHERE\n scheduled_for <= now() - ('3 seconds')::interval AND running = false\n GROUP BY tag", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - null - ] - }, - "hash": "02b516dac764662194db1bc33e365c01f40bae70af3683f1f09748f6020f0d49" -} diff --git a/backend/.sqlx/query-0ee63ef2dd5c88edba2a1f56d31f29876724922f148dad7af35b36efbf70207a.json b/backend/.sqlx/query-0ee63ef2dd5c88edba2a1f56d31f29876724922f148dad7af35b36efbf70207a.json new file mode 100644 index 0000000000..5cb52b6ead --- /dev/null +++ b/backend/.sqlx/query-0ee63ef2dd5c88edba2a1f56d31f29876724922f148dad7af35b36efbf70207a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM healthchecks WHERE check_type = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "0ee63ef2dd5c88edba2a1f56d31f29876724922f148dad7af35b36efbf70207a" +} diff --git a/backend/.sqlx/query-12a86755706ce030a0a9142da78d035d0c7d361240b60005cc864e4345eb0bc7.json b/backend/.sqlx/query-12a86755706ce030a0a9142da78d035d0c7d361240b60005cc864e4345eb0bc7.json new file mode 100644 index 0000000000..dc357c2169 --- /dev/null +++ b/backend/.sqlx/query-12a86755706ce030a0a9142da78d035d0c7d361240b60005cc864e4345eb0bc7.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE websocket_trigger SET enabled = $1, email = $2, edited_by = $3, edited_at = now(), server_id = NULL, last_server_ping = NULL, error = NULL\n WHERE path = $4 AND workspace_id = $5 RETURNING 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Bool", + "Varchar", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "12a86755706ce030a0a9142da78d035d0c7d361240b60005cc864e4345eb0bc7" +} diff --git a/backend/.sqlx/query-14abf759dae7ba5c38017ba6001927c6df0653a02b87bcea939066e39ebcf24d.json b/backend/.sqlx/query-14abf759dae7ba5c38017ba6001927c6df0653a02b87bcea939066e39ebcf24d.json new file mode 100644 index 0000000000..29aaf47e88 --- /dev/null +++ b/backend/.sqlx/query-14abf759dae7ba5c38017ba6001927c6df0653a02b87bcea939066e39ebcf24d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO concurrency_locks (id, last_locked_at, owner)\n VALUES ($1, now(), $2)\n ON CONFLICT (id)\n DO UPDATE SET\n last_locked_at = now(),\n owner = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "14abf759dae7ba5c38017ba6001927c6df0653a02b87bcea939066e39ebcf24d" +} diff --git a/backend/.sqlx/query-c2060e8cacef6c3b5ce51ed203a2dbafc18d66f2924d1fe518c6728997647db2.json b/backend/.sqlx/query-1a4d291c2f239f7b50c116594cebb031862e1a18ad9204e02a0194817db26d6a.json similarity index 75% rename from backend/.sqlx/query-c2060e8cacef6c3b5ce51ed203a2dbafc18d66f2924d1fe518c6728997647db2.json rename to backend/.sqlx/query-1a4d291c2f239f7b50c116594cebb031862e1a18ad9204e02a0194817db26d6a.json index 6a08bbeab3..b8c5b89128 100644 --- a/backend/.sqlx/query-c2060e8cacef6c3b5ce51ed203a2dbafc18d66f2924d1fe518c6728997647db2.json +++ b/backend/.sqlx/query-1a4d291c2f239f7b50c116594cebb031862e1a18ad9204e02a0194817db26d6a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT email, login_type::text, verified, super_admin, name, company, username from password ORDER BY super_admin DESC, email LIMIT $1 OFFSET $2", + "query": "SELECT email, login_type::text, verified, super_admin, name, company, username, NULL::bool as operator_only FROM password ORDER BY super_admin DESC, email LIMIT $1 OFFSET $2", "describe": { "columns": [ { @@ -37,6 +37,11 @@ "ordinal": 6, "name": "username", "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "operator_only", + "type_info": "Bool" } ], "parameters": { @@ -52,8 +57,9 @@ false, true, true, - true + true, + null ] }, - "hash": "c2060e8cacef6c3b5ce51ed203a2dbafc18d66f2924d1fe518c6728997647db2" + "hash": "1a4d291c2f239f7b50c116594cebb031862e1a18ad9204e02a0194817db26d6a" } diff --git a/backend/.sqlx/query-1ca5bc2d35c0498b587fd0618434def64233dc4f8fc3344d8d74be8e96ded659.json b/backend/.sqlx/query-1ca5bc2d35c0498b587fd0618434def64233dc4f8fc3344d8d74be8e96ded659.json new file mode 100644 index 0000000000..3a2e44bdba --- /dev/null +++ b/backend/.sqlx/query-1ca5bc2d35c0498b587fd0618434def64233dc4f8fc3344d8d74be8e96ded659.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM schedule WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1ca5bc2d35c0498b587fd0618434def64233dc4f8fc3344d8d74be8e96ded659" +} diff --git a/backend/.sqlx/query-1f5f0858909eb5bac63c4e3b1add95226bd94ca3facb92a620ffa59dacad6705.json b/backend/.sqlx/query-1f5f0858909eb5bac63c4e3b1add95226bd94ca3facb92a620ffa59dacad6705.json new file mode 100644 index 0000000000..0adff2fe2c --- /dev/null +++ b/backend/.sqlx/query-1f5f0858909eb5bac63c4e3b1add95226bd94ca3facb92a620ffa59dacad6705.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO autoscaling_event (worker_group, event_type, desired_workers, reason) VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "autoscaling_event_type", + "kind": { + "Enum": [ + "full_scaleout", + "scalein", + "scaleout" + ] + } + } + }, + "Int4", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1f5f0858909eb5bac63c4e3b1add95226bd94ca3facb92a620ffa59dacad6705" +} diff --git a/backend/.sqlx/query-2041526bc58872d71f91f7698144039bd67f8e37895befa94a15b7e4019e114b.json b/backend/.sqlx/query-2041526bc58872d71f91f7698144039bd67f8e37895befa94a15b7e4019e114b.json new file mode 100644 index 0000000000..cbe5ec7607 --- /dev/null +++ b/backend/.sqlx/query-2041526bc58872d71f91f7698144039bd67f8e37895befa94a15b7e4019e114b.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM healthchecks WHERE healthy = true AND created_at < NOW() - INTERVAL '14 days'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "2041526bc58872d71f91f7698144039bd67f8e37895befa94a15b7e4019e114b" +} diff --git a/backend/.sqlx/query-2238aaed46031f71b17c1cabcef935a0d3b2ca6d057f446ed9b02d4386e2ddd9.json b/backend/.sqlx/query-2238aaed46031f71b17c1cabcef935a0d3b2ca6d057f446ed9b02d4386e2ddd9.json new file mode 100644 index 0000000000..aa32886ba8 --- /dev/null +++ b/backend/.sqlx/query-2238aaed46031f71b17c1cabcef935a0d3b2ca6d057f446ed9b02d4386e2ddd9.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM websocket_trigger WHERE path = $1 AND workspace_id = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "2238aaed46031f71b17c1cabcef935a0d3b2ca6d057f446ed9b02d4386e2ddd9" +} diff --git a/backend/.sqlx/query-27920aaa55666ffc14a36a247f89ff7994ee40d3953b9f772d0e0ab999bccb7b.json b/backend/.sqlx/query-27920aaa55666ffc14a36a247f89ff7994ee40d3953b9f772d0e0ab999bccb7b.json new file mode 100644 index 0000000000..26a8ba2ee2 --- /dev/null +++ b/backend/.sqlx/query-27920aaa55666ffc14a36a247f89ff7994ee40d3953b9f772d0e0ab999bccb7b.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO healthchecks (check_type, healthy) VALUES ($1, false)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "27920aaa55666ffc14a36a247f89ff7994ee40d3953b9f772d0e0ab999bccb7b" +} diff --git a/backend/.sqlx/query-8813665f8adfcab0daefbac2cc6b50e427dfd1c12d12895451affd307dc59c37.json b/backend/.sqlx/query-2c14d3a88193f16ad3b8cd590749cb5537995f2499f6cb8f0f316fb62902d542.json similarity index 77% rename from backend/.sqlx/query-8813665f8adfcab0daefbac2cc6b50e427dfd1c12d12895451affd307dc59c37.json rename to backend/.sqlx/query-2c14d3a88193f16ad3b8cd590749cb5537995f2499f6cb8f0f316fb62902d542.json index ce8adcd5b7..93017f5aec 100644 --- a/backend/.sqlx/query-8813665f8adfcab0daefbac2cc6b50e427dfd1c12d12895451affd307dc59c37.json +++ b/backend/.sqlx/query-2c14d3a88193f16ad3b8cd590749cb5537995f2499f6cb8f0f316fb62902d542.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT email, login_type::TEXT, super_admin, verified, name, company, username FROM password WHERE email = $1", + "query": "SELECT email, login_type::TEXT, super_admin, verified, name, company, username, NULL::bool as operator_only FROM password WHERE email = $1", "describe": { "columns": [ { @@ -37,6 +37,11 @@ "ordinal": 6, "name": "username", "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "operator_only", + "type_info": "Bool" } ], "parameters": { @@ -51,8 +56,9 @@ false, true, true, - true + true, + null ] }, - "hash": "8813665f8adfcab0daefbac2cc6b50e427dfd1c12d12895451affd307dc59c37" + "hash": "2c14d3a88193f16ad3b8cd590749cb5537995f2499f6cb8f0f316fb62902d542" } diff --git a/backend/.sqlx/query-2e6165543e34216dfaedf6e10729f733cff812e62dea1c1bce8cefd3a3979b14.json b/backend/.sqlx/query-2e6165543e34216dfaedf6e10729f733cff812e62dea1c1bce8cefd3a3979b14.json new file mode 100644 index 0000000000..4a15e2d4fa --- /dev/null +++ b/backend/.sqlx/query-2e6165543e34216dfaedf6e10729f733cff812e62dea1c1bce8cefd3a3979b14.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM websocket_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "2e6165543e34216dfaedf6e10729f733cff812e62dea1c1bce8cefd3a3979b14" +} diff --git a/backend/.sqlx/query-2eec077cc9e27d7ccd160cbaac118c321c422705f79e69550bb60f377083bcef.json b/backend/.sqlx/query-2eec077cc9e27d7ccd160cbaac118c321c422705f79e69550bb60f377083bcef.json new file mode 100644 index 0000000000..f3eecb9006 --- /dev/null +++ b/backend/.sqlx/query-2eec077cc9e27d7ccd160cbaac118c321c422705f79e69550bb60f377083bcef.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, name, company, username\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC\n LIMIT $1 OFFSET $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "operator_only", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "login_type", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "verified", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "super_admin", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "company", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "username", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + null, + null, + false, + false, + true, + true, + true + ] + }, + "hash": "2eec077cc9e27d7ccd160cbaac118c321c422705f79e69550bb60f377083bcef" +} diff --git a/backend/.sqlx/query-31bc3dcea29be9cc0242771d25a232f173446d29c08fc29ddb8d55294f2c070e.json b/backend/.sqlx/query-31bc3dcea29be9cc0242771d25a232f173446d29c08fc29ddb8d55294f2c070e.json new file mode 100644 index 0000000000..49fc84c349 --- /dev/null +++ b/backend/.sqlx/query-31bc3dcea29be9cc0242771d25a232f173446d29c08fc29ddb8d55294f2c070e.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM http_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "31bc3dcea29be9cc0242771d25a232f173446d29c08fc29ddb8d55294f2c070e" +} diff --git a/backend/.sqlx/query-9ed77e78e6295c62745ac3ac3b7e5f544f654d7cddc13acb5d3f4fdc12a8875f.json b/backend/.sqlx/query-34a45763bb4d14162f4cd3fa07cd8020f1f6085f4ee85f5eab3458637edf26cd.json similarity index 52% rename from backend/.sqlx/query-9ed77e78e6295c62745ac3ac3b7e5f544f654d7cddc13acb5d3f4fdc12a8875f.json rename to backend/.sqlx/query-34a45763bb4d14162f4cd3fa07cd8020f1f6085f4ee85f5eab3458637edf26cd.json index 9c7af05c42..f9dc00c28c 100644 --- a/backend/.sqlx/query-9ed77e78e6295c62745ac3ac3b7e5f544f654d7cddc13acb5d3f4fdc12a8875f.json +++ b/backend/.sqlx/query-34a45763bb4d14162f4cd3fa07cd8020f1f6085f4ee85f5eab3458637edf26cd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT created_at FROM metrics WHERE id = 'license_key_renewal' ORDER BY created_at DESC LIMIT 1", + "query": "SELECT created_at FROM healthchecks WHERE check_type = $1 ORDER BY created_at DESC LIMIT 1", "describe": { "columns": [ { @@ -10,11 +10,13 @@ } ], "parameters": { - "Left": [] + "Left": [ + "Text" + ] }, "nullable": [ false ] }, - "hash": "9ed77e78e6295c62745ac3ac3b7e5f544f654d7cddc13acb5d3f4fdc12a8875f" + "hash": "34a45763bb4d14162f4cd3fa07cd8020f1f6085f4ee85f5eab3458637edf26cd" } diff --git a/backend/.sqlx/query-3ecb25b05d6c14b499f9b00af42ae74134728899f6b59c68b246979bc5143e30.json b/backend/.sqlx/query-3ecb25b05d6c14b499f9b00af42ae74134728899f6b59c68b246979bc5143e30.json new file mode 100644 index 0000000000..a652b56bad --- /dev/null +++ b/backend/.sqlx/query-3ecb25b05d6c14b499f9b00af42ae74134728899f6b59c68b246979bc5143e30.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT COUNT(*) as count, \n MIN(scheduled_for) as oldest_job\n FROM queue \n WHERE tag = $1 \n AND scheduled_for <= NOW() - $2::interval \n AND running = false\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "oldest_job", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Interval" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "3ecb25b05d6c14b499f9b00af42ae74134728899f6b59c68b246979bc5143e30" +} diff --git a/backend/.sqlx/query-4e9668a46bad9e82baa51422946d373b18b6577198df7545c94bd19be3446775.json b/backend/.sqlx/query-4e9668a46bad9e82baa51422946d373b18b6577198df7545c94bd19be3446775.json new file mode 100644 index 0000000000..2f96d31ecb --- /dev/null +++ b/backend/.sqlx/query-4e9668a46bad9e82baa51422946d373b18b6577198df7545c94bd19be3446775.json @@ -0,0 +1,108 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO websocket_trigger (workspace_id, path, url, script_path, is_flow, enabled, filters, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, now()) RETURNING *", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "url", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 9, + "name": "server_id", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "last_server_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 11, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "enabled", + "type_info": "Bool" + }, + { + "ordinal": 13, + "name": "filters", + "type_info": "JsonbArray" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Bool", + "JsonbArray", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false + ] + }, + "hash": "4e9668a46bad9e82baa51422946d373b18b6577198df7545c94bd19be3446775" +} diff --git a/backend/.sqlx/query-4eca060026a0cb19c5794cd56ace89fc04765191f251945d14bbe78718714f6e.json b/backend/.sqlx/query-4eca060026a0cb19c5794cd56ace89fc04765191f251945d14bbe78718714f6e.json new file mode 100644 index 0000000000..3b812a53db --- /dev/null +++ b/backend/.sqlx/query-4eca060026a0cb19c5794cd56ace89fc04765191f251945d14bbe78718714f6e.json @@ -0,0 +1,39 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT event_type::AUTOSCALING_EVENT_TYPE AS \"event_type: _\", EXTRACT(EPOCH FROM (NOW() - applied_at))::int as seconds_ago FROM autoscaling_event WHERE worker_group = $1 ORDER BY applied_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "event_type: _", + "type_info": { + "Custom": { + "name": "autoscaling_event_type", + "kind": { + "Enum": [ + "full_scaleout", + "scalein", + "scaleout" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "seconds_ago", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "4eca060026a0cb19c5794cd56ace89fc04765191f251945d14bbe78718714f6e" +} diff --git a/backend/.sqlx/query-5303cb9dd5903aa4791ef8e5e5881a50a832e65c8c9632e2e12cd9c2747f2fc7.json b/backend/.sqlx/query-5303cb9dd5903aa4791ef8e5e5881a50a832e65c8c9632e2e12cd9c2747f2fc7.json new file mode 100644 index 0000000000..a95b00b8c8 --- /dev/null +++ b/backend/.sqlx/query-5303cb9dd5903aa4791ef8e5e5881a50a832e65c8c9632e2e12cd9c2747f2fc7.json @@ -0,0 +1,98 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT *\n FROM websocket_trigger\n WHERE enabled IS TRUE AND (server_id IS NULL OR last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "url", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 9, + "name": "server_id", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "last_server_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 11, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "enabled", + "type_info": "Bool" + }, + { + "ordinal": 13, + "name": "filters", + "type_info": "JsonbArray" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false + ] + }, + "hash": "5303cb9dd5903aa4791ef8e5e5881a50a832e65c8c9632e2e12cd9c2747f2fc7" +} diff --git a/backend/.sqlx/query-57e270e032e8c04dda7b5c1ca949861756b3ad367a4a500728332a7cb91560a4.json b/backend/.sqlx/query-57e270e032e8c04dda7b5c1ca949861756b3ad367a4a500728332a7cb91560a4.json new file mode 100644 index 0000000000..ccf6b0ad8f --- /dev/null +++ b/backend/.sqlx/query-57e270e032e8c04dda7b5c1ca949861756b3ad367a4a500728332a7cb91560a4.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE concurrency_locks SET\n last_locked_at = now()\n WHERE id = $1 AND owner = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "57e270e032e8c04dda7b5c1ca949861756b3ad367a4a500728332a7cb91560a4" +} diff --git a/backend/.sqlx/query-5fc6b4a4dbb7875bdec76f876c18543435a95b019b20081f52f6ed6f4457e3c7.json b/backend/.sqlx/query-5fc6b4a4dbb7875bdec76f876c18543435a95b019b20081f52f6ed6f4457e3c7.json new file mode 100644 index 0000000000..d4725c224e --- /dev/null +++ b/backend/.sqlx/query-5fc6b4a4dbb7875bdec76f876c18543435a95b019b20081f52f6ed6f4457e3c7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:' || $2]::text[]", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "5fc6b4a4dbb7875bdec76f876c18543435a95b019b20081f52f6ed6f4457e3c7" +} diff --git a/backend/.sqlx/query-5fd70c70ce52cbc51fa9124cb05f82b5951f17d1b7eade53c6d89253d55f8b9f.json b/backend/.sqlx/query-5fd70c70ce52cbc51fa9124cb05f82b5951f17d1b7eade53c6d89253d55f8b9f.json new file mode 100644 index 0000000000..51f58cb81c --- /dev/null +++ b/backend/.sqlx/query-5fd70c70ce52cbc51fa9124cb05f82b5951f17d1b7eade53c6d89253d55f8b9f.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT owner FROM concurrency_locks WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "owner", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "5fd70c70ce52cbc51fa9124cb05f82b5951f17d1b7eade53c6d89253d55f8b9f" +} diff --git a/backend/.sqlx/query-6d134b137ae81534e145fc5b6474cf963ee26a3ad3a0a3d8dc064cb14c8fd9a6.json b/backend/.sqlx/query-6d134b137ae81534e145fc5b6474cf963ee26a3ad3a0a3d8dc064cb14c8fd9a6.json new file mode 100644 index 0000000000..03351cdeab --- /dev/null +++ b/backend/.sqlx/query-6d134b137ae81534e145fc5b6474cf963ee26a3ad3a0a3d8dc064cb14c8fd9a6.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, worker_group, event_type::text, desired_workers, reason, applied_at FROM autoscaling_event WHERE worker_group = $1 ORDER BY applied_at DESC LIMIT 5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "worker_group", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "event_type", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "desired_workers", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "reason", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "applied_at", + "type_info": "Timestamp" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + null, + false, + true, + false + ] + }, + "hash": "6d134b137ae81534e145fc5b6474cf963ee26a3ad3a0a3d8dc064cb14c8fd9a6" +} diff --git a/backend/.sqlx/query-722a3096f03d25ef94292d53801d41037de4bc69dd434232029c731cbbcbc22f.json b/backend/.sqlx/query-722a3096f03d25ef94292d53801d41037de4bc69dd434232029c731cbbcbc22f.json deleted file mode 100644 index 2e4dee2a30..0000000000 --- a/backend/.sqlx/query-722a3096f03d25ef94292d53801d41037de4bc69dd434232029c731cbbcbc22f.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO windmill_migrations (name) VALUES ('bypassrls_1-2')", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "722a3096f03d25ef94292d53801d41037de4bc69dd434232029c731cbbcbc22f" -} diff --git a/backend/.sqlx/query-829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3.json b/backend/.sqlx/query-829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3.json new file mode 100644 index 0000000000..e00aba3aab --- /dev/null +++ b/backend/.sqlx/query-829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT job_kind = 'identity' FROM completed_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "829130d74c107e4e1a86f3e772657ae030463c139c39072777e453bfb7e9c0c3" +} diff --git a/backend/.sqlx/query-900ac59515e4283f4b57516210575dfe92f74a7220ed69e61899a6e0f053d9cd.json b/backend/.sqlx/query-900ac59515e4283f4b57516210575dfe92f74a7220ed69e61899a6e0f053d9cd.json new file mode 100644 index 0000000000..0bcd4d447f --- /dev/null +++ b/backend/.sqlx/query-900ac59515e4283f4b57516210575dfe92f74a7220ed69e61899a6e0f053d9cd.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO concurrency_locks (id, last_locked_at) VALUES ($1, NOW()) ON CONFLICT (id) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "900ac59515e4283f4b57516210575dfe92f74a7220ed69e61899a6e0f053d9cd" +} diff --git a/backend/.sqlx/query-a7f5431e3b8960e9dc46fae69dd4391516d8b169186548ec44528c84078b80d8.json b/backend/.sqlx/query-a7f5431e3b8960e9dc46fae69dd4391516d8b169186548ec44528c84078b80d8.json new file mode 100644 index 0000000000..7dc8e3efa7 --- /dev/null +++ b/backend/.sqlx/query-a7f5431e3b8960e9dc46fae69dd4391516d8b169186548ec44528c84078b80d8.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM token WHERE label LIKE 'email-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a7f5431e3b8960e9dc46fae69dd4391516d8b169186548ec44528c84078b80d8" +} diff --git a/backend/.sqlx/query-acbf74cf3302bfcf7615285070d3f8958932bb8a2dda715f1b9152ab44442780.json b/backend/.sqlx/query-acbf74cf3302bfcf7615285070d3f8958932bb8a2dda715f1b9152ab44442780.json new file mode 100644 index 0000000000..eef46ce41d --- /dev/null +++ b/backend/.sqlx/query-acbf74cf3302bfcf7615285070d3f8958932bb8a2dda715f1b9152ab44442780.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE websocket_trigger SET url = $1, script_path = $2, path = $3, is_flow = $4, filters = $5, edited_by = $6, email = $7, edited_at = now(), server_id = NULL, last_server_ping = NULL, error = NULL\n WHERE workspace_id = $8 AND path = $9", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Bool", + "JsonbArray", + "Varchar", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "acbf74cf3302bfcf7615285070d3f8958932bb8a2dda715f1b9152ab44442780" +} diff --git a/backend/.sqlx/query-ad42118ccf6a9d2d1e072c4df064ddf964a5b3cd088fc162d0d8222325d4a5ea.json b/backend/.sqlx/query-ad42118ccf6a9d2d1e072c4df064ddf964a5b3cd088fc162d0d8222325d4a5ea.json new file mode 100644 index 0000000000..c43c2a2a65 --- /dev/null +++ b/backend/.sqlx/query-ad42118ccf6a9d2d1e072c4df064ddf964a5b3cd088fc162d0d8222325d4a5ea.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE healthchecks SET healthy = true WHERE check_type = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "ad42118ccf6a9d2d1e072c4df064ddf964a5b3cd088fc162d0d8222325d4a5ea" +} diff --git a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json index 99269c9851..54e94cfb8f 100644 --- a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json +++ b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json @@ -18,8 +18,8 @@ "Left": [] }, "nullable": [ - true, - false + false, + true ] }, "hash": "b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76" diff --git a/backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json b/backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json index 71aaf4ba90..cc825ff56f 100644 --- a/backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json +++ b/backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json @@ -52,7 +52,8 @@ "trigger", "failure", "command", - "approval" + "approval", + "preprocessor" ] } } diff --git a/backend/.sqlx/query-c060b8bbc5af7d2e7d0aaff64f0f62ec9db58611a99b0ba7f0375638b128ab89.json b/backend/.sqlx/query-c060b8bbc5af7d2e7d0aaff64f0f62ec9db58611a99b0ba7f0375638b128ab89.json new file mode 100644 index 0000000000..1a37220559 --- /dev/null +++ b/backend/.sqlx/query-c060b8bbc5af7d2e7d0aaff64f0f62ec9db58611a99b0ba7f0375638b128ab89.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT schedule FROM schedule WHERE path = $1 AND script_path = $1 AND is_flow = $2 AND workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "schedule", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c060b8bbc5af7d2e7d0aaff64f0f62ec9db58611a99b0ba7f0375638b128ab89" +} diff --git a/backend/.sqlx/query-c4e1873bfc7b905e7299a021f4baa2a97e95f4797c5e11f37822e19828422b7e.json b/backend/.sqlx/query-c4e1873bfc7b905e7299a021f4baa2a97e95f4797c5e11f37822e19828422b7e.json new file mode 100644 index 0000000000..d3c16a5275 --- /dev/null +++ b/backend/.sqlx/query-c4e1873bfc7b905e7299a021f4baa2a97e95f4797c5e11f37822e19828422b7e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE concurrency_locks SET last_locked_at = NOW() WHERE id = $1 AND last_locked_at < NOW() - INTERVAL '1 second' * $2 RETURNING 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Float8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c4e1873bfc7b905e7299a021f4baa2a97e95f4797c5e11f37822e19828422b7e" +} diff --git a/backend/.sqlx/query-34ad8a2a5bd89b9b8e25847a7e5e94ef99e35a178ad6328c1bcde2a6d6f88cb5.json b/backend/.sqlx/query-c624f15f3e321b1eecf123da9bf0b18e8c1d16ef25ffb9d04e5447d0d583d55c.json similarity index 58% rename from backend/.sqlx/query-34ad8a2a5bd89b9b8e25847a7e5e94ef99e35a178ad6328c1bcde2a6d6f88cb5.json rename to backend/.sqlx/query-c624f15f3e321b1eecf123da9bf0b18e8c1d16ef25ffb9d04e5447d0d583d55c.json index 43d0f596ae..edb2fc7f49 100644 --- a/backend/.sqlx/query-34ad8a2a5bd89b9b8e25847a7e5e94ef99e35a178ad6328c1bcde2a6d6f88cb5.json +++ b/backend/.sqlx/query-c624f15f3e321b1eecf123da9bf0b18e8c1d16ef25ffb9d04e5447d0d583d55c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO token\n (token, email, label, expiration, super_admin, scopes)\n VALUES ($1, $2, $3, $4, $5, $6)", + "query": "INSERT INTO token\n (token, email, label, expiration, super_admin, scopes, workspace_id)\n VALUES ($1, $2, $3, $4, $5, $6, $7)", "describe": { "columns": [], "parameters": { @@ -10,10 +10,11 @@ "Varchar", "Timestamptz", "Bool", - "TextArray" + "TextArray", + "Varchar" ] }, "nullable": [] }, - "hash": "34ad8a2a5bd89b9b8e25847a7e5e94ef99e35a178ad6328c1bcde2a6d6f88cb5" + "hash": "c624f15f3e321b1eecf123da9bf0b18e8c1d16ef25ffb9d04e5447d0d583d55c" } diff --git a/backend/.sqlx/query-c7ee7ce64686cef41cebd99ad7ef31572fc1bf12e6ae473fd58fafb025989965.json b/backend/.sqlx/query-c7ee7ce64686cef41cebd99ad7ef31572fc1bf12e6ae473fd58fafb025989965.json new file mode 100644 index 0000000000..7dd454bbbe --- /dev/null +++ b/backend/.sqlx/query-c7ee7ce64686cef41cebd99ad7ef31572fc1bf12e6ae473fd58fafb025989965.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at, scopes, email FROM token WHERE workspace_id = $1 AND scopes @> ARRAY['run:script/' || $2]::text[]", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "label", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "token_prefix", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "expiration", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "last_used_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "scopes", + "type_info": "TextArray" + }, + { + "ordinal": 6, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true, + null, + true, + false, + false, + true, + true + ] + }, + "hash": "c7ee7ce64686cef41cebd99ad7ef31572fc1bf12e6ae473fd58fafb025989965" +} diff --git a/backend/.sqlx/query-cecf1addc4aecb087a14786b2a9165895ca61ef042947c7314f66514d7f29edc.json b/backend/.sqlx/query-cecf1addc4aecb087a14786b2a9165895ca61ef042947c7314f66514d7f29edc.json new file mode 100644 index 0000000000..361f5ebd78 --- /dev/null +++ b/backend/.sqlx/query-cecf1addc4aecb087a14786b2a9165895ca61ef042947c7314f66514d7f29edc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT last_locked_at\n FROM concurrency_locks\n WHERE id = $1\n FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_locked_at", + "type_info": "Timestamp" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "cecf1addc4aecb087a14786b2a9165895ca61ef042947c7314f66514d7f29edc" +} diff --git a/backend/.sqlx/query-d54840373df5da9662ed11ed0a605cddac517bdc06dca0a8be071330338e948a.json b/backend/.sqlx/query-d54840373df5da9662ed11ed0a605cddac517bdc06dca0a8be071330338e948a.json new file mode 100644 index 0000000000..5b298f8db3 --- /dev/null +++ b/backend/.sqlx/query-d54840373df5da9662ed11ed0a605cddac517bdc06dca0a8be071330338e948a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM websocket_trigger WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d54840373df5da9662ed11ed0a605cddac517bdc06dca0a8be071330338e948a" +} diff --git a/backend/.sqlx/query-ec47955683d811b12e82ff3e6aeafa0df46a320c436bad5eb1acc127df138a61.json b/backend/.sqlx/query-d697b7311430e7bd5375ec5494179f4071e3bfe123d9600249cfa80c1103edd8.json similarity index 51% rename from backend/.sqlx/query-ec47955683d811b12e82ff3e6aeafa0df46a320c436bad5eb1acc127df138a61.json rename to backend/.sqlx/query-d697b7311430e7bd5375ec5494179f4071e3bfe123d9600249cfa80c1103edd8.json index 2ed5e047db..e35c4e2730 100644 --- a/backend/.sqlx/query-ec47955683d811b12e82ff3e6aeafa0df46a320c436bad5eb1acc127df138a61.json +++ b/backend/.sqlx/query-d697b7311430e7bd5375ec5494179f4071e3bfe123d9600249cfa80c1103edd8.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE script SET lock = $1, created_at = now() WHERE hash = $2 AND workspace_id = $3", + "query": "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "ec47955683d811b12e82ff3e6aeafa0df46a320c436bad5eb1acc127df138a61" + "hash": "d697b7311430e7bd5375ec5494179f4071e3bfe123d9600249cfa80c1103edd8" } diff --git a/backend/.sqlx/query-d7a0f19f9e18d2ea49316012375ad78b69292ba091d69880945e42bebe890d66.json b/backend/.sqlx/query-d7a0f19f9e18d2ea49316012375ad78b69292ba091d69880945e42bebe890d66.json new file mode 100644 index 0000000000..f619796a27 --- /dev/null +++ b/backend/.sqlx/query-d7a0f19f9e18d2ea49316012375ad78b69292ba091d69880945e42bebe890d66.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d7a0f19f9e18d2ea49316012375ad78b69292ba091d69880945e42bebe890d66" +} diff --git a/backend/.sqlx/query-db24f1f6ef1e26b7de7926bd3f32d1fa673c4970c25ff9ad98f0fdf199801b53.json b/backend/.sqlx/query-db24f1f6ef1e26b7de7926bd3f32d1fa673c4970c25ff9ad98f0fdf199801b53.json new file mode 100644 index 0000000000..70f4b91b71 --- /dev/null +++ b/backend/.sqlx/query-db24f1f6ef1e26b7de7926bd3f32d1fa673c4970c25ff9ad98f0fdf199801b53.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) as \"websocket_used!\", EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) as \"http_routes_used!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "websocket_used!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "http_routes_used!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "db24f1f6ef1e26b7de7926bd3f32d1fa673c4970c25ff9ad98f0fdf199801b53" +} diff --git a/backend/.sqlx/query-eb1f916f9beea3eea83ce359f5305d0cfb0d6cdba9cc56c6139f57e46345f843.json b/backend/.sqlx/query-eb1f916f9beea3eea83ce359f5305d0cfb0d6cdba9cc56c6139f57e46345f843.json deleted file mode 100644 index ecd83ae2e3..0000000000 --- a/backend/.sqlx/query-eb1f916f9beea3eea83ce359f5305d0cfb0d6cdba9cc56c6139f57e46345f843.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'bypassrls_1-2')", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "eb1f916f9beea3eea83ce359f5305d0cfb0d6cdba9cc56c6139f57e46345f843" -} diff --git a/backend/.sqlx/query-9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef.json b/backend/.sqlx/query-eb932b613a6dbb2cdff97e5512d42b538ba83115c0ea798be00b01659600f45a.json similarity index 60% rename from backend/.sqlx/query-9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef.json rename to backend/.sqlx/query-eb932b613a6dbb2cdff97e5512d42b538ba83115c0ea798be00b01659600f45a.json index ae30d3bd98..7717a34760 100644 --- a/backend/.sqlx/query-9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef.json +++ b/backend/.sqlx/query-eb932b613a6dbb2cdff97e5512d42b538ba83115c0ea798be00b01659600f45a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1)", + "query": "SELECT EXISTS(SELECT 1 FROM healthchecks WHERE check_type = $1 AND healthy = false)", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef" + "hash": "eb932b613a6dbb2cdff97e5512d42b538ba83115c0ea798be00b01659600f45a" } diff --git a/backend/.sqlx/query-eff32aeac25a75d06f73e08c26dd3fd25f6b85cbea870505751c6a82457ae1da.json b/backend/.sqlx/query-eff32aeac25a75d06f73e08c26dd3fd25f6b85cbea870505751c6a82457ae1da.json new file mode 100644 index 0000000000..2190ea06eb --- /dev/null +++ b/backend/.sqlx/query-eff32aeac25a75d06f73e08c26dd3fd25f6b85cbea870505751c6a82457ae1da.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at, scopes, email FROM token WHERE workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "label", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "token_prefix", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "expiration", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "last_used_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "scopes", + "type_info": "TextArray" + }, + { + "ordinal": 6, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true, + null, + true, + false, + false, + true, + true + ] + }, + "hash": "eff32aeac25a75d06f73e08c26dd3fd25f6b85cbea870505751c6a82457ae1da" +} diff --git a/backend/.sqlx/query-f06e0e4fa358b26792df22fff48b71a6fcfa1e5603ea472892917c1accd1aafb.json b/backend/.sqlx/query-f06e0e4fa358b26792df22fff48b71a6fcfa1e5603ea472892917c1accd1aafb.json new file mode 100644 index 0000000000..71cdfe60bf --- /dev/null +++ b/backend/.sqlx/query-f06e0e4fa358b26792df22fff48b71a6fcfa1e5603ea472892917c1accd1aafb.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM token WHERE label LIKE 'email-%' AND workspace_id = $1 AND scopes @> ARRAY['run:script/' || $2]::text[]", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f06e0e4fa358b26792df22fff48b71a6fcfa1e5603ea472892917c1accd1aafb" +} diff --git a/backend/.sqlx/query-f2baee15e6d1fecd6d2d7b39fda1b50a15ec8bd349f1081af54da5dc2f5e3021.json b/backend/.sqlx/query-f2baee15e6d1fecd6d2d7b39fda1b50a15ec8bd349f1081af54da5dc2f5e3021.json new file mode 100644 index 0000000000..2e75b4e74b --- /dev/null +++ b/backend/.sqlx/query-f2baee15e6d1fecd6d2d7b39fda1b50a15ec8bd349f1081af54da5dc2f5e3021.json @@ -0,0 +1,101 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT *\n FROM websocket_trigger\n WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "url", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 8, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 9, + "name": "server_id", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "last_server_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 11, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 12, + "name": "enabled", + "type_info": "Bool" + }, + { + "ordinal": 13, + "name": "filters", + "type_info": "JsonbArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + false, + false + ] + }, + "hash": "f2baee15e6d1fecd6d2d7b39fda1b50a15ec8bd349f1081af54da5dc2f5e3021" +} diff --git a/backend/.sqlx/query-febd70e6d5304ad912c232d3d8fc3f8ce0133d8a1a9e5ca1124eff9a988dc2d7.json b/backend/.sqlx/query-febd70e6d5304ad912c232d3d8fc3f8ce0133d8a1a9e5ca1124eff9a988dc2d7.json new file mode 100644 index 0000000000..7958f9d2f3 --- /dev/null +++ b/backend/.sqlx/query-febd70e6d5304ad912c232d3d8fc3f8ce0133d8a1a9e5ca1124eff9a988dc2d7.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE websocket_trigger SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND server_id = $4 AND enabled IS TRUE RETURNING 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "febd70e6d5304ad912c232d3d8fc3f8ce0133d8a1a9e5ca1124eff9a988dc2d7" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 0ee755b3ec..d12b1b2698 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14,9 +14,9 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] @@ -40,22 +40,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ "cfg-if", - "cipher 0.3.0", + "cipher", "cpufeatures", "opaque-debug", ] -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher 0.4.4", - "cpufeatures", -] - [[package]] name = "ahash" version = "0.7.8" @@ -128,9 +117,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.15" +version = "0.6.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +checksum = "23a1e53f0f5d86382dafe1cf314783b2044280f406e7e1506368220ad11b1338" dependencies = [ "anstyle", "anstyle-parse", @@ -143,43 +132,43 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +checksum = "74f37166d7d48a0284b99dd824694c26119c700b53bf0d1540cdb147dbdaaf13" [[package]] name = "arc-swap" @@ -187,19 +176,6 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" -[[package]] -name = "archiver-rs" -version = "0.5.1" -source = "git+https://github.com/gz/archiver-rs.git?branch=patch-1#a73cef92c2a5b8f48c2a4a9e889952072e03b4b7" -dependencies = [ - "bzip2", - "flate2", - "tar", - "thiserror", - "xz2", - "zip", -] - [[package]] name = "argon2" version = "0.5.3" @@ -209,7 +185,7 @@ dependencies = [ "base64ct", "blake2", "cpufeatures", - "password-hash 0.5.0", + "password-hash", ] [[package]] @@ -383,7 +359,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.5.0", + "indexmap 2.6.0", "lexical-core", "num", "serde", @@ -465,7 +441,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -510,11 +486,11 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.12" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" +checksum = "0cb8f1d480b0ea3783ab015936d2a55c87e219676f0c0b7dec61494043f21857" dependencies = [ - "brotli", + "brotli 7.0.0", "bzip2", "flate2", "futures-core", @@ -536,7 +512,7 @@ dependencies = [ "async-task", "concurrent-queue", "fastrand 2.1.1", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "slab", ] @@ -551,7 +527,7 @@ dependencies = [ "async-io", "async-lock", "blocking", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "once_cell", ] @@ -565,7 +541,7 @@ dependencies = [ "cfg-if", "concurrent-queue", "futures-io", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "parking", "polling", "rustix", @@ -595,7 +571,7 @@ dependencies = [ "bytes", "http 1.1.0", "rand 0.8.5", - "reqwest 0.12.7", + "reqwest 0.12.9", "serde", "serde-aux", "serde_json", @@ -612,7 +588,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -629,7 +605,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-io", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "gloo-timers", "kv-log-macro", "log", @@ -643,9 +619,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", @@ -654,27 +630,27 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] name = "async-stripe" -version = "0.34.1" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109605d984dd71a9a278e2d43a2831a20e80f25c3bcc25174096d12352cfc469" +checksum = "58d670cf4d47a1b8ffef54286a5625382e360a34ee76902fd93ad8c7032a0c30" dependencies = [ "chrono", "futures-util", "hex", "hmac", "http-types", - "hyper 0.14.30", + "hyper 0.14.31", "hyper-tls 0.5.0", "serde", "serde_json", @@ -702,7 +678,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -756,9 +732,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.7" +version = "1.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8191fb3091fa0561d1379ef80333c3c7191c6f0435d986e85821bcf7acbd1126" +checksum = "2d6448cfb224dd6a9b9ac734f58622dd0d4751f3589f3b777345745f46b2eb14" dependencies = [ "aws-credential-types", "aws-runtime", @@ -818,14 +794,14 @@ dependencies = [ "percent-encoding", "pin-project-lite", "tracing", - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] name = "aws-sdk-sso" -version = "1.44.0" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b90cfe6504115e13c41d3ea90286ede5aa14da294f3fe077027a6e83850843c" +checksum = "ded855583fa1d22e88fe39fd6062b062376e50a8211989e07cf5e38d52eb3453" dependencies = [ "aws-credential-types", "aws-runtime", @@ -845,9 +821,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.45.0" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167c0fad1f212952084137308359e8e4c4724d1c643038ce163f06de9662c1d0" +checksum = "9177ea1192e6601ae16c7273385690d88a7ed386a00b74a6bc894d12103cd933" dependencies = [ "aws-credential-types", "aws-runtime", @@ -867,9 +843,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.44.0" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb5f98188ec1435b68097daa2a37d74b9d17c9caa799466338a8d1544e71b9d" +checksum = "823ef553cf36713c97453e2ddff1eb8f62be7f4523544e2a5db64caf80100f0a" dependencies = [ "aws-credential-types", "aws-runtime", @@ -890,9 +866,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.4" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc8db6904450bafe7473c6ca9123f88cc11089e41a025408f992db4e22d3be68" +checksum = "5619742a0d8f253be760bfbb8e8e8368c69e3587e4637af5754e488a611499b1" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -963,9 +939,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87" +checksum = "be28bd063fa91fd871d131fc8b68d7cd4c5fa0869bea68daca50dcb1cbd76be2" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -978,7 +954,7 @@ dependencies = [ "http-body 0.4.6", "http-body 1.0.1", "httparse", - "hyper 0.14.30", + "hyper 0.14.31", "hyper-rustls 0.24.2", "once_cell", "pin-project-lite", @@ -1007,9 +983,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.7" +version = "1.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147100a7bea70fa20ef224a6bad700358305f5dc0f84649c53769761395b355b" +checksum = "07c9cdc179e6afbf5d391ab08c85eac817b51c87e1892a5edb5f7bbdc64314b4" dependencies = [ "base64-simd 0.8.0", "bytes", @@ -1067,7 +1043,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-util", "itoa", "matchit", @@ -1182,9 +1158,9 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bb8" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b10cf871f3ff2ce56432fddc2615ac7acc3aa22ca321f8fea800846fbb32f188" +checksum = "d89aabfae550a5c44b43ab941844ffcd2e993cb6900b342debf59e9ea74acdb8" dependencies = [ "async-trait", "futures-util", @@ -1194,18 +1170,18 @@ dependencies = [ [[package]] name = "better_scoped_tls" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "794edcc9b3fb07bb4aecaa11f093fd45663b4feadb782d68303a2268bc2701de" +checksum = "297b153aa5e573b5863108a6ddc9d5c968bd0b20e75cc614ee9821d2f45679c7" dependencies = [ "scoped-tls", ] [[package]] name = "bigdecimal" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d712318a27c7150326677b321a5fa91b55f6d9034ffd67f20319e147d40cee" +checksum = "8f850665a0385e070b64c38d2354e6c104c8479c59868d1e48a0c13ee2c7a1c1" dependencies = [ "autocfg", "libm", @@ -1225,9 +1201,9 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.4" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ "bitflags 2.6.0", "cexpr", @@ -1236,13 +1212,13 @@ dependencies = [ "lazy_static", "lazycell", "log", - "prettyplease 0.2.22", + "prettyplease 0.2.25", "proc-macro2", "quote", "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.79", + "syn 2.0.86", "which 4.4.2", ] @@ -1261,7 +1237,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -1334,7 +1310,7 @@ dependencies = [ "arrayvec", "cc", "cfg-if", - "constant_time_eq 0.3.1", + "constant_time_eq", ] [[package]] @@ -1363,7 +1339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cb03d1bed155d89dce0f845b7899b18a9a163e148fd004e1c28421a783e2d8e" dependencies = [ "block-padding", - "cipher 0.3.0", + "cipher", ] [[package]] @@ -1381,7 +1357,7 @@ dependencies = [ "async-channel 2.3.1", "async-task", "futures-io", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "piper", ] @@ -1405,7 +1381,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", "syn_derive", ] @@ -1420,6 +1396,17 @@ dependencies = [ "brotli-decompressor", ] +[[package]] +name = "brotli" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + [[package]] name = "brotli-decompressor" version = "4.0.1" @@ -1454,6 +1441,9 @@ name = "bumpalo" version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +dependencies = [ + "allocator-api2", +] [[package]] name = "bytecheck" @@ -1479,22 +1469,22 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" +checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc8b54b395f2fcfbb3d90c47b01c7f444d94d05bdeb775811dec868ac3bbc26" +checksum = "bcfcc3cd946cb52f0bbfdbbcfa2f4e24f75ebb6c0e1002f7c25904fada18b9ec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -1505,9 +1495,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "bytes-utils" @@ -1609,9 +1599,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.22" +version = "1.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0" +checksum = "e3788d6ac30243803df38a3e9991cf37e41210232916d41a8222ae378f912624" dependencies = [ "jobserver", "libc", @@ -1712,16 +1702,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - [[package]] name = "clang-sys" version = "1.8.1" @@ -1735,9 +1715,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.18" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -1745,9 +1725,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.18" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -1764,7 +1744,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -1784,9 +1764,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "combine" @@ -1898,12 +1878,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - [[package]] name = "constant_time_eq" version = "0.3.1" @@ -2150,7 +2124,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -2198,7 +2172,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -2220,7 +2194,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -2259,7 +2233,7 @@ dependencies = [ "arrow-array", "arrow-ipc", "arrow-schema", - "async-compression 0.4.12", + "async-compression 0.4.17", "async-trait", "bytes", "bzip2", @@ -2282,7 +2256,7 @@ dependencies = [ "glob", "half", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.12.1", "log", "num_cpus", @@ -2297,7 +2271,7 @@ dependencies = [ "tokio", "tokio-util", "url", - "uuid 1.10.0", + "uuid 1.11.0", "xz2", "zstd 0.13.2", ] @@ -2397,7 +2371,7 @@ dependencies = [ "regex", "sha2 0.10.8", "unicode-segmentation", - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] @@ -2451,7 +2425,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.12.1", "log", "regex-syntax 0.8.5", @@ -2480,7 +2454,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "hex", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.12.1", "log", "paste", @@ -2524,7 +2498,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.12.1", "log", "once_cell", @@ -2564,16 +2538,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ "serde", - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] name = "deno_ast" -version = "0.40.0" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d08372522975cce97fe0efbe42fea508c76eea4421619de6d63baae32792f7d" +checksum = "b2b9d03b1bbeeecdac54367f075d572131736d06c5be3bc49037855bc5ab1bbb" dependencies = [ - "anyhow", "base64 0.21.7", "deno_media_type", "deno_terminal 0.1.1", @@ -2581,15 +2554,16 @@ dependencies = [ "once_cell", "percent-encoding", "serde", + "sourcemap 9.0.0", "swc_atoms", - "swc_common 0.34.4", + "swc_common", "swc_config", "swc_config_macro", - "swc_ecma_ast 0.115.1", + "swc_ecma_ast", "swc_ecma_codegen", "swc_ecma_codegen_macros", "swc_ecma_loader", - "swc_ecma_parser 0.146.12", + "swc_ecma_parser", "swc_ecma_transforms_base", "swc_ecma_transforms_classes", "swc_ecma_transforms_macros", @@ -2597,7 +2571,7 @@ dependencies = [ "swc_ecma_transforms_react", "swc_ecma_transforms_typescript", "swc_ecma_utils", - "swc_ecma_visit 0.101.0", + "swc_ecma_visit", "swc_eq_ignore_macros", "swc_macros_common", "swc_visit", @@ -2610,18 +2584,18 @@ dependencies = [ [[package]] name = "deno_console" -version = "0.163.0" +version = "0.171.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4eea4d3eeb96874c63fde5001d0242042957b600000f179af4ad3235a411139" +checksum = "144108c8bb93b1df2cda4583d9beb8cd4e18798d3e030af8b07d2c55c1e3259b" dependencies = [ "deno_core", ] [[package]] name = "deno_core" -version = "0.299.0" +version = "0.311.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428488cc6b392a199a159054da754f56a1fdc63ee03f16be2c21cbd22e936e7b" +checksum = "5e09bd55da542fa1fde753aff617c355b5d782e763ab2a19e4371a56d7844cac" dependencies = [ "anyhow", "bincode", @@ -2642,7 +2616,7 @@ dependencies = [ "serde_json", "serde_v8", "smallvec", - "sourcemap", + "sourcemap 8.0.1", "static_assertions", "tokio", "url", @@ -2657,9 +2631,9 @@ checksum = "a13951ea98c0a4c372f162d669193b4c9d991512de9f2381dd161027f34b26b1" [[package]] name = "deno_fetch" -version = "0.187.0" +version = "0.195.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f11f1ca946366c42e7daaab0d747720710dbb11511d9bfce1b328dd71b58f87" +checksum = "8a91340a1d60cebe1392e4b6e1614709ab5fdcfc1f55842561e498d7ef9b2cb9" dependencies = [ "base64 0.21.7", "bytes", @@ -2668,9 +2642,10 @@ dependencies = [ "deno_permissions", "deno_tls", "dyn-clone", + "error_reporter", "http 1.1.0", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-rustls 0.27.3", "hyper-util", "ipnet", @@ -2683,7 +2658,7 @@ dependencies = [ "tokio-socks", "tokio-util", "tower 0.4.13", - "tower-http", + "tower-http 0.6.1", "tower-service", ] @@ -2708,14 +2683,14 @@ dependencies = [ "dlopen2_derive", "once_cell", "rustls-native-certs 0.7.3", - "rustls-pemfile 2.1.3", + "rustls-pemfile 2.2.0", ] [[package]] name = "deno_net" -version = "0.155.0" +version = "0.163.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3f337ffe00186bc67f989f5ba2759418bb43e1352b045aa9e715e216c632dc" +checksum = "5b769fd37232a38bf15a3834c9da8cc99409b52d0cf4a936f3117744369f8063" dependencies = [ "deno_core", "deno_permissions", @@ -2731,31 +2706,44 @@ dependencies = [ [[package]] name = "deno_ops" -version = "0.175.0" +version = "0.187.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2b71759647722be6ae051919b75cb66b3dccafe61b53c75ad5a6fad9d0ee4a" +checksum = "e040fd4def8a67538fe38c9955fd970efc9f44284bd69d44f8992a456afd665d" dependencies = [ "proc-macro-rules", "proc-macro2", "quote", "strum 0.25.0", "strum_macros 0.25.3", - "syn 2.0.79", + "syn 2.0.86", "thiserror", ] [[package]] -name = "deno_permissions" -version = "0.23.0" +name = "deno_path_util" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e67d40735d56445409349cec3eafe25668814c7f3794158c62dfd1b147b13c" +checksum = "4889646c1ce8437a6fde3acb057fd7e2d039e62c61f5063fc125ed1ede114dc6" +dependencies = [ + "percent-encoding", + "thiserror", + "url", +] + +[[package]] +name = "deno_permissions" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1328c2d1d26cd066ba9d7d5fb3451081219e373342423f78d7a1b73bfac9849" dependencies = [ "deno_core", + "deno_path_util", "deno_terminal 0.2.0", "fqdn", "libc", "log", "once_cell", + "percent-encoding", "serde", "which 4.4.2", "winapi", @@ -2783,14 +2771,14 @@ dependencies = [ [[package]] name = "deno_tls" -version = "0.150.0" +version = "0.158.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b995ce6016bc4f05453c8632a9d42a58536c1cbb7ece09f1aa66dda3f8d474b" +checksum = "22a1abc6bd8af41aa2496ceceef94f4277092c781a9495c437327a17659553a2" dependencies = [ "deno_core", "deno_native_certs", - "rustls 0.23.13", - "rustls-pemfile 2.1.3", + "rustls 0.23.16", + "rustls-pemfile 2.2.0", "rustls-tokio-stream", "rustls-webpki 0.102.8", "serde", @@ -2810,9 +2798,9 @@ dependencies = [ [[package]] name = "deno_url" -version = "0.163.0" +version = "0.171.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a05423871cf79fc0e73b9117b114045b7936a60b87a188b673a2b377a65eaa1" +checksum = "68eb6834d66ff13c19633b0e1b621292461b744858130b0a1c51f34a57ba1a03" dependencies = [ "deno_core", "urlpattern", @@ -2820,9 +2808,9 @@ dependencies = [ [[package]] name = "deno_web" -version = "0.194.0" +version = "0.202.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "884761df6d0bbe2869a8bfb3f4437aef75686dcbedd82dd4d23d1e7b06d50b14" +checksum = "e0862246372f5b559b788aa07ca1b9385909d59630251156a54c4d2c5342d1eb" dependencies = [ "async-trait", "base64-simd 0.8.0", @@ -2834,14 +2822,14 @@ dependencies = [ "futures", "serde", "tokio", - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] name = "deno_webidl" -version = "0.163.0" +version = "0.171.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a564c240839f20b8808d0182a0c1a85428a92fb7fe6cc6ea256adda11ad08a8d" +checksum = "5e969b61b740479379eaf303a6900d11f162f9de610640398f89f0aa58abb7c4" dependencies = [ "deno_core", ] @@ -2919,7 +2907,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -2929,7 +2917,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac41dd49fb554432020d52c875fc290e110113f864c6b1b525cd62c7e7747a5d" dependencies = [ "byteorder", - "cipher 0.3.0", + "cipher", "opaque-debug", ] @@ -3016,7 +3004,7 @@ checksum = "f2b99bf03862d7f545ebc28ddd33a665b50865f4dfd84031a393823879bd4c54" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -3045,16 +3033,16 @@ checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" [[package]] name = "dprint-swc-ext" -version = "0.17.1" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b909f9f9b22a6265839887544dce97b0b8e2b2635abf622f45613deb3de63e0" +checksum = "0ba28c12892aadb751c2ba7001d8460faee4748a04b4edc51c7121cc67ee03db" dependencies = [ "num-bigint", "rustc-hash 1.1.0", "swc_atoms", - "swc_common 0.34.4", - "swc_ecma_ast 0.115.1", - "swc_ecma_parser 0.146.12", + "swc_common", + "swc_ecma_ast", + "swc_ecma_parser", "text_lines", ] @@ -3172,7 +3160,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -3192,7 +3180,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -3211,6 +3199,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "error_reporter" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31ae425815400e5ed474178a7a22e275a9687086a12ca63ec793ff292d8fdae8" + [[package]] name = "esaxx-rs" version = "0.1.10" @@ -3266,9 +3260,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastdivide" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59668941c55e5c186b8b58c391629af56774ec768f73c08bbcd56f09348eb00b" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" [[package]] name = "fastrand" @@ -3342,9 +3336,9 @@ dependencies = [ [[package]] name = "flume" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", @@ -3357,6 +3351,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -3395,7 +3395,7 @@ checksum = "32016f1242eb82af5474752d00fd8ebcd9004bd69b462b1c91de833972d08ed4" dependencies = [ "proc-macro2", "swc_macros_common", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -3427,7 +3427,7 @@ checksum = "e99b8b3c28ae0e84b604c75f721c21dc77afb3706076af5e8216d15fd1deaae3" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -3439,7 +3439,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -3451,7 +3451,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -3482,9 +3482,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -3497,9 +3497,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -3507,15 +3507,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -3535,9 +3535,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" @@ -3556,9 +3556,9 @@ dependencies = [ [[package]] name = "futures-lite" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" +checksum = "3f1fa2f9765705486b33fd2acf1577f8ec449c2ba1f318ae5447697b7c08d210" dependencies = [ "fastrand 2.1.1", "futures-core", @@ -3569,32 +3569,32 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -3617,7 +3617,7 @@ dependencies = [ "async-trait", "base64 0.21.7", "dirs-next", - "hyper 0.14.30", + "hyper 0.14.31", "hyper-rustls 0.24.2", "ring 0.16.20", "rustls 0.21.12", @@ -3807,9 +3807,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "git-version" @@ -3828,7 +3828,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -3906,7 +3906,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.5.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -3925,7 +3925,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.5.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -3974,6 +3974,17 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashlink" version = "0.9.1" @@ -4167,9 +4178,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -4185,9 +4196,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.30" +version = "0.14.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" +checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" dependencies = [ "bytes", "futures-channel", @@ -4209,9 +4220,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" dependencies = [ "bytes", "futures-channel", @@ -4236,7 +4247,7 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.30", + "hyper 0.14.31", "log", "rustls 0.21.12", "rustls-native-certs 0.6.3", @@ -4252,9 +4263,9 @@ checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-util", - "rustls 0.23.13", + "rustls 0.23.16", "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", @@ -4269,7 +4280,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper 0.14.30", + "hyper 0.14.31", "native-tls", "tokio", "tokio-native-tls", @@ -4283,7 +4294,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-util", "native-tls", "tokio", @@ -4293,16 +4304,16 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", "http-body 1.0.1", - "hyper 1.4.1", + "hyper 1.5.0", "pin-project-lite", "socket2 0.5.7", "tokio", @@ -4379,12 +4390,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "serde", ] @@ -4407,15 +4418,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" -[[package]] -name = "inout" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "generic-array", -] - [[package]] name = "instant" version = "0.1.13" @@ -4448,9 +4450,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "is-macro" @@ -4461,7 +4463,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -4550,7 +4552,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", ] [[package]] @@ -4670,9 +4672,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.159" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "libgit2-sys" @@ -4698,9 +4700,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "libredox" @@ -4790,11 +4792,11 @@ dependencies = [ [[package]] name = "lru" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.0", ] [[package]] @@ -4848,7 +4850,7 @@ version = "3.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c42f95f9d296f2dcb50665f507ed5a68a171453142663ce44d77a4eb217b053" dependencies = [ - "aes 0.7.5", + "aes", "base64 0.21.7", "block-modes", "crc-any", @@ -4887,7 +4889,7 @@ dependencies = [ "base64 0.22.1", "gethostname", "mail-builder", - "rustls 0.23.13", + "rustls 0.23.16", "rustls-pki-types", "smtp-proto", "tokio", @@ -5084,14 +5086,13 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.2" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ - "hermit-abi 0.3.9", "libc", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] @@ -5112,7 +5113,7 @@ checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -5151,7 +5152,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", "termcolor", "thiserror", ] @@ -5222,7 +5223,7 @@ dependencies = [ "subprocess", "thiserror", "time", - "uuid 1.10.0", + "uuid 1.11.0", "zstd 0.13.2", ] @@ -5418,9 +5419,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] @@ -5437,14 +5438,14 @@ dependencies = [ "chrono", "futures", "humantime", - "hyper 1.4.1", + "hyper 1.5.0", "itertools 0.13.0", "md-5 0.10.6", "parking_lot", "percent-encoding", "quick-xml 0.36.2", "rand 0.8.5", - "reqwest 0.12.7", + "reqwest 0.12.9", "ring 0.17.8", "serde", "serde_json", @@ -5457,12 +5458,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.1" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" -dependencies = [ - "portable-atomic", -] +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oneshot" @@ -5543,9 +5541,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.66" +version = "0.10.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ "bitflags 2.6.0", "cfg-if", @@ -5564,7 +5562,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -5575,9 +5573,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.103" +version = "0.9.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" dependencies = [ "cc", "libc", @@ -5695,7 +5693,7 @@ dependencies = [ "arrow-schema", "arrow-select", "base64 0.22.1", - "brotli", + "brotli 6.0.0", "bytes", "chrono", "flate2", @@ -5725,17 +5723,6 @@ dependencies = [ "regex", ] -[[package]] -name = "password-hash" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" -dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "password-hash" version = "0.5.0" @@ -5755,21 +5742,9 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pathdiff" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" - -[[package]] -name = "pbkdf2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" -dependencies = [ - "digest 0.10.7", - "hmac", - "password-hash 0.4.2", - "sha2 0.10.8", -] +checksum = "d61c5ce1153ab5b689d0c074c4e7fc613e942dfb7dd9eea5ab202d2ad91fe361" [[package]] name = "pem" @@ -5821,25 +5796,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.5.0", -] - -[[package]] -name = "pg-embed" -version = "0.7.2" -source = "git+https://github.com/faokunega/pg-embed#72db5e053f0afac6eee51d3baa2fd5c90803e02d" -dependencies = [ - "archiver-rs", - "async-trait", - "bytes", - "dirs", - "futures", - "lazy_static", - "log", - "reqwest 0.11.27", - "thiserror", - "tokio", - "zip", + "indexmap 2.6.0", ] [[package]] @@ -5882,7 +5839,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -5908,29 +5865,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -6064,7 +6021,7 @@ dependencies = [ "postgres-protocol", "serde", "serde_json", - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] @@ -6100,12 +6057,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.22" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -6158,7 +6115,7 @@ checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" dependencies = [ "proc-macro-rules-macros", "proc-macro2", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -6170,14 +6127,14 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -6231,7 +6188,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "syn 2.0.79", + "syn 2.0.86", "thiserror", "typify", "unicode-ident", @@ -6251,7 +6208,7 @@ dependencies = [ "serde_json", "serde_tokenstream", "serde_yaml", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -6294,7 +6251,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -6408,7 +6365,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.0.0", - "rustls 0.23.13", + "rustls 0.23.16", "socket2 0.5.7", "thiserror", "tokio", @@ -6425,7 +6382,7 @@ dependencies = [ "rand 0.8.5", "ring 0.17.8", "rustc-hash 2.0.0", - "rustls 0.23.13", + "rustls 0.23.16", "slab", "thiserror", "tinyvec", @@ -6434,10 +6391,11 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +checksum = "e346e016eacfff12233c243718197ca12f148c84e1e84268a896699b41c71780" dependencies = [ + "cfg_aliases", "libc", "once_cell", "socket2 0.5.7", @@ -6646,9 +6604,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -6733,7 +6691,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.30", + "hyper 0.14.31", "hyper-rustls 0.24.2", "hyper-tls 0.5.0", "ipnet", @@ -6767,11 +6725,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.7" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ - "async-compression 0.4.12", + "async-compression 0.4.17", "base64 0.22.1", "bytes", "encoding_rs", @@ -6781,7 +6739,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "http-body-util", - "hyper 1.4.1", + "hyper 1.5.0", "hyper-rustls 0.27.3", "hyper-tls 0.6.0", "hyper-util", @@ -6794,9 +6752,9 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.13", - "rustls-native-certs 0.7.3", - "rustls-pemfile 2.1.3", + "rustls 0.23.16", + "rustls-native-certs 0.8.0", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", @@ -6887,7 +6845,7 @@ dependencies = [ "rkyv_derive", "seahash", "tinyvec", - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] @@ -6977,7 +6935,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.79", + "syn 2.0.86", "walkdir", ] @@ -7069,9 +7027,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.37" +version = "0.38.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" dependencies = [ "bitflags 2.6.0", "errno", @@ -7094,9 +7052,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.13" +version = "0.23.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" +checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" dependencies = [ "log", "once_cell", @@ -7126,7 +7084,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" dependencies = [ "openssl-probe", - "rustls-pemfile 2.1.3", + "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", "security-framework", @@ -7139,7 +7097,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" dependencies = [ "openssl-probe", - "rustls-pemfile 2.1.3", + "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", "security-framework", @@ -7156,19 +7114,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-tokio-stream" @@ -7177,7 +7134,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22557157d7395bc30727745b365d923f1ecc230c4c80b176545f3f4f08c46e33" dependencies = [ "futures", - "rustls 0.23.13", + "rustls 0.23.16", "socket2 0.5.7", "tokio", ] @@ -7258,9 +7215,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "ryu" @@ -7291,7 +7248,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b75583aad4a51c50fc0af69c230d18078c9d5a69a98d0f6013d01053acf744f4" dependencies = [ "base64 0.21.7", - "bindgen 0.69.4", + "bindgen 0.69.5", "chrono", "data-encoding", "derive_builder", @@ -7308,7 +7265,7 @@ dependencies = [ "serde", "thiserror", "url", - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] @@ -7328,9 +7285,9 @@ checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" [[package]] name = "schannel" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ "windows-sys 0.59.0", ] @@ -7346,7 +7303,7 @@ dependencies = [ "schemars_derive", "serde", "serde_json", - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] @@ -7358,7 +7315,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -7458,9 +7415,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" dependencies = [ "serde_derive", ] @@ -7499,13 +7456,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -7516,16 +7473,16 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "memchr", "ryu", @@ -7591,7 +7548,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -7608,9 +7565,9 @@ dependencies = [ [[package]] name = "serde_v8" -version = "0.208.0" +version = "0.220.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583f3c71a6f7acc1711ad718a33f6e799bacdc711d297b15bb28533f32264c58" +checksum = "6e7a65d91d79acc82aa229aeb084f4a39bda269069bc1520df40f679495388e4" dependencies = [ "num-bigint", "serde", @@ -7621,15 +7578,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_derive", "serde_json", @@ -7639,14 +7596,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -7655,7 +7612,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "ryu", "serde", @@ -7911,6 +7868,25 @@ dependencies = [ "url", ] +[[package]] +name = "sourcemap" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab08a862c70980b8e23698b507e272317ae52a608a164a844111f5372374f1f" +dependencies = [ + "base64-simd 0.7.0", + "bitvec", + "data-encoding", + "debugid", + "if_chain", + "rustc-hash 1.1.0", + "rustc_version 0.2.3", + "serde", + "serde_json", + "unicode-id-start", + "url", +] + [[package]] name = "spin" version = "0.5.2" @@ -7996,7 +7972,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -8035,14 +8011,14 @@ dependencies = [ "hashbrown 0.14.5", "hashlink", "hex", - "indexmap 2.5.0", + "indexmap 2.6.0", "log", "memchr", "once_cell", "paste", "percent-encoding", - "rustls 0.23.13", - "rustls-pemfile 2.1.3", + "rustls 0.23.16", + "rustls-pemfile 2.2.0", "serde", "serde_json", "sha2 0.10.8", @@ -8053,7 +8029,7 @@ dependencies = [ "tokio-stream", "tracing", "url", - "uuid 1.10.0", + "uuid 1.11.0", "webpki-roots 0.26.6", ] @@ -8067,7 +8043,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -8090,7 +8066,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.79", + "syn 2.0.86", "tempfile", "tokio", "url", @@ -8137,7 +8113,7 @@ dependencies = [ "stringprep", "thiserror", "tracing", - "uuid 1.10.0", + "uuid 1.11.0", "whoami", ] @@ -8179,7 +8155,7 @@ dependencies = [ "stringprep", "thiserror", "tracing", - "uuid 1.10.0", + "uuid 1.11.0", "whoami", ] @@ -8205,7 +8181,7 @@ dependencies = [ "sqlx-core", "tracing", "url", - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] @@ -8242,7 +8218,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -8296,7 +8272,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -8309,7 +8285,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -8328,6 +8304,19 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "swc_allocator" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76aa0eb65c0f39f9b6d82a7e5192c30f7ac9a78f084a21f270de1d8c600ca388" +dependencies = [ + "bumpalo", + "hashbrown 0.14.5", + "ptr_meta", + "rustc-hash 1.1.0", + "triomphe", +] + [[package]] name = "swc_atoms" version = "0.6.7" @@ -8356,9 +8345,9 @@ dependencies = [ [[package]] name = "swc_common" -version = "0.33.26" +version = "0.37.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2f9706038906e66f3919028f9f7a37f3ed552f1b85578e93f4468742e2da438" +checksum = "12d0a8eaaf1606c9207077d75828008cb2dfb51b095a766bd2b72ef893576e31" dependencies = [ "ast_node", "better_scoped_tls", @@ -8371,32 +8360,8 @@ dependencies = [ "rustc-hash 1.1.0", "serde", "siphasher", - "swc_atoms", - "swc_eq_ignore_macros", - "swc_visit", - "tracing", - "unicode-width", - "url", -] - -[[package]] -name = "swc_common" -version = "0.34.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9087befec6b63911f9d2f239e4f91c9b21589c169b86ed2d616944d23cf4a243" -dependencies = [ - "ast_node", - "better_scoped_tls", - "cfg-if", - "either", - "from_variant", - "new_debug_unreachable", - "num-bigint", - "once_cell", - "rustc-hash 1.1.0", - "serde", - "siphasher", - "sourcemap", + "sourcemap 9.0.0", + "swc_allocator", "swc_atoms", "swc_eq_ignore_macros", "swc_visit", @@ -8407,12 +8372,12 @@ dependencies = [ [[package]] name = "swc_config" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b67e115ab136fe0eb03558bb0508ca7782eeb446a96d165508c48617e3fd94" +checksum = "4740e53eaf68b101203c1df0937d5161a29f3c13bceed0836ddfe245b72dd000" dependencies = [ "anyhow", - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_json", "swc_cached", @@ -8428,31 +8393,14 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] name = "swc_ecma_ast" -version = "0.113.7" +version = "0.118.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98a534a8360a076a030989f6d121ba6044345594bdf0457c4629f432742026b8" -dependencies = [ - "bitflags 2.6.0", - "is-macro", - "num-bigint", - "phf", - "scoped-tls", - "string_enum", - "swc_atoms", - "swc_common 0.33.26", - "unicode-id-start", -] - -[[package]] -name = "swc_ecma_ast" -version = "0.115.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7be1306930c235435a892104c00c2b5e16231043c085d5a10bd3e7537b15659b" +checksum = "a6f866d12e4d519052b92a0a86d1ac7ff17570da1272ca0c89b3d6f802cd79df" dependencies = [ "bitflags 2.6.0", "is-macro", @@ -8462,60 +8410,60 @@ dependencies = [ "serde", "string_enum", "swc_atoms", - "swc_common 0.34.4", + "swc_common", "unicode-id-start", ] [[package]] name = "swc_ecma_codegen" -version = "0.151.1" +version = "0.155.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5141a8cb4eb69e090e6aea5d49061b46919be5210f3d084f9d9ad63d30f5cff" +checksum = "cc7641608ef117cfbef9581a99d02059b522fcca75e5244fa0cbbd8606689c6f" dependencies = [ "memchr", "num-bigint", "once_cell", - "rustc-hash 1.1.0", "serde", - "sourcemap", + "sourcemap 9.0.0", + "swc_allocator", "swc_atoms", - "swc_common 0.34.4", - "swc_ecma_ast 0.115.1", + "swc_common", + "swc_ecma_ast", "swc_ecma_codegen_macros", "tracing", ] [[package]] name = "swc_ecma_codegen_macros" -version = "0.7.6" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090e409af49c8d1a3c13b3aab1ed09dd4eda982207eb3e63c2ad342f072b49c8" +checksum = "859fabde36db38634f3fad548dd5e3410c1aebba1b67a3c63e67018fa57a0bca" dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] name = "swc_ecma_loader" -version = "0.46.1" +version = "0.49.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9febebf047d1286e7b723fa2758f3229da2c103834f3eaee69833f46692612" +checksum = "55fa3d55045b97894bfb04d38aff6d6302ac8a6a38e3bb3dfb0d20475c4974a9" dependencies = [ "anyhow", "pathdiff", "serde", "swc_atoms", - "swc_common 0.34.4", + "swc_common", "tracing", ] [[package]] name = "swc_ecma_parser" -version = "0.144.3" +version = "0.149.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0b4193b9c127db1990a5a08111aafe0122bc8b138646807c63f2a6521b7da4" +checksum = "683dada14722714588b56481399c699378b35b2ba4deb5c4db2fb627a97fb54b" dependencies = [ "either", "new_debug_unreachable", @@ -8527,69 +8475,47 @@ dependencies = [ "smartstring", "stacker", "swc_atoms", - "swc_common 0.33.26", - "swc_ecma_ast 0.113.7", - "tracing", - "typed-arena", -] - -[[package]] -name = "swc_ecma_parser" -version = "0.146.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4e0c2e85f12c63b85c805e923079b04d1fb3e25edd069d638eed5f2098de74" -dependencies = [ - "either", - "new_debug_unreachable", - "num-bigint", - "num-traits", - "phf", - "serde", - "smallvec", - "smartstring", - "stacker", - "swc_atoms", - "swc_common 0.34.4", - "swc_ecma_ast 0.115.1", + "swc_common", + "swc_ecma_ast", "tracing", "typed-arena", ] [[package]] name = "swc_ecma_transforms_base" -version = "0.140.3" +version = "0.145.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d37dc505c92af56d0f77cf6f31a6ccd37ac40cad1e01ff77277e0b1c70e8f8ff" +checksum = "65f21494e75d0bd8ef42010b47cabab9caaed8f2207570e809f6f4eb51a710d1" dependencies = [ "better_scoped_tls", "bitflags 2.6.0", - "indexmap 2.5.0", + "indexmap 2.6.0", "once_cell", "phf", "rustc-hash 1.1.0", "serde", "smallvec", "swc_atoms", - "swc_common 0.34.4", - "swc_ecma_ast 0.115.1", - "swc_ecma_parser 0.146.12", + "swc_common", + "swc_ecma_ast", + "swc_ecma_parser", "swc_ecma_utils", - "swc_ecma_visit 0.101.0", + "swc_ecma_visit", "tracing", ] [[package]] name = "swc_ecma_transforms_classes" -version = "0.129.0" +version = "0.134.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3eab5f8179e5b0aedf385eacc2c033691c6d211a7babd1bbbff12cf794a824e" +checksum = "3c3d884594385bea9405a2e1721151470d9a14d3ceec5dd773c0ca6894791601" dependencies = [ "swc_atoms", - "swc_common 0.34.4", - "swc_ecma_ast 0.115.1", + "swc_common", + "swc_ecma_ast", "swc_ecma_transforms_base", "swc_ecma_utils", - "swc_ecma_visit 0.101.0", + "swc_ecma_visit", ] [[package]] @@ -8601,160 +8527,148 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] name = "swc_ecma_transforms_proposal" -version = "0.174.3" +version = "0.179.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6df8aa6752cc2fcf3d78ac67827542fb666e52283f2b26802aa058906bb750d3" +checksum = "79938ff510fc647febd8c6c3ef4143d099fdad87a223680e632623d056dae2dd" dependencies = [ "either", "rustc-hash 1.1.0", "serde", "smallvec", "swc_atoms", - "swc_common 0.34.4", - "swc_ecma_ast 0.115.1", + "swc_common", + "swc_ecma_ast", "swc_ecma_transforms_base", "swc_ecma_transforms_classes", "swc_ecma_transforms_macros", "swc_ecma_utils", - "swc_ecma_visit 0.101.0", + "swc_ecma_visit", ] [[package]] name = "swc_ecma_transforms_react" -version = "0.186.2" +version = "0.191.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446da32cac8299973aaf1d37496562bfd0c1e4f3c3ab5d0af6f07f42e8184102" +checksum = "76c76d8b9792ce51401d38da0fa62158d61f6d80d16d68fe5b03ce4bf5fba383" dependencies = [ "base64 0.21.7", "dashmap", - "indexmap 2.5.0", + "indexmap 2.6.0", "once_cell", "serde", "sha1", "string_enum", + "swc_allocator", "swc_atoms", - "swc_common 0.34.4", + "swc_common", "swc_config", - "swc_ecma_ast 0.115.1", - "swc_ecma_parser 0.146.12", + "swc_ecma_ast", + "swc_ecma_parser", "swc_ecma_transforms_base", "swc_ecma_transforms_macros", "swc_ecma_utils", - "swc_ecma_visit 0.101.0", + "swc_ecma_visit", ] [[package]] name = "swc_ecma_transforms_typescript" -version = "0.191.2" +version = "0.198.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1ce8af2865449e714ae56dacb6b54b3f6dc4cc25074da4e39b878bd93c5e39c" +checksum = "15455da4768f97186c40523e83600495210c11825d3a44db43383fd81eace88d" dependencies = [ "ryu-js", "serde", "swc_atoms", - "swc_common 0.34.4", - "swc_ecma_ast 0.115.1", + "swc_common", + "swc_ecma_ast", "swc_ecma_transforms_base", "swc_ecma_transforms_react", "swc_ecma_utils", - "swc_ecma_visit 0.101.0", + "swc_ecma_visit", ] [[package]] name = "swc_ecma_utils" -version = "0.130.3" +version = "0.134.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e62b199454a576c5fdbd7e1bef8ab88a395427456d8a713d994b7d469833aa" +checksum = "029eec7dd485923a75b5a45befd04510288870250270292fc2c1b3a9e7547408" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "num_cpus", "once_cell", "rustc-hash 1.1.0", "ryu-js", "swc_atoms", - "swc_common 0.34.4", - "swc_ecma_ast 0.115.1", - "swc_ecma_visit 0.101.0", + "swc_common", + "swc_ecma_ast", + "swc_ecma_visit", "tracing", "unicode-id", ] [[package]] name = "swc_ecma_visit" -version = "0.99.1" +version = "0.104.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28a6ce28ad8e591f8d627f1f9cb26b25e5d83052a9bc1b674d95fc28040cfa98" +checksum = "5b1c6802e68e51f336e8bc9644e9ff9da75d7da9c1a6247d532f2e908aa33e81" dependencies = [ + "new_debug_unreachable", "num-bigint", "swc_atoms", - "swc_common 0.33.26", - "swc_ecma_ast 0.113.7", - "swc_visit", - "tracing", -] - -[[package]] -name = "swc_ecma_visit" -version = "0.101.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce0d997f0c9b4e181225f603d161f6757c2a97022258170982cfe005ec69ec92" -dependencies = [ - "num-bigint", - "swc_atoms", - "swc_common 0.34.4", - "swc_ecma_ast 0.115.1", + "swc_common", + "swc_ecma_ast", "swc_visit", "tracing", ] [[package]] name = "swc_eq_ignore_macros" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "695a1d8b461033d32429b5befbf0ad4d7a2c4d6ba9cd5ba4e0645c615839e8e4" +checksum = "63db0adcff29d220c3d151c5b25c0eabe7e32dd936212b84cdaa1392e3130497" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] name = "swc_macros_common" -version = "0.3.11" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91745f3561057493d2da768437c427c0e979dff7396507ae02f16c981c4a8466" +checksum = "f486687bfb7b5c560868f69ed2d458b880cebc9babebcb67e49f31b55c5bf847" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] name = "swc_visit" -version = "0.5.14" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043d11fe683dcb934583ead49405c0896a5af5face522e4682c16971ef7871b9" +checksum = "1ceb044142ba2719ef9eb3b6b454fce61ab849eb696c34d190f04651955c613d" dependencies = [ "either", - "swc_visit_macros", + "new_debug_unreachable", ] [[package]] name = "swc_visit_macros" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae9ef18ff8daffa999f729db056d2821cd2f790f3a11e46422d19f46bb193e7" +checksum = "92807d840959f39c60ce8a774a3f83e8193c658068e6d270dbe0a05e40e90b41" dependencies = [ "Inflector", "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -8770,9 +8684,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.79" +version = "2.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "e89275301d38033efb81a6e60e3497e734dfcc62571f2854bf4b16690398824c" dependencies = [ "proc-macro2", "quote", @@ -8788,7 +8702,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -8814,7 +8728,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -8920,7 +8834,7 @@ dependencies = [ "tempfile", "thiserror", "time", - "uuid 1.10.0", + "uuid 1.11.0", "winapi", ] @@ -9022,9 +8936,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020" +checksum = "c65998313f8e17d0d553d28f91a0df93e4dbbbf770279c7bc21ca0f09ea1a1f6" dependencies = [ "filetime", "libc", @@ -9064,22 +8978,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "5d171f59dbaa811dbbb1aee1e73db92ec2b122911a48e1390dfe327a821ddede" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "b08be0f17bd307950653ce45db00cd31200d82b624b36e181337d9c7d92765b5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -9128,7 +9042,7 @@ dependencies = [ "tokio-rustls 0.24.1", "tokio-util", "tracing", - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] @@ -9278,32 +9192,33 @@ dependencies = [ [[package]] name = "tokio" -version = "1.40.0" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", "libc", "mio", + "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2 0.5.7", "tokio-macros", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -9358,7 +9273,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.13", + "rustls 0.23.16", "rustls-pki-types", "tokio", ] @@ -9401,6 +9316,20 @@ dependencies = [ "xattr", ] +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "native-tls", + "tokio", + "tokio-native-tls", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.12" @@ -9454,7 +9383,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", @@ -9467,7 +9396,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", @@ -9541,7 +9470,24 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ - "async-compression 0.4.12", + "bitflags 2.6.0", + "bytes", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" +dependencies = [ + "async-compression 0.4.17", "bitflags 2.6.0", "bytes", "futures-core", @@ -9553,7 +9499,6 @@ dependencies = [ "tokio-util", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -9600,7 +9545,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -9663,7 +9608,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea6023f9fe4b69267ccd3ed7d203d931c43c5f82dbaa0f07202bc17193a5f43" dependencies = [ "loki-api", - "reqwest 0.12.7", + "reqwest 0.12.9", "serde", "serde_json", "snap", @@ -9710,9 +9655,9 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" +checksum = "ef8f7726da4807b58ea5c96fdc122f80702030edc33b35aff9190a51148ccc85" dependencies = [ "serde", "stable_deref_trait", @@ -9772,6 +9717,25 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http 1.1.0", + "httparse", + "log", + "native-tls", + "rand 0.8.5", + "sha1", + "thiserror", + "utf-8", +] + [[package]] name = "twox-hash" version = "1.6.3" @@ -9818,7 +9782,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.79", + "syn 2.0.86", "thiserror", "unicode-ident", ] @@ -9835,7 +9799,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.79", + "syn 2.0.86", "typify-impl", ] @@ -9847,7 +9811,7 @@ checksum = "04f903f293d11f31c0c29e4148f6dc0d033a7f80cebc0282bea147611667d289" dependencies = [ "getrandom 0.2.15", "rand 0.8.5", - "uuid 1.10.0", + "uuid 1.11.0", "web-time", ] @@ -9917,18 +9881,15 @@ dependencies = [ [[package]] name = "unicase" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] +checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-id" @@ -9938,9 +9899,9 @@ checksum = "10103c57044730945224467c09f71a4db0071c123a0648cc3e818913bde6b561" [[package]] name = "unicode-id-start" -version = "1.0.4" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02aebfa694eccbbbffdd92922c7de136b9fe764396d2f10e21bce1681477cfc1" +checksum = "2f322b60f6b9736017344fa0635d64be2f458fbc04eef65f6be22976dd1ffd5b" [[package]] name = "unicode-ident" @@ -9968,9 +9929,9 @@ dependencies = [ [[package]] name = "unicode-properties" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" [[package]] name = "unicode-segmentation" @@ -10047,7 +10008,7 @@ dependencies = [ "log", "native-tls", "once_cell", - "rustls 0.23.13", + "rustls 0.23.16", "rustls-pki-types", "serde", "serde_json", @@ -10075,17 +10036,22 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "urlpattern" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9bd5ff03aea02fa45b13a7980151fe45009af1980ba69f651ec367121a31609" +checksum = "70acd30e3aa1450bc2eece896ce2ad0d178e9c079493819301573dae3c37ba6d" dependencies = [ - "derive_more", "regex", "serde", "unic-ucd-ident", "url", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8-ranges" version = "1.0.5" @@ -10109,9 +10075,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom 0.2.15", "serde", @@ -10119,11 +10085,11 @@ dependencies = [ [[package]] name = "v8" -version = "0.99.0" +version = "0.106.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa3fc0608a78f0c7d4ec88025759cb78c90a29984b48540060355a626ae329c1" +checksum = "a381badc47c6f15acb5fe0b5b40234162349ed9d4e4fd7c83a7f5547c0fc69c5" dependencies = [ - "bindgen 0.69.4", + "bindgen 0.69.5", "bitflags 2.6.0", "fslock", "gzip-header", @@ -10142,9 +10108,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a84c137d37ab0142f0f2ddfe332651fdbf252e7b7dbb4e67b6c1f1b2e925101" +checksum = "3ef4c4aa54d5d05a279399bfa921ec387b7aba77caf7a682ae8d86785b8fdad2" [[package]] name = "vcpkg" @@ -10228,7 +10194,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", "wasm-bindgen-shared", ] @@ -10262,7 +10228,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -10295,7 +10261,7 @@ checksum = "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -10429,7 +10395,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "axum", @@ -10443,11 +10409,10 @@ dependencies = [ "lazy_static", "object_store", "once_cell", - "pg-embed", "prometheus", "quote", "rand 0.8.5", - "reqwest 0.12.7", + "reqwest 0.12.9", "rsmq_async", "serde", "serde_json", @@ -10459,9 +10424,10 @@ dependencies = [ "tokio", "tracing", "url", - "uuid 1.10.0", + "uuid 1.11.0", "windmill-api", "windmill-api-client", + "windmill-autoscaling", "windmill-common", "windmill-git-sync", "windmill-indexer", @@ -10471,7 +10437,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "argon2", @@ -10490,7 +10456,6 @@ dependencies = [ "chrono", "chrono-tz 0.10.0", "cookie 0.17.0", - "crc", "cron", "datafusion", "futures", @@ -10499,7 +10464,7 @@ dependencies = [ "hf-hub", "hmac", "http 1.1.0", - "hyper 1.4.1", + "hyper 1.5.0", "itertools 0.13.0", "jsonwebtoken", "lazy_static", @@ -10516,7 +10481,7 @@ dependencies = [ "quick_cache", "rand 0.8.5", "regex", - "reqwest 0.12.7", + "reqwest 0.12.9", "rsa 0.7.2", "rsmq_async", "rust-embed", @@ -10534,16 +10499,17 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-tar", + "tokio-tungstenite", "tokio-util", "tower 0.5.1", "tower-cookies", - "tower-http", + "tower-http 0.5.2", "tracing", "tracing-subscriber", "ulid", "url", "urlencoding", - "uuid 1.10.0", + "uuid 1.11.0", "windmill-audit", "windmill-common", "windmill-git-sync", @@ -10556,7 +10522,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.402.3" +version = "1.416.2" dependencies = [ "base64 0.21.7", "chrono", @@ -10569,12 +10535,12 @@ dependencies = [ "serde", "serde_json", "syn 1.0.109", - "uuid 1.10.0", + "uuid 1.11.0", ] [[package]] name = "windmill-audit" -version = "1.402.3" +version = "1.416.2" dependencies = [ "chrono", "serde", @@ -10585,9 +10551,24 @@ dependencies = [ "windmill-common", ] +[[package]] +name = "windmill-autoscaling" +version = "1.416.2" +dependencies = [ + "anyhow", + "rsmq_async", + "serde", + "serde_json", + "sqlx", + "tracing", + "uuid 1.11.0", + "windmill-common", + "windmill-queue", +] + [[package]] name = "windmill-common" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "async-stream", @@ -10597,14 +10578,15 @@ dependencies = [ "bytes", "chrono", "const_format", + "crc", "cron", "futures-core", "gethostname", "git-version", "hex", "hmac", - "hyper 1.4.1", - "indexmap 2.5.0", + "hyper 1.5.0", + "indexmap 2.6.0", "itertools 0.13.0", "lazy_static", "magic-crypt", @@ -10613,7 +10595,7 @@ dependencies = [ "prometheus", "rand 0.8.5", "regex", - "reqwest 0.12.7", + "reqwest 0.12.9", "serde", "serde_json", "sha2 0.10.8", @@ -10626,12 +10608,13 @@ dependencies = [ "tracing-flame", "tracing-loki", "tracing-subscriber", - "uuid 1.10.0", + "uuid 1.11.0", + "windmill-macros", ] [[package]] name = "windmill-git-sync" -version = "1.402.3" +version = "1.416.2" dependencies = [ "regex", "rsmq_async", @@ -10639,19 +10622,20 @@ dependencies = [ "serde_json", "sqlx", "tracing", - "uuid 1.10.0", + "uuid 1.11.0", "windmill-common", "windmill-queue", ] [[package]] name = "windmill-indexer" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "bytes", "chrono", "futures", + "lazy_static", "object_store", "serde", "serde_json", @@ -10661,13 +10645,25 @@ dependencies = [ "tokio", "tokio-tar", "tracing", - "uuid 1.10.0", + "uuid 1.11.0", "windmill-common", ] +[[package]] +name = "windmill-macros" +version = "1.416.2" +dependencies = [ + "itertools 0.13.0", + "lazy_static", + "proc-macro2", + "quote", + "regex", + "syn 2.0.86", +] + [[package]] name = "windmill-parser" -version = "1.402.3" +version = "1.416.2" dependencies = [ "convert_case 0.6.0", "serde", @@ -10676,7 +10672,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "lazy_static", @@ -10688,7 +10684,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "gosyn", @@ -10700,7 +10696,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "lazy_static", @@ -10712,7 +10708,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10723,7 +10719,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10734,7 +10730,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "async-recursion", @@ -10752,7 +10748,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10762,14 +10758,14 @@ dependencies = [ "quote", "regex", "serde_json", - "syn 2.0.79", + "syn 2.0.86", "toml 0.7.8", "windmill-parser", ] [[package]] name = "windmill-parser-sql" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "lazy_static", @@ -10781,17 +10777,17 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "lazy_static", "regex", "serde-wasm-bindgen", "serde_json", - "swc_common 0.33.26", - "swc_ecma_ast 0.113.7", - "swc_ecma_parser 0.144.3", - "swc_ecma_visit 0.99.1", + "swc_common", + "swc_ecma_ast", + "swc_ecma_parser", + "swc_ecma_visit", "triomphe", "wasm-bindgen", "windmill-parser", @@ -10799,7 +10795,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10820,7 +10816,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "serde_json", @@ -10830,7 +10826,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "async-recursion", @@ -10846,7 +10842,7 @@ dependencies = [ "lazy_static", "prometheus", "regex", - "reqwest 0.12.7", + "reqwest 0.12.9", "rsmq_async", "serde", "serde_json", @@ -10856,14 +10852,14 @@ dependencies = [ "tokio", "tracing", "ulid", - "uuid 1.10.0", + "uuid 1.11.0", "windmill-audit", "windmill-common", ] [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.402.3" +version = "1.416.2" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10873,7 +10869,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.402.3" +version = "1.416.2" dependencies = [ "anyhow", "async-recursion", @@ -10913,13 +10909,14 @@ dependencies = [ "prometheus", "rand 0.8.5", "regex", - "reqwest 0.12.7", + "reqwest 0.12.9", "rsmq_async", "rust_decimal", "serde", "serde_json", "sha2 0.10.8", "sqlx", + "swc_ecma_parser", "tar", "tiberius", "tokio", @@ -10927,7 +10924,7 @@ dependencies = [ "tokio-util", "tracing", "urlencoding", - "uuid 1.10.0", + "uuid 1.11.0", "windmill-audit", "windmill-common", "windmill-git-sync", @@ -11238,7 +11235,7 @@ checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", "synstructure", ] @@ -11260,7 +11257,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", ] [[package]] @@ -11280,7 +11277,7 @@ checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.86", "synstructure", ] @@ -11296,18 +11293,9 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ - "aes 0.8.4", "byteorder", - "bzip2", - "constant_time_eq 0.1.5", "crc32fast", "crossbeam-utils", - "flate2", - "hmac", - "pbkdf2", - "sha1", - "time", - "zstd 0.11.2+zstd.1.5.2", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 9b7a0123c8..1c0abc299c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.402.3" +version = "1.416.2" authors.workspace = true edition.workspace = true @@ -13,7 +13,9 @@ members = [ "./windmill-common", "./windmill-audit", "./windmill-git-sync", + "./windmill-autoscaling", "./windmill-indexer", + "./windmill-macros", "./parsers/windmill-parser", "./parsers/windmill-parser-ts", "./parsers/windmill-parser-wasm", @@ -23,11 +25,11 @@ members = [ "./parsers/windmill-parser-py", "./parsers/windmill-parser-py-imports", "./parsers/windmill-sql-datatype-parser-wasm", - "./parsers/windmill-parser-yaml", + "./parsers/windmill-parser-yaml", "windmill-macros", ] [workspace.package] -version = "1.402.3" +version = "1.416.2" authors = ["Ruben Fiszel "] edition = "2021" @@ -39,15 +41,17 @@ path = "./src/main.rs" opt-level = 0 incremental = true +[profile.release] +lto = "thin" + [features] default = [] -enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"] +enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"] enterprise_saml = ["windmill-api/enterprise_saml"] stripe = ["windmill-api/stripe"] benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"] flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"] loki = ["windmill-common/loki"] -pg_embed = ["dep:pg-embed"] embedding = ["windmill-api/embedding"] parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "windmill-indexer/parquet", "dep:object_store"] prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus"] @@ -57,6 +61,7 @@ cloud = ["windmill-queue/cloud", "windmill-worker/cloud"] jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"] tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"] sqlx = ["windmill-worker/sqlx"] +deno_core = ["windmill-worker/deno_core", "dep:deno_core"] [dependencies] anyhow.workspace = true @@ -68,6 +73,7 @@ windmill-git-sync.workspace = true windmill-api = { workspace = true, default-features = false } windmill-worker.workspace = true windmill-indexer = { workspace = true, optional = true } +windmill-autoscaling = { workspace = true, optional = true } futures.workspace = true tracing.workspace = true sqlx.workspace = true @@ -85,9 +91,8 @@ uuid.workspace = true gethostname.workspace = true serde_json.workspace = true serde.workspace = true -deno_core.workspace = true +deno_core = { workspace = true, optional = true } object_store = { workspace = true, optional = true } -pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false, features = ['rt_tokio']} quote.workspace = true @@ -105,6 +110,7 @@ serde.workspace = true windmill-api-client.workspace = true deno_core = { workspace = true, features = ["include_js_files_for_snapshotting", "unsafe_use_unprotected_platform"] } + [workspace.dependencies] windmill-api = { path = "./windmill-api", default-features = false } windmill-queue = { path = "./windmill-queue" } @@ -112,7 +118,9 @@ windmill-worker = { path = "./windmill-worker" } windmill-common = { path = "./windmill-common", default-features = false } windmill-audit = { path = "./windmill-audit" } windmill-git-sync = { path = "./windmill-git-sync" } +windmill-autoscaling = { path = "./windmill-autoscaling" } windmill-indexer = {path = "./windmill-indexer"} +windmill-macros = {path = "./windmill-macros"} windmill-parser = { path = "./parsers/windmill-parser" } windmill-parser-ts = { path = "./parsers/windmill-parser-ts" } windmill-parser-py = { path = "./parsers/windmill-parser-py" } @@ -152,7 +160,7 @@ hex = "^0" sql-builder = "^3" argon2 = "^0" quick_cache = "^0" -rand = "0.8.5" +rand = "^0" rand_core = { version = "^0", features = ["std"] } magic-crypt = "^3" git-version = "^0" @@ -164,7 +172,7 @@ urlencoding = "^2" url = "^2" async-oauth2 = "^0" reqwest = { version = "^0.12", features = ["json", "stream", "gzip"] } -time = "0.3.16" +time = "^0" serde_urlencoded = "^0" tokio-tar = "^0" tempfile = "^3" @@ -172,21 +180,25 @@ tokio-util = { version = "^0", features = ["io"] } json-pointer = "^0" itertools = "^0" regex = "^1" -deno_fetch = "0.187.0" -deno_tls = "0.150.0" -deno_console = "0.163.0" -deno_url = "0.163.0" -deno_webidl = "0.163.0" -deno_web = "0.194.0" -deno_net = "0.155.0" -deno_core = "0.299.0" -deno_ast = { version = "=0.40.0", features = ["transpiling"] } + +deno_fetch = "0.195.0" +deno_tls = "0.158.0" +deno_console = "0.171.0" +deno_url = "0.171.0" +deno_webidl = "0.171.0" +deno_web = "0.202.0" +deno_net = "0.163.0" +deno_core = "0.311.0" +deno_ast = { version = "=0.42.2", features = ["transpiling"] } + +swc_common = "=0.37.5" +swc_ecma_parser = "=0.149.1" +swc_ecma_ast = "=0.118.2" +swc_ecma_visit = "=0.104.8" + async-recursion = "^1" -swc_common = "=0.33.26" -swc_ecma_parser = "=0.144.3" -swc_ecma_ast = "=0.113.7" -swc_ecma_visit = "=0.99.1" -base64 = "0.21.0" + +base64 = "^0" base32 = "^0" hmac = "0.12.1" sha2 = "0.10.6" @@ -210,7 +222,7 @@ serde_derive = "1.0.147" const_format = { version = "0.2", features = ["rust_1_64", "rust_1_51"] } dyn-iter = "0.2.0" rsa = "0.7.2" -async-stripe = { version = "0.34.1", features = [ +async-stripe = { version = "0.39.1", features = [ "runtime-tokio-hyper", "checkout", "billing", @@ -270,14 +282,16 @@ tikv-jemallocator = { version = "0.5" } tikv-jemalloc-sys = { version = "^0.5" } tikv-jemalloc-ctl = { version = "^0.5" } -# 0.1.12 broken (nested dependency of swc_common) -triomphe = "<0.1.12" +triomphe = "^0" tantivy = "0.22.0" +# Macro-related +proc-macro2 = "1.0" pulldown-cmark = "0.9" toml = "0.7" syn = { version = "2.0.74", features = ["full"] } quote = "1.0.36" regex-lite = "0.1.6" yaml-rust = "0.4.5" +tokio-tungstenite = { version = "0.24.0", features = ["native-tls"] } \ No newline at end of file diff --git a/backend/custom_migrations/bypassrls_1.sql b/backend/custom_migrations/bypassrls_1.sql deleted file mode 100644 index 1918b26b0b..0000000000 --- a/backend/custom_migrations/bypassrls_1.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE POLICY admin_policy ON account TO windmill_admin USING (true); -CREATE POLICY admin_policy ON app TO windmill_admin USING (true); -CREATE POLICY admin_policy ON audit TO windmill_admin USING (true); -CREATE POLICY admin_policy ON capture TO windmill_admin USING (true); -CREATE POLICY admin_policy ON completed_job TO windmill_admin USING (true); -CREATE POLICY admin_policy ON flow TO windmill_admin USING (true); -CREATE POLICY admin_policy ON folder TO windmill_admin USING (true); -CREATE POLICY admin_policy ON queue TO windmill_admin USING (true); -CREATE POLICY admin_policy ON raw_app TO windmill_admin USING (true); -CREATE POLICY admin_policy ON resource TO windmill_admin USING (true); -CREATE POLICY admin_policy ON schedule TO windmill_admin USING (true); -CREATE POLICY admin_policy ON script TO windmill_admin USING (true); -CREATE POLICY admin_policy ON usr_to_group TO windmill_admin USING (true); -CREATE POLICY admin_policy ON variable TO windmill_admin USING (true); \ No newline at end of file diff --git a/backend/custom_migrations/create_workspace_without_md5.sql b/backend/custom_migrations/create_workspace_without_md5.sql new file mode 100644 index 0000000000..fc140d9e57 --- /dev/null +++ b/backend/custom_migrations/create_workspace_without_md5.sql @@ -0,0 +1,16 @@ +INSERT INTO workspace(id, name, owner) VALUES + ('admins', 'Admins', 'admin@windmill.dev') ON CONFLICT DO NOTHING; + +INSERT INTO workspace_settings (workspace_id) VALUES + ('admins') ON CONFLICT DO NOTHING; + +INSERT INTO workspace_key + (workspace_id, kind, key) + VALUES ('admins', 'cloud', array_to_string( + array( + SELECT chr( (trunc(65 + random() * 25)::int) + + CASE WHEN random() > 0.5 THEN 32 ELSE 0 END ) -- generates random uppercase/lowercase letters + FROM generate_series(1, 32) -- generates 32 characters + ), + '' +)) ON CONFLICT DO NOTHING; \ No newline at end of file diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 994e59747a..e7dd9365ec 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -3d37b6c31155265d8d026ae9d6ced0b433078f87 \ No newline at end of file +f136a2f499e0fe7c10c54c79488851980d796eb2 \ No newline at end of file diff --git a/backend/migrations/20240930183601_add_preprocessor_kind.down.sql b/backend/migrations/20240930183601_add_preprocessor_kind.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20240930183601_add_preprocessor_kind.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20240930183601_add_preprocessor_kind.up.sql b/backend/migrations/20240930183601_add_preprocessor_kind.up.sql new file mode 100644 index 0000000000..d70dc181aa --- /dev/null +++ b/backend/migrations/20240930183601_add_preprocessor_kind.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TYPE SCRIPT_KIND ADD VALUE IF NOT EXISTS 'preprocessor'; \ No newline at end of file diff --git a/backend/migrations/20241002163207_add_websocket_triggers.down.sql b/backend/migrations/20241002163207_add_websocket_triggers.down.sql new file mode 100644 index 0000000000..bb579574a8 --- /dev/null +++ b/backend/migrations/20241002163207_add_websocket_triggers.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +DROP TABLE websocket_trigger; \ No newline at end of file diff --git a/backend/migrations/20241002163207_add_websocket_triggers.up.sql b/backend/migrations/20241002163207_add_websocket_triggers.up.sql new file mode 100644 index 0000000000..4d6b44cb8f --- /dev/null +++ b/backend/migrations/20241002163207_add_websocket_triggers.up.sql @@ -0,0 +1,67 @@ +-- Add up migration script here + +CREATE TABLE websocket_trigger ( + path VARCHAR(255) NOT NULL, + url VARCHAR(255) NOT NULL, + script_path VARCHAR(255) NOT NULL, + is_flow BOOLEAN NOT NULL, + workspace_id VARCHAR(50) NOT NULL, + edited_by VARCHAR(50) NOT NULL, + email VARCHAR(255) NOT NULL, + edited_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + extra_perms JSONB NOT NULL DEFAULT '{}', + server_id VARCHAR(50) NULL, + last_server_ping TIMESTAMPTZ NULL, + error TEXT NULL, + enabled BOOLEAN NOT NULL, + filters JSONB[] NOT NULL DEFAULT '{}', + PRIMARY KEY (path, workspace_id) +); + +GRANT ALL ON websocket_trigger TO windmill_user; +GRANT ALL ON websocket_trigger TO windmill_admin; + +ALTER TABLE websocket_trigger ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON websocket_trigger FOR ALL TO windmill_admin USING (true); + +CREATE POLICY see_folder_extra_perms_user_select ON websocket_trigger FOR SELECT TO windmill_user +USING (SPLIT_PART(websocket_trigger.path, '/', 1) = 'f' AND SPLIT_PART(websocket_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[])); +CREATE POLICY see_folder_extra_perms_user_insert ON websocket_trigger FOR INSERT TO windmill_user +WITH CHECK (SPLIT_PART(websocket_trigger.path, '/', 1) = 'f' AND SPLIT_PART(websocket_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[])); +CREATE POLICY see_folder_extra_perms_user_update ON websocket_trigger FOR UPDATE TO windmill_user +USING (SPLIT_PART(websocket_trigger.path, '/', 1) = 'f' AND SPLIT_PART(websocket_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[])); +CREATE POLICY see_folder_extra_perms_user_delete ON websocket_trigger FOR DELETE TO windmill_user +USING (SPLIT_PART(websocket_trigger.path, '/', 1) = 'f' AND SPLIT_PART(websocket_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[])); + +CREATE POLICY see_own ON websocket_trigger FOR ALL TO windmill_user +USING (SPLIT_PART(websocket_trigger.path, '/', 1) = 'u' AND SPLIT_PART(websocket_trigger.path, '/', 2) = current_setting('session.user')); +CREATE POLICY see_member ON websocket_trigger FOR ALL TO windmill_user +USING (SPLIT_PART(websocket_trigger.path, '/', 1) = 'g' AND SPLIT_PART(websocket_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[])); + +CREATE POLICY see_extra_perms_user_select ON websocket_trigger FOR SELECT TO windmill_user +USING (extra_perms ? CONCAT('u/', current_setting('session.user'))); +CREATE POLICY see_extra_perms_user_insert ON websocket_trigger FOR INSERT TO windmill_user +WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean); +CREATE POLICY see_extra_perms_user_update ON websocket_trigger FOR UPDATE TO windmill_user +USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean); +CREATE POLICY see_extra_perms_user_delete ON websocket_trigger FOR DELETE TO windmill_user +USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean); + +CREATE POLICY see_extra_perms_groups_select ON websocket_trigger FOR SELECT TO windmill_user +USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]); +CREATE POLICY see_extra_perms_groups_insert ON websocket_trigger FOR INSERT TO windmill_user +WITH CHECK (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]) + AND value::boolean)); +CREATE POLICY see_extra_perms_groups_update ON websocket_trigger FOR UPDATE TO windmill_user +USING (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]) + AND value::boolean)); +CREATE POLICY see_extra_perms_groups_delete ON websocket_trigger FOR DELETE TO windmill_user +USING (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]) + AND value::boolean)); \ No newline at end of file diff --git a/backend/migrations/20241006144414_admin_policy.down.sql b/backend/migrations/20241006144414_admin_policy.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20241006144414_admin_policy.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20241006144414_admin_policy.up.sql b/backend/migrations/20241006144414_admin_policy.up.sql new file mode 100644 index 0000000000..e6a3f3807e --- /dev/null +++ b/backend/migrations/20241006144414_admin_policy.up.sql @@ -0,0 +1,24 @@ +-- Add up migration script here +DO +$$ +DECLARE + tbl_name text; + policy_exists boolean; + tbl_names text[] := ARRAY['account', 'app', 'audit', 'capture', 'completed_job', 'flow', 'folder', 'http_trigger', 'queue', 'raw_app', 'resource', 'schedule', 'script', 'usr_to_group', 'variable']; +BEGIN + FOR tbl_name IN SELECT unnest(tbl_names) + LOOP + SELECT EXISTS ( + SELECT 1 + FROM pg_policies + WHERE schemaname = 'public' + AND tablename = tbl_name + AND policyname = 'admin_policy' + ) INTO policy_exists; + + IF NOT policy_exists THEN + EXECUTE format('CREATE POLICY admin_policy ON %I TO windmill_admin USING (true);', tbl_name); + END IF; + END LOOP; +END; +$$; \ No newline at end of file diff --git a/backend/migrations/20241008155800_concurrency_lock_table.down.sql b/backend/migrations/20241008155800_concurrency_lock_table.down.sql new file mode 100644 index 0000000000..2c7ec9a8f8 --- /dev/null +++ b/backend/migrations/20241008155800_concurrency_lock_table.down.sql @@ -0,0 +1,2 @@ +-- Drop the alert_locks table +DROP TABLE IF EXISTS concurrency_locks; diff --git a/backend/migrations/20241008155800_concurrency_lock_table.up.sql b/backend/migrations/20241008155800_concurrency_lock_table.up.sql new file mode 100644 index 0000000000..8449de152a --- /dev/null +++ b/backend/migrations/20241008155800_concurrency_lock_table.up.sql @@ -0,0 +1,6 @@ +-- Create the alert_locks table +CREATE TABLE concurrency_locks ( + id VARCHAR PRIMARY KEY, + last_locked_at TIMESTAMP NOT NULL, + owner VARCHAR NULL +); diff --git a/backend/migrations/20241024125924_autoscaling.down.sql b/backend/migrations/20241024125924_autoscaling.down.sql new file mode 100644 index 0000000000..8f536a7553 --- /dev/null +++ b/backend/migrations/20241024125924_autoscaling.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +DROP TABLE autoscaling_event; +DROP TYPE autoscaling_event_type; diff --git a/backend/migrations/20241024125924_autoscaling.up.sql b/backend/migrations/20241024125924_autoscaling.up.sql new file mode 100644 index 0000000000..fa3b161d56 --- /dev/null +++ b/backend/migrations/20241024125924_autoscaling.up.sql @@ -0,0 +1,13 @@ +-- Add up migration script here +CREATE TYPE AUTOSCALING_EVENT_TYPE AS ENUM ('full_scaleout', 'scalein', 'scaleout'); + +CREATE TABLE autoscaling_event ( + id SERIAL PRIMARY KEY, + worker_group TEXT NOT NULL, + event_type AUTOSCALING_EVENT_TYPE NOT NULL, + desired_workers INTEGER NOT NULL, + applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + reason TEXT +); + +CREATE INDEX autoscaling_event_worker_group_idx ON autoscaling_event (worker_group, applied_at); \ No newline at end of file diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index 034aece2f4..994d9999fd 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -152,5 +152,24 @@ "vismanet_erp_interactive_api:read", "vismanet_erp_interactive_api:update" ] + }, + "spotify": { + "auth_url": "https://accounts.spotify.com/authorize", + "token_url": "https://accounts.spotify.com/api/token", + "scopes": [ + "user-read-playback-state", + "user-modify-playback-state", + "user-read-currently-playing", + "playlist-read-private", + "playlist-read-collaborative", + "playlist-modify-private", + "playlist-modify-public", + "user-follow-read", + "user-read-playback-position", + "user-read-recently-played", + "user-top-read", + "user-library-modify", + "user-library-read" + ] } } \ No newline at end of file diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 34487e04ef..841a763ec2 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -61,6 +61,7 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma "opensearchpy" => "opensearch-py", "lokalise" => "python-lokalise-api", "msgraph" => "msgraph-sdk", + "pythonjsonlogger" => "python-json-logger", }; fn replace_import(x: String) -> String { diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index 742a840802..d68b2eeb6e 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -15,13 +15,13 @@ use windmill_parser::{ use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Span, Spanned}; use swc_ecma_ast::{ - ArrayLit, AssignPat, BigInt, BindingIdent, Bool, Decl, ExportDecl, Expr, FnDecl, Ident, Lit, - MemberExpr, MemberProp, ModuleDecl, ModuleItem, Number, ObjectLit, ObjectPat, Param, Pat, Str, - TsArrayType, TsEntityName, TsKeywordType, TsKeywordTypeKind, TsLit, TsLitType, TsOptionalType, - TsParenthesizedType, TsPropertySignature, TsType, TsTypeAnn, TsTypeElement, TsTypeLit, - TsTypeRef, TsUnionOrIntersectionType, TsUnionType, + ArrayLit, AssignPat, BigInt, BindingIdent, Bool, Decl, ExportDecl, Expr, FnDecl, Ident, + IdentName, Lit, MemberExpr, MemberProp, ModuleDecl, ModuleItem, Number, ObjectLit, ObjectPat, + Param, Pat, Str, TsArrayType, TsEntityName, TsKeywordType, TsKeywordTypeKind, TsLit, TsLitType, + TsOptionalType, TsParenthesizedType, TsPropertySignature, TsType, TsTypeAnn, TsTypeElement, + TsTypeLit, TsTypeRef, TsUnionOrIntersectionType, TsUnionType, }; -use swc_ecma_parser::{lexer::Lexer, EsConfig, Parser, StringInput, Syntax, TsConfig}; +use swc_ecma_parser::{lexer::Lexer, EsSyntax, Parser, StringInput, Syntax, TsSyntax}; use regex::Regex; #[cfg(target_arch = "wasm32")] @@ -48,9 +48,9 @@ impl Visit for ImportsFinder { pub fn parse_expr_for_imports(code: &str) -> anyhow::Result> { let cm: Lrc = Default::default(); - let fm = cm.new_source_file(FileName::Custom("main.d.ts".into()), code.into()); + let fm = cm.new_source_file(FileName::Custom("main.d.ts".into()).into(), code.into()); let lexer = Lexer::new( - Syntax::Typescript(TsConfig::default()), + Syntax::Typescript(TsSyntax::default()), // EsVersion defaults to es5 Default::default(), StringInput::from(&*fm), @@ -69,7 +69,7 @@ pub fn parse_expr_for_imports(code: &str) -> anyhow::Result> { })?; let mut visitor = ImportsFinder { imports: HashSet::new() }; - swc_ecma_visit::visit_module(&mut visitor, &expr); + visitor.visit_module(&expr); Ok(visitor.imports.into_iter().collect()) } @@ -87,7 +87,7 @@ impl Visit for OutputFinder { c.visit_with(self); } match m { - MemberExpr { obj, prop: MemberProp::Ident(Ident { sym, .. }), .. } => { + MemberExpr { obj, prop: MemberProp::Ident(IdentName { sym, .. }), .. } => { match *obj.to_owned() { Expr::Ident(Ident { sym: sym_i, .. }) => { self.idents.insert((sym_i.to_string(), sym.to_string())); @@ -102,10 +102,10 @@ impl Visit for OutputFinder { pub fn parse_expr_for_ids(code: &str) -> anyhow::Result> { let cm: Lrc = Default::default(); - let fm = cm.new_source_file(FileName::Custom("main.ts".into()), code.into()); + let fm = cm.new_source_file(FileName::Custom("main.ts".into()).into(), code.into()); let lexer = Lexer::new( // We want to parse ecmascript - Syntax::Es(EsConfig { jsx: false, ..Default::default() }), + Syntax::Es(EsSyntax { jsx: false, ..Default::default() }), // EsVersion defaults to es5 Default::default(), StringInput::from(&*fm), @@ -124,7 +124,7 @@ pub fn parse_expr_for_ids(code: &str) -> anyhow::Result> { })?; let mut visitor = OutputFinder { idents: HashSet::new() }; - swc_ecma_visit::visit_module(&mut visitor, &expr); + visitor.visit_module(&expr); Ok(visitor.idents.into_iter().collect()) } @@ -135,10 +135,10 @@ pub fn parse_deno_signature( main_override: Option, ) -> anyhow::Result { let cm: Lrc = Default::default(); - let fm = cm.new_source_file(FileName::Custom("main.ts".into()), code.into()); + let fm = cm.new_source_file(FileName::Custom("main.ts".into()).into(), code.into()); let lexer = Lexer::new( // We want to parse ecmascript - Syntax::Typescript(TsConfig::default()), + Syntax::Typescript(TsSyntax::default()), // EsVersion defaults to es5 Default::default(), StringInput::from(&*fm), diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index 151d6eb562..8c29c97b7e 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -188,9 +188,15 @@ pub struct AnsiblePlaybookOptions { pub force_handlers: Option<()>, } +#[derive(Debug, Clone)] +pub enum ResourceOrVariablePath { + Resource(String), + Variable(String), +} + #[derive(Debug, Clone)] pub struct FileResource { - pub resource_path: String, + pub resource_path: ResourceOrVariablePath, pub target_path: String, } @@ -309,7 +315,7 @@ pub fn parse_ansible_reqs( } } } - Yaml::String(key) if key == "file_resources" => { + Yaml::String(key) if key == "files" || key == "file_resources" => { if let Yaml::Array(file_resources) = value { let resources: anyhow::Result> = file_resources.iter().map(parse_file_resource).collect(); @@ -399,15 +405,11 @@ fn parse_ansible_options(opts: &Vec) -> AnsiblePlaybookOptions { if c > 0 && c <= 6 { ret.verbosity = Some("v".repeat(c.min(6))); } - } } - _ => () - + _ => (), } } - - } } @@ -422,10 +424,10 @@ fn count_consecutive_vs(s: &str) -> usize { if c == 'v' { current_count += 1; if current_count == 6 { - return 6; // Stop early if we reach 6 + return 6; // Stop early if we reach 6 } } else { - current_count = 0; // Reset count if the character is not 'v' + current_count = 0; // Reset count if the character is not 'v' } max_count = max_count.max(current_count); } @@ -444,7 +446,24 @@ fn parse_file_resource(yaml: &Yaml) -> anyhow::Result { "No `target` provided for file resource {}. Please input a target relative path for the ansible playbook to see this file.", resource_path ))?; - return Ok(FileResource { resource_path: resource_path.clone(), target_path }); + return Ok(FileResource { + resource_path: ResourceOrVariablePath::Resource(resource_path.clone()), + target_path, + }); + } + if let Some(Yaml::String(resource_path)) = f.get(&Yaml::String("variable".to_string())) { + let target_path = f + .get(&Yaml::String("target".to_string())) + .and_then(|x| x.as_str()) + .map(|x| x.to_string()) + .ok_or(anyhow!( + "No `target` provided for file resource {}. Please input a target relative path for the ansible playbook to see this file.", + resource_path + ))?; + return Ok(FileResource { + resource_path: ResourceOrVariablePath::Variable(resource_path.clone()), + target_path, + }); } return Err(anyhow!( "File resource should have a `resource` field, linking to a text file resource" diff --git a/backend/src/ee.rs b/backend/src/ee.rs index ef944b984b..91816cd1ba 100644 --- a/backend/src/ee.rs +++ b/backend/src/ee.rs @@ -1,16 +1,8 @@ -use anyhow::anyhow; -#[cfg(feature = "enterprise")] -use windmill_common::error::{Error, Result}; - -pub async fn set_license_key(_license_key: String) -> anyhow::Result<()> { +pub async fn set_license_key(_license_key: String) -> () { // Implementation is not open source - Err(anyhow!("License cannot be set in Windmill CE")) } #[cfg(feature = "enterprise")] -pub async fn verify_license_key() -> Result<()> { +pub async fn verify_license_key() -> () { // Implementation is not open source - Err(Error::InternalErr( - "License always invalid in Windmill CE".to_string(), - )) } diff --git a/backend/src/main.rs b/backend/src/main.rs index aa6993cf92..11d12f9dab 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -27,7 +27,7 @@ use uuid::Uuid; use windmill_api::HTTP_CLIENT; #[cfg(feature = "enterprise")] -use windmill_common::ee::schedule_key_renewal; +use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICENSE_KEY_VALID}; use windmill_common::{ global_settings::{ @@ -67,7 +67,7 @@ use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, - RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, + RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -92,9 +92,6 @@ const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0); mod ee; mod monitor; -#[cfg(feature = "pg_embed")] -mod pg_embed; - #[inline(always)] fn create_and_run_current_thread_inner(future: F) -> R where @@ -118,7 +115,8 @@ where } pub fn main() -> anyhow::Result<()> { - deno_core::JsRuntime::init_platform(None); + #[cfg(feature = "deno_core")] + deno_core::JsRuntime::init_platform(None, false); create_and_run_current_thread_inner(windmill_main()) } @@ -137,6 +135,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { })?; create_dir_all(HUB_CACHE_DIR).await?; + create_dir_all(BUN_BUNDLE_CACHE_DIR).await?; for path in paths.values() { tracing::info!("Caching hub script at {path}"); @@ -168,7 +167,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { create_dir_all(&job_dir).await?; if let Some(lockfile) = res.lockfile { let _ = windmill_worker::prepare_job_dir(&lockfile, &job_dir).await?; - + let envs = windmill_worker::get_common_bun_proc_envs(None).await; let _ = windmill_worker::install_bun_lockfile( &mut 0, &mut None, @@ -177,11 +176,31 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { None, &job_dir, "cache_init", - windmill_worker::get_common_bun_proc_envs(None).await, + envs.clone(), false, &mut None, ) .await?; + + let _ = windmill_common::worker::write_file(&job_dir, "main.js", &res.content)?; + + if let Err(e) = windmill_worker::prebundle_bun_script( + &res.content, + Some(lockfile), + &path, + &job_id, + "admins", + None, + &job_dir, + "", + "cache_init", + "", + &mut None, + ) + .await + { + panic!("Error prebundling bun script: {e:#}"); + } } else { tracing::warn!("No lockfile found for bun script {path}, skipping..."); } @@ -265,7 +284,8 @@ async fn windmill_main() -> anyhow::Result<()> { tracing::info!("Binary is in 'indexer' mode"); #[cfg(not(feature = "tantivy"))] { - panic!("Indexer mode requires the tantivy feature flag"); + tracing::error!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features."); + panic!("Indexer mode requires compiling with the tantivy feature flag."); } #[cfg(feature = "tantivy")] Mode::Indexer @@ -288,7 +308,8 @@ async fn windmill_main() -> anyhow::Result<()> { Mode::Standalone }); - let num_workers = if mode == Mode::Server || mode == Mode::Indexer { + #[allow(unused_mut)] + let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer { 0 } else { std::env::var("NUM_WORKERS") @@ -341,14 +362,6 @@ async fn windmill_main() -> anyhow::Result<()> { config }); - #[cfg(feature = "pg_embed")] - let _pg = { - let (db_url, pg) = pg_embed::start().await.expect("pg embed"); - tracing::info!("Use embedded pg: {db_url}"); - std::env::set_var("DATABASE_URL", db_url); - pg - }; - tracing::info!("Connecting to database..."); let db = windmill_common::connect_db(server_mode, indexer_mode).await?; tracing::info!("Database connected"); @@ -373,8 +386,16 @@ async fn windmill_main() -> anyhow::Result<()> { let is_agent = mode == Mode::Agent; if !is_agent { - // migration code to avoid break - windmill_api::migrate_db(&db).await?; + let skip_migration = std::env::var("SKIP_MIGRATION") + .map(|val| val == "true") + .unwrap_or(false); + + if !skip_migration { + // migration code to avoid break + windmill_api::migrate_db(&db).await?; + } else { + tracing::info!("SKIP_MIGRATION set, skipping db migration...") + } } let (killpill_tx, mut killpill_rx) = tokio::sync::broadcast::channel::<()>(2); @@ -403,6 +424,50 @@ Windmill Community Edition {GIT_VERSION} display_config(&ENV_SETTINGS); + if let Err(e) = reload_base_url_setting(&db).await { + tracing::error!("Error loading base url: {:?}", e) + } + + if let Err(e) = reload_critical_error_channels_setting(&db).await { + tracing::error!("Could loading critical error emails setting: {:?}", e); + } + + #[cfg(feature = "enterprise")] + { + // load the license key and check if it's valid + // if not valid and not server mode just quit + // if not expired and server mode then force renewal + // if key still invalid and num_workers > 0, set to 0 + if let Err(err) = reload_license_key(&db).await { + tracing::error!("Failed to reload license key: {err:#}"); + } + let valid_key = *LICENSE_KEY_VALID.read().await; + if !valid_key && !server_mode { + panic!("Invalid license key, workers require a valid license key"); + } + if server_mode { + // only force renewal if invalid but not empty (= expired) + let renewed_now = maybe_renew_license_key_on_start( + &HTTP_CLIENT, + &db, + !valid_key && !LICENSE_KEY_ID.read().await.is_empty(), + ) + .await; + if renewed_now { + if let Err(err) = reload_license_key(&db).await { + tracing::error!("Failed to reload license key: {err:#}"); + } + } + if num_workers > 0 { + let valid_key = *LICENSE_KEY_VALID.read().await; + if !valid_key { + tracing::warn!("License key invalid, setting num_workers to 0"); + num_workers = 0; + } + } + } + } + let worker_mode = num_workers > 0; if server_mode || worker_mode || indexer_mode { @@ -429,7 +494,16 @@ Windmill Community Edition {GIT_VERSION} initial_load(&db, killpill_tx.clone(), worker_mode, server_mode, is_agent).await; - monitor_db(&db, &base_internal_url, rsmq.clone(), server_mode, true).await; + monitor_db( + &db, + &base_internal_url, + rsmq.clone(), + server_mode, + worker_mode, + true, + killpill_tx.clone(), + ) + .await; monitor_pool(&db).await; @@ -457,7 +531,7 @@ Windmill Community Edition {GIT_VERSION} #[cfg(feature = "tantivy")] let (index_reader, index_writer) = if should_index_jobs { - let (r, w) = windmill_indexer::indexer_ee::init_index().await?; + let (r, w) = windmill_indexer::indexer_ee::init_index(&db).await?; (Some(r), Some(w)) } else { (None, None) @@ -529,8 +603,11 @@ Windmill Community Edition {GIT_VERSION} rx.recv().await?; } } - tracing::info!("Starting phase 2 of shutdown"); - killpill_phase2_tx.send(())?; + if killpill_phase2_tx.receiver_count() > 0 { + tracing::info!("Starting phase 2 of shutdown"); + killpill_phase2_tx.send(())?; + tracing::info!("Phase 2 of shutdown completed"); + } Ok(()) as anyhow::Result<()> }; @@ -556,7 +633,9 @@ Windmill Community Edition {GIT_VERSION} &base_internal_url, rsmq.clone(), server_mode, - false + worker_mode, + false, + tx.clone(), ) .await; }, @@ -599,7 +678,15 @@ Windmill Community Edition {GIT_VERSION} }, LICENSE_KEY_SETTING => { if let Err(e) = reload_license_key(&db).await { - tracing::error!(error = %e, "Could not reload license key setting"); + tracing::error!("Failed to reload license key: {e:#}"); + } + #[cfg(feature = "enterprise")] + if worker_mode { + let valid_key = *LICENSE_KEY_VALID.read().await; + if !valid_key { + tracing::error!("Invalid license key, exiting..."); + tx.send(()).expect("send"); + } } }, DEFAULT_TAGS_PER_WORKSPACE_SETTING => { @@ -745,14 +832,8 @@ Windmill Community Edition {GIT_VERSION} Ok(()) as anyhow::Result<()> }; - let instance_name = rd_string(8); - if mode == Mode::Server || mode == Mode::Standalone { - schedule_stats(instance_name, &db, &HTTP_CLIENT).await; - } - - #[cfg(feature = "enterprise")] - if mode == Mode::Server || mode == Mode::Standalone { - schedule_key_renewal(&HTTP_CLIENT, &db).await; + if server_mode { + schedule_stats(&db, &HTTP_CLIENT).await; } futures::try_join!( @@ -875,6 +956,7 @@ pub async fn run_workers error::Result<()> { - let q = load_value_from_global_settings(db, LICENSE_KEY_SETTING).await?; +pub async fn reload_license_key(db: &DB) -> anyhow::Result<()> { + let q = load_value_from_global_settings(db, LICENSE_KEY_SETTING) + .await + .map_err(|err| anyhow::anyhow!("Error reloading license key: {}", err.to_string()))?; let mut value = std::env::var("LICENSE_KEY") .ok() @@ -873,9 +859,7 @@ pub async fn reload_license_key(db: &DB) -> error::Result<()> { tracing::error!("Could not parse LICENSE_KEY found: {:#?}", &q); } }; - - set_license_key(value).await?; - + set_license_key(value).await; Ok(()) } @@ -1014,7 +998,9 @@ pub async fn monitor_db( base_internal_url: &str, rsmq: Option, server_mode: bool, + _worker_mode: bool, initial_load: bool, + _killpill_tx: tokio::sync::broadcast::Sender<()>, ) { let zombie_jobs_f = async { if server_mode && !initial_load { @@ -1035,15 +1021,14 @@ pub async fn monitor_db( let verify_license_key_f = async { #[cfg(feature = "enterprise")] - if let Err(e) = verify_license_key().await { - tracing::error!("Error verifying license key: {:?}", e); - let mut l = LICENSE_KEY_VALID.write().await; - *l = false; - } else { - let is_valid = LICENSE_KEY_VALID.read().await.clone(); - if !is_valid { - let mut l = LICENSE_KEY_VALID.write().await; - *l = true; + if !initial_load { + verify_license_key().await; + if _worker_mode { + let valid_key = *LICENSE_KEY_VALID.read().await; + if !valid_key { + tracing::error!("Invalid license key, exiting..."); + _killpill_tx.send(()).expect("send"); + } } } }; @@ -1061,12 +1046,30 @@ pub async fn monitor_db( } }; + let jobs_waiting_alerts_f = async { + #[cfg(feature = "enterprise")] + if server_mode { + jobs_waiting_alerts(&db).await; + } + }; + + let apply_autoscaling_f = async { + #[cfg(feature = "enterprise")] + if server_mode && !initial_load { + if let Err(e) = windmill_autoscaling::apply_all_autoscaling(db).await { + tracing::error!("Error applying autoscaling: {:?}", e); + } + } + }; + join!( expired_items_f, zombie_jobs_f, expose_queue_metrics_f, verify_license_key_f, - worker_groups_alerts_f + worker_groups_alerts_f, + jobs_waiting_alerts_f, + apply_autoscaling_f, ); } @@ -1084,19 +1087,11 @@ pub async fn expose_queue_metrics(db: &Pool) { .unwrap_or(true); if metrics_enabled || save_metrics { - let queue_counts = sqlx::query!( - "SELECT tag, count(*) as count FROM queue WHERE - scheduled_for <= now() - ('3 seconds')::interval AND running = false - GROUP BY tag" - ) - .fetch_all(db) - .await - .ok() - .unwrap_or_else(|| vec![]); + let queue_counts = windmill_common::queue::get_queue_counts(db).await; for q in queue_counts { - let count = q.count.unwrap_or(0); - let tag = q.tag; + let count = q.1; + let tag = q.0; if metrics_enabled { let metric = (*QUEUE_COUNT).with_label_values(&[&tag]); metric.set(count as i64); diff --git a/backend/src/pg_embed.rs b/backend/src/pg_embed.rs deleted file mode 100644 index ad5529dc02..0000000000 --- a/backend/src/pg_embed.rs +++ /dev/null @@ -1,46 +0,0 @@ -use pg_embed::pg_enums::PgAuthMethod; -use pg_embed::pg_fetch::PgFetchSettings; -use pg_embed::postgres::{PgEmbed, PgSettings}; -use std::path::PathBuf; -use std::time::Duration; - -pub async fn start() -> anyhow::Result<(String, PgEmbed)> { - let pg_settings = PgSettings { - database_dir: PathBuf::from("/tmp/db"), - port: 6543, - user: "postgres".to_string(), - password: "password".to_string(), - auth_method: PgAuthMethod::Plain, - persistent: false, - timeout: Some(Duration::from_secs(15)), - migration_dir: None, - }; - - let fetch_settings = PgFetchSettings { - version: pg_embed::pg_fetch::PostgresVersion("15.3.0"), - - ..Default::default() - }; - - tracing::info!( - "Fetch settings: {:?} {:?}", - fetch_settings.operating_system, - fetch_settings.architecture - ); - - let mut pg = PgEmbed::new(pg_settings, fetch_settings).await?; - - pg.setup().await.expect("pg setup"); - - pg.start_db().await.expect("pg start db"); - - //TODO: re-enable this to make it work - // if !pg.database_exists("windmill").await.expect("db exists") { - // pg.create_database("windmill") - // .await - // .expect("pg create database"); - // } - - let uri = pg.full_db_uri("windmill"); - Ok((uri, pg)) -} diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index e97c53700a..a93c6eef1d 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -1126,6 +1126,7 @@ async fn test_deno_flow(db: Pool) { priority: None, delete_after_use: None, continue_on_error: None, + skip_if: None, }, FlowModule { id: "b".to_string(), @@ -1166,6 +1167,7 @@ async fn test_deno_flow(db: Pool) { priority: None, delete_after_use: None, continue_on_error: None, + skip_if: None, }], } .into(), @@ -1181,6 +1183,7 @@ async fn test_deno_flow(db: Pool) { priority: None, delete_after_use: None, continue_on_error: None, + skip_if: None, }, ], same_worker: false, @@ -1286,6 +1289,7 @@ async fn test_deno_flow_same_worker(db: Pool) { priority: None, delete_after_use: None, continue_on_error: None, + skip_if: None, }, FlowModule { id: "b".to_string(), @@ -1336,6 +1340,7 @@ async fn test_deno_flow_same_worker(db: Pool) { priority: None, delete_after_use: None, continue_on_error: None, + skip_if: None, }, FlowModule { id: "e".to_string(), @@ -1372,7 +1377,7 @@ async fn test_deno_flow_same_worker(db: Pool) { priority: None, delete_after_use: None, continue_on_error: None, - + skip_if: None, }, ], }.into(), @@ -1388,6 +1393,7 @@ async fn test_deno_flow_same_worker(db: Pool) { priority: None, delete_after_use: None, continue_on_error: None, + skip_if: None, }, FlowModule { id: "c".to_string(), @@ -1431,6 +1437,7 @@ async fn test_deno_flow_same_worker(db: Pool) { priority: None, delete_after_use: None, continue_on_error: None, + skip_if: None, }, ], same_worker: true, @@ -2738,7 +2745,7 @@ async fn test_flow_lock_all(db: Pool) { "lock": null, "path": null, "type": "rawscript", - "content": "import * as wmill from \"https://deno.land/x/windmill@v1.50.0/mod.ts\"\n\nexport async function main() {\n return \"Hello\"\n}\n", + "content": "import * as wmill from \"https://deno.land/x/windmill@v1.50.0/mod.ts\"\n\nexport async function main() {\n return wmill\n}\n", "language": "deno", "input_transforms": {} }, diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index fd8bedf0f9..b7c4a41aed 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -95,9 +95,9 @@ openidconnect = { workspace = true, optional = true} url = { workspace = true, optional = true} jsonwebtoken = { workspace = true } matchit.workspace = true +tokio-tungstenite.workspace = true pin-project.workspace = true -crc.workspace = true http.workspace = true async-stream.workspace = true ulid.workspace = true diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 23e8f019df..db789c2163 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -1,7 +1,7 @@ { "openapi": "3.0.3", "info": { - "version": "1.304.2", + "version": "1.409.0", "title": "Windmill API", "contact": { "name": "Windmill Team", @@ -176,6 +176,22 @@ { "$ref": "#/components/parameters/Operation" }, + { + "name": "operations", + "in": "query", + "description": "comma separated list of exact operations to include", + "schema": { + "type": "string" + } + }, + { + "name": "exclude_operations", + "in": "query", + "description": "comma separated list of operations to exclude", + "schema": { + "type": "string" + } + }, { "$ref": "#/components/parameters/ResourceName" }, @@ -501,6 +517,9 @@ "properties": { "is_super_admin": { "type": "boolean" + }, + "name": { + "type": "string" } } } @@ -660,6 +679,65 @@ } } }, + "/users/overwrite": { + "post": { + "summary": "global overwrite users (require super admin and EE)", + "operationId": "globalUsersOverwrite", + "tags": [ + "user" + ], + "requestBody": { + "description": "List of users", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ExportedUser" + } + } + } + } + }, + "responses": { + "200": { + "description": "Success message", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/users/export": { + "get": { + "summary": "global export users (require super admin and EE)", + "operationId": "globalUsersExport", + "tags": [ + "user" + ], + "responses": { + "200": { + "description": "exported users", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ExportedUser" + } + } + } + } + } + } + } + }, "/w/{workspace}/users/delete/{username}": { "delete": { "summary": "delete user (require admin privilege)", @@ -1061,6 +1139,49 @@ } } }, + "/settings/test_critical_channels": { + "post": { + "summary": "test critical channels", + "operationId": "testCriticalChannels", + "tags": [ + "setting" + ], + "requestBody": { + "description": "test critical channel payload", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "email": { + "type": "string" + }, + "slack_channel": { + "type": "string" + } + } + } + } + } + } + }, + "responses": { + "200": { + "description": "status", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/settings/test_license_key": { "post": { "summary": "test license key", @@ -1155,6 +1276,103 @@ } } }, + "/settings/latest_key_renewal_attempt": { + "get": { + "summary": "get latest key renewal attempt", + "operationId": "getLatestKeyRenewalAttempt", + "tags": [ + "setting" + ], + "responses": { + "200": { + "description": "status", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "result": { + "type": "string" + }, + "attempted_at": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "result", + "attempted_at" + ], + "nullable": true + } + } + } + } + } + } + }, + "/settings/renew_license_key": { + "post": { + "summary": "renew license key", + "operationId": "renewLicenseKey", + "tags": [ + "setting" + ], + "parameters": [ + { + "name": "license_key", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "status", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/settings/customer_portal": { + "post": { + "summary": "create customer portal session", + "operationId": "createCustomerPortalSession", + "tags": [ + "setting" + ], + "parameters": [ + { + "name": "license_key", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "url to portal", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/saml/test_metadata": { "post": { "summary": "test metadata", @@ -1187,6 +1405,30 @@ } } }, + "/settings/list_global": { + "get": { + "summary": "list global settings", + "operationId": "listGlobalSettings", + "tags": [ + "setting" + ], + "responses": { + "200": { + "description": "list of settings", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/GlobalSetting" + } + } + } + } + } + } + } + }, "/users/email": { "get": { "summary": "get current user email (if logged in)", @@ -2122,6 +2364,9 @@ "git_sync": { "$ref": "#/components/schemas/WorkspaceGitSyncSettings" }, + "deploy_ui": { + "$ref": "#/components/schemas/WorkspaceDeployUISettings" + }, "default_app": { "type": "string" }, @@ -2730,6 +2975,46 @@ } } }, + "/w/{workspace}/workspaces/edit_deploy_ui_config": { + "post": { + "summary": "edit workspace deploy ui settings", + "operationId": "editWorkspaceDeployUISettings", + "tags": [ + "workspace" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "Workspace deploy UI settings", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "deploy_ui_settings": { + "$ref": "#/components/schemas/WorkspaceDeployUISettings" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "status", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, "/w/{workspace}/workspaces/edit_default_app": { "post": { "summary": "edit default app for workspace", @@ -2934,6 +3219,9 @@ "properties": { "new_key": { "type": "string" + }, + "skip_reencrypt": { + "type": "boolean" } }, "required": [ @@ -3127,6 +3415,40 @@ } } }, + "/w/{workspace}/users/username_to_email/{username}": { + "get": { + "summary": "get email from username", + "operationId": "usernameToEmail", + "tags": [ + "user" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "name": "username", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "email", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/users/tokens/create": { "post": { "summary": "create token", @@ -3236,6 +3558,12 @@ "schema": { "type": "boolean" } + }, + { + "$ref": "#/components/parameters/Page" + }, + { + "$ref": "#/components/parameters/PerPage" } ], "responses": { @@ -3559,6 +3887,19 @@ "parameters": [ { "$ref": "#/components/parameters/WorkspaceId" + }, + { + "name": "path_start", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "$ref": "#/components/parameters/Page" + }, + { + "$ref": "#/components/parameters/PerPage" } ], "responses": { @@ -3710,6 +4051,50 @@ } } }, + "/oauth/connect_slack_callback": { + "post": { + "summary": "connect slack callback instance", + "operationId": "connectSlackCallbackInstance", + "tags": [ + "oauth" + ], + "requestBody": { + "description": "code endpoint", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "state": { + "type": "string" + } + }, + "required": [ + "code", + "state" + ] + } + } + } + }, + "responses": { + "200": { + "description": "success message", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/oauth/connect_callback/{client_name}": { "post": { "summary": "connect callback", @@ -3932,7 +4317,18 @@ "oauth": { "type": "array", "items": { - "type": "string" + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "display_name": { + "type": "string" + } + }, + "required": [ + "type" + ] } }, "saml": { @@ -3962,19 +4358,50 @@ "content": { "application/json": { "schema": { - "additionalProperties": { - "type": "object", - "properties": { - "extra_params": { - "additionalProperties": { - "type": "string" - } - }, - "scopes": { - "type": "array", - "items": { - "type": "string" - } + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "/oauth/get_connect/{client}": { + "get": { + "summary": "get oauth connect", + "operationId": "getOAuthConnect", + "tags": [ + "oauth" + ], + "parameters": [ + { + "name": "client", + "description": "client name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "get", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "extra_params": { + "type": "object" + }, + "scopes": { + "type": "array", + "items": { + "type": "string" } } } @@ -4294,6 +4721,13 @@ "schema": { "type": "string" } + }, + { + "name": "path_start", + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -4433,6 +4867,30 @@ } } }, + "/w/{workspace}/resources/file_resource_type_to_file_ext_map": { + "get": { + "summary": "get map from resource type to format extension", + "operationId": "fileResourceTypeToFileExtMap", + "tags": [ + "resource" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "responses": { + "200": { + "description": "map from resource type to file ext", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, "/w/{workspace}/resources/type/delete/{path}": { "delete": { "summary": "delete resource_type", @@ -5281,7 +5739,23 @@ }, { "name": "show_archived", - "description": "(default false)\nshow also the archived files.\nwhen multiple archived hash share the same path, only the ones with the latest create_at\nare \ned.\n", + "description": "(default false)\nshow only the archived files.\nwhen multiple archived hash share the same path, only the ones with the latest create_at\nare \ned.\n", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "include_without_main", + "description": "(default false)\ninclude scripts without an exported main function\n", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "include_draft_only", + "description": "(default false)\ninclude scripts that have no deployed version\n", "in": "query", "schema": { "type": "boolean" @@ -5310,6 +5784,14 @@ "schema": { "type": "boolean" } + }, + { + "name": "with_deployment_msg", + "description": "(default false)\ninclude deployment message\n", + "in": "query", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -5735,6 +6217,13 @@ }, { "$ref": "#/components/parameters/ScriptPath" + }, + { + "name": "with_starred_info", + "in": "query", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -5751,6 +6240,67 @@ } } }, + "/w/{workspace}/scripts/get_triggers_count/{path}": { + "get": { + "summary": "get triggers count of script", + "operationId": "getTriggersCountOfScript", + "tags": [ + "script" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/ScriptPath" + } + ], + "responses": { + "200": { + "description": "triggers count", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TriggersCount" + } + } + } + } + } + } + }, + "/w/{workspace}/scripts/list_tokens/{path}": { + "get": { + "summary": "get tokens with script scope", + "operationId": "listTokensOfScript", + "tags": [ + "script" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/ScriptPath" + } + ], + "responses": { + "200": { + "description": "tokens list", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/TruncatedToken" + } + } + } + } + } + } + } + }, "/w/{workspace}/scripts/get/draft/{path}": { "get": { "summary": "get script by path with draft", @@ -5963,6 +6513,13 @@ }, { "$ref": "#/components/parameters/ScriptHash" + }, + { + "name": "with_starred_info", + "in": "query", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -6076,6 +6633,14 @@ "type": "integer" } }, + { + "name": "skip_preprocessor", + "description": "skip the preprocessor", + "in": "query", + "schema": { + "type": "boolean" + } + }, { "$ref": "#/components/parameters/ParentJob" }, @@ -6518,7 +7083,7 @@ }, { "name": "show_archived", - "description": "(default false)\nshow also the archived files.\nwhen multiple archived hash share the same path, only the ones with the latest create_at\nare displayed.\n", + "description": "(default false)\nshow only the archived files.\nwhen multiple archived hash share the same path, only the ones with the latest create_at\nare displayed.\n", "in": "query", "schema": { "type": "boolean" @@ -6531,6 +7096,22 @@ "schema": { "type": "boolean" } + }, + { + "name": "include_draft_only", + "description": "(default false)\ninclude items that have no deployed version\n", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "with_deployment_msg", + "description": "(default false)\ninclude deployment message\n", + "in": "query", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -6565,6 +7146,133 @@ } } }, + "/w/{workspace}/flows/history/p/{path}": { + "get": { + "summary": "get flow history by path", + "operationId": "getFlowHistory", + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/ScriptPath" + } + ], + "tags": [ + "flow" + ], + "responses": { + "200": { + "description": "Flow history", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/FlowVersion" + } + } + } + } + } + } + } + }, + "/w/{workspace}/flows/get/v/{version}/p/{path}": { + "get": { + "summary": "get flow version", + "operationId": "getFlowVersion", + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "type": "string", + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "number" + } + }, + { + "$ref": "#/components/parameters/ScriptPath" + } + ], + "tags": [ + "flow" + ], + "responses": { + "200": { + "description": "flow details", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Flow" + } + } + } + } + } + } + }, + "/w/{workspace}/flows/history_update/v/{version}/p/{path}": { + "post": { + "summary": "update flow history", + "operationId": "updateFlowHistory", + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "type": "string", + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "number" + } + }, + { + "$ref": "#/components/parameters/ScriptPath" + } + ], + "requestBody": { + "description": "Flow deployment message", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "deployment_msg": { + "type": "string" + } + }, + "required": [ + "deployment_msg" + ] + } + } + } + }, + "tags": [ + "flow" + ], + "responses": { + "200": { + "description": "success", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/w/{workspace}/flows/get/{path}": { "get": { "summary": "get flow by path", @@ -6578,6 +7286,13 @@ }, { "$ref": "#/components/parameters/ScriptPath" + }, + { + "name": "with_starred_info", + "in": "query", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -6594,6 +7309,67 @@ } } }, + "/w/{workspace}/flows/get_triggers_count/{path}": { + "get": { + "summary": "get triggers count of flow", + "operationId": "getTriggersCountOfFlow", + "tags": [ + "flow" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/ScriptPath" + } + ], + "responses": { + "200": { + "description": "triggers count", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TriggersCount" + } + } + } + } + } + } + }, + "/w/{workspace}/flows/list_tokens/{path}": { + "get": { + "summary": "get tokens with flow scope", + "operationId": "listTokensOfFlow", + "tags": [ + "flow" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/ScriptPath" + } + ], + "responses": { + "200": { + "description": "tokens list", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/TruncatedToken" + } + } + } + } + } + } + } + }, "/w/{workspace}/flows/toggle_workspace_error_handler/{path}": { "post": { "summary": "Toggle ON and OFF the workspace error handler for a given flow", @@ -6887,44 +7663,6 @@ } } }, - "/w/{workspace}/flows/input_history/p/{path}": { - "get": { - "summary": "list inputs for previous completed flow jobs", - "operationId": "getFlowInputHistoryByPath", - "tags": [ - "flow" - ], - "parameters": [ - { - "$ref": "#/components/parameters/WorkspaceId" - }, - { - "$ref": "#/components/parameters/ScriptPath" - }, - { - "$ref": "#/components/parameters/Page" - }, - { - "$ref": "#/components/parameters/PerPage" - } - ], - "responses": { - "200": { - "description": "input history for completed jobs with this flow path", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Input" - } - } - } - } - } - } - } - }, "/w/{workspace}/raw_apps/list": { "get": { "summary": "list all raw apps", @@ -7136,6 +7874,22 @@ "schema": { "type": "boolean" } + }, + { + "name": "include_draft_only", + "description": "(default false)\ninclude items that have no deployed version\n", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "with_deployment_msg", + "description": "(default false)\ninclude deployment message\n", + "in": "query", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -7258,6 +8012,13 @@ }, { "$ref": "#/components/parameters/ScriptPath" + }, + { + "name": "with_starred_info", + "in": "query", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -7756,6 +8517,9 @@ "path": { "type": "string" }, + "lock": { + "type": "string" + }, "cache_ttl": { "type": "integer" } @@ -7767,6 +8531,15 @@ }, "force_viewer_static_fields": { "type": "object" + }, + "force_viewer_one_of_fields": { + "type": "object" + }, + "force_viewer_allow_user_resources": { + "type": "array", + "items": { + "type": "string" + } } }, "required": [ @@ -7822,6 +8595,14 @@ "type": "integer" } }, + { + "name": "skip_preprocessor", + "description": "skip the preprocessor", + "in": "query", + "schema": { + "type": "boolean" + } + }, { "$ref": "#/components/parameters/ParentJob" }, @@ -7996,6 +8777,14 @@ "type": "integer" } }, + { + "name": "skip_preprocessor", + "description": "skip the preprocessor", + "in": "query", + "schema": { + "type": "boolean" + } + }, { "$ref": "#/components/parameters/ParentJob" }, @@ -8339,6 +9128,14 @@ "schema": { "type": "boolean" } + }, + { + "name": "is_not_schedule", + "description": "is not a scheduled job", + "in": "query", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -8388,6 +9185,9 @@ "properties": { "database_length": { "type": "integer" + }, + "suspended": { + "type": "integer" } }, "required": [ @@ -8434,10 +9234,120 @@ } } }, - "/w/{workspace}/jobs/queue/cancel_all": { + "/w/{workspace}/jobs/queue/list_filtered_uuids": { + "get": { + "summary": "get the ids of all jobs matching the given filters", + "operationId": "listFilteredUuids", + "tags": [ + "job" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/OrderDesc" + }, + { + "$ref": "#/components/parameters/CreatedBy" + }, + { + "$ref": "#/components/parameters/ParentJob" + }, + { + "$ref": "#/components/parameters/ScriptExactPath" + }, + { + "$ref": "#/components/parameters/ScriptStartPath" + }, + { + "$ref": "#/components/parameters/SchedulePath" + }, + { + "$ref": "#/components/parameters/ScriptExactHash" + }, + { + "$ref": "#/components/parameters/StartedBefore" + }, + { + "$ref": "#/components/parameters/StartedAfter" + }, + { + "$ref": "#/components/parameters/Success" + }, + { + "$ref": "#/components/parameters/ScheduledForBeforeNow" + }, + { + "$ref": "#/components/parameters/JobKinds" + }, + { + "$ref": "#/components/parameters/Suspended" + }, + { + "$ref": "#/components/parameters/Running" + }, + { + "$ref": "#/components/parameters/ArgsFilter" + }, + { + "$ref": "#/components/parameters/ResultFilter" + }, + { + "$ref": "#/components/parameters/Tag" + }, + { + "$ref": "#/components/parameters/Page" + }, + { + "$ref": "#/components/parameters/PerPage" + }, + { + "name": "concurrency_key", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "all_workspaces", + "description": "get jobs from all workspaces (only valid if request come from the `admins` workspace)", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "is_not_schedule", + "description": "is not a scheduled job", + "in": "query", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "uuids of jobs", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "/w/{workspace}/jobs/queue/cancel_selection": { "post": { - "summary": "cancel all jobs", - "operationId": "cancelAll", + "summary": "cancel jobs based on the given uuids", + "operationId": "cancelSelection", "tags": [ "job" ], @@ -8446,6 +9356,20 @@ "$ref": "#/components/parameters/WorkspaceId" } ], + "requestBody": { + "description": "uuids of the jobs to cancel", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + }, "responses": { "200": { "description": "uuids of canceled jobs", @@ -8480,6 +9404,9 @@ { "$ref": "#/components/parameters/CreatedBy" }, + { + "$ref": "#/components/parameters/Label" + }, { "$ref": "#/components/parameters/ParentJob" }, @@ -8545,6 +9472,14 @@ "schema": { "type": "boolean" } + }, + { + "name": "is_not_schedule", + "description": "is not a scheduled job", + "in": "query", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -8578,6 +9513,9 @@ { "$ref": "#/components/parameters/CreatedBy" }, + { + "$ref": "#/components/parameters/Label" + }, { "$ref": "#/components/parameters/ParentJob" }, @@ -8599,6 +9537,12 @@ { "$ref": "#/components/parameters/StartedAfter" }, + { + "$ref": "#/components/parameters/CreatedBefore" + }, + { + "$ref": "#/components/parameters/CreatedAfter" + }, { "$ref": "#/components/parameters/CreatedOrStartedBefore" }, @@ -8611,9 +9555,15 @@ { "$ref": "#/components/parameters/CreatedOrStartedAfter" }, + { + "$ref": "#/components/parameters/CreatedOrStartedAfterCompletedJob" + }, { "$ref": "#/components/parameters/JobKinds" }, + { + "$ref": "#/components/parameters/Suspended" + }, { "$ref": "#/components/parameters/ArgsFilter" }, @@ -8668,6 +9618,14 @@ "schema": { "type": "boolean" } + }, + { + "name": "is_not_schedule", + "description": "is not a scheduled job", + "in": "query", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -8708,6 +9666,62 @@ } } }, + "/jobs/completed/count_by_tag": { + "get": { + "summary": "Count jobs by tag", + "operationId": "countJobsByTag", + "tags": [ + "job" + ], + "parameters": [ + { + "name": "horizon_secs", + "in": "query", + "description": "Past Time horizon in seconds (when to start the count = now - horizon) (default is 3600)", + "required": false, + "schema": { + "type": "integer" + } + }, + { + "name": "workspace_id", + "in": "query", + "description": "Specific workspace ID to filter results (optional)", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Job counts by tag", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "tag": { + "type": "string" + }, + "count": { + "type": "integer" + } + }, + "required": [ + "tag", + "count" + ] + } + } + } + } + } + } + } + }, "/w/{workspace}/jobs_u/get/{id}": { "get": { "summary": "get job", @@ -8802,6 +9816,33 @@ } } }, + "/w/{workspace}/jobs_u/get_args/{id}": { + "get": { + "summary": "get job args", + "operationId": "getJobArgs", + "tags": [ + "job" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/JobId" + } + ], + "responses": { + "200": { + "description": "job args", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, "/w/{workspace}/jobs_u/getupdate/{id}": { "get": { "summary": "get job updates", @@ -8829,6 +9870,13 @@ "schema": { "type": "integer" } + }, + { + "name": "get_progress", + "in": "query", + "schema": { + "type": "boolean" + } } ], "responses": { @@ -8854,6 +9902,9 @@ "mem_peak": { "type": "integer" }, + "progress": { + "type": "integer" + }, "flow_status": { "$ref": "#/components/schemas/WorkflowStatusRecord" } @@ -8865,6 +9916,38 @@ } } }, + "/w/{workspace}/jobs_u/get_log_file/{path}": { + "get": { + "summary": "get log file from object store", + "operationId": "getLogFileFromStore", + "tags": [ + "job" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "name": "path", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "job log", + "content": { + "text/plain": { + "type": "string" + } + } + } + } + } + }, "/w/{workspace}/jobs_u/get_flow_debug_info/{id}": { "get": { "summary": "get flow debug info", @@ -8934,6 +10017,34 @@ }, { "$ref": "#/components/parameters/JobId" + }, + { + "name": "suspended_job", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "resume_id", + "in": "query", + "schema": { + "type": "integer" + } + }, + { + "name": "secret", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "approver", + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -9027,7 +10138,7 @@ }, "/w/{workspace}/jobs_u/queue/cancel/{id}": { "post": { - "summary": "cancel queued job", + "summary": "cancel queued or running job", "operationId": "cancelQueuedJob", "tags": [ "job" @@ -9982,6 +11093,13 @@ "schema": { "type": "boolean" } + }, + { + "name": "path_start", + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -10060,7 +11178,8 @@ "type": "string", "enum": [ "error", - "recovery" + "recovery", + "success" ] }, "override_existing": { @@ -10097,6 +11216,311 @@ } } }, + "/w/{workspace}/http_triggers/create": { + "post": { + "summary": "create http trigger", + "operationId": "createHttpTrigger", + "tags": [ + "http_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "new http trigger", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NewHttpTrigger" + } + } + } + }, + "responses": { + "201": { + "description": "http trigger created", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/http_triggers/update/{path}": { + "post": { + "summary": "update http trigger", + "operationId": "updateHttpTrigger", + "tags": [ + "http_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "requestBody": { + "description": "updated trigger", + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EditHttpTrigger" + } + } + } + }, + "responses": { + "200": { + "description": "http trigger updated", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/http_triggers/delete/{path}": { + "delete": { + "summary": "delete http trigger", + "operationId": "deleteHttpTrigger", + "tags": [ + "http_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "http trigger deleted", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/http_triggers/get/{path}": { + "get": { + "summary": "get http trigger", + "operationId": "getHttpTrigger", + "tags": [ + "http_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "http trigger deleted", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HttpTrigger" + } + } + } + } + } + } + }, + "/w/{workspace}/http_triggers/list": { + "get": { + "summary": "list http triggers", + "operationId": "listHttpTriggers", + "tags": [ + "http_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId", + "required": true + }, + { + "$ref": "#/components/parameters/Page" + }, + { + "$ref": "#/components/parameters/PerPage" + }, + { + "name": "path", + "description": "filter by path", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "is_flow", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "path_start", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "http trigger list", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/HttpTrigger" + } + } + } + } + } + } + } + }, + "/w/{workspace}/http_triggers/exists/{path}": { + "get": { + "summary": "does http trigger exists", + "operationId": "existsHttpTrigger", + "tags": [ + "http_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "http trigger exists", + "content": { + "application/json": { + "schema": { + "type": "boolean" + } + } + } + } + } + } + }, + "/w/{workspace}/http_triggers/route_exists": { + "post": { + "summary": "does route exists", + "operationId": "existsRoute", + "tags": [ + "http_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "requestBody": { + "description": "route exists request", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "route_path": { + "type": "string" + }, + "http_method": { + "type": "string", + "enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + }, + "required": [ + "kind", + "route_path", + "http_method" + ] + } + } + } + }, + "responses": { + "200": { + "description": "route exists", + "content": { + "application/json": { + "schema": { + "type": "boolean" + } + } + } + } + } + } + }, + "/w/{workspace}/http_triggers/used": { + "get": { + "summary": "whether http triggers are used", + "operationId": "used", + "tags": [ + "http_trigger" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + } + ], + "responses": { + "200": { + "description": "whether http triggers are used", + "content": { + "application/json": { + "schema": { + "type": "boolean" + } + } + } + } + } + } + }, "/groups/list": { "get": { "summary": "list instance groups", @@ -10351,6 +11775,65 @@ } } }, + "/groups/export": { + "get": { + "summary": "export instance groups", + "operationId": "exportInstanceGroups", + "tags": [ + "group" + ], + "responses": { + "200": { + "description": "exported instance groups", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ExportedInstanceGroup" + } + } + } + } + } + } + } + }, + "/groups/overwrite": { + "post": { + "summary": "overwrite instance groups", + "operationId": "overwriteInstanceGroups", + "tags": [ + "group" + ], + "requestBody": { + "description": "overwrite instance groups", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ExportedInstanceGroup" + } + } + } + } + }, + "responses": { + "200": { + "description": "success message", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/w/{workspace}/groups/list": { "get": { "summary": "list groups", @@ -10759,6 +12242,9 @@ "name": { "type": "string" }, + "summary": { + "type": "string" + }, "owners": { "type": "array", "items": { @@ -10815,6 +12301,9 @@ "schema": { "type": "object", "properties": { + "summary": { + "type": "string" + }, "owners": { "type": "array", "items": { @@ -11131,6 +12620,57 @@ } } }, + "/workers/queue_metrics": { + "get": { + "summary": "get queue metrics", + "operationId": "getQueueMetrics", + "tags": [ + "worker" + ], + "responses": { + "200": { + "description": "metrics", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "values": { + "type": "array", + "items": { + "type": "object", + "properties": { + "created_at": { + "type": "string" + }, + "value": { + "type": "number" + } + }, + "required": [ + "created_at", + "value" + ] + } + } + }, + "required": [ + "id", + "values" + ] + } + } + } + } + } + } + } + }, "/configs/list_worker_groups": { "get": { "summary": "list worker groups", @@ -11248,6 +12788,30 @@ } } }, + "/configs/list": { + "get": { + "summary": "list configs", + "operationId": "listConfigs", + "tags": [ + "config" + ], + "responses": { + "200": { + "description": "list of configs", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Config" + } + } + } + } + } + } + } + }, "/w/{workspace}/acls/get/{kind}/{path}": { "get": { "summary": "get granular acls", @@ -11277,7 +12841,8 @@ "flow", "folder", "app", - "raw_app" + "raw_app", + "http_trigger" ] } } @@ -11328,7 +12893,8 @@ "flow", "folder", "app", - "raw_app" + "raw_app", + "http_trigger" ] } } @@ -11398,7 +12964,8 @@ "flow", "folder", "app", - "raw_app" + "raw_app", + "http_trigger" ] } } @@ -11633,6 +13200,52 @@ } } }, + "/w/{workspace}/inputs/{jobOrInputId}/args": { + "get": { + "summary": "Get args from history or saved input", + "operationId": "getArgsFromHistoryOrSavedInput", + "tags": [ + "input" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "name": "jobOrInputId", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "input", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "allow_large", + "in": "query", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "args", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, "/w/{workspace}/inputs/list": { "get": { "summary": "List saved Inputs for a Runnable", @@ -12097,7 +13710,7 @@ }, "/w/{workspace}/job_helpers/test_connection": { "get": { - "summary": "Test connection to the workspace datasets storage", + "summary": "Test connection to the workspace object storage", "operationId": "datasetStorageTestConnection", "tags": [ "helpers" @@ -12105,6 +13718,13 @@ "parameters": [ { "$ref": "#/components/parameters/WorkspaceId" + }, + { + "name": "storage", + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -12121,7 +13741,7 @@ }, "/w/{workspace}/job_helpers/list_stored_files": { "get": { - "summary": "List the file keys available in the workspace files storage (S3)", + "summary": "List the file keys available in a workspace object storage", "operationId": "listStoredFiles", "tags": [ "helpers" @@ -12151,6 +13771,13 @@ "schema": { "type": "string" } + }, + { + "name": "storage", + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -12202,6 +13829,13 @@ "schema": { "type": "string" } + }, + { + "name": "storage", + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -12278,6 +13912,13 @@ "schema": { "type": "integer" } + }, + { + "name": "storage", + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -12349,6 +13990,13 @@ "schema": { "type": "string" } + }, + { + "name": "storage", + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -12363,6 +14011,144 @@ } } }, + "/w/{workspace}/job_helpers/load_table_count/{path}": { + "get": { + "summary": "Load the table row count", + "operationId": "loadTableRowCount", + "tags": [ + "helpers" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + }, + { + "name": "search_col", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "search_term", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "storage", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Table count", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "count": { + "type": "number" + } + } + } + } + } + } + } + } + }, + "/w/{workspace}/job_helpers/load_csv_preview/{path}": { + "get": { + "summary": "Load a preview of a csv file", + "operationId": "loadCsvPreview", + "tags": [ + "helpers" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/Path" + }, + { + "name": "offset", + "in": "query", + "schema": { + "type": "number" + } + }, + { + "name": "limit", + "in": "query", + "schema": { + "type": "number" + } + }, + { + "name": "sort_col", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "sort_desc", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "search_col", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "search_term", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "storage", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "csv_separator", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Csv Preview", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, "/w/{workspace}/job_helpers/delete_s3_file": { "delete": { "summary": "Permanently delete file from S3", @@ -12381,6 +14167,13 @@ "schema": { "type": "string" } + }, + { + "name": "storage", + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -12421,6 +14214,13 @@ "schema": { "type": "string" } + }, + { + "name": "storage", + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -12477,6 +14277,13 @@ "schema": { "type": "string" } + }, + { + "name": "storage", + "in": "query", + "schema": { + "type": "string" + } } ], "requestBody": { @@ -12547,6 +14354,13 @@ "schema": { "type": "string" } + }, + { + "name": "storage", + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -12564,6 +14378,56 @@ } } }, + "/w/{workspace}/job_helpers/download_s3_parquet_file_as_csv": { + "get": { + "summary": "Download file to S3 bucket", + "operationId": "fileDownloadParquetAsCsv", + "tags": [ + "helpers" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "name": "file_key", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "s3_resource_path", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "resource_type", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "The downloaded file", + "content": { + "text/csv": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/w/{workspace}/job_metrics/get/{id}": { "post": { "summary": "get job metrics", @@ -12637,6 +14501,182 @@ } } }, + "/w/{workspace}/job_metrics/set_progress/{id}": { + "post": { + "summary": "set job metrics", + "operationId": "setJobProgress", + "tags": [ + "metrics" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/JobId" + } + ], + "requestBody": { + "description": "parameters for statistics retrieval", + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "percent": { + "type": "integer" + }, + "flow_job_id": { + "type": "string", + "format": "uuid" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Job progress updated", + "content": { + "application/json": { + "schema": {} + } + } + } + } + } + }, + "/w/{workspace}/job_metrics/get_progress/{id}": { + "get": { + "summary": "get job progress", + "operationId": "getJobProgress", + "tags": [ + "metrics" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/JobId" + } + ], + "responses": { + "200": { + "description": "job progress between 0 and 99", + "content": { + "application/json": { + "schema": { + "type": "integer" + } + } + } + } + } + } + }, + "/service_logs/list_files": { + "get": { + "summary": "list log files ordered by timestamp", + "operationId": "listLogFiles", + "tags": [ + "service_logs" + ], + "parameters": [ + { + "$ref": "#/components/parameters/Before" + }, + { + "$ref": "#/components/parameters/After" + }, + { + "name": "with_error", + "in": "query", + "required": false, + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "time", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "hostname": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "worker_group": { + "type": "string" + }, + "log_ts": { + "type": "string", + "format": "date-time" + }, + "file_path": { + "type": "string" + }, + "ok_lines": { + "type": "integer" + }, + "err_lines": { + "type": "integer" + }, + "json_fmt": { + "type": "boolean" + } + }, + "required": [ + "hostname", + "mode", + "log_ts", + "file_path", + "json_fmt" + ] + } + } + } + } + } + } + } + }, + "/service_logs/get_log_file/{path}": { + "get": { + "summary": "get log file by path", + "operationId": "getLogFile", + "tags": [ + "service_logs" + ], + "parameters": [ + { + "$ref": "#/components/parameters/Path" + } + ], + "responses": { + "200": { + "description": "log stream", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/concurrency_groups/list": { "get": { "summary": "List all concurrency groups", @@ -12661,7 +14701,7 @@ } } }, - "/concurrency_groups/{concurrency_id}": { + "/concurrency_groups/prune/{concurrency_id}": { "delete": { "summary": "Delete concurrency group", "operationId": "deleteConcurrencyGroup", @@ -12687,6 +14727,238 @@ } } } + }, + "/concurrency_groups/{id}/key": { + "get": { + "summary": "Get the concurrency key for a job that has concurrency limits enabled", + "operationId": "getConcurrencyKey", + "tags": [ + "concurrencyGroups" + ], + "parameters": [ + { + "$ref": "#/components/parameters/JobId" + } + ], + "responses": { + "200": { + "description": "concurrency key for given job", + "content": { + "application/json": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/w/{workspace}/concurrency_groups/list_jobs": { + "get": { + "summary": "Get intervals of job runtime concurrency", + "operationId": "listExtendedJobs", + "tags": [ + "concurrencyGroups", + "job" + ], + "parameters": [ + { + "name": "concurrency_key", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "row_limit", + "in": "query", + "required": false, + "schema": { + "type": "number" + } + }, + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "$ref": "#/components/parameters/CreatedBy" + }, + { + "$ref": "#/components/parameters/Label" + }, + { + "$ref": "#/components/parameters/ParentJob" + }, + { + "$ref": "#/components/parameters/ScriptExactPath" + }, + { + "$ref": "#/components/parameters/ScriptStartPath" + }, + { + "$ref": "#/components/parameters/SchedulePath" + }, + { + "$ref": "#/components/parameters/ScriptExactHash" + }, + { + "$ref": "#/components/parameters/StartedBefore" + }, + { + "$ref": "#/components/parameters/StartedAfter" + }, + { + "$ref": "#/components/parameters/CreatedOrStartedBefore" + }, + { + "$ref": "#/components/parameters/Running" + }, + { + "$ref": "#/components/parameters/ScheduledForBeforeNow" + }, + { + "$ref": "#/components/parameters/CreatedOrStartedAfter" + }, + { + "$ref": "#/components/parameters/CreatedOrStartedAfterCompletedJob" + }, + { + "$ref": "#/components/parameters/JobKinds" + }, + { + "$ref": "#/components/parameters/ArgsFilter" + }, + { + "$ref": "#/components/parameters/Tag" + }, + { + "$ref": "#/components/parameters/ResultFilter" + }, + { + "$ref": "#/components/parameters/Page" + }, + { + "$ref": "#/components/parameters/PerPage" + }, + { + "name": "is_skipped", + "description": "is the job skipped", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "is_flow_step", + "description": "is the job a flow step", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "has_null_parent", + "description": "has null parent", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "success", + "description": "filter on successful jobs", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "all_workspaces", + "description": "get jobs from all workspaces (only valid if request come from the `admins` workspace)", + "in": "query", + "schema": { + "type": "boolean" + } + }, + { + "name": "is_not_schedule", + "description": "is not a scheduled job", + "in": "query", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "time", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExtendedJobs" + } + } + } + } + } + } + }, + "/srch/w/{workspace}/index/search/job": { + "get": { + "summary": "Search through jobs with a string query", + "operationId": "searchJobsIndex", + "tags": [ + "indexSearch" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "name": "search_query", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "search results", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "query_parse_errors": { + "description": "a list of the terms that couldn't be parsed (and thus ignored)", + "type": "array", + "items": { + "type": "object", + "properties": { + "dancer": { + "type": "string" + } + } + } + }, + "hits": { + "description": "the jobs that matched the query", + "type": "array", + "items": { + "$ref": "#/components/schemas/JobSearchHit" + } + } + } + } + } + } + } + } + } } }, "components": { @@ -12839,6 +15111,14 @@ "type": "string" } }, + "Label": { + "name": "label", + "description": "mask to filter exact matching job's label (job labels are completed jobs with as a result an object containing a string in the array at key 'wm_labels')", + "in": "query", + "schema": { + "type": "string" + } + }, "ParentJob": { "name": "parent_job", "description": "The parent job that is at the origin and responsible for the execution of this script if any", @@ -12929,6 +15209,24 @@ "type": "string" } }, + "CreatedBefore": { + "name": "created_before", + "description": "filter on created before (inclusive) timestamp", + "in": "query", + "schema": { + "type": "string", + "format": "date-time" + } + }, + "CreatedAfter": { + "name": "created_after", + "description": "filter on created after (exclusive) timestamp", + "in": "query", + "schema": { + "type": "string", + "format": "date-time" + } + }, "StartedBefore": { "name": "started_before", "description": "filter on started before (inclusive) timestamp", @@ -12947,6 +15245,15 @@ "format": "date-time" } }, + "Before": { + "name": "before", + "description": "filter on started before (inclusive) timestamp", + "in": "query", + "schema": { + "type": "string", + "format": "date-time" + } + }, "CreatedOrStartedAfter": { "name": "created_or_started_after", "description": "filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp", @@ -12956,6 +15263,15 @@ "format": "date-time" } }, + "CreatedOrStartedAfterCompletedJob": { + "name": "created_or_started_after_completed_jobs", + "description": "filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp but only for the completed jobs", + "in": "query", + "schema": { + "type": "string", + "format": "date-time" + } + }, "CreatedOrStartedBefore": { "name": "created_or_started_before", "description": "filter on created_at for non non started job and started_at otherwise before (inclusive) timestamp", @@ -13030,15 +15346,6 @@ "format": "date-time" } }, - "Before": { - "name": "before", - "description": "filter on created before (exclusive) timestamp", - "in": "query", - "schema": { - "type": "string", - "format": "date-time" - } - }, "Username": { "name": "username", "description": "filter on exact username of user", @@ -13198,7 +15505,10 @@ "mssql", "graphql", "nativets", - "bun" + "bun", + "php", + "rust", + "ansible" ] }, "kind": { @@ -13235,6 +15545,9 @@ "concurrency_time_window_s": { "type": "integer" }, + "concurrency_key": { + "type": "string" + }, "cache_ttl": { "type": "number" }, @@ -13258,6 +15571,15 @@ }, "visible_to_runner_only": { "type": "boolean" + }, + "no_main_func": { + "type": "boolean" + }, + "codebase": { + "type": "string" + }, + "has_preprocessor": { + "type": "boolean" } }, "required": [ @@ -13274,7 +15596,9 @@ "extra_perms", "language", "kind", - "starred" + "starred", + "no_main_func", + "has_preprocessor" ] }, "NewScript": { @@ -13319,7 +15643,10 @@ "mssql", "graphql", "nativets", - "bun" + "bun", + "php", + "rust", + "ansible" ] }, "kind": { @@ -13379,6 +15706,15 @@ }, "visible_to_runner_only": { "type": "boolean" + }, + "no_main_func": { + "type": "boolean" + }, + "codebase": { + "type": "string" + }, + "has_preprocessor": { + "type": "boolean" } }, "required": [ @@ -13437,9 +15773,6 @@ "name": { "type": "string" }, - "args": { - "type": "object" - }, "created_by": { "type": "string" }, @@ -13613,7 +15946,10 @@ "mssql", "graphql", "nativets", - "bun" + "bun", + "php", + "rust", + "ansible" ] }, "email": { @@ -13630,6 +15966,15 @@ }, "priority": { "type": "integer" + }, + "self_wait_time_ms": { + "type": "number" + }, + "aggregate_wait_time_ms": { + "type": "number" + }, + "suspend": { + "type": "number" } }, "required": [ @@ -13750,7 +16095,10 @@ "mssql", "graphql", "nativets", - "bun" + "bun", + "php", + "rust", + "ansible" ] }, "is_skipped": { @@ -13770,6 +16118,18 @@ }, "priority": { "type": "integer" + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "self_wait_time_ms": { + "type": "number" + }, + "aggregate_wait_time_ms": { + "type": "number" } }, "required": [ @@ -13789,29 +16149,58 @@ "tag" ] }, + "ObscuredJob": { + "type": "object", + "properties": { + "typ": { + "type": "string" + }, + "started_at": { + "type": "string", + "format": "date-time" + }, + "duration_ms": { + "type": "number" + } + } + }, "Job": { - "allOf": [ + "oneOf": [ { - "oneOf": [ + "allOf": [ { "$ref": "#/components/schemas/CompletedJob" }, { - "$ref": "#/components/schemas/QueuedJob" + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "CompletedJob" + ] + } + } } ] }, { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "CompletedJob", - "QueuedJob" - ] + "allOf": [ + { + "$ref": "#/components/schemas/QueuedJob" + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "QueuedJob" + ] + } + } } - } + ] } ], "discriminator": { @@ -13940,6 +16329,9 @@ "items": { "type": "string" } + }, + "email": { + "type": "string" } }, "required": [ @@ -13963,6 +16355,9 @@ "items": { "type": "string" } + }, + "workspace_id": { + "type": "string" } } }, @@ -13978,6 +16373,9 @@ }, "impersonate_email": { "type": "string" + }, + "workspace_id": { + "type": "string" } }, "required": [ @@ -14025,6 +16423,10 @@ }, "is_refreshed": { "type": "boolean" + }, + "expires_at": { + "type": "string", + "format": "date-time" } }, "required": [ @@ -14077,6 +16479,10 @@ }, "is_oauth": { "type": "boolean" + }, + "expires_at": { + "type": "string", + "format": "date-time" } }, "required": [ @@ -14396,6 +16802,14 @@ "typ" ] } + }, + "no_main_func": { + "type": "boolean", + "nullable": true + }, + "has_preprocessor": { + "type": "boolean", + "nullable": true } }, "required": [ @@ -14403,7 +16817,9 @@ "start_kwargs", "args", "type", - "error" + "error", + "no_main_func", + "has_preprocessor" ] }, "Preview": { @@ -14433,7 +16849,10 @@ "mssql", "graphql", "nativets", - "bun" + "bun", + "php", + "rust", + "ansible" ] }, "tag": { @@ -14550,6 +16969,13 @@ "additionalProperties": { "type": "boolean" } + }, + "created_by": { + "type": "string" + }, + "edited_at": { + "type": "string", + "format": "date-time" } }, "required": [ @@ -14597,6 +17023,13 @@ }, "account": { "type": "number" + }, + "created_by": { + "type": "string" + }, + "edited_at": { + "type": "string", + "format": "date-time" } }, "required": [ @@ -14619,6 +17052,16 @@ "schema": {}, "description": { "type": "string" + }, + "created_by": { + "type": "string" + }, + "edited_at": { + "type": "string", + "format": "date-time" + }, + "format_extension": { + "type": "string" } }, "required": [ @@ -14698,6 +17141,12 @@ "on_recovery_extra_args": { "$ref": "#/components/schemas/ScriptArgs" }, + "on_success": { + "type": "string" + }, + "on_success_extra_args": { + "$ref": "#/components/schemas/ScriptArgs" + }, "ws_error_handler_muted": { "type": "boolean" }, @@ -14712,6 +17161,10 @@ }, "tag": { "type": "string" + }, + "paused_until": { + "type": "string", + "format": "date-time" } }, "required": [ @@ -14806,6 +17259,12 @@ "on_recovery_extra_args": { "$ref": "#/components/schemas/ScriptArgs" }, + "on_success": { + "type": "string" + }, + "on_success_extra_args": { + "$ref": "#/components/schemas/ScriptArgs" + }, "ws_error_handler_muted": { "type": "boolean" }, @@ -14820,6 +17279,10 @@ }, "tag": { "type": "string" + }, + "paused_until": { + "type": "string", + "format": "date-time" } }, "required": [ @@ -14864,6 +17327,12 @@ "on_recovery_extra_args": { "$ref": "#/components/schemas/ScriptArgs" }, + "on_success": { + "type": "string" + }, + "on_success_extra_args": { + "$ref": "#/components/schemas/ScriptArgs" + }, "ws_error_handler_muted": { "type": "boolean" }, @@ -14878,6 +17347,10 @@ }, "tag": { "type": "string" + }, + "paused_until": { + "type": "string", + "format": "date-time" } }, "required": [ @@ -14888,6 +17361,181 @@ "args" ] }, + "HttpTrigger": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "edited_by": { + "type": "string" + }, + "edited_at": { + "type": "string", + "format": "date-time" + }, + "script_path": { + "type": "string" + }, + "route_path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + }, + "extra_perms": { + "type": "object", + "additionalProperties": { + "type": "boolean" + } + }, + "email": { + "type": "string" + }, + "workspace_id": { + "type": "string" + }, + "http_method": { + "type": "string", + "enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + }, + "is_async": { + "type": "boolean" + }, + "requires_auth": { + "type": "boolean" + } + }, + "required": [ + "path", + "edited_by", + "edited_at", + "script_path", + "route_path", + "extra_perms", + "is_flow", + "email", + "workspace_id", + "is_async", + "requires_auth", + "http_method" + ] + }, + "NewHttpTrigger": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "script_path": { + "type": "string" + }, + "route_path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + }, + "http_method": { + "type": "string", + "enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + }, + "is_async": { + "type": "boolean" + }, + "requires_auth": { + "type": "boolean" + } + }, + "required": [ + "path", + "script_path", + "route_path", + "is_flow", + "is_async", + "requires_auth", + "http_method" + ] + }, + "EditHttpTrigger": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "script_path": { + "type": "string" + }, + "route_path": { + "type": "string" + }, + "is_flow": { + "type": "boolean" + }, + "http_method": { + "type": "string", + "enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + }, + "is_async": { + "type": "boolean" + }, + "requires_auth": { + "type": "boolean" + } + }, + "required": [ + "path", + "script_path", + "is_flow", + "kind", + "is_async", + "requires_auth", + "http_method" + ] + }, + "TriggersCount": { + "type": "object", + "properties": { + "primary_schedule": { + "type": "object", + "properties": { + "schedule": { + "type": "string" + } + } + }, + "schedule_count": { + "type": "number" + }, + "http_routes_count": { + "type": "number" + }, + "webhook_count": { + "type": "number" + }, + "email_count": { + "type": "number" + } + } + }, "Group": { "type": "object", "properties": { @@ -14951,6 +17599,16 @@ "additionalProperties": { "type": "boolean" } + }, + "summary": { + "type": "string" + }, + "created_by": { + "type": "string" + }, + "edited_at": { + "type": "string", + "format": "date-time" } }, "required": [ @@ -14992,6 +17650,36 @@ }, "wm_version": { "type": "string" + }, + "last_job_id": { + "type": "string" + }, + "last_job_workspace_id": { + "type": "string" + }, + "occupancy_rate": { + "type": "number" + }, + "occupancy_rate_15s": { + "type": "number" + }, + "occupancy_rate_5m": { + "type": "number" + }, + "occupancy_rate_30m": { + "type": "number" + }, + "memory": { + "type": "number" + }, + "vcpus": { + "type": "number" + }, + "memory_usage": { + "type": "number" + }, + "wm_memory_usage": { + "type": "number" } }, "required": [ @@ -15148,6 +17836,12 @@ } ] }, + "ExtraPerms": { + "type": "object", + "additionalProperties": { + "type": "boolean" + } + }, "FlowMetadata": { "type": "object", "properties": { @@ -15168,10 +17862,7 @@ "type": "boolean" }, "extra_perms": { - "type": "object" - }, - "additionalProperties": { - "type": "boolean" + "$ref": "#/components/schemas/ExtraPerms" }, "starred": { "type": "boolean" @@ -15291,6 +17982,12 @@ "type": "object" } }, + "triggerables_v2": { + "type": "object", + "additionalProperties": { + "type": "object" + } + }, "execution_mode": { "type": "string", "enum": [ @@ -15424,7 +18121,9 @@ "type": "string", "format": "date-time" }, - "value": {}, + "value": { + "type": "object" + }, "policy": { "$ref": "#/components/schemas/Policy" }, @@ -15487,6 +18186,25 @@ "version" ] }, + "FlowVersion": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "deployment_msg": { + "type": "string" + } + }, + "required": [ + "id", + "created_at" + ] + }, "SlackToken": { "type": "object", "properties": { @@ -15581,6 +18299,32 @@ }, "public_resource": { "type": "boolean" + }, + "secondary_storage": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "S3Storage", + "AzureBlobStorage", + "AzureWorkloadIdentity", + "S3AwsOidc" + ] + }, + "s3_resource_path": { + "type": "string" + }, + "azure_blob_resource_path": { + "type": "string" + }, + "public_resource": { + "type": "boolean" + } + } + } } } }, @@ -15709,6 +18453,31 @@ } } }, + "WorkspaceDeployUISettings": { + "type": "object", + "properties": { + "include_path": { + "type": "array", + "items": { + "type": "string" + } + }, + "include_type": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "script", + "flow", + "app", + "resource", + "variable", + "secret" + ] + } + } + } + }, "WorkspaceDefaultScripts": { "type": "object", "properties": { @@ -15873,7 +18642,10 @@ "mssql", "graphql", "nativets", - "bun" + "bun", + "php", + "rust", + "ansible" ] } }, @@ -15886,21 +18658,144 @@ "ConcurrencyGroup": { "type": "object", "properties": { - "concurrency_id": { + "concurrency_key": { "type": "string" }, - "job_uuids": { + "total_running": { + "type": "number" + } + }, + "required": [ + "concurrency_key", + "total_running" + ] + }, + "ExtendedJobs": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Job" + } + }, + "obscured_jobs": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ObscuredJob" + } + }, + "omitted_obscured_jobs": { + "description": "Obscured jobs omitted for security because of too specific filtering", + "type": "boolean" + } + }, + "required": [ + "jobs", + "obscured_jobs" + ] + }, + "ExportedUser": { + "type": "object", + "properties": { + "email": { + "type": "string" + }, + "password_hash": { + "type": "string" + }, + "super_admin": { + "type": "boolean" + }, + "verified": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "company": { + "type": "string" + }, + "first_time_user": { + "type": "boolean" + }, + "username": { + "type": "string" + } + }, + "required": [ + "email", + "super_admin", + "verified", + "first_time_user" + ] + }, + "GlobalSetting": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "object" + } + }, + "required": [ + "name", + "value" + ] + }, + "Config": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "config": { + "type": "object" + } + }, + "required": [ + "name" + ] + }, + "ExportedInstanceGroup": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "summary": { + "type": "string" + }, + "emails": { "type": "array", "items": { "type": "string" } + }, + "id": { + "type": "string" + }, + "scim_display_name": { + "type": "string" + }, + "external_id": { + "type": "string" } }, "required": [ - "concurrency_id", - "job_uuids" + "name" ] }, + "JobSearchHit": { + "type": "object", + "properties": { + "dancer": { + "type": "string" + } + } + }, "StaticTransform": { "type": "object", "properties": { @@ -15979,7 +18874,8 @@ "snowflake", "mssql", "graphql", - "nativets" + "nativets", + "php" ] }, "path": { @@ -16002,6 +18898,9 @@ }, "concurrency_time_window_s": { "type": "number" + }, + "custom_concurrency_key": { + "type": "string" } }, "required": [ @@ -16031,6 +18930,9 @@ "enum": [ "script" ] + }, + "tag_override": { + "type": "string" } }, "required": [ @@ -16087,6 +18989,31 @@ "expr" ] }, + "stop_after_all_iters_if": { + "type": "object", + "properties": { + "skip_if_stopped": { + "type": "boolean" + }, + "expr": { + "type": "string" + } + }, + "required": [ + "expr" + ] + }, + "skip_if": { + "type": "object", + "properties": { + "expr": { + "type": "string" + } + }, + "required": [ + "expr" + ] + }, "sleep": { "$ref": "#/components/schemas/InputTransform" }, @@ -16139,6 +19066,9 @@ }, "hide_cancel": { "type": "boolean" + }, + "continue_on_disapprove_timeout": { + "type": "boolean" } } }, @@ -16419,12 +19349,18 @@ "failure_module": { "$ref": "#/components/schemas/FlowModule" }, + "preprocessor_module": { + "$ref": "#/components/schemas/FlowModule" + }, "same_worker": { "type": "boolean" }, "concurrent_limit": { "type": "number" }, + "concurrency_key": { + "type": "string" + }, "concurrency_time_window_s": { "type": "number" }, @@ -16490,6 +19426,9 @@ "count": { "type": "integer" }, + "progress": { + "type": "integer" + }, "iterator": { "type": "object", "properties": { @@ -16509,6 +19448,12 @@ "type": "string" } }, + "flow_jobs_success": { + "type": "array", + "items": { + "type": "boolean" + } + }, "branch_chosen": { "type": "object", "properties": { @@ -16559,6 +19504,16 @@ "approver" ] } + }, + "failed_retries": { + "type": "array", + "items": { + "type": "string", + "format": "uuid" + } + }, + "skipped": { + "type": "boolean" } }, "required": [ @@ -16580,6 +19535,13 @@ "user_states": { "additionalProperties": true }, + "preprocessor_module": { + "allOf": [ + { + "$ref": "#/components/schemas/FlowStatusModule" + } + ] + }, "failure_module": { "allOf": [ { diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 9654b10550..e2a0b00a85 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -1,6 +1,6 @@ openapi: 3.0.3 info: - version: 1.269.0 + version: 1.409.0 title: Windmill API contact: name: Windmill Team @@ -87,7 +87,7 @@ paths: - name: id in: path required: true - schema: &ref_24 + schema: &ref_28 type: integer responses: '200': @@ -221,44 +221,54 @@ paths: - name: page description: which page to return (start at 1, default 1) in: query - schema: &ref_3 + schema: &ref_5 type: integer - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: &ref_4 + schema: &ref_6 type: integer - name: before - description: filter on created before (exclusive) timestamp + description: filter on started before (inclusive) timestamp in: query - schema: &ref_102 + schema: &ref_109 type: string format: date-time - name: after description: filter on created after (exclusive) timestamp in: query - schema: &ref_101 + schema: &ref_110 type: string format: date-time - name: username description: filter on exact username of user in: query - schema: &ref_103 + schema: &ref_119 type: string - name: operation description: filter on exact or prefix name of operation in: query - schema: &ref_104 + schema: &ref_120 + type: string + - name: operations + in: query + description: comma separated list of exact operations to include + schema: + type: string + - name: exclude_operations + in: query + description: comma separated list of operations to exclude + schema: type: string - name: resource description: filter on exact or prefix name of resource in: query - schema: &ref_105 + schema: &ref_121 type: string - name: action_kind description: filter on type of operation in: query - schema: &ref_106 + schema: &ref_122 type: string enum: - Create @@ -290,12 +300,12 @@ paths: application/json: schema: type: object - properties: &ref_119 + properties: &ref_136 email: type: string password: type: string - required: &ref_120 + required: &ref_137 - email - password responses: @@ -332,10 +342,10 @@ paths: text/plain: schema: type: string - /w/{workspace}/users/add: - post: - summary: create user (require admin privilege) - operationId: createUser + /w/{workspace}/users/{username}: + get: + summary: get user (require admin privilege) + operationId: getUser tags: - user - admin @@ -344,31 +354,56 @@ paths: in: path required: true schema: *ref_0 - requestBody: - description: new user - required: true - content: - application/json: - schema: - type: object - properties: &ref_121 - email: - type: string - username: - type: string - is_admin: - type: boolean - required: &ref_122 - - email - - username - - is_admin + - name: username + in: path + required: true + schema: + type: string responses: - '201': + '200': description: user created content: - text/plain: + application/json: schema: - type: string + type: object + properties: &ref_10 + email: + type: string + username: + type: string + is_admin: + type: boolean + is_super_admin: + type: boolean + created_at: + type: string + format: date-time + operator: + type: boolean + disabled: + type: boolean + groups: + type: array + items: + type: string + folders: + type: array + items: + type: string + folders_owners: + type: array + items: + type: string + required: &ref_11 + - email + - username + - is_admin + - is_super_admin + - created_at + - operator + - disabled + - folders + - folders_owners /w/{workspace}/users/update/{username}: post: summary: update user (require admin privilege) @@ -393,7 +428,7 @@ paths: application/json: schema: type: object - properties: &ref_123 + properties: &ref_138 is_admin: type: boolean operator: @@ -421,7 +456,7 @@ paths: - name: path in: path required: true - schema: &ref_17 + schema: &ref_21 type: string responses: '200': @@ -512,6 +547,8 @@ paths: properties: is_super_admin: type: boolean + name: + type: string responses: '200': description: user updated @@ -519,6 +556,74 @@ paths: text/plain: schema: type: string + /users/username_info/{email}: + get: + summary: global username info (require super admin) + operationId: globalUsernameInfo + tags: + - user + parameters: + - name: email + in: path + required: true + schema: + type: string + responses: + '200': + description: user renamed + content: + application/json: + schema: + type: object + properties: + username: + type: string + workspace_usernames: + type: array + items: + type: object + properties: + workspace_id: + type: string + username: + type: string + required: + - workspace_id + - username + required: + - username + - workspace_usernames + /users/rename/{email}: + post: + summary: global rename user (require super admin) + operationId: globalUserRename + tags: + - user + parameters: + - name: email + in: path + required: true + schema: + type: string + requestBody: + description: new username + required: true + content: + application/json: + schema: + type: object + properties: + new_username: + type: string + required: + - new_username + responses: + '200': + description: user renamed + content: + text/plain: + schema: + type: string /users/delete/{email}: delete: summary: global delete user (require super admin) @@ -538,6 +643,67 @@ paths: text/plain: schema: type: string + /users/overwrite: + post: + summary: global overwrite users (require super admin and EE) + operationId: globalUsersOverwrite + tags: + - user + requestBody: + description: List of users + required: true + content: + application/json: + schema: + type: array + items: + type: object + properties: &ref_3 + email: + type: string + password_hash: + type: string + super_admin: + type: boolean + verified: + type: boolean + name: + type: string + company: + type: string + first_time_user: + type: boolean + username: + type: string + required: &ref_4 + - email + - super_admin + - verified + - first_time_user + responses: + '200': + description: Success message + content: + text/plain: + schema: + type: string + /users/export: + get: + summary: global export users (require super admin and EE) + operationId: globalUsersExport + tags: + - user + responses: + '200': + description: exported users + content: + application/json: + schema: + type: array + items: + type: object + properties: *ref_3 + required: *ref_4 /w/{workspace}/users/delete/{username}: delete: summary: delete user (require admin privilege) @@ -577,7 +743,7 @@ paths: type: array items: type: object - properties: &ref_5 + properties: &ref_7 id: type: string name: @@ -586,7 +752,7 @@ paths: type: string domain: type: string - required: &ref_6 + required: &ref_8 - id - name - owner @@ -616,7 +782,7 @@ paths: application/json: schema: type: object - properties: &ref_151 + properties: &ref_172 email: type: string workspaces: @@ -634,7 +800,7 @@ paths: - id - name - username - required: &ref_152 + required: &ref_173 - email - workspaces /workspaces/list_as_superadmin: @@ -647,11 +813,11 @@ paths: - name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 responses: '200': description: workspaces @@ -661,8 +827,8 @@ paths: type: array items: type: object - properties: *ref_5 - required: *ref_6 + properties: *ref_7 + required: *ref_8 /workspaces/create: post: summary: create workspace @@ -676,17 +842,16 @@ paths: application/json: schema: type: object - properties: &ref_153 + properties: &ref_174 id: type: string name: type: string username: type: string - required: &ref_154 + required: &ref_175 - id - name - - username responses: '201': description: token created @@ -756,7 +921,7 @@ paths: - name: key in: path required: true - schema: &ref_7 + schema: &ref_9 type: string responses: '200': @@ -773,7 +938,7 @@ paths: - name: key in: path required: true - schema: *ref_7 + schema: *ref_9 requestBody: description: value set required: true @@ -850,6 +1015,33 @@ paths: text/plain: schema: type: string + /settings/test_critical_channels: + post: + summary: test critical channels + operationId: testCriticalChannels + tags: + - setting + requestBody: + description: test critical channel payload + required: true + content: + application/json: + schema: + type: array + items: + type: object + properties: + email: + type: string + slack_channel: + type: string + responses: + '200': + description: status + content: + text/plain: + schema: + type: string /settings/test_license_key: post: summary: test license key @@ -875,6 +1067,27 @@ paths: text/plain: schema: type: string + /settings/test_object_storage_config: + post: + summary: test object storage config + operationId: testObjectStorageConfig + tags: + - setting + requestBody: + description: test object storage config + required: true + content: + application/json: + schema: + type: object + additionalProperties: true + responses: + '200': + description: status + content: + text/plain: + schema: + type: string /settings/send_stats: post: summary: send stats @@ -888,6 +1101,110 @@ paths: text/plain: schema: type: string + /settings/latest_key_renewal_attempt: + get: + summary: get latest key renewal attempt + operationId: getLatestKeyRenewalAttempt + tags: + - setting + responses: + '200': + description: status + content: + application/json: + schema: + type: object + properties: + result: + type: string + attempted_at: + type: string + format: date-time + required: + - result + - attempted_at + nullable: true + /settings/renew_license_key: + post: + summary: renew license key + operationId: renewLicenseKey + tags: + - setting + parameters: + - name: license_key + in: query + required: false + schema: + type: string + responses: + '200': + description: status + content: + text/plain: + schema: + type: string + /settings/customer_portal: + post: + summary: create customer portal session + operationId: createCustomerPortalSession + tags: + - setting + parameters: + - name: license_key + in: query + required: false + schema: + type: string + responses: + '200': + description: url to portal + content: + text/plain: + schema: + type: string + /saml/test_metadata: + post: + summary: test metadata + operationId: testMetadata + tags: + - setting + requestBody: + description: test metadata + required: true + content: + application/json: + schema: + type: string + responses: + '200': + description: status + content: + text/plain: + schema: + type: string + /settings/list_global: + get: + summary: list global settings + operationId: listGlobalSettings + tags: + - setting + responses: + '200': + description: list of settings + content: + application/json: + schema: + type: array + items: + type: object + properties: &ref_214 + name: + type: string + value: + type: object + required: &ref_215 + - name + - value /users/email: get: summary: get current user email (if logged in) @@ -1026,7 +1343,7 @@ paths: application/json: schema: type: object - properties: &ref_10 + properties: &ref_12 email: type: string login_type: @@ -1042,7 +1359,9 @@ paths: type: string company: type: string - required: &ref_11 + username: + type: string + required: &ref_13 - email - login_type - super_admin @@ -1062,7 +1381,7 @@ paths: type: array items: type: object - properties: &ref_12 + properties: &ref_14 workspace_id: type: string email: @@ -1071,7 +1390,7 @@ paths: type: boolean operator: type: boolean - required: &ref_13 + required: &ref_15 - workspace_id - email - is_admin @@ -1094,49 +1413,8 @@ paths: application/json: schema: type: object - properties: &ref_8 - email: - type: string - username: - type: string - is_admin: - type: boolean - is_super_admin: - type: boolean - created_at: - type: string - format: date-time - operator: - type: boolean - disabled: - type: boolean - groups: - type: array - items: - type: string - folders: - type: array - items: - type: string - folders_owners: - type: array - items: - type: string - usage: - type: object - properties: &ref_118 - executions: - type: number - required: &ref_9 - - email - - username - - is_admin - - is_super_admin - - created_at - - operator - - disabled - - folders - - folders_owners + properties: *ref_10 + required: *ref_11 /users/accept_invite: post: summary: accept invite to workspace @@ -1157,7 +1435,6 @@ paths: type: string required: - workspace_id - - username responses: '200': description: status @@ -1257,7 +1534,6 @@ paths: - email - is_admin - operator - - username responses: '200': description: status @@ -1373,6 +1649,82 @@ paths: text/plain: schema: type: string + /w/{workspace}/workspaces/get_workspace_name: + get: + summary: get workspace name + operationId: getWorkspaceName + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: status + content: + text/plain: + schema: + type: string + /w/{workspace}/workspaces/change_workspace_name: + post: + summary: change workspace name + operationId: changeWorkspaceName + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + content: + application/json: + schema: + type: object + properties: + new_name: + type: string + required: + - username + responses: + '200': + description: status + content: + text/plain: + schema: + type: string + /w/{workspace}/workspaces/change_workspace_id: + post: + summary: change workspace id + operationId: changeWorkspaceId + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + content: + application/json: + schema: + type: object + properties: + new_id: + type: string + new_name: + type: string + required: + - username + responses: + '200': + description: status + content: + text/plain: + schema: + type: string /w/{workspace}/users/whois/{username}: get: summary: whois @@ -1396,8 +1748,8 @@ paths: application/json: schema: type: object - properties: *ref_8 - required: *ref_9 + properties: *ref_10 + required: *ref_11 /users/exists/{email}: get: summary: exists email @@ -1427,11 +1779,11 @@ paths: - name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 responses: '200': description: user @@ -1441,8 +1793,8 @@ paths: type: array items: type: object - properties: *ref_10 - required: *ref_11 + properties: *ref_12 + required: *ref_13 /w/{workspace}/workspaces/list_pending_invites: get: summary: list pending invites for a workspace @@ -1463,8 +1815,8 @@ paths: type: array items: type: object - properties: *ref_12 - required: *ref_13 + properties: *ref_14 + required: *ref_15 /w/{workspace}/workspaces/get_settings: get: summary: get settings @@ -1500,6 +1852,8 @@ paths: type: boolean plan: type: string + automatic_billing: + type: boolean customer_id: type: string webhook: @@ -1514,26 +1868,46 @@ paths: type: string error_handler_extra_args: type: object - additionalProperties: &ref_14 {} + additionalProperties: &ref_16 {} error_handler_muted_on_cancel: type: boolean large_file_storage: type: object - properties: &ref_15 + properties: &ref_17 type: type: string enum: - S3Storage - AzureBlobStorage + - AzureWorkloadIdentity + - S3AwsOidc s3_resource_path: type: string azure_blob_resource_path: type: string public_resource: type: boolean + secondary_storage: + type: object + additionalProperties: + type: object + properties: + type: + type: string + enum: + - S3Storage + - AzureBlobStorage + - AzureWorkloadIdentity + - S3AwsOidc + s3_resource_path: + type: string + azure_blob_resource_path: + type: string + public_resource: + type: boolean git_sync: type: object - properties: &ref_16 + properties: &ref_18 include_path: type: array items: @@ -1552,24 +1926,78 @@ paths: - secret - resourcetype - schedule + - user + - group repositories: type: array items: type: object - properties: &ref_174 + properties: &ref_198 script_path: type: string git_repo_resource_path: type: string use_individual_branch: type: boolean - required: &ref_175 + group_by_folder: + type: boolean + exclude_types_override: + type: array + items: + type: string + enum: + - script + - flow + - app + - folder + - resource + - variable + - secret + - resourcetype + - schedule + - user + - group + required: &ref_199 - script_path - git_repo_resource_path + deploy_ui: + type: object + properties: &ref_19 + include_path: + type: array + items: + type: string + include_type: + type: array + items: + type: string + enum: + - script + - flow + - app + - resource + - variable + - secret default_app: type: string + default_scripts: + type: object + properties: &ref_20 + order: + type: array + items: + type: string + hidden: + type: array + items: + type: string + default_script_content: + additionalProperties: + type: string required: - code_completion_enabled + - automatic_billing + - error_handler_muted_on_cancel /w/{workspace}/workspaces/get_deploy_to: get: summary: get deploy to @@ -1634,8 +2062,43 @@ paths: type: number seats: type: number + automatic_billing: + type: boolean required: - premium + - automatic_billing + /w/{workspace}/workspaces/set_automatic_billing: + post: + summary: set automatic billing + operationId: setAutomaticBilling + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: automatic billing + required: true + content: + application/json: + schema: + type: object + properties: + automatic_billing: + type: boolean + seats: + type: number + required: + - automatic_billing + responses: + '200': + description: status + content: + text/plain: + schema: + type: string /w/{workspace}/workspaces/edit_slack_command: post: summary: edit slack command @@ -1867,7 +2330,7 @@ paths: type: string error_handler_extra_args: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 error_handler_muted_on_cancel: type: boolean responses: @@ -1898,7 +2361,7 @@ paths: properties: large_file_storage: type: object - properties: *ref_15 + properties: *ref_17 responses: '200': description: status @@ -1926,7 +2389,35 @@ paths: properties: git_sync_settings: type: object - properties: *ref_16 + properties: *ref_18 + responses: + '200': + description: status + content: + application/json: + schema: {} + /w/{workspace}/workspaces/edit_deploy_ui_config: + post: + summary: edit workspace deploy ui settings + operationId: editWorkspaceDeployUISettings + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: Workspace deploy UI settings + required: true + content: + application/json: + schema: + type: object + properties: + deploy_ui_settings: + type: object + properties: *ref_19 responses: '200': description: status @@ -1961,6 +2452,81 @@ paths: text/plain: schema: type: string + /w/{workspace}/workspaces/default_scripts: + post: + summary: edit default scripts for workspace + operationId: editDefaultScripts + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: Workspace default app + content: + application/json: + schema: + type: object + properties: *ref_20 + responses: + '200': + description: status + content: + text/plain: + schema: + type: string + get: + summary: get default scripts for workspace + operationId: get default scripts + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: status + content: + application/json: + schema: + type: object + properties: *ref_20 + /w/{workspace}/workspaces/set_environment_variable: + post: + summary: set environment variable + operationId: setEnvironmentVariable + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: Workspace default app + required: true + content: + application/json: + schema: + type: object + properties: + name: + type: string + value: + type: string + required: + - name + responses: + '200': + description: status + content: + text/plain: + schema: + type: string /w/{workspace}/workspaces/encryption_key: get: summary: retrieves the encryption key for this workspace @@ -2004,6 +2570,8 @@ paths: properties: new_key: type: string + skip_reencrypt: + type: boolean required: - new_key responses: @@ -2052,7 +2620,25 @@ paths: application/json: schema: type: object - properties: *ref_15 + properties: *ref_17 + /w/{workspace}/workspaces/usage: + get: + summary: get usage + operationId: getWorkspaceUsage + tags: + - workspace + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: usage + content: + text/plain: + schema: + type: number /w/{workspace}/users/list: get: summary: list users @@ -2073,8 +2659,33 @@ paths: type: array items: type: object - properties: *ref_8 - required: *ref_9 + properties: *ref_10 + required: *ref_11 + /w/{workspace}/users/list_usage: + get: + summary: list users usage + operationId: listUsersUsage + tags: + - user + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: user + content: + application/json: + schema: + type: array + items: + type: object + properties: &ref_135 + email: + type: string + executions: + type: number /w/{workspace}/users/list_usernames: get: summary: list usernames @@ -2095,6 +2706,29 @@ paths: type: array items: type: string + /w/{workspace}/users/username_to_email/{username}: + get: + summary: get email from username + operationId: usernameToEmail + tags: + - user + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: username + in: path + required: true + schema: + type: string + responses: + '200': + description: email + content: + text/plain: + schema: + type: string /users/tokens/create: post: summary: create token @@ -2108,7 +2742,7 @@ paths: application/json: schema: type: object - properties: &ref_126 + properties: &ref_139 label: type: string expiration: @@ -2118,6 +2752,8 @@ paths: type: array items: type: string + workspace_id: + type: string responses: '201': description: token created @@ -2138,7 +2774,7 @@ paths: application/json: schema: type: object - properties: &ref_127 + properties: &ref_140 label: type: string expiration: @@ -2146,7 +2782,9 @@ paths: format: date-time impersonate_email: type: string - required: &ref_128 + workspace_id: + type: string + required: &ref_141 - impersonate_email responses: '201': @@ -2185,6 +2823,14 @@ paths: in: query schema: type: boolean + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 responses: '200': description: truncated token @@ -2194,7 +2840,7 @@ paths: type: array items: type: object - properties: &ref_124 + properties: &ref_38 label: type: string expiration: @@ -2212,7 +2858,9 @@ paths: type: array items: type: string - required: &ref_125 + email: + type: string + required: &ref_39 - token_prefix - created_at - last_used_at @@ -2261,7 +2909,7 @@ paths: application/json: schema: type: object - properties: &ref_131 + properties: &ref_144 path: type: string value: @@ -2274,7 +2922,10 @@ paths: type: integer is_oauth: type: boolean - required: &ref_132 + expires_at: + type: string + format: date-time + required: &ref_145 - path - value - is_secret @@ -2325,7 +2976,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: variable deleted @@ -2347,7 +2998,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 - name: already_encrypted in: query schema: @@ -2359,7 +3010,7 @@ paths: application/json: schema: type: object - properties: &ref_133 + properties: &ref_146 path: type: string value: @@ -2389,7 +3040,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 - name: decrypt_secret description: | ask to decrypt secret if this variable is secret @@ -2411,7 +3062,7 @@ paths: application/json: schema: type: object - properties: &ref_18 + properties: &ref_22 workspace_id: type: string path: @@ -2438,7 +3089,10 @@ paths: type: boolean is_refreshed: type: boolean - required: &ref_19 + expires_at: + type: string + format: date-time + required: &ref_23 - workspace_id - path - is_secret @@ -2457,7 +3111,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: variable @@ -2479,7 +3133,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: variable @@ -2498,6 +3152,18 @@ paths: in: path required: true schema: *ref_0 + - name: path_start + in: query + schema: + type: string + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 responses: '200': description: variable list @@ -2507,8 +3173,8 @@ paths: type: array items: type: object - properties: *ref_18 - required: *ref_19 + properties: *ref_22 + required: *ref_23 /w/{workspace}/variables/list_contextual: get: summary: list contextual variables @@ -2529,17 +3195,20 @@ paths: type: array items: type: object - properties: &ref_129 + properties: &ref_142 name: type: string value: type: string description: type: string - required: &ref_130 + is_custom: + type: boolean + required: &ref_143 - name - value - description + - is_custom /oauth/login_callback/{client_name}: post: security: [] @@ -2551,7 +3220,7 @@ paths: - name: client_name in: path required: true - schema: &ref_20 + schema: &ref_24 type: string requestBody: description: Partially filled script @@ -2614,6 +3283,34 @@ paths: text/plain: schema: type: string + /oauth/connect_slack_callback: + post: + summary: connect slack callback instance + operationId: connectSlackCallbackInstance + tags: + - oauth + requestBody: + description: code endpoint + required: true + content: + application/json: + schema: + type: object + properties: + code: + type: string + state: + type: string + required: + - code + - state + responses: + '200': + description: success message + content: + text/plain: + schema: + type: string /oauth/connect_callback/{client_name}: post: summary: connect callback @@ -2624,7 +3321,7 @@ paths: - name: client_name in: path required: true - schema: *ref_20 + schema: *ref_24 requestBody: description: code endpoint required: true @@ -2647,7 +3344,7 @@ paths: application/json: schema: type: object - properties: &ref_167 + properties: &ref_191 access_token: type: string expires_in: @@ -2658,7 +3355,7 @@ paths: type: array items: type: string - required: &ref_168 + required: &ref_192 - access_token /w/{workspace}/oauth/create_account: post: @@ -2709,7 +3406,7 @@ paths: - name: id in: path required: true - schema: &ref_21 + schema: &ref_25 type: integer requestBody: description: variable path @@ -2744,7 +3441,7 @@ paths: - name: id in: path required: true - schema: *ref_21 + schema: *ref_25 responses: '200': description: disconnected client @@ -2787,7 +3484,14 @@ paths: oauth: type: array items: - type: string + type: object + properties: + type: + type: string + display_name: + type: string + required: + - type saml: type: string required: @@ -2804,16 +3508,36 @@ paths: content: application/json: schema: - additionalProperties: - type: object - properties: - extra_params: - additionalProperties: - type: string - scopes: - type: array - items: - type: string + type: array + items: + type: string + /oauth/get_connect/{client}: + get: + summary: get oauth connect + operationId: getOAuthConnect + tags: + - oauth + parameters: + - name: client + description: client name + in: path + required: true + schema: + type: string + responses: + '200': + description: get + content: + application/json: + schema: + type: object + properties: + extra_params: + type: object + scopes: + type: array + items: + type: string /w/{workspace}/resources/create: post: summary: create resource @@ -2836,7 +3560,7 @@ paths: application/json: schema: type: object - properties: &ref_136 + properties: &ref_153 path: type: string value: {} @@ -2844,7 +3568,7 @@ paths: type: string resource_type: type: string - required: &ref_137 + required: &ref_154 - path - value - resource_type @@ -2869,7 +3593,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: resource deleted @@ -2891,7 +3615,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 requestBody: description: updated resource required: true @@ -2899,7 +3623,7 @@ paths: application/json: schema: type: object - properties: &ref_138 + properties: &ref_155 path: type: string description: @@ -2926,7 +3650,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 requestBody: description: updated resource required: true @@ -2957,7 +3681,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: resource @@ -2965,7 +3689,7 @@ paths: application/json: schema: type: object - properties: &ref_139 + properties: &ref_156 workspace_id: type: string path: @@ -2981,7 +3705,12 @@ paths: type: object additionalProperties: type: boolean - required: &ref_140 + created_by: + type: string + edited_at: + type: string + format: date-time + required: &ref_157 - path - resource_type - is_oauth @@ -2999,7 +3728,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 - name: job_id description: job id in: query @@ -3026,7 +3755,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: resource value @@ -3047,7 +3776,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: does resource exists @@ -3069,11 +3798,11 @@ paths: - name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 - name: resource_type description: resource_types to list from, separated by ',', in: query @@ -3084,6 +3813,10 @@ paths: in: query schema: type: string + - name: path_start + in: query + schema: + type: string responses: '200': description: resource list @@ -3093,7 +3826,7 @@ paths: type: array items: type: object - properties: &ref_141 + properties: &ref_158 workspace_id: type: string path: @@ -3119,7 +3852,12 @@ paths: type: boolean account: type: number - required: &ref_142 + created_by: + type: string + edited_at: + type: string + format: date-time + required: &ref_159 - path - resource_type - is_oauth @@ -3166,7 +3904,7 @@ paths: - name: name in: path required: true - schema: &ref_80 + schema: &ref_92 type: string responses: '200': @@ -3203,7 +3941,7 @@ paths: application/json: schema: type: object - properties: &ref_22 + properties: &ref_26 workspace_id: type: string name: @@ -3211,7 +3949,14 @@ paths: schema: {} description: type: string - required: &ref_23 + created_by: + type: string + edited_at: + type: string + format: date-time + format_extension: + type: string + required: &ref_27 - name responses: '201': @@ -3220,6 +3965,23 @@ paths: text/plain: schema: type: string + /w/{workspace}/resources/file_resource_type_to_file_ext_map: + get: + summary: get map from resource type to format extension + operationId: fileResourceTypeToFileExtMap + tags: + - resource + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: map from resource type to file ext + content: + application/json: + schema: {} /w/{workspace}/resources/type/delete/{path}: delete: summary: delete resource_type @@ -3234,7 +3996,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: resource_type deleted @@ -3256,7 +4018,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 requestBody: description: updated resource_type required: true @@ -3264,7 +4026,7 @@ paths: application/json: schema: type: object - properties: &ref_143 + properties: &ref_160 schema: {} description: type: string @@ -3289,7 +4051,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: resource_type deleted @@ -3297,8 +4059,8 @@ paths: application/json: schema: type: object - properties: *ref_22 - required: *ref_23 + properties: *ref_26 + required: *ref_27 /w/{workspace}/resources/type/exists/{path}: get: summary: does resource_type exists @@ -3313,7 +4075,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: does resource_type exist @@ -3341,8 +4103,8 @@ paths: type: array items: type: object - properties: *ref_22 - required: *ref_23 + properties: *ref_26 + required: *ref_27 /w/{workspace}/resources/type/listnames: get: summary: list resource_types names @@ -3481,7 +4243,7 @@ paths: - name: id in: path required: true - schema: *ref_24 + schema: *ref_28 responses: '200': description: flow @@ -3492,51 +4254,51 @@ paths: properties: flow: type: object - properties: &ref_43 + properties: &ref_50 summary: type: string description: type: string value: type: object - properties: &ref_50 + properties: &ref_58 modules: type: array items: type: object - properties: &ref_27 + properties: &ref_31 id: type: string value: - oneOf: &ref_206 + oneOf: &ref_239 - type: object - properties: &ref_192 + properties: &ref_223 input_transforms: type: object additionalProperties: - oneOf: &ref_25 + oneOf: &ref_29 - type: object - properties: &ref_188 + properties: &ref_219 value: {} type: type: string enum: - javascript - required: &ref_189 + required: &ref_220 - expr - type - type: object - properties: &ref_190 + properties: &ref_221 expr: type: string type: type: string enum: - javascript - required: &ref_191 + required: &ref_222 - expr - type - discriminator: &ref_26 + discriminator: &ref_30 propertyName: type mapping: static: '#/components/schemas/StaticTransform' @@ -3559,6 +4321,7 @@ paths: - mssql - graphql - nativets + - php path: type: string lock: @@ -3573,18 +4336,20 @@ paths: type: number concurrency_time_window_s: type: number - required: &ref_193 + custom_concurrency_key: + type: string + required: &ref_224 - type - content - language - input_transforms - type: object - properties: &ref_194 + properties: &ref_225 input_transforms: type: object additionalProperties: - oneOf: *ref_25 - discriminator: *ref_26 + oneOf: *ref_29 + discriminator: *ref_30 path: type: string hash: @@ -3593,40 +4358,42 @@ paths: type: string enum: - script - required: &ref_195 + tag_override: + type: string + required: &ref_226 - type - path - input_transforms - type: object - properties: &ref_196 + properties: &ref_227 input_transforms: type: object additionalProperties: - oneOf: *ref_25 - discriminator: *ref_26 + oneOf: *ref_29 + discriminator: *ref_30 path: type: string type: type: string enum: - flow - required: &ref_197 + required: &ref_228 - type - path - input_transforms - type: object - properties: &ref_198 + properties: &ref_229 modules: type: array items: type: object - properties: *ref_27 - required: &ref_28 + properties: *ref_31 + required: &ref_32 - value - id iterator: - oneOf: *ref_25 - discriminator: *ref_26 + oneOf: *ref_29 + discriminator: *ref_30 skip_failures: type: boolean type: @@ -3637,13 +4404,35 @@ paths: type: boolean parallelism: type: integer - required: &ref_199 + required: &ref_230 - modules - iterator - skip_failures - type - type: object - properties: &ref_200 + properties: &ref_231 + modules: + type: array + items: + type: object + properties: *ref_31 + required: *ref_32 + skip_failures: + type: boolean + type: + type: string + enum: + - forloopflow + parallel: + type: boolean + parallelism: + type: integer + required: &ref_232 + - modules + - skip_failures + - type + - type: object + properties: &ref_233 branches: type: array items: @@ -3657,8 +4446,8 @@ paths: type: array items: type: object - properties: *ref_27 - required: *ref_28 + properties: *ref_31 + required: *ref_32 required: - modules - expr @@ -3666,20 +4455,20 @@ paths: type: array items: type: object - properties: *ref_27 - required: *ref_28 + properties: *ref_31 + required: *ref_32 required: - modules type: type: string enum: - branchone - required: &ref_201 + required: &ref_234 - branches - default - type - type: object - properties: &ref_202 + properties: &ref_235 branches: type: array items: @@ -3693,8 +4482,8 @@ paths: type: array items: type: object - properties: *ref_27 - required: *ref_28 + properties: *ref_31 + required: *ref_32 required: - modules - expr @@ -3704,26 +4493,27 @@ paths: - branchall parallel: type: boolean - required: &ref_203 + required: &ref_236 - branches - type - type: object - properties: &ref_204 + properties: &ref_237 type: type: string enum: - identity flow: type: boolean - required: &ref_205 + required: &ref_238 - type - discriminator: &ref_207 + discriminator: &ref_240 propertyName: type mapping: rawscript: '#/components/schemas/RawScript' script: '#/components/schemas/PathScript' flow: '#/components/schemas/PathFlow' forloopflow: '#/components/schemas/ForloopFlow' + whileloopflow: '#/components/schemas/WhileloopFlow' branchone: '#/components/schemas/BranchOne' branchall: '#/components/schemas/BranchAll' identity: '#/components/schemas/Identity' @@ -3736,9 +4526,25 @@ paths: type: string required: - expr + stop_after_all_iters_if: + type: object + properties: + skip_if_stopped: + type: boolean + expr: + type: string + required: + - expr + skip_if: + type: object + properties: + expr: + type: string + required: + - expr sleep: - oneOf: *ref_25 - discriminator: *ref_26 + oneOf: *ref_29 + discriminator: *ref_30 cache_ttl: type: number timeout: @@ -3768,15 +4574,21 @@ paths: user_auth_required: type: boolean user_groups_required: - oneOf: *ref_25 - discriminator: *ref_26 + oneOf: *ref_29 + discriminator: *ref_30 self_approval_disabled: type: boolean + hide_cancel: + type: boolean + continue_on_disapprove_timeout: + type: boolean priority: type: number + continue_on_error: + type: boolean retry: type: object - properties: &ref_77 + properties: &ref_87 constant: type: object properties: @@ -3797,15 +4609,21 @@ paths: type: integer minimum: 0 maximum: 100 - required: *ref_28 + required: *ref_32 failure_module: type: object - properties: *ref_27 - required: *ref_28 + properties: *ref_31 + required: *ref_32 + preprocessor_module: + type: object + properties: *ref_31 + required: *ref_32 same_worker: type: boolean concurrent_limit: type: number + concurrency_key: + type: string concurrency_time_window_s: type: number skip_expr: @@ -3816,11 +4634,11 @@ paths: type: number early_return: type: string - required: &ref_51 + required: &ref_59 - modules schema: type: object - required: &ref_44 + required: &ref_51 - summary - value /apps/hub/list: @@ -3873,7 +4691,7 @@ paths: - name: id in: path required: true - schema: *ref_24 + schema: *ref_28 responses: '200': description: app @@ -3903,7 +4721,7 @@ paths: - name: path in: path required: true - schema: &ref_29 + schema: &ref_33 type: string responses: '200': @@ -3922,7 +4740,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: script details @@ -3993,7 +4811,7 @@ paths: type: number kind: name: kind - schema: &ref_30 + schema: &ref_34 type: string enum: - script @@ -4066,7 +4884,7 @@ paths: type: string kind: name: kind - schema: *ref_30 + schema: *ref_34 score: type: number required: @@ -4119,20 +4937,20 @@ paths: - name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 - name: order_desc description: order by desc order (default true) in: query - schema: &ref_41 + schema: &ref_48 type: boolean - name: created_by description: mask to filter exact matching user creator in: query - schema: &ref_42 + schema: &ref_49 type: string - name: path_start description: mask to filter matching starting path @@ -4182,7 +5000,7 @@ paths: description: > (default false) - show also the archived files. + show only the archived files. when multiple archived hash share the same path, only the ones with the latest create_at @@ -4193,6 +5011,20 @@ paths: in: query schema: type: boolean + - name: include_without_main + description: | + (default false) + include scripts without an exported main function + in: query + schema: + type: boolean + - name: include_draft_only + description: | + (default false) + include scripts that have no deployed version + in: query + schema: + type: boolean - name: is_template description: | (default regardless) @@ -4216,6 +5048,13 @@ paths: in: query schema: type: boolean + - name: with_deployment_msg + description: | + (default false) + include deployment message + in: query + schema: + type: boolean responses: '200': description: All scripts @@ -4225,7 +5064,7 @@ paths: type: array items: type: object - properties: &ref_31 + properties: &ref_35 workspace_id: type: string hash: @@ -4282,6 +5121,9 @@ paths: - graphql - nativets - bun + - php + - rust + - ansible kind: type: string enum: @@ -4306,6 +5148,8 @@ paths: type: integer concurrency_time_window_s: type: integer + concurrency_key: + type: string cache_ttl: type: number dedicated_worker: @@ -4320,7 +5164,15 @@ paths: type: integer delete_after_use: type: boolean - required: &ref_32 + visible_to_runner_only: + type: boolean + no_main_func: + type: boolean + codebase: + type: string + has_preprocessor: + type: boolean + required: &ref_36 - hash - path - summary @@ -4335,6 +5187,8 @@ paths: - language - kind - starred + - no_main_func + - has_preprocessor /w/{workspace}/scripts/list_paths: get: summary: list all scripts paths @@ -4416,7 +5270,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: draft deleted @@ -4442,7 +5296,7 @@ paths: application/json: schema: type: object - properties: &ref_34 + properties: &ref_40 path: type: string parent_hash: @@ -4475,6 +5329,9 @@ paths: - graphql - nativets - bun + - php + - rust + - ansible kind: type: string enum: @@ -4513,7 +5370,15 @@ paths: type: string concurrency_key: type: string - required: &ref_35 + visible_to_runner_only: + type: boolean + no_main_func: + type: boolean + codebase: + type: string + has_preprocessor: + type: boolean + required: &ref_41 - path - summary - description @@ -4540,7 +5405,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 requestBody: description: Workspace error handler enabled required: true @@ -4617,7 +5482,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: script archived @@ -4639,7 +5504,7 @@ paths: - name: hash in: path required: true - schema: &ref_33 + schema: &ref_37 type: string responses: '200': @@ -4648,8 +5513,8 @@ paths: application/json: schema: type: object - properties: *ref_31 - required: *ref_32 + properties: *ref_35 + required: *ref_36 /w/{workspace}/scripts/delete/h/{hash}: post: summary: delete script by hash (erase content but keep hash, require admin) @@ -4664,7 +5529,7 @@ paths: - name: hash in: path required: true - schema: *ref_33 + schema: *ref_37 responses: '200': description: script details @@ -4672,8 +5537,8 @@ paths: application/json: schema: type: object - properties: *ref_31 - required: *ref_32 + properties: *ref_35 + required: *ref_36 /w/{workspace}/scripts/delete/p/{path}: post: summary: delete all scripts at a given path (require admin) @@ -4688,7 +5553,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: script path @@ -4710,7 +5575,11 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 + - name: with_starred_info + in: query + schema: + type: boolean responses: '200': description: script details @@ -4718,8 +5587,70 @@ paths: application/json: schema: type: object - properties: *ref_31 - required: *ref_32 + properties: *ref_35 + required: *ref_36 + /w/{workspace}/scripts/get_triggers_count/{path}: + get: + summary: get triggers count of script + operationId: getTriggersCountOfScript + tags: + - script + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_33 + responses: + '200': + description: triggers count + content: + application/json: + schema: + type: object + properties: &ref_53 + primary_schedule: + type: object + properties: + schedule: + type: string + schedule_count: + type: number + http_routes_count: + type: number + webhook_count: + type: number + email_count: + type: number + /w/{workspace}/scripts/list_tokens/{path}: + get: + summary: get tokens with script scope + operationId: listTokensOfScript + tags: + - script + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_33 + responses: + '200': + description: tokens list + content: + application/json: + schema: + type: array + items: + type: object + properties: *ref_38 + required: *ref_39 /w/{workspace}/scripts/get/draft/{path}: get: summary: get script by path with draft @@ -4734,23 +5665,23 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: script details content: application/json: schema: - allOf: &ref_110 + allOf: &ref_126 - type: object - properties: *ref_34 - required: *ref_35 + properties: *ref_40 + required: *ref_41 - type: object properties: draft: type: object - properties: *ref_34 - required: *ref_35 + properties: *ref_40 + required: *ref_41 hash: type: string required: @@ -4769,7 +5700,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: script history @@ -4779,12 +5710,12 @@ paths: type: array items: type: object - properties: &ref_111 + properties: &ref_127 script_hash: type: string deployment_msg: type: string - required: &ref_112 + required: &ref_128 - script_hash /w/{workspace}/scripts/history_update/h/{hash}/p/{path}: post: @@ -4800,11 +5731,11 @@ paths: - name: hash in: path required: true - schema: *ref_33 + schema: *ref_37 - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 requestBody: description: Script deployment message required: true @@ -4836,7 +5767,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: script content @@ -4860,12 +5791,12 @@ paths: - name: token in: path required: true - schema: &ref_96 + schema: &ref_115 type: string - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: script content @@ -4887,7 +5818,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: does it exists @@ -4909,7 +5840,11 @@ paths: - name: hash in: path required: true - schema: *ref_33 + schema: *ref_37 + - name: with_starred_info + in: query + schema: + type: boolean responses: '200': description: script details @@ -4917,8 +5852,8 @@ paths: application/json: schema: type: object - properties: *ref_31 - required: *ref_32 + properties: *ref_35 + required: *ref_36 /w/{workspace}/scripts/raw/h/{path}: get: summary: raw script by hash @@ -4933,7 +5868,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: script content @@ -4955,7 +5890,7 @@ paths: - name: hash in: path required: true - schema: *ref_33 + schema: *ref_37 responses: '200': description: script details @@ -4982,7 +5917,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 - name: scheduled_for description: when to schedule this job (leave empty for immediate run) in: query @@ -4994,18 +5929,30 @@ paths: in: query schema: type: integer + - name: skip_preprocessor + description: skip the preprocessor + in: query + schema: + type: boolean - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: &ref_36 + schema: &ref_42 type: string format: uuid - name: tag description: Override the tag to use in: query - schema: &ref_38 + schema: &ref_44 + type: string + - name: cache_ttl + description: >- + Override the cache time to live (in seconds). Can not be used to + disable caching, only override with a new cache ttl + in: query + schema: &ref_45 type: string - name: job_id description: >- @@ -5014,7 +5961,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: &ref_37 + schema: &ref_43 type: string format: uuid - name: invisible_to_owner @@ -5029,7 +5976,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 responses: '201': description: job created @@ -5052,13 +5999,13 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_36 + schema: *ref_42 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -5066,7 +6013,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_37 + schema: *ref_43 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -5075,14 +6022,14 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: &ref_39 + schema: &ref_46 type: string - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: &ref_40 + schema: &ref_47 type: string requestBody: description: script args @@ -5091,7 +6038,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 responses: '200': description: job result @@ -5112,17 +6059,23 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_36 + schema: *ref_42 - name: tag description: Override the tag to use in: query - schema: *ref_38 + schema: *ref_44 + - name: cache_ttl + description: >- + Override the cache time to live (in seconds). Can not be used to + disable caching, only override with a new cache ttl + in: query + schema: *ref_45 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -5130,7 +6083,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_37 + schema: *ref_43 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -5139,13 +6092,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_39 + schema: *ref_46 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_40 + schema: *ref_47 requestBody: description: script args required: true @@ -5153,7 +6106,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 responses: '200': description: job result @@ -5173,17 +6126,23 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_36 + schema: *ref_42 - name: tag description: Override the tag to use in: query - schema: *ref_38 + schema: *ref_44 + - name: cache_ttl + description: >- + Override the cache time to live (in seconds). Can not be used to + disable caching, only override with a new cache ttl + in: query + schema: *ref_45 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -5191,7 +6150,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_37 + schema: *ref_43 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -5200,13 +6159,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_39 + schema: *ref_46 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_40 + schema: *ref_47 - name: payload description: > The base64 encoded payload that has been encoded as a JSON. e.g how @@ -5214,7 +6173,7 @@ paths: `encodeURIComponent(btoa(JSON.stringify({a: 2})))` in: query - schema: &ref_76 + schema: &ref_86 type: string responses: '200': @@ -5236,7 +6195,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -5245,13 +6204,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_39 + schema: *ref_46 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_40 + schema: *ref_47 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -5259,7 +6218,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_37 + schema: *ref_43 requestBody: description: script args required: true @@ -5267,7 +6226,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 responses: '200': description: job result @@ -5288,7 +6247,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -5297,13 +6256,13 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_39 + schema: *ref_46 - name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_40 + schema: *ref_47 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -5311,7 +6270,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_37 + schema: *ref_43 requestBody: description: script args required: true @@ -5319,7 +6278,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 responses: '200': description: job result @@ -5414,19 +6373,19 @@ paths: - name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 - name: order_desc description: order by desc order (default true) in: query - schema: *ref_41 + schema: *ref_48 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_42 + schema: *ref_49 - name: path_start description: mask to filter matching starting path in: query @@ -5441,7 +6400,7 @@ paths: description: > (default false) - show also the archived files. + show only the archived files. when multiple archived hash share the same path, only the ones with the latest create_at @@ -5457,6 +6416,20 @@ paths: in: query schema: type: boolean + - name: include_draft_only + description: | + (default false) + include items that have no deployed version + in: query + schema: + type: boolean + - name: with_deployment_msg + description: | + (default false) + include deployment message + in: query + schema: + type: boolean responses: '200': description: All flow @@ -5466,12 +6439,12 @@ paths: type: array items: allOf: - - allOf: &ref_45 + - allOf: &ref_52 - type: object - properties: *ref_43 - required: *ref_44 + properties: *ref_50 + required: *ref_51 - type: object - properties: &ref_155 + properties: &ref_177 workspace_id: type: string path: @@ -5485,8 +6458,8 @@ paths: type: boolean extra_perms: type: object - additionalProperties: - type: boolean + additionalProperties: &ref_176 + type: boolean starred: type: boolean draft_only: @@ -5501,7 +6474,9 @@ paths: type: boolean timeout: type: number - required: &ref_156 + visible_to_runner_only: + type: boolean + required: &ref_178 - path - edited_by - edited_at @@ -5513,6 +6488,109 @@ paths: type: boolean draft_only: type: boolean + /w/{workspace}/flows/history/p/{path}: + get: + summary: get flow history by path + operationId: getFlowHistory + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_33 + tags: + - flow + responses: + '200': + description: Flow history + content: + application/json: + schema: + type: array + items: + type: object + properties: &ref_189 + id: + type: integer + created_at: + type: string + format: date-time + deployment_msg: + type: string + required: &ref_190 + - id + - created_at + /w/{workspace}/flows/get/v/{version}/p/{path}: + get: + summary: get flow version + operationId: getFlowVersion + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - type: string + name: version + in: path + required: true + schema: + type: number + - name: path + in: path + required: true + schema: *ref_33 + tags: + - flow + responses: + '200': + description: flow details + content: + application/json: + schema: + allOf: *ref_52 + /w/{workspace}/flows/history_update/v/{version}/p/{path}: + post: + summary: update flow history + operationId: updateFlowHistory + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - type: string + name: version + in: path + required: true + schema: + type: number + - name: path + in: path + required: true + schema: *ref_33 + requestBody: + description: Flow deployment message + required: true + content: + application/json: + schema: + type: object + properties: + deployment_msg: + type: string + required: + - deployment_msg + tags: + - flow + responses: + '200': + description: success + content: + text/plain: + schema: + type: string /w/{workspace}/flows/get/{path}: get: summary: get flow by path @@ -5527,14 +6605,67 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 + - name: with_starred_info + in: query + schema: + type: boolean responses: '200': description: flow details content: application/json: schema: - allOf: *ref_45 + allOf: *ref_52 + /w/{workspace}/flows/get_triggers_count/{path}: + get: + summary: get triggers count of flow + operationId: getTriggersCountOfFlow + tags: + - flow + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_33 + responses: + '200': + description: triggers count + content: + application/json: + schema: + type: object + properties: *ref_53 + /w/{workspace}/flows/list_tokens/{path}: + get: + summary: get tokens with flow scope + operationId: listTokensOfFlow + tags: + - flow + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_33 + responses: + '200': + description: tokens list + content: + application/json: + schema: + type: array + items: + type: object + properties: *ref_38 + required: *ref_39 /w/{workspace}/flows/toggle_workspace_error_handler/{path}: post: summary: Toggle ON and OFF the workspace error handler for a given flow @@ -5549,7 +6680,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 requestBody: description: Workspace error handler enabled required: true @@ -5581,7 +6712,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: flow details with draft @@ -5589,11 +6720,11 @@ paths: application/json: schema: allOf: - - allOf: *ref_45 + - allOf: *ref_52 - type: object properties: draft: - allOf: *ref_45 + allOf: *ref_52 /w/{workspace}/flows/exists/{path}: get: summary: exists flow by path @@ -5608,7 +6739,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: flow details @@ -5634,10 +6765,10 @@ paths: application/json: schema: allOf: - - allOf: &ref_46 + - allOf: &ref_54 - type: object - properties: *ref_43 - required: *ref_44 + properties: *ref_50 + required: *ref_51 - type: object properties: path: @@ -5652,6 +6783,8 @@ paths: type: boolean timeout: type: number + visible_to_runner_only: + type: boolean required: - path - type: object @@ -5681,7 +6814,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 requestBody: description: Partially filled flow required: true @@ -5689,7 +6822,7 @@ paths: application/json: schema: allOf: - - allOf: *ref_46 + - allOf: *ref_54 - type: object properties: deployment_message: @@ -5715,7 +6848,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 requestBody: description: archiveFlow required: true @@ -5747,7 +6880,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: flow delete @@ -5755,61 +6888,6 @@ paths: text/plain: schema: type: string - /w/{workspace}/flows/input_history/p/{path}: - get: - summary: list inputs for previous completed flow jobs - operationId: getFlowInputHistoryByPath - tags: - - flow - parameters: - - name: workspace - in: path - required: true - schema: *ref_0 - - name: path - in: path - required: true - schema: *ref_29 - - name: page - description: which page to return (start at 1, default 1) - in: query - schema: *ref_3 - - name: per_page - description: number of items to return for a given page (default 30, max 100) - in: query - schema: *ref_4 - responses: - '200': - description: input history for completed jobs with this flow path - content: - application/json: - schema: - type: array - items: - type: object - properties: &ref_87 - id: - type: string - name: - type: string - args: - type: object - created_by: - type: string - created_at: - type: string - format: date-time - is_public: - type: boolean - success: - type: boolean - required: &ref_88 - - id - - name - - args - - created_by - - created_at - - is_public /w/{workspace}/raw_apps/list: get: summary: list all raw apps @@ -5824,19 +6902,19 @@ paths: - name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 - name: order_desc description: order by desc order (default true) in: query - schema: *ref_41 + schema: *ref_48 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_42 + schema: *ref_49 - name: path_start description: mask to filter matching starting path in: query @@ -5863,7 +6941,7 @@ paths: type: array items: type: object - properties: &ref_162 + properties: &ref_184 workspace_id: type: string path: @@ -5881,7 +6959,7 @@ paths: edited_at: type: string format: date-time - required: &ref_163 + required: &ref_185 - workspace_id - path - summary @@ -5902,7 +6980,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: app exists @@ -5924,12 +7002,12 @@ paths: - name: version in: path required: true - schema: &ref_95 + schema: &ref_114 type: number - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: app details @@ -5978,19 +7056,19 @@ paths: - name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 - name: order_desc description: order by desc order (default true) in: query - schema: *ref_41 + schema: *ref_48 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_42 + schema: *ref_49 - name: path_start description: mask to filter matching starting path in: query @@ -6008,6 +7086,20 @@ paths: in: query schema: type: boolean + - name: include_draft_only + description: | + (default false) + include items that have no deployed version + in: query + schema: + type: boolean + - name: with_deployment_msg + description: | + (default false) + include deployment message + in: query + schema: + type: boolean responses: '200': description: All apps @@ -6017,7 +7109,7 @@ paths: type: array items: type: object - properties: &ref_160 + properties: &ref_182 id: type: integer workspace_id: @@ -6043,7 +7135,7 @@ paths: - viewer - publisher - anonymous - required: &ref_161 + required: &ref_183 - id - workspace_id - path @@ -6078,11 +7170,15 @@ paths: type: string policy: type: object - properties: &ref_47 + properties: &ref_55 triggerables: type: object additionalProperties: type: object + triggerables_v2: + type: object + additionalProperties: + type: object execution_mode: type: string enum: @@ -6123,7 +7219,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: app exists @@ -6145,7 +7241,11 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 + - name: with_starred_info + in: query + schema: + type: boolean responses: '200': description: app details @@ -6153,7 +7253,7 @@ paths: application/json: schema: type: object - properties: &ref_48 + properties: &ref_56 id: type: integer workspace_id: @@ -6171,10 +7271,11 @@ paths: created_at: type: string format: date-time - value: {} + value: + type: object policy: type: object - properties: *ref_47 + properties: *ref_55 execution_mode: type: string enum: @@ -6185,7 +7286,7 @@ paths: type: object additionalProperties: type: boolean - required: &ref_49 + required: &ref_57 - id - workspace_id - path @@ -6211,17 +7312,17 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: app details with draft content: application/json: schema: - allOf: &ref_164 + allOf: &ref_186 - type: object - properties: *ref_48 - required: *ref_49 + properties: *ref_56 + required: *ref_57 - type: object properties: draft_only: @@ -6241,7 +7342,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 responses: '200': description: app history @@ -6251,12 +7352,12 @@ paths: type: array items: type: object - properties: &ref_165 + properties: &ref_187 version: type: integer deployment_msg: type: string - required: &ref_166 + required: &ref_188 - version /w/{workspace}/apps/history_update/a/{id}/v/{version}: post: @@ -6272,11 +7373,11 @@ paths: - name: id in: path required: true - schema: *ref_24 + schema: *ref_28 - name: version in: path required: true - schema: &ref_97 + schema: &ref_116 type: integer requestBody: description: App deployment message @@ -6309,7 +7410,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: app details @@ -6317,8 +7418,8 @@ paths: application/json: schema: type: object - properties: *ref_48 - required: *ref_49 + properties: *ref_56 + required: *ref_57 /w/{workspace}/apps_u/public_resource/{path}: get: summary: get public resource @@ -6333,7 +7434,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: resource value @@ -6354,7 +7455,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: app secret @@ -6376,7 +7477,7 @@ paths: - name: id in: path required: true - schema: *ref_24 + schema: *ref_28 responses: '200': description: app details @@ -6384,8 +7485,8 @@ paths: application/json: schema: type: object - properties: *ref_48 - required: *ref_49 + properties: *ref_56 + required: *ref_57 /w/{workspace}/raw_apps/create: post: summary: create raw app @@ -6436,7 +7537,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 requestBody: description: updateraw app required: true @@ -6472,7 +7573,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: app deleted @@ -6494,7 +7595,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: app deleted @@ -6516,7 +7617,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 requestBody: description: update app required: true @@ -6532,7 +7633,7 @@ paths: value: {} policy: type: object - properties: *ref_47 + properties: *ref_55 deployment_message: type: string responses: @@ -6556,7 +7657,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 requestBody: description: update app required: true @@ -6579,6 +7680,8 @@ paths: type: string path: type: string + lock: + type: string cache_ttl: type: integer required: @@ -6586,6 +7689,12 @@ paths: - language force_viewer_static_fields: type: object + force_viewer_one_of_fields: + type: object + force_viewer_allow_user_resources: + type: array + items: + type: string required: - args - component @@ -6610,7 +7719,7 @@ paths: - name: path in: path required: true - schema: *ref_29 + schema: *ref_33 - name: scheduled_for description: when to schedule this job (leave empty for immediate run) in: query @@ -6622,16 +7731,21 @@ paths: in: query schema: type: integer + - name: skip_preprocessor + description: skip the preprocessor + in: query + schema: + type: boolean - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_36 + schema: *ref_42 - name: tag description: Override the tag to use in: query - schema: *ref_38 + schema: *ref_44 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6639,7 +7753,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_37 + schema: *ref_43 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6648,7 +7762,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_39 + schema: *ref_46 - name: invisible_to_owner description: make the run invisible to the the flow owner (default false) in: query @@ -6661,7 +7775,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 responses: '201': description: job created @@ -6684,7 +7798,7 @@ paths: - name: id in: path required: true - schema: &ref_73 + schema: &ref_83 type: string format: uuid - name: step_id @@ -6717,11 +7831,11 @@ paths: The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_36 + schema: *ref_42 - name: tag description: Override the tag to use in: query - schema: *ref_38 + schema: *ref_44 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6729,7 +7843,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_37 + schema: *ref_43 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6738,7 +7852,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_39 + schema: *ref_46 - name: invisible_to_owner description: make the run invisible to the the flow owner (default false) in: query @@ -6751,7 +7865,7 @@ paths: application/json: schema: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 responses: '201': description: job created @@ -6774,7 +7888,7 @@ paths: - name: hash in: path required: true - schema: *ref_33 + schema: *ref_37 - name: scheduled_for description: when to schedule this job (leave empty for immediate run) in: query @@ -6786,16 +7900,27 @@ paths: in: query schema: type: integer + - name: skip_preprocessor + description: skip the preprocessor + in: query + schema: + type: boolean - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_36 + schema: *ref_42 - name: tag description: Override the tag to use in: query - schema: *ref_38 + schema: *ref_44 + - name: cache_ttl + description: >- + Override the cache time to live (in seconds). Can not be used to + disable caching, only override with a new cache ttl + in: query + schema: *ref_45 - name: job_id description: >- The job id to assign to the created job. if missing, job is chosen @@ -6803,7 +7928,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_37 + schema: *ref_43 - name: include_header description: > List of headers's keys (separated with ',') whove value are added to @@ -6812,7 +7937,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_39 + schema: *ref_46 - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -6852,7 +7977,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_39 + schema: *ref_46 - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -6865,7 +7990,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_37 + schema: *ref_43 requestBody: description: preview required: true @@ -6873,14 +7998,14 @@ paths: application/json: schema: type: object - properties: &ref_134 + properties: &ref_147 content: type: string path: type: string args: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 language: type: string enum: @@ -6897,6 +8022,9 @@ paths: - graphql - nativets - bun + - php + - rust + - ansible tag: type: string kind: @@ -6909,7 +8037,49 @@ paths: type: boolean lock: type: string - required: &ref_135 + required: &ref_148 + - args + responses: + '201': + description: job created + content: + text/plain: + schema: + type: string + format: uuid + /w/{workspace}/jobs/workflow_as_code/{job_id}/{entrypoint}: + post: + summary: run code-workflow task + operationId: runCodeWorkflowTask + tags: + - job + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: job_id + in: path + required: true + schema: + type: string + - name: entrypoint + in: path + required: true + schema: + type: string + requestBody: + description: preview + required: true + content: + application/json: + schema: + type: object + properties: &ref_149 + args: + type: object + additionalProperties: *ref_16 + required: &ref_150 - args responses: '201': @@ -6942,7 +8112,7 @@ paths: type: array items: type: object - properties: &ref_184 + properties: &ref_208 raw_code: type: string path: @@ -6963,7 +8133,10 @@ paths: - graphql - nativets - bun - required: &ref_185 + - php + - rust + - ansible + required: &ref_209 - raw_code - path - language @@ -7003,7 +8176,7 @@ paths: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_39 + schema: *ref_46 - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -7016,7 +8189,7 @@ paths: queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_37 + schema: *ref_43 requestBody: description: preview required: true @@ -7024,21 +8197,21 @@ paths: application/json: schema: type: object - properties: &ref_157 + properties: &ref_179 value: type: object - properties: *ref_50 - required: *ref_51 + properties: *ref_58 + required: *ref_59 path: type: string args: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 tag: type: string restarted_from: type: object - properties: &ref_159 + properties: &ref_181 flow_job_id: type: string format: uuid @@ -7046,7 +8219,7 @@ paths: type: string branch_or_iteration_n: type: integer - required: &ref_158 + required: &ref_180 - value - content - args @@ -7072,95 +8245,103 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_41 + schema: *ref_48 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_42 + schema: *ref_49 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_36 + schema: *ref_42 - name: script_path_exact description: mask to filter exact matching path in: query - schema: &ref_54 + schema: &ref_62 type: string - name: script_path_start description: mask to filter matching starting path in: query - schema: &ref_55 + schema: &ref_63 type: string - name: schedule_path description: mask to filter by schedule path in: query - schema: &ref_56 + schema: &ref_64 type: string - name: script_hash description: mask to filter exact matching path in: query - schema: &ref_57 + schema: &ref_65 type: string - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: &ref_58 + schema: &ref_66 type: string format: date-time - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: &ref_59 + schema: &ref_67 type: string format: date-time - name: success description: filter on successful jobs in: query - schema: &ref_60 + schema: &ref_68 type: boolean - name: scheduled_for_before_now description: filter on jobs scheduled_for before now (hence waitinf for a worker) in: query - schema: &ref_68 + schema: &ref_69 type: boolean - name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: &ref_61 + schema: &ref_70 type: string - name: suspended description: filter on suspended jobs in: query - schema: &ref_100 + schema: &ref_71 type: boolean - name: running description: filter on running jobs in: query - schema: &ref_67 + schema: &ref_72 type: boolean - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: &ref_62 + schema: &ref_73 type: string - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: &ref_63 + schema: &ref_74 type: string - name: tag description: filter on jobs with a given tag/worker group in: query - schema: &ref_64 + schema: &ref_75 type: string + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 - name: all_workspaces description: >- get jobs from all workspaces (only valid if request come from the @@ -7168,6 +8349,11 @@ paths: in: query schema: type: boolean + - name: is_not_schedule + description: is not a scheduled job + in: query + schema: + type: boolean responses: '200': description: All queued jobs @@ -7177,7 +8363,7 @@ paths: type: array items: type: object - properties: &ref_71 + properties: &ref_81 workspace_id: type: string id: @@ -7205,7 +8391,7 @@ paths: type: string args: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 logs: type: string raw_code: @@ -7244,14 +8430,14 @@ paths: by extension its DT_TOKEN. flow_status: type: object - properties: &ref_65 + properties: &ref_76 step: type: integer modules: type: array items: type: object - properties: &ref_52 + properties: &ref_60 type: type: string enum: @@ -7268,6 +8454,8 @@ paths: format: uuid count: type: integer + progress: + type: integer iterator: type: object properties: @@ -7281,6 +8469,10 @@ paths: type: array items: type: string + flow_jobs_success: + type: array + items: + type: boolean branch_chosen: type: object properties: @@ -7315,13 +8507,27 @@ paths: required: - resume_id - approver - required: &ref_53 + failed_retries: + type: array + items: + type: string + format: uuid + skipped: + type: boolean + required: &ref_61 - type + user_states: + additionalProperties: true + preprocessor_module: + allOf: + - type: object + properties: *ref_60 + required: *ref_61 failure_module: allOf: - type: object - properties: *ref_52 - required: *ref_53 + properties: *ref_60 + required: *ref_61 - type: object properties: parent_module: @@ -7336,14 +8542,14 @@ paths: items: type: string format: uuid - required: &ref_66 + required: &ref_77 - step - modules - failure_module raw_flow: type: object - properties: *ref_50 - required: *ref_51 + properties: *ref_58 + required: *ref_59 is_flow_step: type: boolean language: @@ -7362,6 +8568,9 @@ paths: - graphql - nativets - bun + - php + - rust + - ansible email: type: string visible_to_owner: @@ -7372,7 +8581,13 @@ paths: type: string priority: type: integer - required: &ref_72 + self_wait_time_ms: + type: number + aggregate_wait_time_ms: + type: number + suspend: + type: number + required: &ref_82 - id - running - canceled @@ -7410,6 +8625,8 @@ paths: properties: database_length: type: integer + suspended: + type: integer required: - database_length /w/{workspace}/jobs/completed/count: @@ -7435,10 +8652,10 @@ paths: type: integer required: - database_length - /w/{workspace}/jobs/queue/cancel_all: - post: - summary: cancel all jobs - operationId: cancelAll + /w/{workspace}/jobs/queue/list_filtered_uuids: + get: + summary: get the ids of all jobs matching the given filters + operationId: listFilteredUuids tags: - job parameters: @@ -7446,6 +8663,136 @@ paths: in: path required: true schema: *ref_0 + - name: order_desc + description: order by desc order (default true) + in: query + schema: *ref_48 + - name: created_by + description: mask to filter exact matching user creator + in: query + schema: *ref_49 + - name: parent_job + description: >- + The parent job that is at the origin and responsible for the + execution of this script if any + in: query + schema: *ref_42 + - name: script_path_exact + description: mask to filter exact matching path + in: query + schema: *ref_62 + - name: script_path_start + description: mask to filter matching starting path + in: query + schema: *ref_63 + - name: schedule_path + description: mask to filter by schedule path + in: query + schema: *ref_64 + - name: script_hash + description: mask to filter exact matching path + in: query + schema: *ref_65 + - name: started_before + description: filter on started before (inclusive) timestamp + in: query + schema: *ref_66 + - name: started_after + description: filter on started after (exclusive) timestamp + in: query + schema: *ref_67 + - name: success + description: filter on successful jobs + in: query + schema: *ref_68 + - name: scheduled_for_before_now + description: filter on jobs scheduled_for before now (hence waitinf for a worker) + in: query + schema: *ref_69 + - name: job_kinds + description: >- + filter on job kind (values 'preview', 'script', 'dependencies', + 'flow') separated by, + in: query + schema: *ref_70 + - name: suspended + description: filter on suspended jobs + in: query + schema: *ref_71 + - name: running + description: filter on running jobs + in: query + schema: *ref_72 + - name: args + description: >- + filter on jobs containing those args as a json subset (@> in + postgres) + in: query + schema: *ref_73 + - name: result + description: >- + filter on jobs containing those result as a json subset (@> in + postgres) + in: query + schema: *ref_74 + - name: tag + description: filter on jobs with a given tag/worker group + in: query + schema: *ref_75 + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 + - name: concurrency_key + in: query + required: false + schema: + type: string + - name: all_workspaces + description: >- + get jobs from all workspaces (only valid if request come from the + `admins` workspace) + in: query + schema: + type: boolean + - name: is_not_schedule + description: is not a scheduled job + in: query + schema: + type: boolean + responses: + '200': + description: uuids of jobs + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/jobs/queue/cancel_selection: + post: + summary: cancel jobs based on the given uuids + operationId: cancelSelection + tags: + - job + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: uuids of the jobs to cancel + required: true + content: + application/json: + schema: + type: array + items: + type: string responses: '200': description: uuids of canceled jobs @@ -7469,67 +8816,83 @@ paths: - name: order_desc description: order by desc order (default true) in: query - schema: *ref_41 + schema: *ref_48 - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_42 + schema: *ref_49 + - name: label + description: >- + mask to filter exact matching job's label (job labels are completed + jobs with as a result an object containing a string in the array at + key 'wm_labels') + in: query + schema: &ref_78 + type: string - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_36 + schema: *ref_42 - name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_54 + schema: *ref_62 - name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_55 + schema: *ref_63 - name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_56 + schema: *ref_64 - name: script_hash description: mask to filter exact matching path in: query - schema: *ref_57 + schema: *ref_65 - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_58 + schema: *ref_66 - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_59 + schema: *ref_67 - name: success description: filter on successful jobs in: query - schema: *ref_60 + schema: *ref_68 - name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_61 + schema: *ref_70 - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_62 + schema: *ref_73 - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_63 + schema: *ref_74 - name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_64 + schema: *ref_75 + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 - name: is_skipped description: is the job skipped in: query @@ -7540,6 +8903,16 @@ paths: in: query schema: type: boolean + - name: has_null_parent + description: has null parent + in: query + schema: + type: boolean + - name: is_not_schedule + description: is not a scheduled job + in: query + schema: + type: boolean responses: '200': description: All completed jobs @@ -7549,7 +8922,7 @@ paths: type: array items: type: object - properties: &ref_69 + properties: &ref_79 workspace_id: type: string id: @@ -7576,7 +8949,7 @@ paths: type: string args: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 result: {} logs: type: string @@ -7615,12 +8988,12 @@ paths: by extension its DT_TOKEN. flow_status: type: object - properties: *ref_65 - required: *ref_66 + properties: *ref_76 + required: *ref_77 raw_flow: type: object - properties: *ref_50 - required: *ref_51 + properties: *ref_58 + required: *ref_59 is_flow_step: type: boolean language: @@ -7639,6 +9012,9 @@ paths: - graphql - nativets - bun + - php + - rust + - ansible is_skipped: type: boolean email: @@ -7651,7 +9027,15 @@ paths: type: string priority: type: integer - required: &ref_70 + labels: + type: array + items: + type: string + self_wait_time_ms: + type: number + aggregate_wait_time_ms: + type: number + required: &ref_80 - id - created_by - duration_ms @@ -7680,59 +9064,87 @@ paths: - name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_42 + schema: *ref_49 + - name: label + description: >- + mask to filter exact matching job's label (job labels are completed + jobs with as a result an object containing a string in the array at + key 'wm_labels') + in: query + schema: *ref_78 - name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_36 + schema: *ref_42 - name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_54 + schema: *ref_62 - name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_55 + schema: *ref_63 - name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_56 + schema: *ref_64 - name: script_hash description: mask to filter exact matching path in: query - schema: *ref_57 + schema: *ref_65 - name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_58 + schema: *ref_66 - name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_59 + schema: *ref_67 + - name: created_before + description: filter on created before (inclusive) timestamp + in: query + schema: &ref_117 + type: string + format: date-time + - name: created_after + description: filter on created after (exclusive) timestamp + in: query + schema: &ref_118 + type: string + format: date-time - name: created_or_started_before description: >- filter on created_at for non non started job and started_at otherwise before (inclusive) timestamp in: query - schema: &ref_99 + schema: &ref_111 type: string format: date-time - name: running description: filter on running jobs in: query - schema: *ref_67 + schema: *ref_72 - name: scheduled_for_before_now description: filter on jobs scheduled_for before now (hence waitinf for a worker) in: query - schema: *ref_68 + schema: *ref_69 - name: created_or_started_after description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp in: query - schema: &ref_98 + schema: &ref_112 + type: string + format: date-time + - name: created_or_started_after_completed_jobs + description: >- + filter on created_at for non non started job and started_at + otherwise after (exclusive) timestamp but only for the completed + jobs + in: query + schema: &ref_113 type: string format: date-time - name: job_kinds @@ -7740,23 +9152,35 @@ paths: filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_61 + schema: *ref_70 + - name: suspended + description: filter on suspended jobs + in: query + schema: *ref_71 - name: args description: >- filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_62 + schema: *ref_73 - name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_64 + schema: *ref_75 - name: result description: >- filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_63 + schema: *ref_74 + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 - name: is_skipped description: is the job skipped in: query @@ -7767,6 +9191,11 @@ paths: in: query schema: type: boolean + - name: has_null_parent + description: has null parent + in: query + schema: + type: boolean - name: success description: filter on successful jobs in: query @@ -7779,6 +9208,11 @@ paths: in: query schema: type: boolean + - name: is_not_schedule + description: is not a scheduled job + in: query + schema: + type: boolean responses: '200': description: All jobs @@ -7787,22 +9221,28 @@ paths: schema: type: array items: - allOf: &ref_74 - - oneOf: + oneOf: &ref_84 + - allOf: - type: object - properties: *ref_69 - required: *ref_70 + properties: *ref_79 + required: *ref_80 - type: object - properties: *ref_71 - required: *ref_72 - - type: object - properties: - type: - type: string - enum: - - CompletedJob - - QueuedJob - discriminator: &ref_75 + properties: + type: + type: string + enum: + - CompletedJob + - allOf: + - type: object + properties: *ref_81 + required: *ref_82 + - type: object + properties: + type: + type: string + enum: + - QueuedJob + discriminator: &ref_85 propertyName: type /jobs/db_clock: get: @@ -7817,6 +9257,44 @@ paths: application/json: schema: type: integer + /jobs/completed/count_by_tag: + get: + summary: Count jobs by tag + operationId: countJobsByTag + tags: + - job + parameters: + - name: horizon_secs + in: query + description: >- + Past Time horizon in seconds (when to start the count = now - + horizon) (default is 3600) + required: false + schema: + type: integer + - name: workspace_id + in: query + description: Specific workspace ID to filter results (optional) + required: false + schema: + type: string + responses: + '200': + description: Job counts by tag + content: + application/json: + schema: + type: array + items: + type: object + properties: + tag: + type: string + count: + type: integer + required: + - tag + - count /w/{workspace}/jobs_u/get/{id}: get: summary: get job @@ -7831,15 +9309,19 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 + - name: no_logs + in: query + schema: + type: boolean responses: '200': description: job details content: application/json: schema: - allOf: *ref_74 - discriminator: *ref_75 + oneOf: *ref_84 + discriminator: *ref_85 /w/{workspace}/jobs_u/get_root_job_id/{id}: get: summary: get root job id @@ -7854,7 +9336,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 responses: '200': description: get root job id @@ -7876,7 +9358,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 responses: '200': description: job details @@ -7884,6 +9366,27 @@ paths: text/plain: schema: type: string + /w/{workspace}/jobs_u/get_args/{id}: + get: + summary: get job args + operationId: getJobArgs + tags: + - job + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: id + in: path + required: true + schema: *ref_83 + responses: + '200': + description: job args + content: + application/json: + schema: {} /w/{workspace}/jobs_u/getupdate/{id}: get: summary: get job updates @@ -7898,7 +9401,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 - name: running in: query schema: @@ -7907,6 +9410,10 @@ paths: in: query schema: type: integer + - name: get_progress + in: query + schema: + type: boolean responses: '200': description: job details @@ -7921,8 +9428,49 @@ paths: type: boolean new_logs: type: string + log_offset: + type: integer mem_peak: type: integer + progress: + type: integer + flow_status: + type: object + additionalProperties: &ref_151 + type: object + properties: &ref_152 + scheduled_for: + type: string + format: date-time + started_at: + type: string + format: date-time + duration_ms: + type: number + name: + type: string + /w/{workspace}/jobs_u/get_log_file/{path}: + get: + summary: get log file from object store + operationId: getLogFileFromStore + tags: + - job + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: + type: string + responses: + '200': + description: job log + content: + text/plain: + type: string /w/{workspace}/jobs_u/get_flow_debug_info/{id}: get: summary: get flow debug info @@ -7937,7 +9485,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 responses: '200': description: flow debug info details @@ -7958,7 +9506,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 responses: '200': description: job details @@ -7966,8 +9514,8 @@ paths: application/json: schema: type: object - properties: *ref_69 - required: *ref_70 + properties: *ref_79 + required: *ref_80 /w/{workspace}/jobs_u/completed/get_result/{id}: get: summary: get completed job result @@ -7982,7 +9530,23 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 + - name: suspended_job + in: query + schema: + type: string + - name: resume_id + in: query + schema: + type: integer + - name: secret + in: query + schema: + type: string + - name: approver + in: query + schema: + type: string responses: '200': description: result @@ -8003,10 +9567,10 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 - name: get_started in: query - schema: &ref_108 + schema: &ref_124 type: boolean responses: '200': @@ -8040,7 +9604,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 responses: '200': description: job details @@ -8048,11 +9612,11 @@ paths: application/json: schema: type: object - properties: *ref_69 - required: *ref_70 + properties: *ref_79 + required: *ref_80 /w/{workspace}/jobs_u/queue/cancel/{id}: post: - summary: cancel queued job + summary: cancel queued or running job operationId: cancelQueuedJob tags: - job @@ -8064,7 +9628,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 requestBody: description: reason required: true @@ -8096,7 +9660,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 requestBody: description: reason required: true @@ -8128,7 +9692,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 requestBody: description: reason required: true @@ -8160,7 +9724,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 - name: resume_id in: path required: true @@ -8191,7 +9755,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 - name: resume_id in: path required: true @@ -8233,7 +9797,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 - name: payload description: > The base64 encoded payload that has been encoded as a JSON. e.g how @@ -8241,7 +9805,7 @@ paths: `encodeURIComponent(btoa(JSON.stringify({a: 2})))` in: query - schema: *ref_76 + schema: *ref_86 - name: resume_id in: path required: true @@ -8276,7 +9840,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 - name: resume_id in: path required: true @@ -8304,6 +9868,64 @@ paths: text/plain: schema: type: string + /w/{workspace}/jobs/flow/user_states/{id}/{key}: + post: + summary: set flow user state at a given key + operationId: setFlowUserState + tags: + - job + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: id + in: path + required: true + schema: *ref_83 + - name: key + in: path + required: true + schema: + type: string + requestBody: + description: new value + required: true + content: + application/json: + schema: {} + responses: + '200': + description: flow user state updated + content: + text/plain: + schema: + type: string + get: + summary: get flow user state at a given key + operationId: getFlowUserState + tags: + - job + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: id + in: path + required: true + schema: *ref_83 + - name: key + in: path + required: true + schema: + type: string + responses: + '200': + description: flow user state updated + content: + application/json: + schema: {} /w/{workspace}/jobs/flow/resume/{id}: post: summary: resume a job for a suspended flow as an owner @@ -8318,7 +9940,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 requestBody: required: true content: @@ -8346,7 +9968,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 - name: resume_id in: path required: true @@ -8381,7 +10003,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 - name: resume_id in: path required: true @@ -8423,7 +10045,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 - name: resume_id in: path required: true @@ -8447,8 +10069,8 @@ paths: type: object properties: job: - allOf: *ref_74 - discriminator: *ref_75 + oneOf: *ref_84 + discriminator: *ref_85 approvers: type: array items: @@ -8513,7 +10135,7 @@ paths: application/json: schema: type: object - properties: &ref_145 + properties: &ref_162 path: type: string schedule: @@ -8526,7 +10148,7 @@ paths: type: boolean args: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 enabled: type: boolean on_failure: @@ -8537,26 +10159,34 @@ paths: type: boolean on_failure_extra_args: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 on_recovery: type: string on_recovery_times: type: number on_recovery_extra_args: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 + on_success: + type: string + on_success_extra_args: + type: object + additionalProperties: *ref_16 ws_error_handler_muted: type: boolean retry: type: object - properties: *ref_77 + properties: *ref_87 no_flow_overlap: type: boolean summary: type: string tag: type: string - required: &ref_146 + paused_until: + type: string + format: date-time + required: &ref_163 - path - schedule - timezone @@ -8584,7 +10214,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 requestBody: description: updated schedule required: true @@ -8592,14 +10222,14 @@ paths: application/json: schema: type: object - properties: &ref_147 + properties: &ref_164 schedule: type: string timezone: type: string args: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 on_failure: type: string on_failure_times: @@ -8608,26 +10238,34 @@ paths: type: boolean on_failure_extra_args: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 on_recovery: type: string on_recovery_times: type: number on_recovery_extra_args: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 + on_success: + type: string + on_success_extra_args: + type: object + additionalProperties: *ref_16 ws_error_handler_muted: type: boolean retry: type: object - properties: *ref_77 + properties: *ref_87 no_flow_overlap: type: boolean summary: type: string tag: type: string - required: &ref_148 + paused_until: + type: string + format: date-time + required: &ref_165 - schedule - timezone - script_path @@ -8654,7 +10292,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 requestBody: description: updated schedule enable required: true @@ -8688,7 +10326,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: schedule deleted @@ -8710,7 +10348,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: schedule deleted @@ -8718,7 +10356,7 @@ paths: application/json: schema: type: object - properties: &ref_78 + properties: &ref_88 path: type: string edited_by: @@ -8738,7 +10376,7 @@ paths: type: boolean args: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 extra_perms: type: object additionalProperties: @@ -8755,26 +10393,34 @@ paths: type: boolean on_failure_extra_args: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 on_recovery: type: string on_recovery_times: type: number on_recovery_extra_args: type: object - additionalProperties: *ref_14 + additionalProperties: *ref_16 + on_success: + type: string + on_success_extra_args: + type: object + additionalProperties: *ref_16 ws_error_handler_muted: type: boolean retry: type: object - properties: *ref_77 + properties: *ref_87 summary: type: string no_flow_overlap: type: boolean tag: type: string - required: &ref_79 + paused_until: + type: string + format: date-time + required: &ref_89 - path - edited_by - edited_at @@ -8799,7 +10445,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: schedule exists @@ -8821,11 +10467,17 @@ paths: - name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 + - name: args + description: >- + filter on jobs containing those args as a json subset (@> in + postgres) + in: query + schema: *ref_73 - name: path description: filter by path in: query @@ -8835,6 +10487,10 @@ paths: in: query schema: type: boolean + - name: path_start + in: query + schema: + type: string responses: '200': description: schedule list @@ -8844,8 +10500,8 @@ paths: type: array items: type: object - properties: *ref_78 - required: *ref_79 + properties: *ref_88 + required: *ref_89 /w/{workspace}/schedules/list_with_jobs: get: summary: list schedules with last 20 jobs @@ -8860,11 +10516,11 @@ paths: - name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 responses: '200': description: schedule list @@ -8873,10 +10529,10 @@ paths: schema: type: array items: - allOf: &ref_144 + allOf: &ref_161 - type: object - properties: *ref_78 - required: *ref_79 + properties: *ref_88 + required: *ref_89 - type: object properties: jobs: @@ -8918,6 +10574,7 @@ paths: enum: - error - recovery + - success override_existing: type: boolean path: @@ -8936,6 +10593,332 @@ paths: responses: '201': description: default error handler set + /w/{workspace}/http_triggers/create: + post: + summary: create http trigger + operationId: createHttpTrigger + tags: + - http_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: new http trigger + required: true + content: + application/json: + schema: + type: object + properties: &ref_166 + path: + type: string + script_path: + type: string + route_path: + type: string + is_flow: + type: boolean + http_method: + type: string + enum: + - get + - post + - put + - delete + - patch + is_async: + type: boolean + requires_auth: + type: boolean + required: &ref_167 + - path + - script_path + - route_path + - is_flow + - is_async + - requires_auth + - http_method + responses: + '201': + description: http trigger created + content: + text/plain: + schema: + type: string + /w/{workspace}/http_triggers/update/{path}: + post: + summary: update http trigger + operationId: updateHttpTrigger + tags: + - http_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_21 + requestBody: + description: updated trigger + required: true + content: + application/json: + schema: + type: object + properties: &ref_168 + path: + type: string + script_path: + type: string + route_path: + type: string + is_flow: + type: boolean + http_method: + type: string + enum: + - get + - post + - put + - delete + - patch + is_async: + type: boolean + requires_auth: + type: boolean + required: &ref_169 + - path + - script_path + - is_flow + - kind + - is_async + - requires_auth + - http_method + responses: + '200': + description: http trigger updated + content: + text/plain: + schema: + type: string + /w/{workspace}/http_triggers/delete/{path}: + delete: + summary: delete http trigger + operationId: deleteHttpTrigger + tags: + - http_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_21 + responses: + '200': + description: http trigger deleted + content: + text/plain: + schema: + type: string + /w/{workspace}/http_triggers/get/{path}: + get: + summary: get http trigger + operationId: getHttpTrigger + tags: + - http_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_21 + responses: + '200': + description: http trigger deleted + content: + application/json: + schema: + type: object + properties: &ref_90 + path: + type: string + edited_by: + type: string + edited_at: + type: string + format: date-time + script_path: + type: string + route_path: + type: string + is_flow: + type: boolean + extra_perms: + type: object + additionalProperties: + type: boolean + email: + type: string + workspace_id: + type: string + http_method: + type: string + enum: + - get + - post + - put + - delete + - patch + is_async: + type: boolean + requires_auth: + type: boolean + required: &ref_91 + - path + - edited_by + - edited_at + - script_path + - route_path + - extra_perms + - is_flow + - email + - workspace_id + - is_async + - requires_auth + - http_method + /w/{workspace}/http_triggers/list: + get: + summary: list http triggers + operationId: listHttpTriggers + tags: + - http_trigger + parameters: + - required: true + name: workspace + in: path + schema: *ref_0 + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 + - name: path + description: filter by path + in: query + schema: + type: string + - name: is_flow + in: query + schema: + type: boolean + - name: path_start + in: query + schema: + type: string + responses: + '200': + description: http trigger list + content: + application/json: + schema: + type: array + items: + type: object + properties: *ref_90 + required: *ref_91 + /w/{workspace}/http_triggers/exists/{path}: + get: + summary: does http trigger exists + operationId: existsHttpTrigger + tags: + - http_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_21 + responses: + '200': + description: http trigger exists + content: + application/json: + schema: + type: boolean + /w/{workspace}/http_triggers/route_exists: + post: + summary: does route exists + operationId: existsRoute + tags: + - http_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: route exists request + required: true + content: + application/json: + schema: + type: object + properties: + route_path: + type: string + http_method: + type: string + enum: + - get + - post + - put + - delete + - patch + required: + - kind + - route_path + - http_method + responses: + '200': + description: route exists + content: + application/json: + schema: + type: boolean + /w/{workspace}/http_triggers/used: + get: + summary: whether http triggers are used + operationId: used + tags: + - http_trigger + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + responses: + '200': + description: whether http triggers are used + content: + application/json: + schema: + type: boolean /groups/list: get: summary: list instance groups @@ -8951,7 +10934,7 @@ paths: type: array items: type: object - properties: &ref_81 + properties: &ref_93 name: type: string summary: @@ -8960,7 +10943,7 @@ paths: type: array items: type: string - required: &ref_82 + required: &ref_94 - name /groups/get/{name}: get: @@ -8972,7 +10955,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 responses: '200': description: instance group @@ -8980,8 +10963,8 @@ paths: application/json: schema: type: object - properties: *ref_81 - required: *ref_82 + properties: *ref_93 + required: *ref_94 /groups/create: post: summary: create instance group @@ -9019,7 +11002,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 requestBody: description: update instance group required: true @@ -9049,7 +11032,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 responses: '200': description: instance group deleted @@ -9067,7 +11050,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 requestBody: description: user to add to instance group required: true @@ -9097,7 +11080,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 requestBody: description: user to remove from instance group required: true @@ -9117,6 +11100,62 @@ paths: text/plain: schema: type: string + /groups/export: + get: + summary: export instance groups + operationId: exportInstanceGroups + tags: + - group + responses: + '200': + description: exported instance groups + content: + application/json: + schema: + type: array + items: + type: object + properties: &ref_95 + name: + type: string + summary: + type: string + emails: + type: array + items: + type: string + id: + type: string + scim_display_name: + type: string + external_id: + type: string + required: &ref_96 + - name + /groups/overwrite: + post: + summary: overwrite instance groups + operationId: overwriteInstanceGroups + tags: + - group + requestBody: + description: overwrite instance groups + required: true + content: + application/json: + schema: + type: array + items: + type: object + properties: *ref_95 + required: *ref_96 + responses: + '200': + description: success message + content: + text/plain: + schema: + type: string /w/{workspace}/groups/list: get: summary: list groups @@ -9131,11 +11170,11 @@ paths: - name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 responses: '200': description: group list @@ -9145,7 +11184,7 @@ paths: type: array items: type: object - properties: &ref_83 + properties: &ref_97 name: type: string summary: @@ -9158,7 +11197,7 @@ paths: type: object additionalProperties: type: boolean - required: &ref_84 + required: &ref_98 - name /w/{workspace}/groups/listnames: get: @@ -9231,7 +11270,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 requestBody: description: updated group required: true @@ -9263,7 +11302,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 responses: '200': description: group deleted @@ -9285,7 +11324,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 responses: '200': description: group @@ -9293,8 +11332,8 @@ paths: application/json: schema: type: object - properties: *ref_83 - required: *ref_84 + properties: *ref_97 + required: *ref_98 /w/{workspace}/groups/adduser/{name}: post: summary: add user to group @@ -9309,7 +11348,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 requestBody: description: added user to group required: true @@ -9341,7 +11380,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 requestBody: description: added user to group required: true @@ -9373,11 +11412,11 @@ paths: - name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 responses: '200': description: folder list @@ -9387,7 +11426,7 @@ paths: type: array items: type: object - properties: &ref_85 + properties: &ref_99 name: type: string owners: @@ -9398,7 +11437,14 @@ paths: type: object additionalProperties: type: boolean - required: &ref_86 + summary: + type: string + created_by: + type: string + edited_at: + type: string + format: date-time + required: &ref_100 - name - owners - extra_perms @@ -9448,6 +11494,8 @@ paths: properties: name: type: string + summary: + type: string owners: type: array items: @@ -9478,7 +11526,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 requestBody: description: update folder required: true @@ -9487,6 +11535,8 @@ paths: schema: type: object properties: + summary: + type: string owners: type: array items: @@ -9515,7 +11565,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 responses: '200': description: folder deleted @@ -9537,7 +11587,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 responses: '200': description: folder @@ -9545,8 +11595,8 @@ paths: application/json: schema: type: object - properties: *ref_85 - required: *ref_86 + properties: *ref_99 + required: *ref_100 /w/{workspace}/folders/getusage/{name}: get: summary: get folder usage @@ -9561,7 +11611,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 responses: '200': description: folder @@ -9603,7 +11653,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 requestBody: description: owner user to folder required: true @@ -9614,6 +11664,8 @@ paths: properties: owner: type: string + required: + - owner responses: '200': description: owner added to folder @@ -9635,7 +11687,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 requestBody: description: added owner to folder required: true @@ -9646,6 +11698,10 @@ paths: properties: owner: type: string + write: + type: boolean + required: + - owner responses: '200': description: owner removed from folder @@ -9663,11 +11719,11 @@ paths: - name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 - name: ping_since in: query required: false @@ -9685,7 +11741,7 @@ paths: type: array items: type: object - properties: &ref_149 + properties: &ref_170 worker: type: string worker_instance: @@ -9707,7 +11763,27 @@ paths: type: string wm_version: type: string - required: &ref_150 + last_job_id: + type: string + last_job_workspace_id: + type: string + occupancy_rate: + type: number + occupancy_rate_15s: + type: number + occupancy_rate_5m: + type: number + occupancy_rate_30m: + type: number + memory: + type: number + vcpus: + type: number + memory_usage: + type: number + wm_memory_usage: + type: number + required: &ref_171 - worker - worker_instance - ping_at @@ -9735,6 +11811,39 @@ paths: application/json: schema: type: boolean + /workers/queue_metrics: + get: + summary: get queue metrics + operationId: getQueueMetrics + tags: + - worker + responses: + '200': + description: metrics + content: + application/json: + schema: + type: array + items: + type: object + properties: + id: + type: string + values: + type: array + items: + type: object + properties: + created_at: + type: string + value: + type: number + required: + - created_at + - value + required: + - id + - values /configs/list_worker_groups: get: summary: list worker groups @@ -9767,7 +11876,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 responses: '200': description: a config @@ -9784,7 +11893,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 requestBody: description: worker group required: true @@ -9807,7 +11916,7 @@ paths: - name: name in: path required: true - schema: *ref_80 + schema: *ref_92 responses: '200': description: Delete config @@ -9815,6 +11924,28 @@ paths: text/plain: schema: type: string + /configs/list: + get: + summary: list configs + operationId: listConfigs + tags: + - config + responses: + '200': + description: list of configs + content: + application/json: + schema: + type: array + items: + type: object + properties: &ref_216 + name: + type: string + config: + type: object + required: &ref_217 + - name /w/{workspace}/acls/get/{kind}/{path}: get: summary: get granular acls @@ -9829,7 +11960,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 - name: kind in: path required: true @@ -9845,6 +11976,7 @@ paths: - folder - app - raw_app + - http_trigger responses: '200': description: acls @@ -9868,7 +12000,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 - name: kind in: path required: true @@ -9884,6 +12016,7 @@ paths: - folder - app - raw_app + - http_trigger requestBody: description: acl to add required: true @@ -9919,7 +12052,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 - name: kind in: path required: true @@ -9935,6 +12068,7 @@ paths: - folder - app - raw_app + - http_trigger requestBody: description: acl to add required: true @@ -9968,7 +12102,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '204': description: flow preview captured @@ -9986,7 +12120,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '201': description: flow preview capture created @@ -10003,7 +12137,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 responses: '200': description: captured flow preview @@ -10083,24 +12217,24 @@ paths: schema: *ref_0 - name: runnable_id in: query - schema: &ref_89 + schema: &ref_101 type: string - name: runnable_type in: query - schema: &ref_90 + schema: &ref_102 type: string - enum: &ref_117 + enum: &ref_133 - ScriptHash - ScriptPath - FlowPath - name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 responses: '200': description: Input history for completed jobs @@ -10110,8 +12244,57 @@ paths: type: array items: type: object - properties: *ref_87 - required: *ref_88 + properties: &ref_103 + id: + type: string + name: + type: string + created_by: + type: string + created_at: + type: string + format: date-time + is_public: + type: boolean + success: + type: boolean + required: &ref_104 + - id + - name + - args + - created_by + - created_at + - is_public + /w/{workspace}/inputs/{jobOrInputId}/args: + get: + summary: Get args from history or saved input + operationId: getArgsFromHistoryOrSavedInput + tags: + - input + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: jobOrInputId + in: path + required: true + schema: + type: string + - name: input + in: query + schema: + type: boolean + - name: allow_large + in: query + schema: + type: boolean + responses: + '200': + description: args + content: + application/json: + schema: {} /w/{workspace}/inputs/list: get: summary: List saved Inputs for a Runnable @@ -10125,18 +12308,18 @@ paths: schema: *ref_0 - name: runnable_id in: query - schema: *ref_89 + schema: *ref_101 - name: runnable_type in: query - schema: *ref_90 + schema: *ref_102 - name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 - name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 responses: '200': description: Saved Inputs for a Runnable @@ -10146,8 +12329,8 @@ paths: type: array items: type: object - properties: *ref_87 - required: *ref_88 + properties: *ref_103 + required: *ref_104 /w/{workspace}/inputs/create: post: summary: Create an Input for future use in a script or flow @@ -10161,10 +12344,10 @@ paths: schema: *ref_0 - name: runnable_id in: query - schema: *ref_89 + schema: *ref_101 - name: runnable_type in: query - schema: *ref_90 + schema: *ref_102 requestBody: description: Input required: true @@ -10172,12 +12355,12 @@ paths: application/json: schema: type: object - properties: &ref_113 + properties: &ref_129 name: type: string args: type: object - required: &ref_114 + required: &ref_130 - name - args - created_by @@ -10207,14 +12390,14 @@ paths: application/json: schema: type: object - properties: &ref_115 + properties: &ref_131 id: type: string name: type: string is_public: type: boolean - required: &ref_116 + required: &ref_132 - id - name - is_public @@ -10240,7 +12423,7 @@ paths: - name: input in: path required: true - schema: &ref_107 + schema: &ref_123 type: string responses: '200': @@ -10273,7 +12456,7 @@ paths: properties: s3_resource: type: object - properties: &ref_91 + properties: &ref_105 bucket: type: string region: @@ -10288,7 +12471,7 @@ paths: type: string pathStyle: type: boolean - required: &ref_92 + required: &ref_106 - bucket - region - endPoint @@ -10364,8 +12547,8 @@ paths: properties: s3_resource: type: object - properties: *ref_91 - required: *ref_92 + properties: *ref_105 + required: *ref_106 responses: '200': description: Connection settings @@ -10386,10 +12569,10 @@ paths: type: boolean client_kwargs: type: object - properties: &ref_93 + properties: &ref_107 region_name: type: string - required: &ref_94 + required: &ref_108 - region_name required: - endpoint_url @@ -10444,8 +12627,8 @@ paths: type: boolean client_kwargs: type: object - properties: *ref_93 - required: *ref_94 + properties: *ref_107 + required: *ref_108 required: - endpoint_url - use_ssl @@ -10503,11 +12686,11 @@ paths: application/json: schema: type: object - properties: *ref_91 - required: *ref_92 + properties: *ref_105 + required: *ref_106 /w/{workspace}/job_helpers/test_connection: get: - summary: Test connection to the workspace datasets storage + summary: Test connection to the workspace object storage operationId: datasetStorageTestConnection tags: - helpers @@ -10516,6 +12699,10 @@ paths: in: path required: true schema: *ref_0 + - name: storage + in: query + schema: + type: string responses: '200': description: Connection settings @@ -10524,7 +12711,7 @@ paths: schema: {} /w/{workspace}/job_helpers/list_stored_files: get: - summary: List the file keys available in the workspace files storage (S3) + summary: List the file keys available in a workspace object storage operationId: listStoredFiles tags: - helpers @@ -10546,6 +12733,10 @@ paths: in: query schema: type: string + - name: storage + in: query + schema: + type: string responses: '200': description: List of file keys @@ -10560,10 +12751,10 @@ paths: type: array items: type: object - properties: &ref_169 + properties: &ref_193 s3: type: string - required: &ref_170 + required: &ref_194 - s3 restricted_access: type: boolean @@ -10585,6 +12776,10 @@ paths: required: true schema: type: string + - name: storage + in: query + schema: + type: string responses: '200': description: FileMetadata @@ -10592,7 +12787,7 @@ paths: application/json: schema: type: object - properties: &ref_171 + properties: &ref_195 mime_type: type: string size_in_bytes: @@ -10645,6 +12840,10 @@ paths: in: query schema: type: integer + - name: storage + in: query + schema: + type: string responses: '200': description: FilePreview @@ -10652,7 +12851,7 @@ paths: application/json: schema: type: object - properties: &ref_172 + properties: &ref_196 msg: type: string content: @@ -10664,7 +12863,7 @@ paths: - Csv - Parquet - Unknown - required: &ref_173 + required: &ref_197 - content_type /w/{workspace}/job_helpers/load_parquet_preview/{path}: get: @@ -10680,7 +12879,7 @@ paths: - name: path in: path required: true - schema: *ref_17 + schema: *ref_21 - name: offset in: query schema: @@ -10705,12 +12904,106 @@ paths: in: query schema: type: string + - name: storage + in: query + schema: + type: string responses: '200': description: Parquet Preview content: application/json: schema: {} + /w/{workspace}/job_helpers/load_table_count/{path}: + get: + summary: Load the table row count + operationId: loadTableRowCount + tags: + - helpers + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_21 + - name: search_col + in: query + schema: + type: string + - name: search_term + in: query + schema: + type: string + - name: storage + in: query + schema: + type: string + responses: + '200': + description: Table count + content: + application/json: + schema: + type: object + properties: + count: + type: number + /w/{workspace}/job_helpers/load_csv_preview/{path}: + get: + summary: Load a preview of a csv file + operationId: loadCsvPreview + tags: + - helpers + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: path + in: path + required: true + schema: *ref_21 + - name: offset + in: query + schema: + type: number + - name: limit + in: query + schema: + type: number + - name: sort_col + in: query + schema: + type: string + - name: sort_desc + in: query + schema: + type: boolean + - name: search_col + in: query + schema: + type: string + - name: search_term + in: query + schema: + type: string + - name: storage + in: query + schema: + type: string + - name: csv_separator + in: query + schema: + type: string + responses: + '200': + description: Csv Preview + content: + application/json: + schema: {} /w/{workspace}/job_helpers/delete_s3_file: delete: summary: Permanently delete file from S3 @@ -10727,6 +13020,10 @@ paths: required: true schema: type: string + - name: storage + in: query + schema: + type: string responses: '200': description: Confirmation @@ -10754,6 +13051,10 @@ paths: required: true schema: type: string + - name: storage + in: query + schema: + type: string responses: '200': description: Confirmation @@ -10791,6 +13092,10 @@ paths: required: false schema: type: string + - name: storage + in: query + schema: + type: string requestBody: description: File content required: true @@ -10837,6 +13142,10 @@ paths: required: false schema: type: string + - name: storage + in: query + schema: + type: string responses: '200': description: Chunk of the downloaded file @@ -10845,6 +13154,39 @@ paths: schema: type: string format: binary + /w/{workspace}/job_helpers/download_s3_parquet_file_as_csv: + get: + summary: Download file to S3 bucket + operationId: fileDownloadParquetAsCsv + tags: + - helpers + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: file_key + in: query + required: true + schema: + type: string + - name: s3_resource_path + in: query + required: false + schema: + type: string + - name: resource_type + in: query + required: false + schema: + type: string + responses: + '200': + description: The downloaded file + content: + text/csv: + schema: + type: string /w/{workspace}/job_metrics/get/{id}: post: summary: get job metrics @@ -10859,7 +13201,7 @@ paths: - name: id in: path required: true - schema: *ref_73 + schema: *ref_83 requestBody: description: parameters for statistics retrieval required: true @@ -10888,48 +13230,175 @@ paths: type: array items: type: object - properties: &ref_176 + properties: &ref_200 id: type: string name: type: string - required: &ref_177 + required: &ref_201 - id scalar_metrics: type: array items: type: object - properties: &ref_178 + properties: &ref_202 metric_id: type: string value: type: number - required: &ref_179 + required: &ref_203 - id - value timeseries_metrics: type: array items: type: object - properties: &ref_180 + properties: &ref_204 metric_id: type: string values: type: array items: type: object - properties: &ref_182 + properties: &ref_206 timestamp: type: string format: date-time value: type: number - required: &ref_183 + required: &ref_207 - timestamp - value - required: &ref_181 + required: &ref_205 - id - values + /w/{workspace}/job_metrics/set_progress/{id}: + post: + summary: set job metrics + operationId: setJobProgress + tags: + - metrics + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: id + in: path + required: true + schema: *ref_83 + requestBody: + description: parameters for statistics retrieval + required: true + content: + application/json: + schema: + type: object + properties: + percent: + type: integer + flow_job_id: + type: string + format: uuid + responses: + '200': + description: Job progress updated + content: + application/json: + schema: {} + /w/{workspace}/job_metrics/get_progress/{id}: + get: + summary: get job progress + operationId: getJobProgress + tags: + - metrics + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: id + in: path + required: true + schema: *ref_83 + responses: + '200': + description: job progress between 0 and 99 + content: + application/json: + schema: + type: integer + /service_logs/list_files: + get: + summary: list log files ordered by timestamp + operationId: listLogFiles + tags: + - service_logs + parameters: + - name: before + description: filter on started before (inclusive) timestamp + in: query + schema: *ref_109 + - name: after + description: filter on created after (exclusive) timestamp + in: query + schema: *ref_110 + - name: with_error + in: query + required: false + schema: + type: boolean + responses: + '200': + description: time + content: + application/json: + schema: + type: array + items: + type: object + properties: + hostname: + type: string + mode: + type: string + worker_group: + type: string + log_ts: + type: string + format: date-time + file_path: + type: string + ok_lines: + type: integer + err_lines: + type: integer + json_fmt: + type: boolean + required: + - hostname + - mode + - log_ts + - file_path + - json_fmt + /service_logs/get_log_file/{path}: + get: + summary: get log file by path + operationId: getLogFile + tags: + - service_logs + parameters: + - name: path + in: path + required: true + schema: *ref_21 + responses: + '200': + description: log stream + content: + text/plain: + schema: + type: string /concurrency_groups/list: get: summary: List all concurrency groups @@ -10945,17 +13414,15 @@ paths: type: array items: type: object - properties: &ref_186 - concurrency_id: + properties: &ref_210 + concurrency_key: type: string - job_uuids: - type: array - items: - type: string - required: &ref_187 - - concurrency_id - - job_uuids - /concurrency_groups/{concurrency_id}: + total_running: + type: number + required: &ref_211 + - concurrency_key + - total_running + /concurrency_groups/prune/{concurrency_id}: delete: summary: Delete concurrency group operationId: deleteConcurrencyGroup @@ -10965,7 +13432,7 @@ paths: - name: concurrency_id in: path required: true - schema: &ref_109 + schema: &ref_125 type: string responses: '200': @@ -10975,6 +13442,251 @@ paths: schema: type: object properties: {} + /concurrency_groups/{id}/key: + get: + summary: Get the concurrency key for a job that has concurrency limits enabled + operationId: getConcurrencyKey + tags: + - concurrencyGroups + parameters: + - name: id + in: path + required: true + schema: *ref_83 + responses: + '200': + description: concurrency key for given job + content: + application/json: + schema: + type: string + /w/{workspace}/concurrency_groups/list_jobs: + get: + summary: Get intervals of job runtime concurrency + operationId: listExtendedJobs + tags: + - concurrencyGroups + - job + parameters: + - name: concurrency_key + in: query + required: false + schema: + type: string + - name: row_limit + in: query + required: false + schema: + type: number + - name: workspace + in: path + required: true + schema: *ref_0 + - name: created_by + description: mask to filter exact matching user creator + in: query + schema: *ref_49 + - name: label + description: >- + mask to filter exact matching job's label (job labels are completed + jobs with as a result an object containing a string in the array at + key 'wm_labels') + in: query + schema: *ref_78 + - name: parent_job + description: >- + The parent job that is at the origin and responsible for the + execution of this script if any + in: query + schema: *ref_42 + - name: script_path_exact + description: mask to filter exact matching path + in: query + schema: *ref_62 + - name: script_path_start + description: mask to filter matching starting path + in: query + schema: *ref_63 + - name: schedule_path + description: mask to filter by schedule path + in: query + schema: *ref_64 + - name: script_hash + description: mask to filter exact matching path + in: query + schema: *ref_65 + - name: started_before + description: filter on started before (inclusive) timestamp + in: query + schema: *ref_66 + - name: started_after + description: filter on started after (exclusive) timestamp + in: query + schema: *ref_67 + - name: created_or_started_before + description: >- + filter on created_at for non non started job and started_at + otherwise before (inclusive) timestamp + in: query + schema: *ref_111 + - name: running + description: filter on running jobs + in: query + schema: *ref_72 + - name: scheduled_for_before_now + description: filter on jobs scheduled_for before now (hence waitinf for a worker) + in: query + schema: *ref_69 + - name: created_or_started_after + description: >- + filter on created_at for non non started job and started_at + otherwise after (exclusive) timestamp + in: query + schema: *ref_112 + - name: created_or_started_after_completed_jobs + description: >- + filter on created_at for non non started job and started_at + otherwise after (exclusive) timestamp but only for the completed + jobs + in: query + schema: *ref_113 + - name: job_kinds + description: >- + filter on job kind (values 'preview', 'script', 'dependencies', + 'flow') separated by, + in: query + schema: *ref_70 + - name: args + description: >- + filter on jobs containing those args as a json subset (@> in + postgres) + in: query + schema: *ref_73 + - name: tag + description: filter on jobs with a given tag/worker group + in: query + schema: *ref_75 + - name: result + description: >- + filter on jobs containing those result as a json subset (@> in + postgres) + in: query + schema: *ref_74 + - name: page + description: which page to return (start at 1, default 1) + in: query + schema: *ref_5 + - name: per_page + description: number of items to return for a given page (default 30, max 100) + in: query + schema: *ref_6 + - name: is_skipped + description: is the job skipped + in: query + schema: + type: boolean + - name: is_flow_step + description: is the job a flow step + in: query + schema: + type: boolean + - name: has_null_parent + description: has null parent + in: query + schema: + type: boolean + - name: success + description: filter on successful jobs + in: query + schema: + type: boolean + - name: all_workspaces + description: >- + get jobs from all workspaces (only valid if request come from the + `admins` workspace) + in: query + schema: + type: boolean + - name: is_not_schedule + description: is not a scheduled job + in: query + schema: + type: boolean + responses: + '200': + description: time + content: + application/json: + schema: + type: object + properties: &ref_212 + jobs: + type: array + items: + oneOf: *ref_84 + discriminator: *ref_85 + obscured_jobs: + type: array + items: + type: object + properties: &ref_134 + typ: + type: string + started_at: + type: string + format: date-time + duration_ms: + type: number + omitted_obscured_jobs: + description: >- + Obscured jobs omitted for security because of too specific + filtering + type: boolean + required: &ref_213 + - jobs + - obscured_jobs + /srch/w/{workspace}/index/search/job: + get: + summary: Search through jobs with a string query + operationId: searchJobsIndex + tags: + - indexSearch + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + - name: search_query + in: query + required: true + schema: + type: string + responses: + '200': + description: search results + content: + application/json: + schema: + type: object + properties: + query_parse_errors: + description: >- + a list of the terms that couldn't be parsed (and thus + ignored) + type: array + items: + type: object + properties: + dancer: + type: string + hits: + description: the jobs that matched the query + type: array + items: + type: object + properties: &ref_218 + dancer: + type: string components: securitySchemes: bearerAuth: @@ -10989,7 +13701,7 @@ components: name: key in: path required: true - schema: *ref_7 + schema: *ref_9 WorkspaceId: name: workspace in: path @@ -10999,89 +13711,104 @@ components: name: version in: path required: true - schema: *ref_95 + schema: *ref_114 Token: name: token in: path required: true - schema: *ref_96 + schema: *ref_115 AccountId: name: id in: path required: true - schema: *ref_21 + schema: *ref_25 ClientName: name: client_name in: path required: true - schema: *ref_20 + schema: *ref_24 ScriptPath: name: path in: path required: true - schema: *ref_29 + schema: *ref_33 ScriptHash: name: hash in: path required: true - schema: *ref_33 + schema: *ref_37 JobId: name: id in: path required: true - schema: *ref_73 + schema: *ref_83 Path: name: path in: path required: true - schema: *ref_17 + schema: *ref_21 PathId: name: id in: path required: true - schema: *ref_24 + schema: *ref_28 PathVersion: name: version in: path required: true - schema: *ref_97 + schema: *ref_116 Name: name: name in: path required: true - schema: *ref_80 + schema: *ref_92 Page: name: page description: which page to return (start at 1, default 1) in: query - schema: *ref_3 + schema: *ref_5 PerPage: name: per_page description: number of items to return for a given page (default 30, max 100) in: query - schema: *ref_4 + schema: *ref_6 OrderDesc: name: order_desc description: order by desc order (default true) in: query - schema: *ref_41 + schema: *ref_48 CreatedBy: name: created_by description: mask to filter exact matching user creator in: query - schema: *ref_42 + schema: *ref_49 + Label: + name: label + description: >- + mask to filter exact matching job's label (job labels are completed jobs + with as a result an object containing a string in the array at key + 'wm_labels') + in: query + schema: *ref_78 ParentJob: name: parent_job description: >- The parent job that is at the origin and responsible for the execution of this script if any in: query - schema: *ref_36 + schema: *ref_42 WorkerTag: name: tag description: Override the tag to use in: query - schema: *ref_38 + schema: *ref_44 + CacheTtl: + name: cache_ttl + description: >- + Override the cache time to live (in seconds). Can not be used to disable + caching, only override with a new cache ttl + in: query + schema: *ref_45 NewJobId: name: job_id description: >- @@ -11089,7 +13816,7 @@ components: randomly using the ULID scheme. If a job id already exists in the queue or as a completed job, the request to create one will fail (Bad Request) in: query - schema: *ref_37 + schema: *ref_43 IncludeHeader: name: include_header description: > @@ -11099,14 +13826,14 @@ components: Header's key lowercased and '-'' replaced to '_' such that 'Content-Type' becomes the 'content_type' arg key in: query - schema: *ref_39 + schema: *ref_46 QueueLimit: name: queue_limit description: > The maximum size of the queue for which the request would get rejected if that job would push it above that limit in: query - schema: *ref_40 + schema: *ref_47 Payload: name: payload description: > @@ -11115,233 +13842,249 @@ components: `encodeURIComponent(btoa(JSON.stringify({a: 2})))` in: query - schema: *ref_76 + schema: *ref_86 ScriptStartPath: name: script_path_start description: mask to filter matching starting path in: query - schema: *ref_55 + schema: *ref_63 SchedulePath: name: schedule_path description: mask to filter by schedule path in: query - schema: *ref_56 + schema: *ref_64 ScriptExactPath: name: script_path_exact description: mask to filter exact matching path in: query - schema: *ref_54 + schema: *ref_62 ScriptExactHash: name: script_hash description: mask to filter exact matching path in: query - schema: *ref_57 + schema: *ref_65 + CreatedBefore: + name: created_before + description: filter on created before (inclusive) timestamp + in: query + schema: *ref_117 + CreatedAfter: + name: created_after + description: filter on created after (exclusive) timestamp + in: query + schema: *ref_118 StartedBefore: name: started_before description: filter on started before (inclusive) timestamp in: query - schema: *ref_58 + schema: *ref_66 StartedAfter: name: started_after description: filter on started after (exclusive) timestamp in: query - schema: *ref_59 + schema: *ref_67 + Before: + name: before + description: filter on started before (inclusive) timestamp + in: query + schema: *ref_109 CreatedOrStartedAfter: name: created_or_started_after description: >- filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp in: query - schema: *ref_98 + schema: *ref_112 + CreatedOrStartedAfterCompletedJob: + name: created_or_started_after_completed_jobs + description: >- + filter on created_at for non non started job and started_at otherwise + after (exclusive) timestamp but only for the completed jobs + in: query + schema: *ref_113 CreatedOrStartedBefore: name: created_or_started_before description: >- filter on created_at for non non started job and started_at otherwise before (inclusive) timestamp in: query - schema: *ref_99 + schema: *ref_111 Success: name: success description: filter on successful jobs in: query - schema: *ref_60 + schema: *ref_68 ScheduledForBeforeNow: name: scheduled_for_before_now description: filter on jobs scheduled_for before now (hence waitinf for a worker) in: query - schema: *ref_68 + schema: *ref_69 Suspended: name: suspended description: filter on suspended jobs in: query - schema: *ref_100 + schema: *ref_71 Running: name: running description: filter on running jobs in: query - schema: *ref_67 + schema: *ref_72 ArgsFilter: name: args description: filter on jobs containing those args as a json subset (@> in postgres) in: query - schema: *ref_62 + schema: *ref_73 Tag: name: tag description: filter on jobs with a given tag/worker group in: query - schema: *ref_64 + schema: *ref_75 ResultFilter: name: result description: filter on jobs containing those result as a json subset (@> in postgres) in: query - schema: *ref_63 + schema: *ref_74 After: name: after description: filter on created after (exclusive) timestamp in: query - schema: *ref_101 - Before: - name: before - description: filter on created before (exclusive) timestamp - in: query - schema: *ref_102 + schema: *ref_110 Username: name: username description: filter on exact username of user in: query - schema: *ref_103 + schema: *ref_119 Operation: name: operation description: filter on exact or prefix name of operation in: query - schema: *ref_104 + schema: *ref_120 ResourceName: name: resource description: filter on exact or prefix name of resource in: query - schema: *ref_105 + schema: *ref_121 ActionKind: name: action_kind description: filter on type of operation in: query - schema: *ref_106 + schema: *ref_122 JobKinds: name: job_kinds description: >- filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query - schema: *ref_61 + schema: *ref_70 RunnableId: name: runnable_id in: query - schema: *ref_89 + schema: *ref_101 RunnableTypeQuery: name: runnable_type in: query - schema: *ref_90 + schema: *ref_102 InputId: name: input in: path required: true - schema: *ref_107 + schema: *ref_123 GetStarted: name: get_started in: query - schema: *ref_108 + schema: *ref_124 ConcurrencyId: name: concurrency_id in: path required: true - schema: *ref_109 + schema: *ref_125 schemas: Script: type: object - properties: *ref_31 - required: *ref_32 + properties: *ref_35 + required: *ref_36 NewScript: type: object - properties: *ref_34 - required: *ref_35 + properties: *ref_40 + required: *ref_41 NewScriptWithDraft: - allOf: *ref_110 + allOf: *ref_126 ScriptHistory: - type: object - properties: *ref_111 - required: *ref_112 - ScriptArgs: - type: object - additionalProperties: *ref_14 - Input: - type: object - properties: *ref_87 - required: *ref_88 - CreateInput: - type: object - properties: *ref_113 - required: *ref_114 - UpdateInput: - type: object - properties: *ref_115 - required: *ref_116 - RunnableType: - type: string - enum: *ref_117 - QueuedJob: - type: object - properties: *ref_71 - required: *ref_72 - CompletedJob: - type: object - properties: *ref_69 - required: *ref_70 - Job: - allOf: *ref_74 - discriminator: *ref_75 - User: - type: object - properties: *ref_8 - required: *ref_9 - Usage: - type: object - properties: *ref_118 - Login: - type: object - properties: *ref_119 - required: *ref_120 - NewUser: - type: object - properties: *ref_121 - required: *ref_122 - EditWorkspaceUser: - type: object - properties: *ref_123 - TruncatedToken: - type: object - properties: *ref_124 - required: *ref_125 - NewToken: - type: object - properties: *ref_126 - NewTokenImpersonate: type: object properties: *ref_127 required: *ref_128 - ListableVariable: + ScriptArgs: type: object - properties: *ref_18 - required: *ref_19 - ContextualVariable: + additionalProperties: *ref_16 + Input: + type: object + properties: *ref_103 + required: *ref_104 + CreateInput: type: object properties: *ref_129 required: *ref_130 - CreateVariable: + UpdateInput: type: object properties: *ref_131 required: *ref_132 + RunnableType: + type: string + enum: *ref_133 + QueuedJob: + type: object + properties: *ref_81 + required: *ref_82 + CompletedJob: + type: object + properties: *ref_79 + required: *ref_80 + ObscuredJob: + type: object + properties: *ref_134 + Job: + oneOf: *ref_84 + discriminator: *ref_85 + User: + type: object + properties: *ref_10 + required: *ref_11 + UserUsage: + type: object + properties: *ref_135 + Login: + type: object + properties: *ref_136 + required: *ref_137 + EditWorkspaceUser: + type: object + properties: *ref_138 + TruncatedToken: + type: object + properties: *ref_38 + required: *ref_39 + NewToken: + type: object + properties: *ref_139 + NewTokenImpersonate: + type: object + properties: *ref_140 + required: *ref_141 + ListableVariable: + type: object + properties: *ref_22 + required: *ref_23 + ContextualVariable: + type: object + properties: *ref_142 + required: *ref_143 + CreateVariable: + type: object + properties: *ref_144 + required: *ref_145 EditVariable: type: object - properties: *ref_133 + properties: *ref_146 AuditLog: type: object properties: *ref_1 @@ -11457,124 +14200,164 @@ components: required: - name - typ + no_main_func: + type: boolean + nullable: true + has_preprocessor: + type: boolean + nullable: true required: - star_args - start_kwargs - args - type - error + - no_main_func + - has_preprocessor Preview: - type: object - properties: *ref_134 - required: *ref_135 - CreateResource: - type: object - properties: *ref_136 - required: *ref_137 - EditResource: - type: object - properties: *ref_138 - Resource: - type: object - properties: *ref_139 - required: *ref_140 - ListableResource: - type: object - properties: *ref_141 - required: *ref_142 - ResourceType: - type: object - properties: *ref_22 - required: *ref_23 - EditResourceType: - type: object - properties: *ref_143 - Schedule: - type: object - properties: *ref_78 - required: *ref_79 - ScheduleWJobs: - allOf: *ref_144 - NewSchedule: - type: object - properties: *ref_145 - required: *ref_146 - EditSchedule: type: object properties: *ref_147 required: *ref_148 - Group: - type: object - properties: *ref_83 - required: *ref_84 - InstanceGroup: - type: object - properties: *ref_81 - required: *ref_82 - Folder: - type: object - properties: *ref_85 - required: *ref_86 - WorkerPing: + WorkflowTask: type: object properties: *ref_149 required: *ref_150 - UserWorkspaceList: + WorkflowStatusRecord: type: object - properties: *ref_151 - required: *ref_152 - CreateWorkspace: + additionalProperties: *ref_151 + WorkflowStatus: + type: object + properties: *ref_152 + CreateResource: type: object properties: *ref_153 required: *ref_154 - Workspace: - type: object - properties: *ref_5 - required: *ref_6 - WorkspaceInvite: - type: object - properties: *ref_12 - required: *ref_13 - GlobalUserInfo: - type: object - properties: *ref_10 - required: *ref_11 - Flow: - allOf: *ref_45 - FlowMetadata: + EditResource: type: object properties: *ref_155 - required: *ref_156 - OpenFlowWPath: - allOf: *ref_46 - FlowPreview: + Resource: type: object - properties: *ref_157 - required: *ref_158 - RestartedFrom: + properties: *ref_156 + required: *ref_157 + ListableResource: type: object - properties: *ref_159 - Policy: + properties: *ref_158 + required: *ref_159 + ResourceType: type: object - properties: *ref_47 - ListableApp: + properties: *ref_26 + required: *ref_27 + EditResourceType: type: object properties: *ref_160 - required: *ref_161 - ListableRawApp: + Schedule: + type: object + properties: *ref_88 + required: *ref_89 + ScheduleWJobs: + allOf: *ref_161 + NewSchedule: type: object properties: *ref_162 required: *ref_163 + EditSchedule: + type: object + properties: *ref_164 + required: *ref_165 + HttpTrigger: + type: object + properties: *ref_90 + required: *ref_91 + NewHttpTrigger: + type: object + properties: *ref_166 + required: *ref_167 + EditHttpTrigger: + type: object + properties: *ref_168 + required: *ref_169 + TriggersCount: + type: object + properties: *ref_53 + Group: + type: object + properties: *ref_97 + required: *ref_98 + InstanceGroup: + type: object + properties: *ref_93 + required: *ref_94 + Folder: + type: object + properties: *ref_99 + required: *ref_100 + WorkerPing: + type: object + properties: *ref_170 + required: *ref_171 + UserWorkspaceList: + type: object + properties: *ref_172 + required: *ref_173 + CreateWorkspace: + type: object + properties: *ref_174 + required: *ref_175 + Workspace: + type: object + properties: *ref_7 + required: *ref_8 + WorkspaceInvite: + type: object + properties: *ref_14 + required: *ref_15 + GlobalUserInfo: + type: object + properties: *ref_12 + required: *ref_13 + Flow: + allOf: *ref_52 + ExtraPerms: + type: object + additionalProperties: *ref_176 + FlowMetadata: + type: object + properties: *ref_177 + required: *ref_178 + OpenFlowWPath: + allOf: *ref_54 + FlowPreview: + type: object + properties: *ref_179 + required: *ref_180 + RestartedFrom: + type: object + properties: *ref_181 + Policy: + type: object + properties: *ref_55 + ListableApp: + type: object + properties: *ref_182 + required: *ref_183 + ListableRawApp: + type: object + properties: *ref_184 + required: *ref_185 AppWithLastVersion: type: object - properties: *ref_48 - required: *ref_49 + properties: *ref_56 + required: *ref_57 AppWithLastVersionWDraft: - allOf: *ref_164 + allOf: *ref_186 AppHistory: type: object - properties: *ref_165 - required: *ref_166 + properties: *ref_187 + required: *ref_188 + FlowVersion: + type: object + properties: *ref_189 + required: *ref_190 SlackToken: type: object properties: @@ -11596,40 +14379,46 @@ components: - bot TokenResponse: type: object - properties: *ref_167 - required: *ref_168 + properties: *ref_191 + required: *ref_192 HubScriptKind: name: kind - schema: *ref_30 + schema: *ref_34 PolarsClientKwargs: type: object - properties: *ref_93 - required: *ref_94 + properties: *ref_107 + required: *ref_108 LargeFileStorage: type: object - properties: *ref_15 + properties: *ref_17 WindmillLargeFile: type: object - properties: *ref_169 - required: *ref_170 + properties: *ref_193 + required: *ref_194 WindmillFileMetadata: type: object - properties: *ref_171 + properties: *ref_195 WindmillFilePreview: type: object - properties: *ref_172 - required: *ref_173 + properties: *ref_196 + required: *ref_197 S3Resource: type: object - properties: *ref_91 - required: *ref_92 + properties: *ref_105 + required: *ref_106 WorkspaceGitSyncSettings: type: object - properties: *ref_16 + properties: *ref_18 + WorkspaceDeployUISettings: + type: object + properties: *ref_19 + WorkspaceDefaultScripts: + type: object + properties: *ref_20 GitRepositorySettings: type: object - properties: *ref_174 - required: *ref_175 + properties: *ref_198 + required: *ref_199 UploadFilePart: type: object properties: @@ -11641,91 +14430,118 @@ components: - part_number - tag MetricMetadata: - type: object - properties: *ref_176 - required: *ref_177 - ScalarMetric: - type: object - properties: *ref_178 - required: *ref_179 - TimeseriesMetric: - type: object - properties: *ref_180 - required: *ref_181 - MetricDataPoint: - type: object - properties: *ref_182 - required: *ref_183 - RawScriptForDependencies: - type: object - properties: *ref_184 - required: *ref_185 - ConcurrencyGroup: - type: object - properties: *ref_186 - required: *ref_187 - StaticTransform: - type: object - properties: *ref_188 - required: *ref_189 - JavascriptTransform: - type: object - properties: *ref_190 - required: *ref_191 - InputTransform: - oneOf: *ref_25 - discriminator: *ref_26 - RawScript: - type: object - properties: *ref_192 - required: *ref_193 - PathScript: - type: object - properties: *ref_194 - required: *ref_195 - PathFlow: - type: object - properties: *ref_196 - required: *ref_197 - FlowModule: - type: object - properties: *ref_27 - required: *ref_28 - ForloopFlow: - type: object - properties: *ref_198 - required: *ref_199 - BranchOne: type: object properties: *ref_200 required: *ref_201 - BranchAll: + ScalarMetric: type: object properties: *ref_202 required: *ref_203 - Identity: + TimeseriesMetric: type: object properties: *ref_204 required: *ref_205 + MetricDataPoint: + type: object + properties: *ref_206 + required: *ref_207 + RawScriptForDependencies: + type: object + properties: *ref_208 + required: *ref_209 + ConcurrencyGroup: + type: object + properties: *ref_210 + required: *ref_211 + ExtendedJobs: + type: object + properties: *ref_212 + required: *ref_213 + ExportedUser: + type: object + properties: *ref_3 + required: *ref_4 + GlobalSetting: + type: object + properties: *ref_214 + required: *ref_215 + Config: + type: object + properties: *ref_216 + required: *ref_217 + ExportedInstanceGroup: + type: object + properties: *ref_95 + required: *ref_96 + JobSearchHit: + type: object + properties: *ref_218 + StaticTransform: + type: object + properties: *ref_219 + required: *ref_220 + JavascriptTransform: + type: object + properties: *ref_221 + required: *ref_222 + InputTransform: + oneOf: *ref_29 + discriminator: *ref_30 + RawScript: + type: object + properties: *ref_223 + required: *ref_224 + PathScript: + type: object + properties: *ref_225 + required: *ref_226 + PathFlow: + type: object + properties: *ref_227 + required: *ref_228 + FlowModule: + type: object + properties: *ref_31 + required: *ref_32 + ForloopFlow: + type: object + properties: *ref_229 + required: *ref_230 + WhileloopFlow: + type: object + properties: *ref_231 + required: *ref_232 + BranchOne: + type: object + properties: *ref_233 + required: *ref_234 + BranchAll: + type: object + properties: *ref_235 + required: *ref_236 + Identity: + type: object + properties: *ref_237 + required: *ref_238 FlowModuleValue: - oneOf: *ref_206 - discriminator: *ref_207 + oneOf: *ref_239 + discriminator: *ref_240 Retry: type: object - properties: *ref_77 + properties: *ref_87 FlowValue: + type: object + properties: *ref_58 + required: *ref_59 + OpenFlow: type: object properties: *ref_50 required: *ref_51 - OpenFlow: - type: object - properties: *ref_43 - required: *ref_44 FlowStatusModule: type: object - properties: *ref_52 - required: *ref_53 + properties: *ref_60 + required: *ref_61 FlowStatus: type: object - properties: *ref_65 - required: *ref_66 + properties: *ref_76 + required: *ref_77 diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c8c30a7db1..eebb8fd0a9 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.402.3 + version: 1.416.2 title: Windmill API contact: @@ -1467,6 +1467,11 @@ paths: parameters: - $ref: "#/components/parameters/Page" - $ref: "#/components/parameters/PerPage" + - name: active_only + in: query + description: filter only active users + schema: + type: boolean responses: "200": description: user @@ -2162,6 +2167,30 @@ paths: schema: type: number + /w/{workspace}/workspaces/used_triggers: + get: + summary: get used triggers + operationId: getUsedTriggers + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: status + content: + application/json: + schema: + type: object + properties: + http_routes_used: + type: boolean + websocket_used: + type: boolean + required: + - http_routes_used + - websocket_used + /w/{workspace}/users/list: get: summary: list users @@ -2789,7 +2818,14 @@ paths: oauth: type: array items: - type: string + type: object + properties: + type: + type: string + display_name: + type: string + required: + - type saml: type: string required: @@ -4013,6 +4049,42 @@ paths: schema: $ref: "#/components/schemas/Script" + /w/{workspace}/scripts/get_triggers_count/{path}: + get: + summary: get triggers count of script + operationId: getTriggersCountOfScript + tags: + - script + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/ScriptPath" + responses: + "200": + description: triggers count + content: + application/json: + schema: + $ref: "#/components/schemas/TriggersCount" + + /w/{workspace}/scripts/list_tokens/{path}: + get: + summary: get tokens with script scope + operationId: listTokensOfScript + tags: + - script + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/ScriptPath" + responses: + "200": + description: tokens list + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/TruncatedToken" + /w/{workspace}/scripts/get/draft/{path}: get: summary: get script by path with draft @@ -4049,6 +4121,26 @@ paths: items: $ref: "#/components/schemas/ScriptHistory" + /w/{workspace}/scripts/get_latest_version/{path}: + get: + summary: get scripts's latest version (hash) + operationId: getScriptLatestVersion + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/ScriptPath" + tags: + - script + responses: + "200": + description: Script version/hash + content: + application/json: + + required: false + + schema: + $ref: "#/components/schemas/ScriptHistory" + /w/{workspace}/scripts/history_update/h/{hash}/p/{path}: post: summary: update history of a script @@ -4537,6 +4629,25 @@ paths: items: $ref: "#/components/schemas/FlowVersion" + /w/{workspace}/flows/get_latest_version/{path}: + get: + summary: get flow's latest version + operationId: getFlowLatestVersion + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/ScriptPath" + tags: + - flow + responses: + "200": + description: Flow version + content: + application/json: + required: false + + schema: + $ref: "#/components/schemas/FlowVersion" + /w/{workspace}/flows/get/v/{version}/p/{path}: get: summary: get flow version @@ -4617,6 +4728,43 @@ paths: schema: $ref: "#/components/schemas/Flow" + /w/{workspace}/flows/get_triggers_count/{path}: + get: + summary: get triggers count of flow + operationId: getTriggersCountOfFlow + tags: + - flow + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/ScriptPath" + responses: + "200": + description: triggers count + content: + application/json: + schema: + $ref: "#/components/schemas/TriggersCount" + + /w/{workspace}/flows/list_tokens/{path}: + get: + summary: get tokens with flow scope + operationId: listTokensOfFlow + tags: + - flow + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/ScriptPath" + responses: + "200": + description: tokens list + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/TruncatedToken" + + /w/{workspace}/flows/toggle_workspace_error_handler/{path}: post: summary: Toggle ON and OFF the workspace error handler for a given flow @@ -5054,6 +5202,24 @@ paths: items: $ref: "#/components/schemas/AppHistory" + /w/{workspace}/apps/get_latest_version/{path}: + get: + summary: get apps's latest version + operationId: getAppLatestVersion + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/ScriptPath" + tags: + - app + responses: + "200": + description: App version + content: + application/json: + required: false + schema: + $ref: "#/components/schemas/AppHistory" + /w/{workspace}/apps/history_update/a/{id}/v/{version}: post: summary: update app history @@ -7072,22 +7238,170 @@ paths: schema: type: boolean - /w/{workspace}/http_triggers/used: - get: - summary: whether http triggers are used - operationId: used + /w/{workspace}/websocket_triggers/create: + post: + summary: create websocket trigger + operationId: createWebsocketTrigger tags: - - http_trigger + - websocket_trigger parameters: - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: new websocket trigger + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/NewWebsocketTrigger" + responses: + "201": + description: websocket trigger created + content: + text/plain: + schema: + type: string + + /w/{workspace}/websocket_triggers/update/{path}: + post: + summary: update websocket trigger + operationId: updateWebsocketTrigger + tags: + - websocket_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + description: updated trigger + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/EditWebsocketTrigger" responses: "200": - description: whether http triggers are used + description: websocket trigger updated + content: + text/plain: + schema: + type: string + + /w/{workspace}/websocket_triggers/delete/{path}: + delete: + summary: delete websocket trigger + operationId: deleteWebsocketTrigger + tags: + - websocket_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: websocket trigger deleted + content: + text/plain: + schema: + type: string + + /w/{workspace}/websocket_triggers/get/{path}: + get: + summary: get websocket trigger + operationId: getWebsocketTrigger + tags: + - websocket_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: websocket trigger deleted + content: + application/json: + schema: + $ref: "#/components/schemas/WebsocketTrigger" + + + /w/{workspace}/websocket_triggers/list: + get: + summary: list websocket triggers + operationId: listWebsocketTriggers + tags: + - websocket_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + required: true + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + - name: path + description: filter by path + in: query + schema: + type: string + - name: is_flow + in: query + schema: + type: boolean + - name: path_start + in: query + schema: + type: string + responses: + "200": + description: websocket trigger list + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/WebsocketTrigger" + + + /w/{workspace}/websocket_triggers/exists/{path}: + get: + summary: does websocket trigger exists + operationId: existsWebsocketTrigger + tags: + - websocket_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: websocket trigger exists content: application/json: schema: type: boolean + /w/{workspace}/websocket_triggers/setenabled/{path}: + post: + summary: set enabled websocket trigger + operationId: setWebsocketTriggerEnabled + tags: + - websocket_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + description: updated websocket trigger enable + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + required: + - enabled + responses: + "200": + description: websocket trigger enabled set + content: + text/plain: + schema: + type: string + + /groups/list: get: summary: list instance groups @@ -7889,6 +8203,29 @@ paths: items: $ref: "#/components/schemas/Config" + /configs/list_autoscaling_events/{worker_group}: + get: + summary: List autoscaling events + operationId: listAutoscalingEvents + tags: + - config + parameters: + - name: worker_group + in: path + required: true + schema: + type: string + responses: + "200": + description: List of autoscaling events + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/AutoscalingEvent" + + /w/{workspace}/acls/get/{kind}/{path}: get: summary: get granular acls @@ -7915,6 +8252,7 @@ paths: app, raw_app, http_trigger, + websocket_trigger, ] responses: "200": @@ -7952,6 +8290,7 @@ paths: app, raw_app, http_trigger, + websocket_trigger, ] requestBody: description: acl to add @@ -8000,6 +8339,7 @@ paths: app, raw_app, http_trigger, + websocket_trigger, ] requestBody: description: acl to add @@ -10247,6 +10587,8 @@ components: type: array items: type: string + email: + type: string required: - token_prefix - created_at @@ -10264,6 +10606,8 @@ components: type: array items: type: string + workspace_id: + type: string NewTokenImpersonate: type: object @@ -10275,6 +10619,8 @@ components: format: date-time impersonate_email: type: string + workspace_id: + type: string required: - impersonate_email @@ -10417,6 +10763,7 @@ components: - "users.delete" - "users.update" - "users.login" + - "users.login_failure" - "users.logout" - "users.accept_invite" - "users.decline_invite" @@ -10428,6 +10775,7 @@ components: - "users.impersonate" - "users.leave_workspace" - "oauth.login" + - "oauth.login_failure" - "oauth.signup" - "variables.create" - "variables.delete" @@ -11111,6 +11459,145 @@ components: - requires_auth - http_method + TriggersCount: + type: object + properties: + primary_schedule: + type: object + properties: + schedule: + type: string + schedule_count: + type: number + http_routes_count: + type: number + webhook_count: + type: number + email_count: + type: number + websocket_count: + type: number + + WebsocketTrigger: + type: object + properties: + path: + type: string + edited_by: + type: string + edited_at: + type: string + format: date-time + script_path: + type: string + url: + type: string + is_flow: + type: boolean + extra_perms: + type: object + additionalProperties: + type: boolean + email: + type: string + workspace_id: + type: string + server_id: + type: string + last_server_ping: + type: string + format: date-time + error: + type: string + enabled: + type: boolean + filters: + type: array + items: + type: object + properties: + key: + type: string + value: {} + required: + - key + - value + + required: + - path + - edited_by + - edited_at + - script_path + - url + - extra_perms + - is_flow + - email + - workspace_id + - enabled + - filters + + NewWebsocketTrigger: + type: object + properties: + path: + type: string + script_path: + type: string + is_flow: + type: boolean + url: + type: string + enabled: + type: boolean + filters: + type: array + items: + type: object + properties: + key: + type: string + value: {} + required: + - key + - value + + required: + - path + - script_path + - url + - is_flow + - filters + + EditWebsocketTrigger: + type: object + properties: + url: + type: string + path: + type: string + script_path: + type: string + is_flow: + type: boolean + filters: + type: array + items: + type: object + properties: + key: + type: string + value: {} + required: + - key + - value + + required: + - path + - script_path + - url + - is_flow + - filters + Group: type: object properties: @@ -11309,6 +11796,8 @@ components: type: string username: type: string + operator_only: + type: boolean required: - email @@ -12010,3 +12499,21 @@ components: properties: dancer: type: string + + AutoscalingEvent: + type: object + properties: + id: + type: integer + format: int64 + worker_group: + type: string + event_type: + type: string + desired_workers: + type: integer + reason: + type: string + applied_at: + type: string + format: date-time diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 7bed72c50e..cd65136a2a 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -62,6 +62,7 @@ pub fn workspaced_service() -> Router { .route("/delete/*path", delete(delete_app)) .route("/create", post(create_app)) .route("/history/p/*path", get(get_app_history)) + .route("/get_latest_version/*path", get(get_latest_version)) .route("/history_update/a/:id/v/:version", post(update_app_history)) } @@ -427,6 +428,38 @@ async fn get_app_history( return Ok(Json(result)); } +async fn get_latest_version( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + +) -> JsonResult> { + + let mut tx = user_db.begin(&authed).await?; + let row = sqlx::query!( + "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg + FROM app a LEFT JOIN app_version av ON a.id = av.app_id LEFT JOIN deployment_metadata dm ON av.id = dm.app_version + WHERE a.workspace_id = $1 AND a.path = $2 + ORDER BY created_at DESC", + w_id, + path.to_path(), + ).fetch_optional(&mut *tx).await?; + tx.commit().await?; + + if let Some(row) = row { + let result = AppHistory { + app_id: row.app_id, + version: row.version_id, + deployment_msg: row.deployment_msg, + }; + + return Ok(Json(Some(result))); + } else { + return Ok(Json(None)); + } + +} + async fn update_app_history( authed: ApiAuthed, Extension(user_db): Extension, diff --git a/backend/windmill-api/src/configs.rs b/backend/windmill-api/src/configs.rs index 64c8a1884f..81b65216e0 100644 --- a/backend/windmill-api/src/configs.rs +++ b/backend/windmill-api/src/configs.rs @@ -29,6 +29,10 @@ pub fn global_service() -> Router { .route("/update/:name", post(update_config).delete(delete_config)) .route("/get/:name", get(get_config)) .route("/list", get(list_configs)) + .route( + "/list_autoscaling_events/:worker_group", + get(list_autoscaling_events), + ) } #[derive(Serialize, Deserialize, FromRow)] @@ -177,6 +181,30 @@ async fn delete_config( Ok(format!("Deleted config {name}")) } +#[derive(Serialize, Deserialize, FromRow)] +struct AutoscalingEvent { + id: i64, + worker_group: String, + event_type: Option, + desired_workers: i32, + reason: Option, + applied_at: chrono::NaiveDateTime, +} + +async fn list_autoscaling_events( + Extension(db): Extension, + Path(worker_group): Path, +) -> error::JsonResult> { + let events = sqlx::query_as!( + AutoscalingEvent, + "SELECT id, worker_group, event_type::text, desired_workers, reason, applied_at FROM autoscaling_event WHERE worker_group = $1 ORDER BY applied_at DESC LIMIT 5", + worker_group + ) + .fetch_all(&db) + .await?; + Ok(Json(events)) +} + #[cfg(feature = "enterprise")] async fn list_configs( authed: ApiAuthed, diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 470a21bccc..6186b0ada1 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -15,6 +15,7 @@ use sqlx::{ PgConnection, Pool, Postgres, }; use windmill_audit::audit_ee::{AuditAuthor, AuditAuthorable}; +use windmill_common::utils::generate_lock_id; use windmill_common::{ db::{Authable, Authed}, error::Error, @@ -29,13 +30,6 @@ async fn current_database(conn: &mut PgConnection) -> Result i64 { - const CRC_IEEE: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); - // 0x3d32ad9e chosen by fair dice roll - 0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64) -} - struct CustomMigrator { inner: PoolConnection, } @@ -136,9 +130,30 @@ impl Migrate for CustomMigrator { migration.version, migration.description ); - let r = self.inner.apply(migration).await; - tracing::info!("Finished applying migration {}", migration.version); - r + if migration.version == 20221207103910 { + tracing::info!("Skipping migration 20221207103910 to avoid using md5"); + self.inner + .execute(include_str!( + "../../custom_migrations/create_workspace_without_md5.sql" + )) + .await?; + let _ = sqlx::query( + r#" + INSERT INTO _sqlx_migrations ( version, description, success, checksum, execution_time ) + VALUES ( $1, $2, TRUE, $3, -1 ) ON CONFLICT DO NOTHING + "#, + ) + .bind(migration.version) + .bind(&*migration.description) + .bind(&*migration.checksum) + .execute(&mut *self.inner) + .await?; + return Ok(std::time::Duration::from_secs(0)); + } else { + let r = self.inner.apply(migration).await; + tracing::info!("Finished applying migration {}", migration.version); + return r; + } } .boxed() } @@ -158,17 +173,6 @@ pub async fn migrate(db: &DB) -> Result<(), Error> { let migrator = db.acquire().await?; let mut custom_migrator = CustomMigrator { inner: migrator }; - if let Err(err) = fix_flow_versioning_migration(&mut custom_migrator, db).await { - tracing::error!("Could not apply flow versioning fix migration: {err:#}"); - } - - let db2 = db.clone(); - let _ = tokio::task::spawn(async move { - if let Err(err) = fix_job_completed_index(&db2).await { - tracing::error!("Could not apply job completed index fix migration: {err:#}"); - } - }); - match sqlx::migrate!("../migrations") .run_direct(&mut custom_migrator) .await @@ -184,11 +188,17 @@ pub async fn migrate(db: &DB) -> Result<(), Error> { Err(err) => Err(err), }?; - #[cfg(feature = "enterprise")] - if let Err(e) = windmill_migrations(&mut custom_migrator, db).await { - tracing::error!("Could not apply windmill custom migrations: {e:#}") + if let Err(err) = fix_flow_versioning_migration(&mut custom_migrator, db).await { + tracing::error!("Could not apply flow versioning fix migration: {err:#}"); } + let db2 = db.clone(); + let _ = tokio::task::spawn(async move { + if let Err(err) = fix_job_completed_index(&db2).await { + tracing::error!("Could not apply job completed index fix migration: {err:#}"); + } + }); + Ok(()) } @@ -292,7 +302,7 @@ macro_rules! run_windmill_migration { .await?; tracing::info!("Finished applying {migration_job_name} migration"); } else { - tracing::info!("migration {migration_job_name} already done"); + tracing::debug!("migration {migration_job_name} already done"); } let _ = sqlx::query("SELECT pg_advisory_unlock(4242)") @@ -301,7 +311,7 @@ macro_rules! run_windmill_migration { tx.commit().await?; tracing::info!("released lock for {migration_job_name}"); } else { - tracing::info!("migration {migration_job_name} already done"); + tracing::debug!("migration {migration_job_name} already done"); } } @@ -482,33 +492,6 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> { Ok(()) } -#[cfg(feature = "enterprise")] -async fn windmill_migrations(migrator: &mut CustomMigrator, db: &DB) -> Result<(), Error> { - if std::env::var("MIGRATION_NO_BYPASSRLS").is_ok() { - migrator.lock().await?; - let has_done_migration = sqlx::query_scalar!( - "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'bypassrls_1-2')", - ) - .fetch_one(db) - .await? - .unwrap_or(false); - - if !has_done_migration { - let query = include_str!("../../custom_migrations/bypassrls_1.sql"); - tracing::info!("Applying bypassrls_1.sql"); - let mut tx: sqlx::Transaction<'_, Postgres> = db.begin().await?; - tx.execute(query).await?; - tracing::info!("Applied bypassrls_1.sql"); - sqlx::query!("INSERT INTO windmill_migrations (name) VALUES ('bypassrls_1-2')") - .execute(&mut *tx) - .await?; - tx.commit().await?; - } - migrator.unlock().await?; - } - Ok(()) -} - #[derive(Clone, Debug)] pub struct ApiAuthed { pub email: String, diff --git a/backend/windmill-api/src/ee.rs b/backend/windmill-api/src/ee.rs index b20521e8b4..cddb639e95 100644 --- a/backend/windmill-api/src/ee.rs +++ b/backend/windmill-api/src/ee.rs @@ -4,7 +4,7 @@ use std::sync::Arc; #[cfg(feature = "enterprise")] use tokio::sync::RwLock; -pub async fn validate_license_key(_license_key: String) -> anyhow::Result { +pub async fn validate_license_key(_license_key: String) -> anyhow::Result<(String, bool)> { // Implementation is not open source Err(anyhow!("License can't be validated in Windmill CE")) } diff --git a/backend/windmill-api/src/embeddings.rs b/backend/windmill-api/src/embeddings.rs index 5cc7226753..fd222bdbfa 100644 --- a/backend/windmill-api/src/embeddings.rs +++ b/backend/windmill-api/src/embeddings.rs @@ -53,6 +53,7 @@ use crate::{resources::ResourceType, HTTP_CLIENT}; lazy_static::lazy_static! { pub static ref EMBEDDINGS_DB: Arc>> = Arc::new(RwLock::new(None)); pub static ref MODEL_INSTANCE: Arc>>> = Arc::new(RwLock::new(None)); + pub static ref HUB_EMBEDDINGS_PULLING_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_PULLING_INTERVAL_SECS").ok().map(|x| x.parse::().ok()).flatten().unwrap_or(3600 * 24); } #[cfg(feature = "embedding")] @@ -607,7 +608,10 @@ pub fn load_embeddings_db(db: &Pool) -> () { drop(model_instance_lock); loop { update_embeddings_db(&db_clone).await; - tokio::time::sleep(std::time::Duration::from_secs(3600 * 24)).await; + tokio::time::sleep(std::time::Duration::from_secs( + *HUB_EMBEDDINGS_PULLING_INTERVAL_SECS, + )) + .await; } } else { tracing::error!( diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 0b5082c6e0..4fc9ce514f 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -9,6 +9,9 @@ use std::collections::HashMap; use crate::db::ApiAuthed; +use crate::triggers::{ + get_triggers_count_internal, list_tokens_internal, TriggersCount, TruncatedTokenWithEmail, +}; use crate::utils::WithStarredInfoQuery; use crate::{ db::DB, @@ -53,11 +56,14 @@ pub fn workspaced_service() -> Router { .route("/update/*path", post(update_flow)) .route("/archive/*path", post(archive_flow_by_path)) .route("/delete/*path", delete(delete_flow_by_path)) + .route("/get_triggers_count/*path", get(get_triggers_count)) + .route("/list_tokens/*path", get(list_tokens)) .route("/get/*path", get(get_flow_by_path)) .route("/get/draft/*path", get(get_flow_by_path_w_draft)) .route("/exists/*path", get(exists_flow_by_path)) .route("/list_paths", get(list_paths)) .route("/history/p/*path", get(get_flow_history)) + .route("/get_latest_version/*path", get(get_latest_version)) .route( "/history_update/v/:version/p/*path", post(update_flow_history), @@ -533,6 +539,30 @@ async fn get_flow_history( Ok(Json(flows)) } +async fn get_latest_version( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + let mut tx = user_db.begin(&authed).await?; + + let version = sqlx::query_as!( + FlowVersion, + "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version + LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version + WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 + ORDER BY flow_version.created_at DESC", + path, + w_id + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json(version)) +} + async fn get_flow_version( authed: ApiAuthed, Extension(user_db): Extension, @@ -874,6 +904,22 @@ async fn update_flow( Ok(nf.path.to_string()) } +async fn get_triggers_count( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + get_triggers_count_internal(&db, &w_id, &path, true).await +} + +async fn list_tokens( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + list_tokens_internal(&db, &w_id, &path, true).await +} + async fn get_flow_by_path( authed: ApiAuthed, Extension(user_db): Extension, @@ -1176,6 +1222,7 @@ mod tests { priority: None, delete_after_use: None, continue_on_error: None, + skip_if: None, }, FlowModule { id: "b".to_string(), @@ -1205,6 +1252,7 @@ mod tests { priority: None, delete_after_use: None, continue_on_error: None, + skip_if: None, }, FlowModule { id: "c".to_string(), @@ -1232,6 +1280,7 @@ mod tests { priority: None, delete_after_use: None, continue_on_error: None, + skip_if: None, }, ], failure_module: Some(Box::new(FlowModule { @@ -1258,6 +1307,7 @@ mod tests { priority: None, delete_after_use: None, continue_on_error: None, + skip_if: None, })), preprocessor_module: None, same_worker: false, @@ -1282,7 +1332,6 @@ mod tests { }, "type": "script", "path": "test", - "tag_override": Option::::None, }, }, { @@ -1326,14 +1375,12 @@ mod tests { "input_transforms": {}, "type": "script", "path": "test", - "tag_override": Option::::None, }, "stop_after_if": { "expr": "previous.isEmpty()", "skip_if_stopped": false } }, - "preprocessor_module": Option::::None }); assert_eq!(dbg!(serde_json::json!(fv)), dbg!(expect)); } diff --git a/backend/windmill-api/src/folders.rs b/backend/windmill-api/src/folders.rs index fef5b7fe07..c9b69836c6 100644 --- a/backend/windmill-api/src/folders.rs +++ b/backend/windmill-api/src/folders.rs @@ -275,8 +275,10 @@ pub fn require_is_owner(authed: &ApiAuthed, name: &str) -> Result<()> { async fn update_folder( authed: ApiAuthed, + Extension(db): Extension, Extension(user_db): Extension, Extension(webhook): Extension, + Extension(rsmq): Extension>, Path((w_id, name)): Path<(String, String)>, Json(mut ng): Json, ) -> Result { @@ -367,6 +369,18 @@ async fn update_folder( } } + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + DeployedObject::Folder { path: format!("f/{}", name) }, + Some(format!("Folder '{}' updated", name)), + rsmq, + true, + ) + .await?; + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api/src/http_triggers.rs b/backend/windmill-api/src/http_triggers.rs index 9e30743fbc..61757cb164 100644 --- a/backend/windmill-api/src/http_triggers.rs +++ b/backend/windmill-api/src/http_triggers.rs @@ -12,10 +12,8 @@ use std::collections::HashMap; use tower_http::cors::CorsLayer; use windmill_audit::{audit_ee::audit_log, ActionKind}; use windmill_common::{ - auth::fetch_authed_from_permissioned_as, db::UserDB, error::{self, JsonResult}, - users::username_to_permissioned_as, utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, worker::{to_raw_value, CLOUD_HOSTED}, }; @@ -27,7 +25,7 @@ use crate::{ run_flow_by_path_inner, run_script_by_path_inner, run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery, }, - users::OptAuthed, + users::{fetch_api_authed, OptAuthed}, }; lazy_static::lazy_static! { @@ -66,7 +64,6 @@ pub fn workspaced_service() -> Router { .route("/update/*path", post(update_trigger)) .route("/delete/*path", delete(delete_trigger)) .route("/exists/*path", get(exists_trigger)) - .route("/used", get(used)) .route("/route_exists", post(exists_route)) } @@ -346,17 +343,6 @@ async fn delete_trigger( Ok(format!("HTTP trigger {path} deleted")) } -async fn used(Extension(db): Extension, Path(w_id): Path) -> JsonResult { - let used = sqlx::query_scalar!( - r#"SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1)"#, - w_id, - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - Ok(Json(used)) -} - async fn exists_trigger( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, @@ -421,28 +407,6 @@ struct TriggerRoute { http_method: HttpMethod, } -async fn fetch_api_authed( - username: String, - email: String, - w_id: &str, - db: &DB, - username_override: String, -) -> error::Result { - let permissioned_as = username_to_permissioned_as(username.as_str()); - let authed = - fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?; - Ok(ApiAuthed { - username: username, - email: email, - is_admin: authed.is_admin, - is_operator: authed.is_operator, - groups: authed.groups, - folders: authed.folders, - scopes: authed.scopes, - username_override: Some(username_override), - }) -} - async fn get_http_route_trigger( route_path: &str, opt_authed: Option, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 156be41b26..7244b2be17 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -11,7 +11,6 @@ use axum::http::HeaderValue; use quick_cache::sync::Cache; use serde_json::value::RawValue; use sqlx::Pool; -use windmill_common::error::JsonResult; use std::collections::HashMap; #[cfg(feature = "prometheus")] use std::sync::atomic::Ordering; @@ -19,12 +18,13 @@ use tokio::io::AsyncReadExt; #[cfg(feature = "prometheus")] use tokio::time::Instant; use tower::ServiceBuilder; +use windmill_common::error::JsonResult; use windmill_common::flow_status::{JobResult, RestartedFrom}; use windmill_common::jobs::{ format_completed_job_result, format_result, CompletedJobWithFormattedResult, FormattedResult, ENTRYPOINT_OVERRIDE, }; -use windmill_common::worker::TMP_DIR; +use windmill_common::worker::{CLOUD_HOSTED, TMP_DIR}; #[cfg(all(feature = "enterprise", feature = "parquet"))] use windmill_common::scripts::PREVIEW_IS_CODEBASE_HASH; @@ -70,7 +70,10 @@ use windmill_common::{ oauth2::HmacSha256, scripts::{ScriptHash, ScriptLang}, users::username_to_permissioned_as, - utils::{not_found_if_none, now_from_db, paginate, paginate_without_limits, require_admin, Pagination, StripPath}, + utils::{ + not_found_if_none, now_from_db, paginate, paginate_without_limits, require_admin, + Pagination, StripPath, + }, }; #[cfg(all(feature = "enterprise", feature = "parquet"))] @@ -81,7 +84,7 @@ use windmill_common::{METRICS_DEBUG_ENABLED, METRICS_ENABLED}; use windmill_common::{get_latest_deployed_hash_for_path, BASE_URL}; use windmill_queue::{ cancel_job, get_queued_job, get_result_by_id_from_running_flow, job_is_complete, push, - DecodeQueries, PushArgs, PushArgsOwned, PushIsolationLevel, QueueTransaction, + DecodeQueries, PushArgs, PushArgsOwned, PushIsolationLevel, }; #[cfg(feature = "prometheus")] @@ -293,11 +296,8 @@ pub fn workspace_unauthed_service() -> Router { pub fn global_root_service() -> Router { Router::new() - .route("/db_clock", get(get_db_clock)) - .route( - "/completed/count_by_tag", - get(count_by_tag), - ) + .route("/db_clock", get(get_db_clock)) + .route("/completed/count_by_tag", get(count_by_tag)) } #[derive(Deserialize)] @@ -547,8 +547,8 @@ pub async fn get_path_for_hash<'c>( Ok(path) } -pub async fn get_path_tag_limits_cache_for_hash<'c, R: rsmq_async::RsmqConnection + Send>( - tx: &mut QueueTransaction<'c, R>, +pub async fn get_path_tag_limits_cache_for_hash( + tx: &DB, w_id: &str, hash: i64, ) -> error::Result<( @@ -1472,6 +1472,8 @@ async fn cancel_jobs( } } + uuids.extend(trivial_jobs); + Ok(Json(uuids)) } @@ -2813,8 +2815,6 @@ pub async fn run_flow_by_path_inner( let flow_path = flow_path.to_path(); check_scopes(&authed, || format!("run:flow/{flow_path}"))?; - let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); - let (tag, dedicated_worker, has_preprocessor) = sqlx::query!( "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor FROM flow @@ -2824,7 +2824,7 @@ pub async fn run_flow_by_path_inner( flow_path, w_id ) - .fetch_optional(&mut tx) + .fetch_optional(&db) .await? .map(|x| (x.tag, x.dedicated_worker, x.has_preprocessor)) .ok_or_else(|| { @@ -2837,7 +2837,7 @@ pub async fn run_flow_by_path_inner( check_tag_available_for_workspace(&w_id, &tag).await?; let scheduled_for = run_query.get_scheduled_for(&db).await?; - let tx = PushIsolationLevel::Transaction(tx); + let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into(), rsmq); let (uuid, tx) = push( &db, tx, @@ -2909,14 +2909,12 @@ pub async fn restart_flow( ) -> error::Result<(StatusCode, String)> { check_license_key_valid().await?; - let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); - let completed_job = sqlx::query_as::<_, CompletedJob>( "SELECT *, result->'wm_labels' as labels from completed_job WHERE id = $1 and workspace_id = $2", ) .bind(job_id) .bind(&w_id) - .fetch_optional(&mut tx) + .fetch_optional(&db) .await? .with_context(|| "Unable to find completed job with the given job UUID")?; @@ -2934,7 +2932,7 @@ pub async fn restart_flow( let scheduled_for = run_query.get_scheduled_for(&db).await?; - let tx = PushIsolationLevel::Transaction(tx); + let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into(), rsmq); let (uuid, tx) = push( &db, @@ -3010,16 +3008,14 @@ pub async fn run_script_by_path_inner( check_scopes(&authed, || format!("run:script/{script_path}"))?; - let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); - let (job_payload, tag, _delete_after_use, timeout) = - script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?; + script_path_to_payload(script_path, &db, &w_id, run_query.skip_preprocessor).await?; let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag).await?; - let tx = PushIsolationLevel::Transaction(tx); + let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into(), rsmq); let (uuid, tx) = push( &db, @@ -3052,6 +3048,11 @@ pub async fn run_script_by_path_inner( Ok((StatusCode::CREATED, uuid.to_string())) } +#[derive(Deserialize)] +pub struct WorkflowAsCodeQuery { + pub skip_update: Option, +} + pub async fn run_workflow_as_code( authed: ApiAuthed, Extension(db): Extension, @@ -3059,15 +3060,32 @@ pub async fn run_workflow_as_code( Extension(rsmq): Extension>, Path((w_id, job_id, entrypoint)): Path<(String, Uuid, String)>, Query(run_query): Query, + Query(wkflow_query): Query, Json(task): Json, ) -> error::Result<(StatusCode, String)> { + let mut i = 1; + + if *CLOUD_HOSTED { + tracing::info!("workflow_as_code_tracing id {i} "); + i += 1; + } + #[cfg(feature = "enterprise")] check_license_key_valid().await?; check_tag_available_for_workspace(&w_id, &run_query.tag).await?; - let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); + if *CLOUD_HOSTED { + tracing::info!("workflow_as_code_tracing id {i} "); + i += 1; + } let job = get_queued_job(&job_id, &w_id, &db).await?; + + if *CLOUD_HOSTED { + tracing::info!("workflow_as_code_tracing id {i} "); + i += 1; + } + let job = not_found_if_none(job, "Queued Job", &job_id.to_string())?; let (job_payload, tag, _delete_after_use, timeout) = match job.job_kind { JobKind::Preview => ( @@ -3090,17 +3108,17 @@ pub async fn run_workflow_as_code( run_query.timeout, ), JobKind::Script => { - script_path_to_payload( - job.script_path(), - &mut tx, - &w_id, - run_query.skip_preprocessor, - ) - .await? + script_path_to_payload(job.script_path(), &db, &w_id, run_query.skip_preprocessor) + .await? } _ => return Err(anyhow::anyhow!("Not supported").into()), }; + if *CLOUD_HOSTED { + tracing::info!("workflow_as_code_tracing id {i} "); + i += 1; + } + let mut extra = HashMap::new(); extra.insert(ENTRYPOINT_OVERRIDE.to_string(), to_raw_value(&entrypoint)); @@ -3109,7 +3127,17 @@ pub async fn run_workflow_as_code( let tag = run_query.tag.clone().or(tag).or(Some(job.tag)); - let tx = PushIsolationLevel::Transaction(tx); + if *CLOUD_HOSTED { + tracing::info!("workflow_as_code_tracing id {i} "); + i += 1; + } + + let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into(), rsmq); + + if *CLOUD_HOSTED { + tracing::info!("workflow_as_code_tracing id {i} "); + i += 1; + } let (uuid, mut tx) = push( &db, @@ -3136,14 +3164,35 @@ pub async fn run_workflow_as_code( Some(&authed.clone().into()), ) .await?; - sqlx::query!( - "UPDATE queue SET flow_status = jsonb_set(COALESCE(flow_status, '{}'::jsonb), array[$1], jsonb_set(jsonb_set('{}'::jsonb, '{scheduled_for}', to_jsonb(now()::text)), '{name}', to_jsonb($4::text))) WHERE id = $2 AND workspace_id = $3", - uuid.to_string(), - job_id, - w_id, - entrypoint - ).execute(&mut tx).await?; + + if *CLOUD_HOSTED { + tracing::info!("workflow_as_code_tracing id {i} "); + i += 1; + } + + if !wkflow_query.skip_update.unwrap_or(false) { + sqlx::query!( + "UPDATE queue SET flow_status = jsonb_set(COALESCE(flow_status, '{}'::jsonb), array[$1], jsonb_set(jsonb_set('{}'::jsonb, '{scheduled_for}', to_jsonb(now()::text)), '{name}', to_jsonb($4::text))) WHERE id = $2 AND workspace_id = $3", + uuid.to_string(), + job_id, + w_id, + entrypoint + ).execute(&mut tx).await?; + } else { + tracing::info!("Skipping update of flow status for job {job_id} in workspace {w_id}"); + } + + if *CLOUD_HOSTED { + tracing::info!("workflow_as_code_tracing id {i} "); + i += 1; + } + tx.commit().await?; + + if *CLOUD_HOSTED { + tracing::info!("workflow_as_code_tracing id {i} "); + } + Ok((StatusCode::CREATED, uuid.to_string())) } @@ -3507,15 +3556,13 @@ pub async fn run_wait_result_job_by_path_get( let script_path = script_path.to_path(); check_scopes(&authed, || format!("run:script/{script_path}"))?; - let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); - let (job_payload, tag, delete_after_use, timeout) = - script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?; + script_path_to_payload(script_path, &db, &w_id, run_query.skip_preprocessor).await?; let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag).await?; - let tx = PushIsolationLevel::Transaction(tx); + let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into(), rsmq); let (uuid, tx) = push( &db, @@ -3632,15 +3679,13 @@ pub async fn run_wait_result_script_by_path_internal( let script_path = script_path.to_path(); check_scopes(&authed, || format!("run:script/{script_path}"))?; - let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); - let (job_payload, tag, delete_after_use, timeout) = - script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?; + script_path_to_payload(script_path, &db, &w_id, run_query.skip_preprocessor).await?; let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag).await?; - let tx = PushIsolationLevel::Transaction(tx); + let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into(), rsmq); let (uuid, tx) = push( &db, @@ -3692,8 +3737,6 @@ pub async fn run_wait_result_script_by_hash( check_queue_too_long(&db, run_query.queue_limit).await?; - let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); - let hash = script_hash.0; let ( path, @@ -3708,7 +3751,7 @@ pub async fn run_wait_result_script_by_hash( delete_after_use, timeout, has_preprocessor, - ) = get_path_tag_limits_cache_for_hash(&mut tx, &w_id, hash).await?; + ) = get_path_tag_limits_cache_for_hash(&db, &w_id, hash).await?; if let Some(run_query_cache_ttl) = run_query.cache_ttl { cache_ttl = Some(run_query_cache_ttl); } @@ -3717,7 +3760,7 @@ pub async fn run_wait_result_script_by_hash( let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag).await?; - let tx = PushIsolationLevel::Transaction(tx); + let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into(), rsmq); let (uuid, tx) = push( &db, @@ -3799,8 +3842,6 @@ pub async fn run_wait_result_flow_by_path_internal( let flow_path = flow_path.to_path(); check_scopes(&authed, || format!("run:flow/{flow_path}"))?; - let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); - let scheduled_for = run_query.get_scheduled_for(&db).await?; let (tag, dedicated_worker, early_return, has_preprocessor) = sqlx::query!( @@ -3812,7 +3853,7 @@ pub async fn run_wait_result_flow_by_path_internal( flow_path, w_id ) - .fetch_optional(&mut tx) + .fetch_optional(&db) .await? .map(|x| (x.tag, x.dedicated_worker, x.early_return, x.has_preprocessor)) .ok_or_else(|| { @@ -3824,7 +3865,7 @@ pub async fn run_wait_result_flow_by_path_internal( let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag).await?; - let tx = PushIsolationLevel::Transaction(tx); + let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into(), rsmq); let (uuid, tx) = push( &db, @@ -4130,10 +4171,10 @@ async fn run_dependencies_job( JsonRawValue::from_string("true".to_string()).unwrap(), ); if language == ScriptLang::Bun { - let annotation = windmill_common::worker::get_annotation(&raw_code); + let annotation = windmill_common::worker::TypeScriptAnnotations::parse(&raw_code); hm.insert( "npm_mode".to_string(), - JsonRawValue::from_string(annotation.npm_mode.to_string()).unwrap(), + JsonRawValue::from_string(annotation.npm.to_string()).unwrap(), ); } (PushArgs { extra: Some(hm), args: &ehm }, deps) @@ -4296,8 +4337,6 @@ async fn add_batch_jobs( } } "flow" => { - let mut tx = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq); - let mut uuids: Vec = Vec::new(); let payload = if let Some(ref fv) = batch_info.flow_value { JobPayload::RawFlow { value: fv.clone(), path: None, restarted_from: None } @@ -4314,6 +4353,7 @@ async fn add_batch_jobs( ))? } }; + let mut tx = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq); for _ in 0..n { let ehm = HashMap::new(); let (uuid, ntx) = push( @@ -4514,8 +4554,6 @@ pub async fn run_job_by_hash_inner( #[cfg(feature = "enterprise")] check_license_key_valid().await?; - let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); - let hash = script_hash.0; let ( path, @@ -4530,7 +4568,7 @@ pub async fn run_job_by_hash_inner( _delete_after_use, // not taken into account in async endpoints timeout, has_preprocessor, - ) = get_path_tag_limits_cache_for_hash(&mut tx, &w_id, hash).await?; + ) = get_path_tag_limits_cache_for_hash(&db, &w_id, hash).await?; check_scopes(&authed, || format!("run:script/{path}"))?; if let Some(run_query_cache_ttl) = run_query.cache_ttl { cache_ttl = Some(run_query_cache_ttl); @@ -4539,7 +4577,7 @@ pub async fn run_job_by_hash_inner( let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag).await?; - let tx = PushIsolationLevel::Transaction(tx); + let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into(), rsmq); let (uuid, tx) = push( &db, @@ -4683,13 +4721,12 @@ async fn get_job_update( .fetch_optional(&db) .await?; - let progress: Option = if get_progress == Some(true){ - sqlx::query_scalar!( + let progress: Option = if get_progress == Some(true) { + sqlx::query_scalar!( "SELECT scalar_int FROM job_stats WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", &w_id, job_id, "progress_perc" - ) .fetch_optional(&db) .await?.and_then(|inner| inner) @@ -5115,8 +5152,6 @@ async fn get_completed_job_result( Ok(Json(result).into_response()) } - - #[derive(Deserialize)] struct CountByTagQuery { horizon_secs: Option, @@ -5130,7 +5165,7 @@ struct TagCount { } async fn count_by_tag( - ApiAuthed { email, ..}: ApiAuthed, + ApiAuthed { email, .. }: ApiAuthed, Extension(db): Extension, Query(query): Query, ) -> JsonResult> { diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 97d2865e2e..3ea78ffd6d 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -39,9 +39,8 @@ use tower_http::{ trace::TraceLayer, }; use windmill_common::db::UserDB; -use windmill_common::utils::rd_string; -use windmill_common::worker::ALL_TAGS; -use windmill_common::BASE_URL; +use windmill_common::worker::{ALL_TAGS, CLOUD_HOSTED}; +use windmill_common::{BASE_URL, INSTANCE_NAME}; use crate::scim_ee::has_scim_token; use windmill_common::error::AppError; @@ -83,12 +82,16 @@ pub mod smtp_server_ee; mod static_assets; mod stripe_ee; mod tracing_init; +mod triggers; mod users; +mod users_ee; mod utils; mod variables; mod webhook_util; +mod websocket_triggers; mod workers; mod workspaces; +mod workspaces_ee; pub const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); @@ -225,7 +228,7 @@ pub async fn run_server( db: db.clone(), user_db: user_db, auth_cache: auth_cache.clone(), - rsmq: rsmq, + rsmq: rsmq.clone(), base_internal_url: base_internal_url.clone(), }); if let Err(err) = smtp_server.start_listener_thread(addr).await { @@ -245,6 +248,11 @@ pub async fn run_server( } }; + if !*CLOUD_HOSTED { + let ws_killpill_rx = rx.resubscribe(); + websocket_triggers::start_websockets(db.clone(), rsmq, ws_killpill_rx).await; + } + // build our application with a route let app = Router::new() .nest( @@ -285,7 +293,11 @@ pub async fn run_server( .nest("/variables", variables::workspaced_service()) .nest("/workspaces", workspaces::workspaced_service()) .nest("/oidc", oidc_ee::workspaced_service()) - .nest("/http_triggers", http_triggers::workspaced_service()), + .nest("/http_triggers", http_triggers::workspaced_service()) + .nest( + "/websocket_triggers", + websocket_triggers::workspaced_service(), + ), ) .nest("/workspaces", workspaces::global_service()) .nest( @@ -373,8 +385,6 @@ pub async fn run_server( ) }; - let instance_name = rd_string(5); - let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); let port = listener.local_addr().map(|x| x.port()).unwrap_or(8000); let ip = listener @@ -385,7 +395,7 @@ pub async fn run_server( let server = axum::serve(listener, app.into_make_service()); tracing::info!( - instance = %instance_name, + instance = %*INSTANCE_NAME, "server started on port={} and addr={}", port, ip @@ -448,9 +458,13 @@ async fn ee_license() -> &'static str { #[cfg(feature = "enterprise")] async fn ee_license() -> String { - use windmill_common::ee::LICENSE_KEY_ID; + use windmill_common::ee::{LICENSE_KEY_ID, LICENSE_KEY_VALID}; - LICENSE_KEY_ID.read().await.clone() + if *LICENSE_KEY_VALID.read().await { + LICENSE_KEY_ID.read().await.clone() + } else { + "".to_string() + } } async fn openapi() -> &'static str { diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 1fb5b12259..bb881d5369 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -58,7 +58,10 @@ pub fn workspaced_service() -> Router { .route("/type/exists/:name", get(exists_resource_type)) .route("/type/update/:name", post(update_resource_type)) .route("/type/delete/:name", delete(delete_resource_type)) - .route("/file_resource_type_to_file_ext_map", get(file_resource_ext_to_resource_type)) + .route( + "/file_resource_type_to_file_ext_map", + get(file_resource_ext_to_resource_type), + ) .route("/type/create", post(create_resource_type)) } diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index bcede4f5e2..56b991713f 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -9,6 +9,9 @@ use crate::{ db::{ApiAuthed, DB}, schedule::clear_schedule, + triggers::{ + get_triggers_count_internal, list_tokens_internal, TriggersCount, TruncatedTokenWithEmail, + }, users::{maybe_refresh_folders, require_owner_of_path, AuthCache}, utils::WithStarredInfoQuery, webhook_util::{WebhookMessage, WebhookShared}, @@ -53,7 +56,7 @@ use windmill_common::{ utils::{ not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath, }, - worker::{get_annotation, to_raw_value}, + worker::to_raw_value, HUB_BASE_URL, }; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; @@ -132,6 +135,8 @@ pub fn workspaced_service() -> Router { .route("/archive/p/*path", post(archive_script_by_path)) .route("/get/draft/*path", get(get_script_by_path_w_draft)) .route("/get/p/*path", get(get_script_by_path)) + .route("/get_triggers_count/*path", get(get_triggers_count)) + .route("/list_tokens/*path", get(list_tokens)) .route("/raw/p/*path", get(raw_script_by_path)) .route("/raw_unpinned/p/*path", get(raw_script_by_path_unpinned)) .route("/exists/p/*path", get(exists_script_by_path)) @@ -147,6 +152,7 @@ pub fn workspaced_service() -> Router { post(toggle_workspace_error_handler), ) .route("/history/p/*path", get(get_script_history)) + .route("/get_latest_version/*path", get(get_latest_version)) .route( "/history_update/h/:hash/p/*path", post(update_script_history), @@ -601,8 +607,8 @@ async fn create_script_internal<'c>( }; let lang = if &ns.language == &ScriptLang::Bun || &ns.language == &ScriptLang::Bunnative { - let anns = get_annotation(&ns.content); - if anns.native_mode { + let anns = windmill_common::worker::TypeScriptAnnotations::parse(&ns.content); + if anns.native { ScriptLang::Bunnative } else { ScriptLang::Bun @@ -874,6 +880,22 @@ async fn get_script_by_path( Ok(Json(script)) } +async fn list_tokens( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let path = path.to_path(); + list_tokens_internal(&db, &w_id, &path, false).await +} + +async fn get_triggers_count( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + get_triggers_count_internal(&db, &w_id, &path, false).await +} + async fn get_script_by_path_w_draft( authed: ApiAuthed, Extension(user_db): Extension, @@ -927,6 +949,38 @@ async fn get_script_history( return Ok(Json(result)); } +async fn get_latest_version( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let row_o = sqlx::query!( + + "SELECT s.hash as hash, dm.deployment_msg as deployment_msg + FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash + WHERE s.workspace_id = $1 AND s.path = $2 + ORDER by created_at DESC", + w_id, + path.to_path(), + ) + + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + if let Some(row) = row_o { + let result = ScriptHistory { + script_hash: ScriptHash(row.hash), + deployment_msg: row.deployment_msg, // + }; + return Ok(Json(Some(result))); + } else { + return Ok(Json(None)); + } + +} + async fn update_script_history( authed: ApiAuthed, Extension(user_db): Extension, diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 395387ebd6..0299145a38 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -24,17 +24,17 @@ use axum::{ #[cfg(feature = "enterprise")] use axum::extract::Query; +use serde::Deserialize; #[cfg(feature = "enterprise")] use windmill_common::ee::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel}; -use serde::Deserialize; use windmill_common::{ + email_ee::send_email, error::{self, JsonResult, Result}, global_settings::{ AUTOMATE_USERNAME_CREATION_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - HUB_BASE_URL_SETTING, + HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, }, server::Smtp, - utils::send_email, }; #[cfg(feature = "parquet")] @@ -163,8 +163,13 @@ pub async fn test_license_key( Json(TestKey { license_key }): Json, ) -> error::Result { require_super_admin(&db, &authed.email).await?; - validate_license_key(license_key).await?; - Ok("Sent test email".to_string()) + let (_, expired) = validate_license_key(license_key).await?; + + if expired { + Err(error::Error::BadRequest("Expired license key".to_string())) + } else { + Ok("Valid license key".to_string()) + } } pub async fn get_local_settings( @@ -257,6 +262,7 @@ pub async fn get_global_setting( && !key.starts_with("default_success_handler_") && key != AUTOMATE_USERNAME_CREATION_SETTING && key != HUB_BASE_URL_SETTING + && key != HUB_ACCESSIBLE_URL_SETTING && key != EMAIL_DOMAIN_SETTING { require_super_admin(&db, &authed.email).await?; @@ -298,7 +304,12 @@ async fn list_global_settings() -> JsonResult { pub async fn send_stats(Extension(db): Extension, authed: ApiAuthed) -> Result { require_super_admin(&db, &authed.email).await?; - windmill_common::stats_ee::send_stats(&"manual".to_string(), &HTTP_CLIENT, &db).await?; + windmill_common::stats_ee::send_stats( + &HTTP_CLIENT, + &db, + windmill_common::stats_ee::SendStatsReason::Manual, + ) + .await?; Ok("Sent stats".to_string()) } @@ -357,8 +368,13 @@ pub async fn renew_license_key( authed: ApiAuthed, ) -> Result { require_super_admin(&db, &authed.email).await?; - windmill_common::stats_ee::send_stats(&"manual".to_string(), &HTTP_CLIENT, &db).await?; - let result = windmill_common::ee::renew_license_key(&HTTP_CLIENT, &db, license_key, true).await; + let result = windmill_common::ee::renew_license_key( + &HTTP_CLIENT, + &db, + license_key, + windmill_common::ee::RenewReason::Manual, + ) + .await; if result != "success" { return Err(error::Error::BadRequest(format!( diff --git a/backend/windmill-api/src/static_assets.rs b/backend/windmill-api/src/static_assets.rs index aa9870907e..783732b1c0 100644 --- a/backend/windmill-api/src/static_assets.rs +++ b/backend/windmill-api/src/static_assets.rs @@ -48,7 +48,10 @@ fn serve_path(path: &str) -> Response { let mut res = Response::builder() .header(header::CONTENT_TYPE, mime.as_ref()) .header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*"); - if mime.as_ref() == mime::APPLICATION_JAVASCRIPT || path.ends_with(".wasm") { + if mime.as_ref() == mime::APPLICATION_JAVASCRIPT + || mime.as_ref() == mime::TEXT_JAVASCRIPT + || path.ends_with(".wasm") + { res = res.header(header::CACHE_CONTROL, "max-age=31536000"); } else if (mime.type_(), mime.subtype()) == (mime::TEXT, mime::CSS) { res = res.header(header::CACHE_CONTROL, "max-age=31536000"); diff --git a/backend/windmill-api/src/tracing_init.rs b/backend/windmill-api/src/tracing_init.rs index 1f93b941cb..60dab73810 100644 --- a/backend/windmill-api/src/tracing_init.rs +++ b/backend/windmill-api/src/tracing_init.rs @@ -9,6 +9,7 @@ use ::tracing::{field, Span}; use hyper::Response; use tower_http::trace::{MakeSpan, OnFailure, OnResponse}; +use uuid::Uuid; lazy_static::lazy_static! { static ref LOG_REQUESTS: bool = std::env::var("LOG_REQUESTS") @@ -45,17 +46,28 @@ impl OnFailure for MyOnFailure { // tracing::error!(latency = latency.as_millis(), "response") } } + +lazy_static::lazy_static! { + static ref TRACING_HEADER: String = std::env::var("TRACING_HEADER") + .ok().unwrap_or_else(|| "x-tracing-id".to_string()); +} #[derive(Clone)] pub struct MyMakeSpan {} impl MakeSpan for MyMakeSpan { fn make_span(&mut self, request: &hyper::Request) -> Span { + let tracing_id = request + .headers() + .get(TRACING_HEADER.as_str()) + .and_then(|x| x.to_str().map(|x| x.to_string()).ok()) + .unwrap_or(Uuid::new_v4().to_string()); tracing::info_span!( "request", method = %request.method(), uri = %request.uri(), username = field::Empty, workspace_id = field::Empty, + trace_id = tracing_id, email = field::Empty, ) } diff --git a/backend/windmill-api/src/triggers.rs b/backend/windmill-api/src/triggers.rs new file mode 100644 index 0000000000..337a20364b --- /dev/null +++ b/backend/windmill-api/src/triggers.rs @@ -0,0 +1,143 @@ +use axum::Json; +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; +use windmill_common::error::JsonResult; + +use crate::db::DB; + +#[derive(Serialize, Deserialize, Debug)] +pub struct TriggerPrimarySchedule { + schedule: String, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct TriggersCount { + primary_schedule: Option, + schedule_count: i64, + http_routes_count: i64, + webhook_count: i64, + email_count: i64, + websocket_count: i64, +} +pub(crate) async fn get_triggers_count_internal( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, +) -> JsonResult { + let primary_schedule = sqlx::query_scalar!( + "SELECT schedule FROM schedule WHERE path = $1 AND script_path = $1 AND is_flow = $2 AND workspace_id = $3", + path, + is_flow, + w_id + ) + .fetch_optional(db) + .await?; + + let schedule_count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM schedule WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + path, + is_flow, + w_id + ) + .fetch_one(db) + .await? + .unwrap_or(0); + + let http_routes_count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM http_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + path, + is_flow, + w_id + ) + .fetch_one(db) + .await? + .unwrap_or(0); + + let websocket_count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM websocket_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + path, + is_flow, + w_id + ) + .fetch_one(db) + .await? + .unwrap_or(0); + + let webhook_count = (if is_flow { + sqlx::query_scalar!( + "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", + w_id, + path, + ) + + } else { + sqlx::query_scalar!( + "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:' || $2]::text[]", + w_id, + path, + ) + }).fetch_one(db) + .await? + .unwrap_or(0); + + let email_count = (if is_flow { + sqlx::query_scalar!( + "SELECT COUNT(*) FROM token WHERE label LIKE 'email-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", + w_id, + path, + ) + + } else { + sqlx::query_scalar!( + "SELECT COUNT(*) FROM token WHERE label LIKE 'email-%' AND workspace_id = $1 AND scopes @> ARRAY['run:script/' || $2]::text[]", + w_id, + path, + ) + }).fetch_one(db) + .await? + .unwrap_or(0); + + Ok(Json(TriggersCount { + primary_schedule: primary_schedule.map(|s| TriggerPrimarySchedule { schedule: s }), + schedule_count, + http_routes_count, + webhook_count, + email_count, + websocket_count, + })) +} + +#[derive(FromRow, Serialize)] +pub struct TruncatedTokenWithEmail { + pub label: Option, + pub token_prefix: Option, + pub expiration: Option>, + pub created_at: chrono::DateTime, + pub last_used_at: chrono::DateTime, + pub scopes: Option>, + pub email: Option, +} + +pub async fn list_tokens_internal( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, +) -> JsonResult> { + let tokens = if is_flow { + sqlx::query_as!( + TruncatedTokenWithEmail, + "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at, scopes, email FROM token WHERE workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", + w_id, path).fetch_all(db) + .await? + } else { + sqlx::query_as!( + TruncatedTokenWithEmail, + "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, last_used_at, scopes, email FROM token WHERE workspace_id = $1 AND scopes @> ARRAY['run:script/' || $2]::text[]", + w_id, path) + .fetch_all(db) + .await? + }; + Ok(Json(tokens)) +} diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs new file mode 100644 index 0000000000..c9a812c4a0 --- /dev/null +++ b/backend/windmill-api/src/users.rs @@ -0,0 +1,3169 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +#![allow(non_snake_case)] + +use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; +use std::sync::Arc; + +use crate::db::ApiAuthed; + +#[cfg(feature = "enterprise")] +use crate::ee::ExternalJwks; +use crate::oauth2_ee::InstanceEvent; +use crate::utils::{ + generate_instance_wide_unique_username, get_instance_username_or_create_pending, +}; +use crate::{ + db::DB, utils::require_super_admin, webhook_util::WebhookShared, COOKIE_DOMAIN, IS_SECURE, +}; +use argon2::{Argon2, PasswordHash, PasswordVerifier}; +use axum::{ + async_trait, + extract::{Extension, FromRequestParts, OriginalUri, Path, Query}, + http::request::Parts, + response::{IntoResponse, Response}, + routing::{delete, get, post}, + Json, Router, +}; +use chrono::TimeZone; +use hyper::{header::LOCATION, StatusCode}; +use lazy_static::lazy_static; +use quick_cache::sync::Cache; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; +use time::OffsetDateTime; +#[cfg(feature = "enterprise")] +use tokio::sync::RwLock; +use tower_cookies::{Cookie, Cookies}; +use tracing::{Instrument, Span}; +use windmill_audit::audit_ee::{audit_log, AuditAuthor}; +use windmill_audit::ActionKind; +use windmill_common::auth::fetch_authed_from_permissioned_as; +use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING; +use windmill_common::users::{truncate_token, username_to_permissioned_as}; +use windmill_common::utils::paginate; +use windmill_common::worker::CLOUD_HOSTED; +use windmill_common::{ + auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, JWT_SECRET}, + db::UserDB, + error::{self, Error, JsonResult, Result}, + users::SUPERADMIN_SECRET_EMAIL, + utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath}, +}; +use windmill_git_sync::handle_deployment_metadata; + +pub const TTL_TOKEN_DB_H: u32 = 72; + +const COOKIE_NAME: &str = "token"; +const COOKIE_PATH: &str = "/"; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_users)) + .route("/list_usage", get(list_user_usage)) + .route("/list_usernames", get(list_usernames)) + .route("/exists", post(exists_username)) + .route("/get/:user", get(get_workspace_user)) + .route("/update/:user", post(update_workspace_user)) + .route("/delete/:user", delete(delete_workspace_user)) + .route("/is_owner/*path", get(is_owner_of_path)) + .route("/whois/:username", get(whois)) + .route("/whoami", get(whoami)) + .route("/leave", post(leave_workspace)) + .route("/username_to_email/:username", get(username_to_email)) +} + +pub fn global_service() -> Router { + Router::new() + .route("/exists/:email", get(exists_email)) + .route("/email", get(get_email)) + .route("/whoami", get(global_whoami)) + .route("/list_invites", get(list_invites)) + .route("/decline_invite", post(decline_invite)) + .route("/accept_invite", post(accept_invite)) + .route("/list_as_super_admin", get(list_users_as_super_admin)) + .route("/setpassword", post(set_password)) + .route("/create", post(create_user)) + .route("/update/:user", post(update_user)) + .route("/delete/:user", delete(delete_user)) + .route("/username_info/:user", get(get_instance_username_info)) + .route("/rename/:user", post(rename_user)) + .route("/tokens/create", post(create_token)) + .route("/tokens/delete/:token_prefix", delete(delete_token)) + .route("/tokens/list", get(list_tokens)) + .route("/tokens/impersonate", post(impersonate)) + .route("/usage", get(get_usage)) + .route("/all_runnables", get(get_all_runnables)) + .route("/refresh_token", get(refresh_token)) + .route( + "/tutorial_progress", + post(update_tutorial_progress).get(get_tutorial_progress), + ) + .route("/leave_instance", post(leave_instance)) + .route("/export", get(export_global_users)) + .route("/overwrite", post(overwrite_global_users)) + + // .route("/list_invite_codes", get(list_invite_codes)) + // .route("/create_invite_code", post(create_invite_code)) + // .route("/signup", post(signup)) + // .route("/lost_password", post(lost_password)) + // .route("/use_magic_link", get(use_magic_link)) +} + +pub fn make_unauthed_service() -> Router { + Router::new() + .route("/login", post(login)) + .route("/logout", post(logout).get(logout)) + .route("/is_first_time_setup", get(is_first_time_setup)) +} + +fn username_override_from_label(label: Option) -> Option { + match label { + Some(label) + if label.starts_with("webhook-") + || label.starts_with("http-") + || label.starts_with("email-") => + { + Some(label) + } + Some(label) if label.starts_with("ephemeral-script-end-user-") => Some( + label + .trim_start_matches("ephemeral-script-end-user-") + .to_string(), + ), + Some(label) if label == "Ephemeral lsp token" => Some("lsp".to_string()), + Some(label) if label != "ephemeral-script" && label != "session" && !label.is_empty() => { + Some(format!("label-{label}")) + } + _ => None, + } +} + +#[derive(Clone)] +pub struct ExpiringAuthCache { + pub authed: ApiAuthed, + pub expiry: chrono::DateTime, +} + +pub struct AuthCache { + cache: Cache<(String, String), ExpiringAuthCache>, + db: DB, + superadmin_secret: Option, + #[cfg(feature = "enterprise")] + ext_jwks: Option>>, +} + +impl AuthCache { + pub fn new( + db: DB, + superadmin_secret: Option, + #[cfg(feature = "enterprise")] ext_jwks: Option>>, + ) -> Self { + AuthCache { + cache: Cache::new(300), + db, + superadmin_secret, + #[cfg(feature = "enterprise")] + ext_jwks, + } + } + + pub async fn invalidate(&self, w_id: &str, token: String) { + self.cache.remove(&(w_id.to_string(), token)); + } + + pub async fn get_authed(&self, w_id: Option, token: &str) -> Option { + let key = ( + w_id.as_ref().unwrap_or(&"".to_string()).to_string(), + token.to_string(), + ); + let s = self.cache.get(&key).map(|c| c.to_owned()); + match s { + Some(ExpiringAuthCache { authed, expiry }) if expiry > chrono::Utc::now() => { + Some(authed) + } + #[cfg(feature = "enterprise")] + _ if token.starts_with("jwt_ext_") => { + let authed_and_exp = match crate::ee::jwt_ext_auth( + w_id.as_ref(), + token.trim_start_matches("jwt_ext_"), + self.ext_jwks.clone(), + ) + .await + { + Ok(r) => Some(r), + Err(e) => { + tracing::error!("JWT_EXT auth error: {:?}", e); + None + } + }; + + if let Some((authed, exp)) = authed_and_exp.clone() { + self.cache.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc.timestamp_nanos(exp as i64 * 1_000_000_000), + }, + ); + + Some(authed) + } else { + None + } + } + _ if token.starts_with("jwt_") => { + let jwt_secret = JWT_SECRET.read().await; + if !jwt_secret.is_empty() { + let jwt_token = token.trim_start_matches("jwt_"); + + let jwt_result = jsonwebtoken::decode::( + jwt_token, + &jsonwebtoken::DecodingKey::from_secret(jwt_secret.as_bytes()), + &jsonwebtoken::Validation::new(jsonwebtoken::Algorithm::HS256), + ); + + match jwt_result { + Ok(payload) => { + if w_id.is_some_and(|w_id| w_id != payload.claims.workspace_id) { + tracing::error!("JWT auth error: workspace_id mismatch"); + return None; + } + + let username_override = + username_override_from_label(payload.claims.label); + let authed = crate::db::ApiAuthed { + email: payload.claims.email, + username: payload.claims.username, + is_admin: payload.claims.is_admin, + is_operator: payload.claims.is_operator, + groups: payload.claims.groups, + folders: payload.claims.folders, + scopes: None, + username_override, + }; + + self.cache.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc + .timestamp_nanos(payload.claims.exp as i64 * 1_000_000_000), + }, + ); + + Some(authed) + } + Err(err) => { + tracing::error!("JWT auth error: {:?}", err); + None + } + } + } else { + tracing::error!("JWT auth error: no jwt secret set"); + None + } + } + _ => { + let user_o = sqlx::query_as::<_, (Option, Option, bool, Option>, Option)>( + "UPDATE token SET last_used_at = now() WHERE token = $1 AND (expiration > NOW() \ + OR expiration IS NULL) AND (workspace_id IS NULL OR workspace_id = $2) RETURNING owner, email, super_admin, scopes, label", + ) + .bind(token) + .bind(w_id.as_ref()) + .fetch_optional(&self.db) + .await + .ok() + .flatten(); + + if let Some(user) = user_o { + let authed_o = { + match user { + (Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => { + let username_override = username_override_from_label(label); + if let Some((prefix, name)) = owner.split_once('/') { + if prefix == "u" { + let (is_admin, is_operator) = if super_admin { + (true, false) + } else { + let r = sqlx::query!( + "SELECT is_admin, operator FROM usr where username = $1 AND \ + workspace_id = $2 AND disabled = false", + name, + &w_id.as_ref().unwrap() + ) + .fetch_one(&self.db) + .await + .ok(); + if let Some(r) = r { + (r.is_admin, r.operator) + } else { + (false, true) + } + }; + + let w_id = &w_id.unwrap(); + let groups = + get_groups_for_user(w_id, &name, &email, &self.db) + .await + .ok() + .unwrap_or_default(); + + let folders = + get_folders_for_user(w_id, &name, &groups, &self.db) + .await + .ok() + .unwrap_or_default(); + + Some(ApiAuthed { + email: email, + username: name.to_string(), + is_admin, + is_operator, + groups, + folders, + scopes: None, + username_override, + }) + } else { + let groups = vec![name.to_string()]; + let folders = get_folders_for_user( + &w_id.unwrap(), + "", + &groups, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + Some(ApiAuthed { + email: email, + username: format!("group-{name}"), + is_admin: false, + groups, + is_operator: false, + folders, + scopes: None, + username_override, + }) + } + } else { + let groups = vec![]; + let folders = vec![]; + Some(ApiAuthed { + email: email, + username: owner, + is_admin: super_admin, + is_operator: true, + groups, + folders, + scopes: None, + username_override, + }) + } + } + (_, Some(email), super_admin, scopes, label) => { + let username_override = username_override_from_label(label); + if w_id.is_some() { + let row_o = sqlx::query_as::<_, (String, bool, bool)>( + "SELECT username, is_admin, operator FROM usr where email = $1 AND \ + workspace_id = $2 AND disabled = false", + ) + .bind(&email) + .bind(&w_id.as_ref().unwrap()) + .fetch_optional(&self.db) + .await + .unwrap_or(Some(("error".to_string(), false, false))); + + match row_o { + Some((username, is_admin, is_operator)) => { + let groups = get_groups_for_user( + &w_id.as_ref().unwrap(), + &username, + &email, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + + let folders = get_folders_for_user( + &w_id.unwrap(), + &username, + &groups, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + Some(ApiAuthed { + email, + username, + is_admin: is_admin || super_admin, + is_operator, + groups, + folders, + scopes, + username_override, + }) + } + None if super_admin => Some(ApiAuthed { + email: email.clone(), + username: email, + is_admin: super_admin, + is_operator: false, + groups: vec![], + folders: vec![], + scopes, + username_override, + }), + None => None, + } + } else { + Some(ApiAuthed { + email: email.to_string(), + username: email, + is_admin: super_admin, + is_operator: true, + groups: Vec::new(), + folders: Vec::new(), + scopes, + username_override, + }) + } + } + _ => None, + } + }; + if let Some(authed) = authed_o.as_ref() { + self.cache.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc::now() + + chrono::Duration::try_seconds(120).unwrap(), + }, + ); + } + authed_o + } else if self + .superadmin_secret + .as_ref() + .map(|x| x == token) + .unwrap_or(false) + { + Some(ApiAuthed { + email: SUPERADMIN_SECRET_EMAIL.to_string(), + username: "superadmin_secret".to_string(), + is_admin: true, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + }) + } else { + None + } + } + } + } +} + +async fn extract_token(parts: &mut Parts, state: &S) -> Option { + let auth_header = parts + .headers + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|s| s.strip_prefix("Bearer ")); + + let from_cookie = match auth_header { + Some(x) => Some(x.to_owned()), + None => Extension::::from_request_parts(parts, state) + .await + .ok() + .and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())), + }; + + #[derive(Deserialize)] + struct Token { + token: Option, + } + match from_cookie { + Some(token) => Some(token), + None => Query::::from_request_parts(parts, state) + .await + .ok() + .and_then(|token| token.token.clone()), + } +} + +#[derive(Clone, Debug)] +pub struct Tokened { + pub token: String, +} + +struct BruteForceCounter { + counter: AtomicU64, + last_reset: AtomicI64, +} + +lazy_static! { + static ref BRUTE_FORCE_COUNTER: BruteForceCounter = + BruteForceCounter { last_reset: AtomicI64::new(0), counter: AtomicU64::new(0) }; +} + +impl BruteForceCounter { + async fn increment(&self) { + let now = time::OffsetDateTime::now_utc().unix_timestamp(); + if self.counter.fetch_add(1, Ordering::Relaxed) > 10000 { + tracing::error!( + "Brute force attack to find valid token detected, sleeping unauthorized response for 2 seconds" + ); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + if now - self.last_reset.load(Ordering::Relaxed) > 60 { + self.counter.store(0, Ordering::Relaxed); + self.last_reset.store(now, Ordering::Relaxed); + } + } +} + +#[async_trait] +impl FromRequestParts for Tokened +where + S: Send + Sync, +{ + type Rejection = (StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method == http::Method::OPTIONS { + return Ok(Tokened { token: "".to_string() }); + }; + let already_tokened = parts.extensions.get::(); + if let Some(tokened) = already_tokened { + Ok(tokened.clone()) + } else { + let token_o = extract_token(parts, state).await; + if let Some(token) = token_o { + let tokened = Self { token }; + parts.extensions.insert(tokened.clone()); + Ok(tokened) + } else { + BRUTE_FORCE_COUNTER.increment().await; + Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) + } + } + } +} + +pub async fn maybe_refresh_folders( + path: &str, + w_id: &str, + authed: ApiAuthed, + db: &DB, +) -> ApiAuthed { + if authed.is_admin { + return authed; + } + let splitted = path.split('/').collect::>(); + if splitted.len() >= 2 + && splitted[0] == "f" + && !authed.folders.iter().any(|(f, _, _)| f == splitted[1]) + { + let name = &authed.username; + let groups = get_groups_for_user(w_id, name, &authed.email, db) + .await + .ok() + .unwrap_or_default(); + + let folders = get_folders_for_user(w_id, name, &groups, db) + .await + .ok() + .unwrap_or_default(); + ApiAuthed { folders, ..authed } + } else { + authed + } +} + +#[async_trait] +impl FromRequestParts for ApiAuthed +where + S: Send + Sync, +{ + type Rejection = (StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method == http::Method::OPTIONS { + return Ok(ApiAuthed { + email: "".to_owned(), + username: "".to_owned(), + is_admin: false, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + }); + }; + let already_authed = parts.extensions.get::(); + if let Some(authed) = already_authed { + Ok(authed.clone()) + } else { + let already_tokened = parts.extensions.get::(); + let token_o = if let Some(token) = already_tokened { + Some(token.token.clone()) + } else { + extract_token(parts, state).await + }; + let original_uri = OriginalUri::from_request_parts(parts, state) + .await + .ok() + .map(|x| x.0) + .unwrap_or_default(); + let path_vec: Vec<&str> = original_uri.path().split("/").collect(); + + let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" { + Some(path_vec[3].to_owned()) + } else { + if path_vec.len() >= 5 + && path_vec[0] == "" + && path_vec[2] == "srch" + && path_vec[3] == "w" + { + Some(path_vec[4].to_string()) + } else { + None + } + }; + if let Some(token) = token_o { + if let Ok(Extension(cache)) = + Extension::>::from_request_parts(parts, state).await + { + if let Some(authed) = cache.get_authed(workspace_id.clone(), &token).await { + parts.extensions.insert(authed.clone()); + if authed.scopes.as_ref().is_some_and(|scopes| { + scopes + .iter() + .any(|s| s.starts_with("jobs:") || s.starts_with("run:")) + }) && (path_vec.len() < 3 + || (path_vec[4] != "jobs" && path_vec[4] != "jobs_u")) + { + BRUTE_FORCE_COUNTER.increment().await; + return Err(( + StatusCode::UNAUTHORIZED, + format!("Unauthorized scoped token: {:?}", authed.scopes), + )); + } + Span::current().record("username", &authed.username.as_str()); + Span::current().record("email", &authed.email); + + if let Some(workspace_id) = workspace_id { + Span::current().record("workspace_id", &workspace_id); + } + return Ok(authed); + } + } + } + BRUTE_FORCE_COUNTER.increment().await; + Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) + } + } +} + +pub fn check_scopes(authed: &ApiAuthed, required: F) -> error::Result<()> +where + F: FnOnce() -> String, +{ + if authed.scopes.as_ref().is_some_and(|scopes| { + scopes + .iter() + .any(|s| s.starts_with("jobs:") || s.starts_with("run:")) + }) { + let req = &required(); + if !authed.scopes.as_ref().unwrap().contains(req) { + return Err(Error::BadRequest(format!("missing required scope: {req}"))); + } + } + Ok(()) +} + +pub fn get_scope_tags(authed: &ApiAuthed) -> Option> { + authed.scopes.as_ref()?.iter().find_map(|s| { + if s.starts_with("if_jobs:filter_tags:") { + Some( + s.trim_start_matches("if_jobs:filter_tags:") + .split(",") + .collect::>(), + ) + } else { + None + } + }) +} + +#[derive(Clone, Debug)] +pub struct OptAuthed(pub Option); + +#[async_trait] +impl FromRequestParts for OptAuthed +where + S: Send + Sync, +{ + type Rejection = (StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + ApiAuthed::from_request_parts(parts, state) + .await + .map(|authed| Self(Some(authed))) + .or_else(|_| Ok(Self(None))) + } +} + +pub async fn fetch_api_authed( + username: String, + email: String, + w_id: &str, + db: &DB, + username_override: String, +) -> error::Result { + let permissioned_as = username_to_permissioned_as(username.as_str()); + let authed = + fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?; + Ok(ApiAuthed { + username: username, + email: email, + is_admin: authed.is_admin, + is_operator: authed.is_operator, + groups: authed.groups, + folders: authed.folders, + scopes: authed.scopes, + username_override: Some(username_override), + }) +} + +#[derive(FromRow, Serialize)] +pub struct User { + pub workspace_id: String, + pub email: String, + pub username: String, + pub is_admin: bool, + pub created_at: chrono::DateTime, + pub operator: bool, + pub disabled: bool, + pub role: Option, +} + +#[derive(Serialize)] +pub struct UserWithUsage { + pub email: String, + pub executions: Option, +} + +#[derive(FromRow, Serialize, Debug)] +pub struct GlobalUserInfo { + email: String, + login_type: Option, + super_admin: bool, + verified: bool, + name: Option, + company: Option, + username: Option, + #[serde(skip_serializing_if = "Option::is_none")] + operator_only: Option, +} + +#[derive(Serialize, Debug)] +pub struct UserInfo { + pub workspace_id: String, + pub email: String, + pub username: String, + pub is_admin: bool, + pub is_super_admin: bool, + pub created_at: chrono::DateTime, + pub groups: Vec, + pub operator: bool, + pub disabled: bool, + pub role: Option, + pub folders_read: Vec, + pub folders: Vec, + pub folders_owners: Vec, +} + +#[derive(FromRow, Serialize)] +pub struct WorkspaceInvite { + pub workspace_id: String, + pub email: String, + pub is_admin: bool, + pub operator: bool, +} + +#[allow(dead_code)] +#[derive(Deserialize)] +pub struct NewUser { + pub email: String, + pub password: String, + pub super_admin: bool, + pub name: Option, + pub company: Option, +} + +#[derive(Deserialize)] +pub struct AcceptInvite { + pub workspace_id: String, + pub username: Option, +} + +#[derive(Deserialize)] +pub struct DeclineInvite { + pub workspace_id: String, +} + +#[derive(Deserialize)] +pub struct EditUser { + pub is_super_admin: Option, + pub name: Option, +} + +#[derive(Deserialize)] +pub struct EditWorkspaceUser { + pub is_admin: Option, + pub operator: Option, + pub disabled: Option, +} + +#[allow(dead_code)] +#[derive(Deserialize)] +pub struct EditPassword { + pub password: String, +} + +#[derive(FromRow, Serialize)] +pub struct TruncatedToken { + pub label: Option, + pub token_prefix: Option, + pub expiration: Option>, + pub created_at: chrono::DateTime, + pub last_used_at: chrono::DateTime, + pub scopes: Option>, +} + +#[derive(Deserialize)] +pub struct NewToken { + pub label: Option, + pub expiration: Option>, + pub impersonate_email: Option, + pub scopes: Option>, + pub workspace_id: Option, +} + +#[derive(Deserialize)] +pub struct Login { + pub email: String, + pub password: String, +} + +lazy_static::lazy_static! { + static ref FIRST_TIME_SETUP: Arc = Arc::new(AtomicBool::new(true)); +} + +pub async fn is_first_time_setup(Extension(db): Extension) -> JsonResult { + if !FIRST_TIME_SETUP.load(std::sync::atomic::Ordering::Relaxed) { + return Ok(Json(false)); + } + let single_user = sqlx::query_scalar!("SELECT 1 FROM password LIMIT 2") + .fetch_all(&db) + .await + .ok() + .unwrap_or_default() + .len() + == 1; + if single_user { + let user_is_admin_and_password_changeme = sqlx::query_scalar!( + "SELECT 1 FROM password WHERE email = 'admin@windmill.dev' AND password_hash = '$argon2id$v=19$m=4096,t=3,p=1$oLJo/lPn/gezXCuFOEyaNw$i0T2tCkw3xUFsrBIKZwr8jVNHlIfoxQe+HfDnLtd12I'" + ).fetch_all(&db) + .await + .ok() + .unwrap_or_default() + .len() == 1; + if user_is_admin_and_password_changeme { + let base_url_is_not_set = + sqlx::query_scalar!("SELECT COUNT(*) FROM global_settings WHERE name = 'base_url'") + .fetch_optional(&db) + .await + .ok() + .flatten() + .flatten() + .unwrap_or(0) + == 0; + if base_url_is_not_set { + return Ok(Json(true)); + } + } + } + FIRST_TIME_SETUP.store(false, std::sync::atomic::Ordering::Relaxed); + Ok(Json(false)) +} + +#[derive(Deserialize)] +struct WorkspaceUsername { + pub username: String, +} + +async fn exists_username( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(WorkspaceUsername { username }): Json, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)", + &w_id, + &username + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + tx.commit().await?; + Ok(Json(exists)) +} + +async fn list_users( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + if *CLOUD_HOSTED && w_id == "demo" { + require_admin(authed.is_admin, &authed.username)?; + } + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query_as!( + User, + " + SELECT * + FROM usr + WHERE workspace_id = $1 + ", + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +async fn list_user_usage( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + if *CLOUD_HOSTED && w_id == "demo" { + require_admin(authed.is_admin, &authed.username)?; + } + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query_as!( + UserWithUsage, + " + SELECT usr.email, usage.executions + FROM usr + , LATERAL ( + SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0)::BIGINT executions + FROM completed_job + WHERE workspace_id = $1 + AND job_kind NOT IN ('flow', 'flowpreview') + AND email = usr.email + AND now() - '1 week'::interval < created_at + ) usage + WHERE workspace_id = $1 + ", + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +#[derive(Deserialize)] +struct ActiveUsersOnly { + active_only: Option, +} + +async fn list_users_as_super_admin( + authed: ApiAuthed, + Extension(db): Extension, + Query(pagination): Query, + Query(ActiveUsersOnly { active_only }): Query, +) -> JsonResult> { + require_super_admin(&db, &authed.email).await?; + let per_page = pagination.per_page.unwrap_or(10000).max(1); + let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page; + + let rows = if active_only.is_some_and(|x| x) { + sqlx::query_as!( + GlobalUserInfo, + "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login')), + authors as (SELECT distinct email FROM usr WHERE usr.operator IS false) + SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, name, company, username + FROM password + WHERE email IN (SELECT email FROM active_users) + ORDER BY super_admin DESC + LIMIT $1 OFFSET $2", + per_page as i32, + offset as i32 + ) + .fetch_all(&db) + .await? + } else { + sqlx::query_as!( + GlobalUserInfo, + "SELECT email, login_type::text, verified, super_admin, name, company, username, NULL::bool as operator_only FROM password ORDER BY super_admin DESC, email LIMIT \ + $1 OFFSET $2", + per_page as i32, + offset as i32 + ) + .fetch_all(&db) + .await? + }; + + Ok(Json(rows)) +} + +#[derive(Serialize, Deserialize)] +struct Progress { + progress: u64, +} +async fn get_tutorial_progress( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult { + let res = sqlx::query_scalar!( + "SELECT progress::bigint FROM tutorial_progress WHERE email = $1", + authed.email + ) + .fetch_optional(&db) + .await? + .flatten() + .unwrap_or_default() as u64; + Ok(Json(Progress { progress: res })) +} + +async fn update_tutorial_progress( + authed: ApiAuthed, + Extension(db): Extension, + Json(progress): Json, +) -> Result { + sqlx::query_scalar!( + "INSERT INTO tutorial_progress VALUES ($2, $1::bigint::bit(64)) ON CONFLICT (email) DO UPDATE SET progress = $1::bigint::bit(64)", + progress.progress as i64, + authed.email + ) + .execute(&db) + .await?; + Ok("tutorial progress updated".to_string()) +} + +async fn list_usernames( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + if *CLOUD_HOSTED && w_id == "demo" { + return Ok(Json(vec![ + authed.username, + "other_usernames_redacted_in_demo_workspace".to_string(), + ])); + } + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query_scalar!("SELECT username from usr WHERE workspace_id = $1", &w_id) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +async fn list_invites( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + let mut tx = db.begin().await?; + let rows = sqlx::query_as!( + WorkspaceInvite, + "SELECT * from workspace_invite WHERE email = $1", + authed.email + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +#[derive(Deserialize)] +struct LogoutQuery { + rd: Option, +} +async fn logout( + Tokened { token }: Tokened, + cookies: Cookies, + Extension(db): Extension, + Query(LogoutQuery { rd }): Query, +) -> Result { + let mut cookie = Cookie::new(COOKIE_NAME, ""); + cookie.set_path(COOKIE_PATH); + if COOKIE_DOMAIN.is_some() { + cookie.set_domain(COOKIE_DOMAIN.clone().unwrap()); + } + cookies.remove(cookie); + let mut tx = db.begin().await?; + let email = sqlx::query_scalar!("DELETE FROM token WHERE token = $1 RETURNING email", token) + .fetch_optional(&mut *tx) + .await?; + if let Some(email) = email { + let email = email.unwrap_or("noemail".to_string()); + audit_log( + &mut *tx, + &AuditAuthor { email: email.clone(), username: email, username_override: None }, + "users.logout", + ActionKind::Delete, + "global", + Some(&truncate_token(&token)), + None, + ) + .await?; + } + tx.commit().await?; + if let Some(rd) = rd { + Ok((StatusCode::TEMPORARY_REDIRECT, [(LOCATION, rd)]).into_response()) + } else { + Ok((StatusCode::OK, "logged out successfully".to_string()).into_response()) + } +} + +async fn whoami( + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { username, email, is_admin, groups, folders, .. }: ApiAuthed, +) -> JsonResult { + let user = get_user(&w_id, &username, &db).await?; + if let Some(user) = user { + Ok(Json(user)) + } else { + Ok(Json(UserInfo { + workspace_id: w_id, + email: email.clone(), + username: email, + is_admin, + is_super_admin: is_admin, + created_at: chrono::Utc::now(), + groups: groups, + operator: false, + disabled: false, + role: Some("superadmin".to_string()), + folders_read: folders.clone().into_iter().map(|x| x.0).collect(), + folders: folders + .clone() + .into_iter() + .filter_map(|x| if x.1 { Some(x.0) } else { None }) + .collect(), + folders_owners: folders + .into_iter() + .filter_map(|x| if x.2 { Some(x.0) } else { None }) + .collect(), + })) + } +} + +async fn global_whoami( + Extension(db): Extension, + ApiAuthed { email, .. }: ApiAuthed, + Tokened { token }: Tokened, +) -> JsonResult { + let user = sqlx::query_as!( + GlobalUserInfo, + "SELECT email, login_type::TEXT, super_admin, verified, name, company, username, NULL::bool as operator_only FROM password WHERE \ + email = $1", + email + ) + .fetch_one(&db) + .await + .map_err(|e| Error::InternalErr(format!("fetching global identity: {e:#}"))); + + if let Ok(user) = user { + Ok(Json(user)) + } else if std::env::var("SUPERADMIN_SECRET").ok() == Some(token) { + Ok(Json(GlobalUserInfo { + email: email.clone(), + login_type: Some("superadmin_secret".to_string()), + super_admin: true, + verified: true, + name: None, + company: None, + username: None, + operator_only: None, + })) + } else { + Err(user.unwrap_err()) + } +} + +async fn exists_email(Extension(db): Extension, Path(email): Path) -> JsonResult { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1)", + email + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + Ok(Json(exists)) +} + +async fn get_email(ApiAuthed { email, .. }: ApiAuthed) -> Result { + Ok(email) +} + +async fn get_usage( + Extension(db): Extension, + ApiAuthed { email, .. }: ApiAuthed, +) -> Result { + let usage = sqlx::query_scalar!( + " + SELECT usage.usage FROM usage + WHERE is_workspace = false + AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date) + AND id = $1", + email + ) + .fetch_optional(&db) + .await? + .unwrap_or(0); + Ok(usage.to_string()) +} + +async fn get_user(w_id: &str, username: &str, db: &DB) -> Result> { + let user = sqlx::query_as!( + User, + "SELECT * FROM usr where username = $1 AND workspace_id = $2", + username, + w_id + ) + .fetch_optional(db) + .await?; + let is_super_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + user.as_ref().map(|x| &x.email) + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + let groups = get_groups_for_user( + &w_id, + username, + &user + .as_ref() + .map(|x| x.email.to_string()) + .unwrap_or_else(|| "".to_string()), + db, + ) + .await?; + let folders = get_folders_for_user(&w_id, username, &groups, db).await?; + + Ok(user.map(|usr| UserInfo { + groups, + workspace_id: usr.workspace_id, + email: usr.email, + username: usr.username, + is_admin: usr.is_admin, + is_super_admin, + created_at: usr.created_at, + operator: usr.operator, + disabled: usr.disabled, + role: usr.role, + folders_read: folders.clone().into_iter().map(|x| x.0).collect(), + folders: folders + .clone() + .into_iter() + .filter_map(|x| if x.1 { Some(x.0) } else { None }) + .collect(), + folders_owners: folders + .into_iter() + .filter_map(|x| if x.2 { Some(x.0) } else { None }) + .collect(), + })) +} + +pub async fn is_owner_of_path( + authed: ApiAuthed, + Path((_w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + if authed.is_admin { + Ok(Json(true)) + } else { + Ok(Json(require_owner_of_path(&authed, path).is_ok())) + } +} + +pub fn require_owner_of_path(authed: &ApiAuthed, path: &str) -> Result<()> { + if authed.is_admin { + return Ok(()); + } + if !path.is_empty() { + let splitted = path.split("/").collect::>(); + if splitted[0] == "u" { + if splitted[1] == authed.username { + Ok(()) + } else { + Err(Error::BadRequest(format!( + "only the owner {} is authorized to perform this operation", + splitted[1] + ))) + } + } else if splitted[0] == "f" { + crate::folders::require_is_owner(authed, splitted[1]) + } else { + Err(Error::BadRequest(format!( + "Not recognized path kind: {}", + path + ))) + } + } else { + Err(Error::BadRequest(format!( + "Cannot be owner of an empty path" + ))) + } +} + +pub fn get_perm_in_extra_perms_for_authed( + v: serde_json::Value, + authed: &ApiAuthed, +) -> Option { + match v { + serde_json::Value::Object(obj) => { + let mut keys = vec![format!("u/{}", authed.username)]; + for g in authed.groups.iter() { + keys.push(format!("g/{}", g)); + } + let mut res = None; + for k in keys { + if let Some(v) = obj.get(&k) { + if let Some(v) = v.as_bool() { + if v { + return Some(true); + } + res = Some(v); + } + } + } + res + } + _ => None, + } +} + +pub async fn require_is_writer( + authed: &ApiAuthed, + path: &str, + w_id: &str, + db: DB, + query: &str, + kind: &str, +) -> Result<()> { + if authed.is_admin { + return Ok(()); + } + if !path.is_empty() { + if require_owner_of_path(authed, path).is_ok() { + return Ok(()); + } + if path.starts_with("f/") && path.split('/').count() >= 2 { + let folder = path.split('/').nth(1).unwrap(); + let extra_perms = sqlx::query_scalar!( + "SELECT extra_perms FROM folder WHERE name = $1 AND workspace_id = $2", + folder, + w_id + ) + .fetch_optional(&db) + .await?; + if let Some(perms) = extra_perms { + let is_folder_writer = + get_perm_in_extra_perms_for_authed(perms, authed).unwrap_or(false); + if is_folder_writer { + return Ok(()); + } + } + } + let extra_perms = sqlx::query_scalar(query) + .bind(path) + .bind(w_id) + .fetch_optional(&db) + .await?; + if let Some(perms) = extra_perms { + let perm = get_perm_in_extra_perms_for_authed(perms, authed); + match perm { + Some(true) => Ok(()), + Some(false) => Err(Error::BadRequest(format!( + "User {} is not a writer of {kind} path {path}", + authed.username + ))), + None => Err(Error::BadRequest(format!( + "User {} has neither read or write permission on {kind} {path}", + authed.username + ))), + } + } else { + Err(Error::BadRequest(format!( + "{path} does not exist yet and user {} is not an owner of the parent folder", + authed.username + ))) + } + } else { + Err(Error::BadRequest(format!( + "Cannot be writer of an empty path" + ))) + } +} +async fn whois( + Extension(db): Extension, + Path((w_id, username)): Path<(String, String)>, +) -> JsonResult { + let user_o = get_user(&w_id, &username, &db).await?; + let user = not_found_if_none(user_o, "User", username)?; + Ok(Json(user)) +} + +// async fn create_invite_code( +// ApiAuthed { email, .. }: ApiAuthed, +// Extension(db): Extension, +// Json(nu): Json, +// ) -> Result<(StatusCode, String)> { + +// let mut tx = db.begin().await?; +// require_super_admin(&mut tx, email).await?; + +// sqlx::query!( +// "INSERT INTO invite_code +// (code, seats_left) +// VALUES ($1, $2)", +// nu.code, +// nu.seats +// ) +// .execute(&mut tx) +// .await?; + +// tx.commit().await?; + +// Ok(( +// StatusCode::CREATED, +// format!("new invite code {}", nu.code), +// )) +// } + +async fn decline_invite( + authed: ApiAuthed, + Extension(db): Extension, + Json(nu): Json, +) -> Result<(StatusCode, String)> { + let mut tx = db.begin().await?; + + let is_admin = sqlx::query_scalar!( + "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2 RETURNING is_admin", + nu.workspace_id, + authed.email, + ) + .fetch_optional(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.decline_invite", + ActionKind::Delete, + &nu.workspace_id, + Some(&authed.email), + None, + ) + .await?; + tx.commit().await?; + + if is_admin.is_some() { + Ok(( + StatusCode::OK, + format!( + "user {} declined invite to workspace {}", + &authed.email, nu.workspace_id + ), + )) + } else { + Err(Error::NotFound(format!( + "invite for {} not found", + authed.email + ))) + } +} + +lazy_static! { + pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z][a-zA-Z_0-9]*$"#).unwrap(); +} + +async fn accept_invite( + authed: ApiAuthed, + Extension(webhook): Extension, + Extension(db): Extension, + Extension(rsmq): Extension>, + Json(nu): Json, +) -> Result<(StatusCode, String)> { + let mut tx = db.begin().await?; + + let r = sqlx::query!( + "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2 RETURNING is_admin, operator", + nu.workspace_id, + authed.email, + ) + .fetch_optional(&mut *tx) + .await?; + + if let Some(r) = r { + let already_in_workspace = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)", + &nu.workspace_id, + &authed.email, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + if already_in_workspace { + tx.commit().await?; + return Ok(( + StatusCode::CREATED, + format!( + "user {} accepted invite to workspace {}", + &authed.email, nu.workspace_id + ), + )); + } + let username; + (tx, username) = join_workspace( + &nu.workspace_id, + &authed, + nu.username, + r.is_admin, + r.operator, + tx, + ) + .await?; + + audit_log( + &mut *tx, + &ApiAuthed { username: username.clone(), ..authed.clone() }, + "users.accept_invite", + ActionKind::Create, + &nu.workspace_id, + Some(&authed.email), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &username, + &db, + &nu.workspace_id, + windmill_git_sync::DeployedObject::User { email: authed.email.clone() }, + Some(format!("User '{}' accepted invite", &authed.email)), + rsmq, + true, + ) + .await?; + webhook.send_instance_event(InstanceEvent::UserJoinedWorkspace { + email: authed.email.clone(), + workspace: nu.workspace_id.clone(), + username: username, + }); + Ok(( + StatusCode::CREATED, + format!( + "user {} accepted invite to workspace {}", + &authed.email, nu.workspace_id + ), + )) + } else { + Err(Error::NotFound(format!( + "invite for {} not found", + authed.email + ))) + } +} + +async fn join_workspace<'c>( + w_id: &str, + authed: &ApiAuthed, + username: Option, + is_admin: bool, + operator: bool, + mut tx: sqlx::Transaction<'c, sqlx::Postgres>, +) -> error::Result<(sqlx::Transaction<'c, sqlx::Postgres>, String)> { + let automate_username_creation = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = $1", + AUTOMATE_USERNAME_CREATION_SETTING, + ) + .fetch_optional(&mut *tx) + .await? + .map(|v| v.as_bool()) + .flatten() + .unwrap_or(false); + + let username = if automate_username_creation { + if username.is_some() && username.unwrap().len() > 0 { + return Err(Error::BadRequest( + "username is not allowed when username creation is automated".to_string(), + )); + } + get_instance_username_or_create_pending(&mut tx, &authed.email).await? + } else { + let username = username.ok_or(Error::BadRequest("username is required".to_string()))?; + let already_exists_username = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)", + &w_id, + username, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + if already_exists_username { + return Err(Error::BadRequest(format!( + "user with username {} already exists in workspace {}", + username, w_id + ))); + } + + if !VALID_USERNAME.is_match(&username) { + return Err(windmill_common::error::Error::BadRequest(format!( + "Usermame can only contain alphanumeric characters and underscores and must start with a letter" + ))); + } + username.to_string() + }; + + let already_exists_email = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)", + &w_id, + authed.email, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + if already_exists_email { + return Err(Error::BadRequest(format!( + "user with email {} already exists in workspace {}", + authed.email, w_id + ))); + } + + sqlx::query!( + "INSERT INTO usr + (workspace_id, email, username, is_admin, operator) + VALUES ($1, $2, $3, $4, $5)", + &w_id, + authed.email, + username, + is_admin, + operator + ) + .execute(&mut *tx) + .await?; + sqlx::query_as!( + Group, + "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + &w_id, + username, + "all", + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &AuditAuthor { username: username.clone(), ..authed.into() }, + "users.add_to_workspace", + ActionKind::Create, + &w_id, + Some(&authed.email), + None, + ) + .await?; + Ok((tx, username)) +} + +async fn leave_instance(Extension(db): Extension, authed: ApiAuthed) -> Result { + let mut tx = db.begin().await?; + sqlx::query!("DELETE FROM password WHERE email = $1", &authed.email) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.leave", + ActionKind::Delete, + "global", + Some(&authed.email), + None, + ) + .await?; + tx.commit().await?; + + Ok(format!("Left instance",)) +} + +async fn get_workspace_user( + ApiAuthed { username, is_admin, .. }: ApiAuthed, + Extension(db): Extension, + Path((w_id, username_to_update)): Path<(String, String)>, +) -> Result> { + require_admin(is_admin, &username)?; + + let user = sqlx::query_as!( + User, + "SELECT * FROM usr WHERE username = $1 AND workspace_id = $2", + &username_to_update, + &w_id + ) + .fetch_optional(&db) + .await?; + + let user = not_found_if_none(user, "User", username_to_update)?; + + Ok(Json(user)) +} + +async fn update_workspace_user( + authed: ApiAuthed, + Extension(db): Extension, + Extension(rsmq): Extension>, + Path((w_id, username_to_update)): Path<(String, String)>, + Json(eu): Json, +) -> Result { + let mut tx = db.begin().await?; + + require_admin(authed.is_admin, &authed.username)?; + + if let Some(a) = eu.is_admin { + sqlx::query_scalar!( + "UPDATE usr SET is_admin = $1 WHERE username = $2 AND workspace_id = $3", + a, + &username_to_update, + &w_id + ) + .execute(&mut *tx) + .await?; + } + + if let Some(a) = eu.operator { + sqlx::query_scalar!( + "UPDATE usr SET operator = $1 WHERE username = $2 AND workspace_id = $3", + a, + &username_to_update, + &w_id + ) + .execute(&mut *tx) + .await?; + } + + if let Some(a) = eu.disabled { + sqlx::query_scalar!( + "UPDATE usr SET disabled = $1 WHERE username = $2 AND workspace_id = $3", + a, + &username_to_update, + &w_id + ) + .execute(&mut *tx) + .await?; + } + + audit_log( + &mut *tx, + &authed, + "users.update", + ActionKind::Update, + &w_id, + Some(&username_to_update), + None, + ) + .await?; + + let user_email = sqlx::query_scalar!( + "SELECT email FROM usr WHERE username = $1 AND workspace_id = $2", + &username_to_update, + &w_id + ) + .fetch_one(&mut *tx) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::User { email: user_email.clone() }, + Some(format!("Updated user '{}'", &user_email)), + rsmq, + true, + ) + .await?; + + Ok(format!("user {} updated", user_email)) +} + +async fn update_user( + authed: ApiAuthed, + Path(email_to_update): Path, + Extension(db): Extension, + Json(eu): Json, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + + if let Some(sa) = eu.is_super_admin { + sqlx::query_scalar!( + "UPDATE password SET super_admin = $1 WHERE email = $2", + sa, + &email_to_update + ) + .execute(&mut *tx) + .await?; + } + + if let Some(n) = eu.name { + sqlx::query_scalar!( + "UPDATE password SET name = $1 WHERE email = $2", + n, + &email_to_update + ) + .execute(&mut *tx) + .await?; + } + + audit_log( + &mut *tx, + &authed, + "users.update", + ActionKind::Update, + "global", + Some(&email_to_update), + None, + ) + .await?; + tx.commit().await?; + Ok(format!("email {} updated", &email_to_update)) +} + +async fn delete_user( + authed: ApiAuthed, + Path(email_to_delete): Path, + Extension(db): Extension, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + + sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) + .execute(&mut *tx) + .await?; + + let usernames = sqlx::query_scalar!( + "DELETE FROM usr WHERE email = $1 RETURNING username", + &email_to_delete + ) + .fetch_all(&mut *tx) + .await?; + + for username in usernames { + sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) + .execute(&mut *tx) + .await?; + + sqlx::query!("DELETE FROM usr_to_group WHERE usr = $1", &username) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "DELETE FROM workspace_invite WHERE email = $1", + &email_to_delete + ) + .execute(&mut *tx) + .await?; + } + audit_log( + &mut *tx, + &authed, + "users.delete", + ActionKind::Delete, + "global", + Some(&email_to_delete), + None, + ) + .await?; + tx.commit().await?; + Ok(format!("email {} deleted", &email_to_delete)) +} + +lazy_static::lazy_static! { + pub static ref NEW_USER_WEBHOOK: Option = std::env::var("NEW_USER_WEBHOOK").ok(); + +} + +async fn create_user( + authed: ApiAuthed, + Extension(db): Extension, + Extension(webhook): Extension, + Extension(argon2): Extension>>, + Extension(rsmq): Extension>, + Json(nu): Json, +) -> Result<(StatusCode, String)> { + crate::users_ee::create_user(authed, db, webhook, argon2, rsmq, nu).await +} + +async fn delete_workspace_user( + authed: ApiAuthed, + Extension(db): Extension, + Extension(rsmq): Extension>, + Path((w_id, username_to_delete)): Path<(String, String)>, +) -> Result { + let mut tx = db.begin().await?; + + require_admin(authed.is_admin, &authed.username)?; + + let email_to_delete_o = sqlx::query_scalar!( + "SELECT email FROM usr where username = $1 AND workspace_id = $2", + username_to_delete, + &w_id, + ) + .fetch_optional(&db) + .await?; + + let email_to_delete = not_found_if_none(email_to_delete_o, "User", &username_to_delete)?; + + sqlx::query_scalar!( + "DELETE FROM usr WHERE email = $1 AND workspace_id = $2", + email_to_delete, + &w_id + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "DELETE FROM usr_to_group WHERE usr = $1 AND workspace_id = $2", + &username_to_delete, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.delete", + ActionKind::Delete, + &w_id, + Some(&username_to_delete), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::User { email: email_to_delete.clone() }, + Some(format!( + "Removed user '{}' from workspace", + &email_to_delete + )), + rsmq, + true, + ) + .await?; + + Ok(format!("username {} deleted", username_to_delete)) +} + +async fn set_password( + Extension(db): Extension, + Extension(argon2): Extension>>, + authed: ApiAuthed, + Json(ep): Json, +) -> Result { + crate::users_ee::set_password(db, argon2, authed, ep).await +} + +async fn login( + cookies: Cookies, + Extension(db): Extension, + Extension(argon2): Extension>>, + Json(Login { email, password }): Json, +) -> Result { + let mut tx = db.begin().await?; + let email = email.to_lowercase(); + let audit_author = + AuditAuthor { email: email.clone(), username: email.clone(), username_override: None }; + let email_w_h: Option<(String, String, bool, bool)> = sqlx::query_as( + "SELECT email, password_hash, super_admin, first_time_user FROM password WHERE email = $1 AND login_type = \ + 'password'", + ) + .bind(&email) + .fetch_optional(&mut *tx) + .await?; + + if let Some((email, hash, super_admin, first_time_user)) = email_w_h { + let parsed_hash = + PasswordHash::new(&hash).map_err(|e| Error::InternalErr(e.to_string()))?; + if argon2 + .verify_password(password.as_bytes(), &parsed_hash) + .is_err() + { + audit_log( + &mut *tx, + &audit_author, + "users.login_failure", + ActionKind::Create, + "global", + None, + None, + ) + .await?; + Err(Error::BadRequest("Invalid login".to_string())) + } else { + if first_time_user { + sqlx::query_scalar!( + "UPDATE password SET first_time_user = false WHERE email = $1", + &email + ) + .execute(&mut *tx) + .await?; + let mut c = Cookie::new("first_time", "1"); + if let Some(domain) = COOKIE_DOMAIN.as_ref() { + c.set_domain(domain); + } + c.set_secure(false); + c.set_expires(time::OffsetDateTime::now_utc() + time::Duration::minutes(15)); + c.set_http_only(false); + c.set_path("/"); + + cookies.add(c); + } + + let token = create_session_token(&email, super_admin, &mut tx, cookies).await?; + + audit_log( + &mut *tx, + &audit_author, + "users.login", + ActionKind::Create, + "global", + Some(&truncate_token(&token)), + None, + ) + .await?; + + tx.commit().await?; + Ok(token) + } + } else { + audit_log( + &mut *tx, + &audit_author, + "users.login_failure", + ActionKind::Create, + "global", + None, + None, + ) + .await?; + Err(Error::BadRequest("Invalid login".to_string())) + } +} + +async fn refresh_token( + Extension(db): Extension, + authed: ApiAuthed, + cookies: Cookies, +) -> Result { + let mut tx = db.begin().await?; + + let super_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + &authed.email + ) + .fetch_optional(&mut *tx) + .await? + .unwrap_or(false); + + let _ = create_session_token(&authed.email, super_admin, &mut tx, cookies).await?; + + tx.commit().await?; + Ok("token refreshed".to_string()) +} + +pub async fn create_session_token<'c>( + email: &str, + super_admin: bool, + tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, + cookies: Cookies, +) -> Result { + let token = rd_string(32); + sqlx::query!( + "INSERT INTO token + (token, email, label, expiration, super_admin) + VALUES ($1, $2, $3, now() + ($4 || ' hours')::interval, $5)", + token, + email, + "session", + TTL_TOKEN_DB_H.to_string(), + super_admin + ) + .execute(&mut **tx) + .await?; + let mut cookie = Cookie::new(COOKIE_NAME, token.clone()); + cookie.set_secure(IS_SECURE.read().await.clone()); + cookie.set_same_site(Some(tower_cookies::cookie::SameSite::Lax)); + cookie.set_http_only(true); + cookie.set_path(COOKIE_PATH); + if COOKIE_DOMAIN.is_some() { + cookie.set_domain(COOKIE_DOMAIN.clone().unwrap()); + } + let mut expire: OffsetDateTime = time::OffsetDateTime::now_utc(); + expire += time::Duration::days(3); + cookie.set_expires(expire); + cookies.add(cookie); + Ok(token) +} + +async fn create_token( + Extension(db): Extension, + authed: ApiAuthed, + Json(new_token): Json, +) -> Result<(StatusCode, String)> { + let token = rd_string(32); + let mut tx = db.begin().await?; + + let is_super_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + authed.email + ) + .fetch_optional(&mut *tx) + .await? + .unwrap_or(false); + sqlx::query!( + "INSERT INTO token + (token, email, label, expiration, super_admin, scopes, workspace_id) + VALUES ($1, $2, $3, $4, $5, $6, $7)", + token, + authed.email, + new_token.label, + new_token.expiration, + is_super_admin, + new_token.scopes.as_ref().map(|x| x.as_slice()), + new_token.workspace_id, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.token.create", + ActionKind::Create, + &"global", + Some(&token[0..10]), + None, + ) + .instrument(tracing::info_span!("token", email = &authed.email)) + .await?; + tx.commit().await?; + Ok((StatusCode::CREATED, token)) +} + +async fn impersonate( + Extension(db): Extension, + authed: ApiAuthed, + Json(new_token): Json, +) -> Result<(StatusCode, String)> { + let token = rd_string(32); + require_super_admin(&db, &authed.email).await?; + + if new_token.impersonate_email.is_none() { + return Err(Error::BadRequest( + "impersonate_username is required".to_string(), + )); + } + + let impersonated = new_token.impersonate_email.unwrap(); + + let is_super_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + impersonated + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + let mut tx = db.begin().await?; + + sqlx::query!( + "INSERT INTO token + (token, email, label, expiration, super_admin) + VALUES ($1, $2, $3, $4, $5)", + token, + impersonated, + new_token.label, + new_token.expiration, + is_super_admin + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.impersonate", + ActionKind::Delete, + &"global", + Some(&token[0..10]), + Some([("impersonated", &format!("{impersonated}")[..])].into()), + ) + .instrument(tracing::info_span!("token", email = &impersonated)) + .await?; + tx.commit().await?; + Ok((StatusCode::CREATED, token)) +} + +#[derive(Deserialize)] +struct ListTokenQuery { + exclude_ephemeral: Option, +} + +async fn list_tokens( + Extension(db): Extension, + ApiAuthed { email, .. }: ApiAuthed, + Query(query): Query, + Query(pagination): Query, +) -> JsonResult> { + let (per_page, offset) = paginate(pagination); + let rows = if query.exclude_ephemeral.unwrap_or(false) { + sqlx::query_as!( + TruncatedToken, + "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, \ + last_used_at, scopes FROM token WHERE email = $1 AND label != 'ephemeral-script' + ORDER BY created_at DESC LIMIT $2 OFFSET $3", + email, + per_page as i64, + offset as i64, + ) + .fetch_all(&db) + .await? + } else { + sqlx::query_as!( + TruncatedToken, + "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, \ + last_used_at, scopes FROM token WHERE email = $1 + ORDER BY created_at DESC LIMIT $2 OFFSET $3", + email, + per_page as i64, + offset as i64, + ) + .fetch_all(&db) + .await? + }; + Ok(Json(rows)) +} + +async fn delete_token( + Extension(db): Extension, + authed: ApiAuthed, + Path(token_prefix): Path, +) -> Result { + let mut tx = db.begin().await?; + + let tokens_deleted: Vec = sqlx::query_scalar( + "DELETE FROM token + WHERE email = $1 + AND token LIKE concat($2::text, '%') + RETURNING concat(substring(token for 10), '*****')", + ) + .bind(&authed.email) + .bind(&token_prefix) + .fetch_all(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.token.delete", + ActionKind::Delete, + &"global", + Some(&token_prefix), + None, + ) + .await?; + tx.commit().await?; + + Ok(format!( + "deleted {} tokens {:?} with prefix {}", + tokens_deleted.len(), + tokens_deleted, + token_prefix + )) +} + +async fn leave_workspace( + Extension(db): Extension, + Path(w_id): Path, + authed: ApiAuthed, +) -> Result { + let mut tx = db.begin().await?; + sqlx::query!( + "DELETE FROM usr WHERE workspace_id = $1 AND username = $2", + &w_id, + authed.username + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.leave_workspace", + ActionKind::Delete, + &w_id, + None, + None, + ) + .await?; + tx.commit().await?; + + Ok(format!("left workspace {w_id}")) +} + +#[derive(Serialize)] +struct Runnable { + workspace: String, + endpoint_async: String, + endpoint_sync: String, + endpoint_openai_sync: String, + summary: String, + description: String, + schema: Option, + kind: String, + path: String, +} + +async fn get_all_runnables( + Extension(db): Extension, + authed: ApiAuthed, + Tokened { token }: Tokened, + Extension(cache): Extension>, +) -> JsonResult> { + let mut tx = db.clone().begin(&authed).await?; + let mut runnables = Vec::new(); + let workspaces = sqlx::query_scalar!( + "SELECT workspace.id as id FROM workspace, usr WHERE usr.workspace_id = workspace.id AND \ + usr.email = $1 AND deleted = false", + authed.email + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + for workspace in workspaces { + let nauthed = cache + .get_authed(Some(workspace.clone()), &token) + .await + .ok_or_else(|| { + Error::BadRequest(format!("not authorized to access workspace: {workspace}")) + })?; + let mut tx = db.clone().begin(&nauthed).await?; + let flows = sqlx::query!( + "SELECT flow.workspace_id as workspace, flow.path, summary, description, flow_version.schema + FROM flow + LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] + WHERE flow.workspace_id = $1", + workspace + ) + .fetch_all(&mut *tx) + .await?; + runnables.extend( + flows + .into_iter() + .map(|f| Runnable { + workspace: f.workspace.clone(), + endpoint_async: format!("/w/{}/jobs/run/f/{}", &f.workspace, &f.path), + endpoint_sync: format!( + "/w/{}/jobs/run_wait_result/f/{}", + &f.workspace, &f.path + ), + endpoint_openai_sync: format!( + "/w/{}/jobs/openai_sync/f/{}", + &f.workspace, &f.path + ), + summary: f.summary, + description: f.description, + schema: f.schema, + kind: "flow".to_string(), + path: f.path, + }) + .collect::>(), + ); + let scripts = sqlx::query!( + "SELECT workspace_id as workspace, path, summary, description, schema FROM script as o WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1) and workspace_id = $1", workspace + ) + .fetch_all(&mut *tx) + .await?; + runnables.extend( + scripts + .into_iter() + .map(|s| Runnable { + workspace: s.workspace.clone(), + endpoint_async: format!("/w/{}/jobs/run/p/{}", &s.workspace, &s.path), + endpoint_sync: format!( + "/w/{}/jobs/run_wait_result/p/{}", + &s.workspace, &s.path + ), + endpoint_openai_sync: format!( + "/w/{}/jobs/openai_sync/p/{}", + &s.workspace, &s.path + ), + summary: s.summary, + description: s.description, + schema: s.schema, + kind: "script".to_string(), + path: s.path, + }) + .collect::>(), + ); + tx.commit().await?; + } + Ok(Json(runnables)) +} + +//used by oauth +#[allow(dead_code)] +#[derive(Deserialize, Debug, Clone)] +pub struct LoginUserInfo { + pub email: Option, + pub name: Option, + pub company: Option, + + pub displayName: Option, +} + +#[derive(Serialize)] +struct InstanceUsernameInfo { + username: String, + workspace_usernames: Vec, +} + +#[derive(Serialize)] +struct WorkspaceUsernameInfo { + workspace_id: String, + username: String, +} +async fn get_instance_username_info( + ApiAuthed { email, .. }: ApiAuthed, + Path(user_email): Path, + Extension(db): Extension, +) -> JsonResult { + require_super_admin(&db, &email).await?; + let mut tx = db.begin().await?; + let instance_username = match sqlx::query_scalar!( + "SELECT username FROM password WHERE email = $1", + &user_email + ) + .fetch_one(&mut *tx) + .await? + { + Some(username) => username, + None => generate_instance_wide_unique_username(&mut tx, &user_email).await?, + }; + + let workspace_usernames = sqlx::query_as!( + WorkspaceUsernameInfo, + "SELECT workspace_id, username FROM usr WHERE email = $1", + &user_email + ) + .fetch_all(&mut *tx) + .await?; + + Ok(Json(InstanceUsernameInfo { + username: instance_username, + workspace_usernames: workspace_usernames, + })) +} + +async fn username_to_email( + Path((w_id, username)): Path<(String, String)>, + Extension(db): Extension, +) -> Result { + let email = sqlx::query_scalar!( + "SELECT email FROM usr WHERE username = $1 AND workspace_id = $2", + &username, + &w_id + ) + .fetch_optional(&db) + .await?; + + let email = not_found_if_none(email, "user", username)?; + + Ok(email) +} + +#[cfg(feature = "enterprise")] +#[derive(Serialize, Deserialize)] +struct ExportedGlobalUser { + email: String, + password_hash: Option, + login_type: String, + super_admin: bool, + verified: bool, + name: Option, + company: Option, + first_time_user: bool, + username: Option, +} + +#[cfg(feature = "enterprise")] +async fn export_global_users( + Extension(db): Extension, + authed: ApiAuthed, +) -> JsonResult> { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + let users = sqlx::query_as!( + ExportedGlobalUser, + "SELECT email, password_hash, login_type, super_admin, verified, name, company, first_time_user, username FROM password" + ) + .fetch_all(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.export_export", + ActionKind::Execute, + "global", + None, + None, + ) + .await?; + + tx.commit().await?; + + Ok(Json(users)) +} + +#[cfg(not(feature = "enterprise"))] +async fn export_global_users() -> JsonResult { + Err(Error::BadRequest( + "This feature is only available in the enterprise version".to_string(), + )) +} + +#[cfg(feature = "enterprise")] +async fn overwrite_global_users( + Extension(db): Extension, + authed: ApiAuthed, + Json(users): Json>, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + sqlx::query!("DELETE FROM password") + .execute(&mut *tx) + .await?; + for user in users { + sqlx::query!( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, company, first_time_user, username) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + user.email, + user.password_hash, + user.login_type, + user.super_admin, + user.verified, + user.name, + user.company, + user.first_time_user, + user.username + ) + .execute(&mut *tx) + .await?; + } + audit_log( + &mut *tx, + &authed, + "users.import_global", + ActionKind::Create, + "global", + None, + None, + ) + .await?; + tx.commit().await?; + Ok("loaded global users".to_string()) +} + +#[cfg(not(feature = "enterprise"))] +async fn overwrite_global_users() -> JsonResult { + Err(Error::BadRequest( + "This feature is only available in the enterprise version".to_string(), + )) +} + +#[derive(Deserialize)] +struct RenameUser { + new_username: String, +} + +async fn rename_user( + authed: ApiAuthed, + Path(user_email): Path, + Extension(db): Extension, + Json(ru): Json, +) -> Result { + require_super_admin(&db, &authed.email).await?; + + let mut tx = db.begin().await?; + + let username_conflict = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 and email != $2 UNION SELECT 1 FROM password WHERE username = $1 UNION SELECT 1 FROM pending_user WHERE username = $1)", + &ru.new_username, + &user_email + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + if username_conflict { + return Err(Error::BadRequest(format!( + "username {} already used by another user", + &ru.new_username + ))); + } + + if !VALID_USERNAME.is_match(&ru.new_username) { + return Err(windmill_common::error::Error::BadRequest(format!( + "Usermame can only contain alphanumeric characters and underscores and must start with a letter" + ))); + } + + sqlx::query!( + "UPDATE password SET username = $1 WHERE email = $2", + ru.new_username, + user_email + ) + .execute(&mut *tx) + .await?; + + let workspace_usernames = sqlx::query!( + "SELECT workspace_id, username FROM usr WHERE email = $1", + &user_email + ) + .fetch_all(&mut *tx) + .await?; + + for w_u in workspace_usernames { + if ru.new_username == w_u.username { + continue; + } + update_username_in_workpsace( + &mut tx, + &user_email, + &w_u.username, + &ru.new_username, + &w_u.workspace_id, + ) + .await?; + } + + audit_log( + &mut *tx, + &authed, + "users.rename", + ActionKind::Update, + "global", + Some(&user_email), + None, + ) + .await?; + tx.commit().await?; + Ok(format!( + "updated username of user {} to {}", + &user_email, &ru.new_username + )) +} + +async fn update_username_in_workpsace<'c>( + tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, + email: &str, + old_username: &str, + new_username: &str, + w_id: &str, +) -> error::Result<()> { + // ---- instance and workspace users ---- + sqlx::query!( + "UPDATE usr SET username = $1 WHERE email = $2", + new_username, + email + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE usr_to_group SET usr = $1 WHERE usr = $2", + new_username, + old_username + ) + .execute(&mut **tx) + .await?; + + // ---- queue ---- + sqlx::query!( + r#"UPDATE queue SET script_path = REGEXP_REPLACE(script_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE script_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE queue SET schedule_path = REGEXP_REPLACE(schedule_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE schedule_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE queue SET permissioned_as = ('u/' || $1) WHERE permissioned_as = ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE queue SET canceled_by = $1 WHERE canceled_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + sqlx::query!( + "UPDATE queue SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + // ---- completed_job ---- + sqlx::query!( + r#"UPDATE completed_job SET script_path = REGEXP_REPLACE(script_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE script_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE completed_job SET schedule_path = REGEXP_REPLACE(schedule_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE schedule_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE completed_job SET permissioned_as = ('u/' || $1) WHERE permissioned_as = ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE completed_job SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + sqlx::query!( + "UPDATE completed_job SET canceled_by = $1 WHERE canceled_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + // ---- resources---- + sqlx::query!( + r#"UPDATE resource SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE resource_type SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE resource SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE resource SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- variables ---- + + sqlx::query!( + r#"UPDATE variable SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE variable SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- scripts ---- + sqlx::query!( + r#"UPDATE script SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE script SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + sqlx::query!( + "UPDATE script SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- flows ---- + sqlx::query!( + r#"INSERT INTO flow + (workspace_id, path, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, concurrency_key, versions, value, schema, edited_by, edited_at) + SELECT workspace_id, REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1'), summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, concurrency_key, versions, value, schema, edited_by, edited_at + FROM flow + WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE flow_version SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "DELETE FROM flow WHERE path LIKE ('u/' || $1 || '/%') AND workspace_id = $2", + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE flow SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- draft ---- + sqlx::query!( + r#"UPDATE draft SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['path'], to_jsonb(REGEXP_REPLACE(value->>'path','u/' || $2 || '/(.*)','u/' || $1 || '/\1')))) WHERE value->>'path' LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + // ---- app ---- + sqlx::query!( + r#"UPDATE app SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb('u/' || $1)) WHERE policy->>'on_behalf_of' = ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE app SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- app_version ---- + + sqlx::query!( + "UPDATE app_version SET created_by = $1 WHERE created_by = $2 AND EXISTS (SELECT 1 FROM app WHERE workspace_id = $3 AND app.id = app_version.app_id)", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- schedules ---- + + sqlx::query!( + r#"UPDATE schedule SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE schedule SET script_path = REGEXP_REPLACE(script_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE script_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE schedule SET edited_by = $1 WHERE edited_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + sqlx::query!( + "UPDATE schedule SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- group_ ---- + + sqlx::query!( + "UPDATE group_ SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- folders ---- + + sqlx::query!( + "UPDATE folder SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE folder SET owners = ARRAY_REPLACE(owners, 'u/' || $2, 'u/' || $1) WHERE ('u/' || $2) = ANY(owners) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE folder SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- input ---- + + sqlx::query!( + "UPDATE input SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + // ---- favorite ---- + + sqlx::query!( + "UPDATE favorite SET usr = $1 WHERE usr = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + // ---- capture ---- + + sqlx::query!( + "UPDATE capture SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + // ---- token ---- + + sqlx::query!( + "UPDATE token SET owner = ('u/' || $1) WHERE owner = ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + sqlx::query!( + r#"UPDATE token SET scopes = array(select regexp_replace(unnest(scopes), 'run:([^/]+)/u/' || $2 || '/(.+)', 'run:\1/u/' || $1 || '/\2')) WHERE EXISTS (SELECT 1 FROM UNNEST(scopes) scope WHERE scope LIKE ('run:%/u/' || $2 || '/%')) AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- raw_app ---- + + sqlx::query!( + "UPDATE raw_app SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} diff --git a/backend/windmill-api/src/users_ee.rs b/backend/windmill-api/src/users_ee.rs new file mode 100644 index 0000000000..e0f15a9317 --- /dev/null +++ b/backend/windmill-api/src/users_ee.rs @@ -0,0 +1,41 @@ +use std::sync::Arc; + +use crate::db::ApiAuthed; + +use crate::users::{EditPassword, NewUser}; +use crate::{db::DB, webhook_util::WebhookShared}; +use argon2::Argon2; + +use http::StatusCode; + +use windmill_common::error::{Error, Result}; + +pub async fn create_user( + _authed: ApiAuthed, + _db: DB, + _webhook: WebhookShared, + _argon2: Arc>, + _rsmq: Option, + mut _nu: NewUser, +) -> Result<(StatusCode, String)> { + Err(Error::InternalErr( + "Not implemented in Windmill's Open Source repository".to_string(), + )) +} + +pub async fn set_password( + _db: DB, + _argon2: Arc>, + _authed: ApiAuthed, + _ep: EditPassword, +) -> Result { + Err(Error::InternalErr( + "Not implemented in Windmill's Open Source repository".to_string(), + )) +} + +pub fn send_email_if_possible(_subject: &str, _content: &str, _to: &str) { + tracing::warn!( + "send_email_if_possible is not implemented in Windmill's Open Source repository" + ); +} diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index d8a81ab741..ea4fccc160 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -155,25 +155,6 @@ pub async fn get_instance_username_or_create_pending<'c>( } } -pub async fn get_and_delete_pending_username_or_generate<'c>( - tx: &mut Transaction<'c, Postgres>, - email: &str, -) -> error::Result { - let username = sqlx::query_scalar!("SELECT username FROM pending_user WHERE email = $1", email) - .fetch_optional(&mut **tx) - .await?; - - if let Some(username) = username { - sqlx::query!("DELETE FROM pending_user WHERE email = $1", email) - .execute(&mut **tx) - .await?; - Ok(username) - } else { - let username = generate_instance_wide_unique_username(&mut *tx, email).await?; - Ok(username) - } -} - pub fn content_plain(body: Body) -> Response { use axum::http::header; Response::builder() diff --git a/backend/windmill-api/src/websocket_triggers.rs b/backend/windmill-api/src/websocket_triggers.rs new file mode 100644 index 0000000000..b94ed85461 --- /dev/null +++ b/backend/windmill-api/src/websocket_triggers.rs @@ -0,0 +1,684 @@ +use axum::{ + extract::{Path, Query}, + routing::{delete, get, post}, + Extension, Json, Router, +}; +use futures::StreamExt; +use http::StatusCode; +use itertools::Itertools; +use rand::seq::SliceRandom; +use serde::{ + de::{self, MapAccess, Visitor}, + Deserialize, Deserializer, Serialize, +}; +use serde_json::Value; +use sql_builder::{bind::Bind, SqlBuilder}; +use sqlx::prelude::FromRow; +use std::{collections::HashMap, fmt}; +use tokio_tungstenite::connect_async; +use windmill_audit::{audit_ee::audit_log, ActionKind}; +use windmill_common::{ + db::UserDB, + error::{self, JsonResult}, + utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, + worker::{to_raw_value, CLOUD_HOSTED}, + INSTANCE_NAME, +}; +use windmill_queue::PushArgsOwned; + +use crate::{ + db::{ApiAuthed, DB}, + jobs::{ + run_wait_result_flow_by_path_internal, run_wait_result_script_by_path_internal, RunJobQuery, + }, + users::fetch_api_authed, +}; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/create", post(create_websocket_trigger)) + .route("/list", get(list_websocket_triggers)) + .route("/get/*path", get(get_websocket_trigger)) + .route("/update/*path", post(update_websocket_trigger)) + .route("/delete/*path", delete(delete_websocket_trigger)) + .route("/exists/*path", get(exists_websocket_trigger)) + .route("/setenabled/*path", post(set_enabled)) +} + +#[derive(Deserialize)] +struct NewWebsocketTrigger { + path: String, + url: String, + script_path: String, + is_flow: bool, + enabled: Option, + filters: Vec, +} + +#[derive(FromRow, Serialize, Clone)] +pub struct WebsocketTrigger { + workspace_id: String, + path: String, + url: String, + script_path: String, + is_flow: bool, + edited_by: String, + email: String, + edited_at: chrono::DateTime, + server_id: Option, + last_server_ping: Option>, + extra_perms: serde_json::Value, + error: Option, + enabled: bool, + filters: Vec, +} + +#[derive(Deserialize)] +struct EditWebsocketTrigger { + path: String, + url: String, + script_path: String, + is_flow: bool, + filters: Vec, +} + +#[derive(Deserialize)] +pub struct ListWebsocketTriggerQuery { + pub page: Option, + pub per_page: Option, + pub path: Option, + pub is_flow: Option, + pub path_start: Option, +} + +async fn list_websocket_triggers( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(lst): Query, +) -> error::JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let (per_page, offset) = paginate(Pagination { per_page: lst.per_page, page: lst.page }); + let mut sqlb = SqlBuilder::select_from("websocket_trigger") + .field("*") + .order_by("edited_at", true) + .and_where("workspace_id = ?".bind(&w_id)) + .offset(offset) + .limit(per_page) + .clone(); + if let Some(path) = lst.path { + sqlb.and_where_eq("script_path", "?".bind(&path)); + } + if let Some(is_flow) = lst.is_flow { + sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); + } + if let Some(path_start) = &lst.path_start { + sqlb.and_where_like_left("path", path_start); + } + let sql = sqlb + .sql() + .map_err(|e| error::Error::InternalErr(e.to_string()))?; + let rows = sqlx::query_as::<_, WebsocketTrigger>(&sql) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json(rows)) +} + +async fn get_websocket_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> error::JsonResult { + let mut tx = user_db.begin(&authed).await?; + let path = path.to_path(); + let trigger = sqlx::query_as!( + WebsocketTrigger, + r#"SELECT * + FROM websocket_trigger + WHERE workspace_id = $1 AND path = $2"#, + w_id, + path, + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + let trigger = not_found_if_none(trigger, "Trigger", path)?; + + Ok(Json(trigger)) +} + +async fn create_websocket_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(ct): Json, +) -> error::Result<(StatusCode, String)> { + if *CLOUD_HOSTED { + return Err(error::Error::BadRequest( + "Websocket triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host".to_string(), + )); + } + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = user_db.begin(&authed).await?; + sqlx::query_as!( + WebsocketTrigger, + "INSERT INTO websocket_trigger (workspace_id, path, url, script_path, is_flow, enabled, filters, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, now()) RETURNING *", + w_id, + ct.path, + ct.url, + ct.script_path, + ct.is_flow, + ct.enabled.unwrap_or(true), + &ct.filters, + &authed.username, + &authed.email + ) + .fetch_one(&mut *tx).await?; + + audit_log( + &mut *tx, + &authed, + "websocket_triggers.create", + ActionKind::Create, + &w_id, + Some(ct.path.as_str()), + None, + ) + .await?; + + tx.commit().await?; + + Ok((StatusCode::CREATED, format!("{}", ct.path))) +} + +async fn update_websocket_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(ct): Json, +) -> error::Result { + let path = path.to_path(); + let mut tx = user_db.begin(&authed).await?; + + // important to update server_id, last_server_ping and error to NULL to stop current websocket listener + sqlx::query!( + "UPDATE websocket_trigger SET url = $1, script_path = $2, path = $3, is_flow = $4, filters = $5, edited_by = $6, email = $7, edited_at = now(), server_id = NULL, last_server_ping = NULL, error = NULL + WHERE workspace_id = $8 AND path = $9", + ct.url, + ct.script_path, + ct.path, + ct.is_flow, + &ct.filters, + &authed.username, + &authed.email, + w_id, + path, + ) + .execute(&mut *tx).await?; + + audit_log( + &mut *tx, + &authed, + "websocket_triggers.update", + ActionKind::Create, + &w_id, + Some(path), + None, + ) + .await?; + + tx.commit().await?; + + Ok(path.to_string()) +} + +#[derive(Deserialize)] +pub struct SetEnabled { + pub enabled: bool, +} + +pub async fn set_enabled( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(payload): Json, +) -> error::Result { + let mut tx = user_db.begin(&authed).await?; + let path = path.to_path(); + + // important to set server_id, last_server_ping and error to NULL to stop current websocket listener + let one_o = sqlx::query_scalar!( + "UPDATE websocket_trigger SET enabled = $1, email = $2, edited_by = $3, edited_at = now(), server_id = NULL, last_server_ping = NULL, error = NULL + WHERE path = $4 AND workspace_id = $5 RETURNING 1", + payload.enabled, + &authed.email, + &authed.username, + path, + w_id, + ).fetch_optional(&mut *tx).await?; + + not_found_if_none(one_o.flatten(), "Websocket trigger", path)?; + + audit_log( + &mut *tx, + &authed, + "websocket_triggers.setenabled", + ActionKind::Update, + &w_id, + Some(path), + Some([("enabled", payload.enabled.to_string().as_ref())].into()), + ) + .await?; + + tx.commit().await?; + + Ok(format!( + "succesfully updated websocket trigger at path {} to status {}", + path, payload.enabled + )) +} + +async fn delete_websocket_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> error::Result { + require_admin(authed.is_admin, &authed.username)?; + let path = path.to_path(); + let mut tx = user_db.begin(&authed).await?; + sqlx::query!( + "DELETE FROM websocket_trigger WHERE workspace_id = $1 AND path = $2", + w_id, + path, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "websocket_triggers.delete", + ActionKind::Delete, + &w_id, + Some(path), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!("Websocket trigger {path} deleted")) +} + +async fn exists_websocket_trigger( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM websocket_trigger WHERE path = $1 AND workspace_id = $2)", + path, + w_id, + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + Ok(Json(exists)) +} + +async fn listen_to_unlistened_websockets( + db: &DB, + rsmq: &Option, + killpill_rx: &tokio::sync::broadcast::Receiver<()>, +) -> () { + match sqlx::query_as!( + WebsocketTrigger, + r#"SELECT * + FROM websocket_trigger + WHERE enabled IS TRUE AND (server_id IS NULL OR last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')"# + ) + .fetch_all(db) + .await + { + Ok(mut triggers) => { + triggers.shuffle(&mut rand::thread_rng()); + for trigger in triggers { + maybe_listen_to_websocket(trigger, db.clone(), rsmq.clone(), killpill_rx.resubscribe()).await; + } + } + Err(err) => { + tracing::error!("Error fetching websocket triggers: {:?}", err); + } + }; +} + +pub async fn start_websockets( + db: DB, + rsmq: Option, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> () { + tokio::spawn(async move { + listen_to_unlistened_websockets(&db, &rsmq, &killpill_rx).await; + loop { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + } + _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { + listen_to_unlistened_websockets(&db, &rsmq, &killpill_rx).await; + } + } + } + }); +} + +async fn maybe_listen_to_websocket( + ws_trigger: WebsocketTrigger, + db: DB, + rsmq: Option, + killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> () { + match sqlx::query_scalar!( + "UPDATE websocket_trigger SET server_id = $1, last_server_ping = now() WHERE enabled IS TRUE AND workspace_id = $2 AND path = $3 AND (server_id IS NULL OR last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", + *INSTANCE_NAME, + ws_trigger.workspace_id, + ws_trigger.path, + ).fetch_optional(&db).await { + Ok(has_lock) => { + if has_lock.flatten().unwrap_or(false) { + tokio::spawn(listen_to_websocket(ws_trigger, db, rsmq, killpill_rx)); + } else { + tracing::info!("Websocket {} already being listened to", ws_trigger.url); + } + }, + Err(err) => { + tracing::error!("Error acquiring lock for websocket {}: {:?}", ws_trigger.path, err); + } + }; +} + +struct SupersetVisitor<'a> { + key: &'a str, + value_to_check: &'a Value, +} + +impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> { + type Value = bool; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a JSON object with a specific key at the top level") + } + + fn visit_map(self, mut map: V) -> Result + where + V: MapAccess<'de>, + { + while let Some(key) = map.next_key::()? { + if key == self.key { + // Deserialize the value for the key and check if it's a superset + let json_value: Value = map.next_value()?; + tracing::info!("json_value: {:?}", json_value); + tracing::info!("value_to_check: {:?}", self.value_to_check); + return Ok(is_superset(&json_value, self.value_to_check)); + } else { + // Skip the value if it's not the one we're interested in + let _ = map.next_value::()?; + } + } + // If the key was not found, return false + Ok(false) + } +} + +// Function to check if json_value is a superset of value_to_check +fn is_superset(json_value: &Value, value_to_check: &Value) -> bool { + match (json_value, value_to_check) { + (Value::Object(json_map), Value::Object(check_map)) => { + // Check that all keys and values in check_map exist and match in json_map + check_map.iter().all(|(k, v)| { + json_map + .get(k) + .map_or(false, |json_val| is_superset(json_val, v)) + }) + } + (Value::Array(json_array), Value::Array(check_array)) => { + // Check that all elements in check_array exist in json_array + check_array.iter().all(|check_item| { + json_array + .iter() + .any(|json_item| is_superset(json_item, check_item)) + }) + } + _ => json_value == value_to_check, + } +} + +// A function to deserialize and check if the value at the given key is a superset of a passed value +fn is_value_superset<'a, 'de, D>( + deserializer: D, + key: &'a str, + value_to_check: &'a Value, +) -> Result +where + D: Deserializer<'de>, +{ + deserializer.deserialize_map(SupersetVisitor { key, value_to_check }) +} + +async fn listen_to_websocket( + ws_trigger: WebsocketTrigger, + db: DB, + rsmq: Option, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> () { + async fn update_ping(db: DB, ws_trigger: &WebsocketTrigger, error: Option<&str>) -> Option<()> { + match sqlx::query_scalar!( + "UPDATE websocket_trigger SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND server_id = $4 AND enabled IS TRUE RETURNING 1", + error, + ws_trigger.workspace_id, + ws_trigger.path, + *INSTANCE_NAME + ).fetch_optional(&db).await { + Ok(updated) => { + if updated.flatten().is_none() { + tracing::info!("Websocket {} changed, disabled, or deleted, stopping...", ws_trigger.url); + return None; + } + }, + Err(err) => { + tracing::warn!("Error updating ping of websocket {}: {:?}", ws_trigger.url, err); + } + }; + + Some(()) + } + + let url = ws_trigger.url.as_str(); + + #[derive(Deserialize)] + struct JsonFilter { + key: String, + value: serde_json::Value, + } + + #[derive(Deserialize)] + #[serde(untagged)] + enum Filter { + JsonFilter(JsonFilter), + } + let filters: Vec = ws_trigger + .filters + .iter() + .filter_map(|m| serde_json::from_value(m.clone()).ok()) + .collect_vec(); + + loop { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + }, + connection = connect_async(url) => { + match connection { + Ok((ws_stream, _)) => { + tracing::info!("Listening to websocket {}", url); + if let None = update_ping(db.clone(), &ws_trigger, None).await { + return; + } + let mut last_ping = tokio::time::Instant::now(); + let (_, mut read) = ws_stream.split(); + loop { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + } + msg = read.next() => { + if let Some(msg) = msg { + if last_ping.elapsed() > tokio::time::Duration::from_secs(5) { + if let None = update_ping(db.clone(), &ws_trigger, None).await { + return; + } + last_ping = tokio::time::Instant::now(); + } + match msg { + Ok(msg) => { + match msg { + tokio_tungstenite::tungstenite::Message::Text(text) => { + let mut should_handle = true; + for filter in &filters { + match filter { + Filter::JsonFilter(JsonFilter { key, value }) => { + let mut deserializer = serde_json::Deserializer::from_str(text.as_str()); + should_handle = match is_value_superset(&mut deserializer, key, &value) { + Ok(filter_match) => { + filter_match + }, + Err(err) => { + tracing::warn!("Error deserializing filter for websocket {}: {:?}", url, err); + false + } + }; + } + } + if !should_handle { + break; + } + } + if should_handle { + let db_ = db.clone(); + let rsmq_ = rsmq.clone(); + let ws_trigger_ = ws_trigger.clone(); + tokio::spawn(async move { + let url = ws_trigger_.url.clone(); + if let Err(err) = run_job(db_, rsmq_, ws_trigger_, text).await { + tracing::error!("Error running job on websocket {}: {:?}", url, err); + }; + }); + } + }, + _ => {} + } + }, + Err(err) => { + tracing::error!("Error reading from websocket {}: {:?}", url, err); + } + } + } else { + tracing::error!("Websocket {} closed", url); + if let None = + update_ping(db.clone(), &ws_trigger, Some("Websocket closed")).await + { + return; + } + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + break; + } + }, + _ = tokio::time::sleep(tokio::time::Duration::from_secs(5)) => { + if let None = update_ping(db.clone(), &ws_trigger, None).await { + return; + } + last_ping = tokio::time::Instant::now(); + }, + } + } + } + Err(err) => { + tracing::error!("Error connecting to websocket {}: {:?}", url, err); + if let None = + update_ping(db.clone(), &ws_trigger, Some(err.to_string().as_str())).await + { + return; + } + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + } + } + } + } + } +} + +async fn run_job( + db: DB, + rsmq: Option, + trigger: WebsocketTrigger, + msg: String, +) -> anyhow::Result<()> { + let args = PushArgsOwned { + args: HashMap::from([("msg".to_string(), to_raw_value(&msg))]), + extra: Some(HashMap::from([( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({"kind": "websocket"})), + )])), + }; + let label_prefix = Some(format!("ws-{}-", trigger.path)); + + let authed = fetch_api_authed( + trigger.edited_by.clone(), + trigger.email.clone(), + &trigger.workspace_id, + &db, + "anonymous".to_string(), + ) + .await?; + + let user_db = UserDB::new(db.clone()); + + let run_query = RunJobQuery::default(); + + if trigger.is_flow { + run_wait_result_flow_by_path_internal( + db, + run_query, + StripPath(trigger.script_path.to_owned()), + authed, + rsmq, + user_db, + args, + trigger.workspace_id.clone(), + label_prefix, + ) + .await?; + } else { + run_wait_result_script_by_path_internal( + db, + run_query, + StripPath(trigger.script_path.to_owned()), + authed, + rsmq, + user_db, + trigger.workspace_id.clone(), + args, + label_prefix, + ) + .await?; + } + + Ok(()) +} diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 95cd97fad0..018ac8de5f 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -9,14 +9,15 @@ use std::collections::HashMap; use crate::db::ApiAuthed; -use crate::utils::{get_instance_username_or_create_pending, INVALID_USERNAME_CHARS}; +use crate::users_ee::send_email_if_possible; +use crate::utils::get_instance_username_or_create_pending; use crate::BASE_URL; use crate::{ apps::AppWithLastVersion, db::DB, folders::Folder, resources::{Resource, ResourceType}, - users::{send_email_if_possible, WorkspaceInvite, VALID_USERNAME}, + users::{WorkspaceInvite, VALID_USERNAME}, utils::require_super_admin, webhook_util::WebhookShared, }; @@ -34,7 +35,7 @@ use itertools::Itertools; use regex::Regex; use uuid::Uuid; -use windmill_audit::audit_ee::{audit_log, AuditAuthor, AuditAuthorable}; +use windmill_audit::audit_ee::audit_log; use windmill_audit::ActionKind; use windmill_common::db::UserDB; use windmill_common::s3_helpers::LargeFileStorage; @@ -116,7 +117,8 @@ pub fn workspaced_service() -> Router { .route("/get_workspace_name", get(get_workspace_name)) .route("/change_workspace_name", post(change_workspace_name)) .route("/change_workspace_id", post(change_workspace_id)) - .route("/usage", get(get_usage)); + .route("/usage", get(get_usage)) + .route("/used_triggers", get(get_used_triggers)); #[cfg(feature = "stripe")] { @@ -217,11 +219,12 @@ struct EditDeployTo { deploy_to: Option, } +#[allow(dead_code)] #[derive(Deserialize)] -struct EditAutoInvite { - operator: Option, - invite_all: Option, - auto_add: Option, +pub struct EditAutoInvite { + pub operator: Option, + pub invite_all: Option, + pub auto_add: Option, } #[derive(Deserialize)] @@ -572,250 +575,21 @@ async fn edit_deploy_to() -> Result { )); } -const BANNED_DOMAINS: &str = include_str!("../banned_domains.txt"); +pub const BANNED_DOMAINS: &str = include_str!("../banned_domains.txt"); async fn is_allowed_auto_domain(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult { let domain = email.split('@').last().unwrap(); return Ok(Json(!BANNED_DOMAINS.contains(domain))); } -async fn auto_add_user( - email: &str, - w_id: &str, - operator: &bool, - tx: &mut Transaction<'_, Postgres>, - authorable: &impl AuditAuthorable, -) -> Result { - let automate_username_creation = sqlx::query_scalar!( - "SELECT value FROM global_settings WHERE name = $1", - AUTOMATE_USERNAME_CREATION_SETTING, - ) - .fetch_optional(&mut **tx) - .await? - .map(|v| v.as_bool()) - .flatten() - .unwrap_or(false); - - let username = if automate_username_creation { - get_instance_username_or_create_pending(&mut *tx, &email).await? - } else { - let mut username = email - .split('@') - .next() - .unwrap() - .to_string() - .replace(".", ""); - - username = INVALID_USERNAME_CHARS - .replace_all(&mut username, "") - .to_string(); - - if username.is_empty() { - username = "user".to_string() - } - - let base_username = username.clone(); - let mut username_conflict = true; - let mut i = 1; - while username_conflict { - if i > 1000 { - return Err(Error::InternalErr(format!( - "too many username conflicts for {}", - email - ))); - } - if i > 1 { - username = format!("{}{}", base_username, i) - } - username_conflict = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 AND workspace_id = $2)", - &username, - &w_id - ) - .fetch_one(&mut **tx) - .await? - .unwrap_or(false); - i += 1; - } - username - }; - - sqlx::query!( - "INSERT INTO usr (workspace_id, username, email, is_admin, operator) VALUES ($1, $2, $3, false, $4) ON CONFLICT DO NOTHING", - &w_id, - &username, - &email, - &operator - ) - .execute(&mut **tx) - .await?; - - sqlx::query_as!( - Group, - "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", - &w_id, - username, - "all", - ) - .execute(&mut **tx) - .await?; - let audit_author = if authorable.username() == authorable.email() && authorable.email() == email - { - // if the user is auto adding themselves (e.g. by joining the instance), we use their newly created workspace username for audit logs - AuditAuthor { - username: username.clone(), - email: email.to_string(), - username_override: None, - } - } else { - AuditAuthor { - username: authorable.username().to_string(), - email: authorable.email().to_string(), - username_override: authorable.username_override().map(|x| x.to_string()), - } - }; - audit_log( - &mut **tx, - &audit_author, - "users.auto_invite_add", - ActionKind::Create, - &w_id, - Some(email), - None, - ) - .await?; - Ok(username) -} - async fn edit_auto_invite( authed: ApiAuthed, Extension(db): Extension, Extension(rsmq): Extension>, Path(w_id): Path, - ApiAuthed { is_admin, email, username, .. }: ApiAuthed, Json(ea): Json, ) -> Result { - require_admin(is_admin, &username)?; - - // #[cfg(not(feature = "enterprise"))] - // { - // return Err(Error::BadRequest( - // "Auto-invite is only available on enterprise".to_string(), - // )); - // } - - let domain = if ea.invite_all.is_some_and(|x| x) { - if *CLOUD_HOSTED { - return Err(Error::BadRequest( - "invite_all is only available locally".to_string(), - )); - } else { - "*" - } - } else { - email.split('@').last().unwrap() - }; - - let mut tx = db.begin().await?; - - let mut users_to_auto_add = Option::None; - - if let (Some(operator), Some(auto_add)) = (ea.operator, ea.auto_add) { - if BANNED_DOMAINS.contains(domain) { - return Err(Error::BadRequest(format!( - "Domain {} is not allowed", - domain - ))); - } - - sqlx::query!( - "UPDATE workspace_settings SET auto_invite_domain = $1, auto_invite_operator = $2, auto_add = $4 WHERE workspace_id = $3", - domain, - operator, - &w_id, - auto_add, - ) - .execute(&mut *tx) - .await?; - - if auto_add { - users_to_auto_add = Some(sqlx::query!( - "SELECT email FROM password WHERE ($2::text = '*' OR email LIKE CONCAT('%', $2::text)) AND NOT EXISTS ( - SELECT 1 FROM usr WHERE workspace_id = $1::text AND email = password.email - )", - &w_id, - domain - ) - .fetch_all(&mut *tx).await?); - - for user in users_to_auto_add.as_ref().unwrap() { - auto_add_user(&user.email, &w_id, &operator, &mut tx, &authed).await?; - send_email_if_possible( - &format!("Added to Windmill's workspace: {w_id}"), - &format!( - "You have been granted access to Windmill's workspace {w_id} by {email}. - - Access the workspace at {}/?workspace={w_id}", - BASE_URL.read().await.clone() - ), - &user.email, - ); - } - } else { - sqlx::query!( - "INSERT INTO workspace_invite - (workspace_id, email, is_admin, operator) - SELECT $1::text, email, false, $3 FROM password WHERE ($2::text = '*' OR email LIKE CONCAT('%', $2::text)) AND NOT EXISTS ( - SELECT 1 FROM usr WHERE workspace_id = $1::text AND email = password.email - ) - ON CONFLICT DO NOTHING", - &w_id, - domain, - operator - ) - .execute(&mut *tx) - .await?; - } - } else { - sqlx::query!( - "UPDATE workspace_settings SET auto_invite_domain = NULL, auto_invite_operator = NULL, auto_add = NULL WHERE workspace_id = $1", - &w_id, - ) - .execute(&mut *tx) - .await?; - } - audit_log( - &mut *tx, - &authed, - "workspaces.edit_auto_invite_domain", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some([("operator", &format!("{:?}", ea.operator)[..])].into()), - ) - .await?; - tx.commit().await?; - - if let Some(users) = users_to_auto_add { - for user in users { - handle_deployment_metadata( - &email, - &username, - &db, - &w_id, - windmill_git_sync::DeployedObject::User { email: user.email.clone() }, - Some(format!("Auto-added user '{}' to workspace", &user.email)), - rsmq.clone(), - true, - ) - .await?; - } - } - - Ok(format!( - "Edit auto-invite for workspace {} to {}", - &w_id, domain - )) + crate::workspaces_ee::edit_auto_invite(authed, db, rsmq, w_id, ea).await } async fn edit_webhook( @@ -1488,6 +1262,30 @@ async fn set_encryption_key( return Ok(()); } +#[derive(Serialize)] +struct UsedTriggers { + pub websocket_used: bool, + pub http_routes_used: bool, +} + +async fn get_used_triggers( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + let websocket_used = sqlx::query_as!( + UsedTriggers, + r#"SELECT EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) as "websocket_used!", EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) as "http_routes_used!""#, + w_id, + ) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json(websocket_used)) +} + async fn list_workspaces_as_super_admin( authed: ApiAuthed, Extension(db): Extension, @@ -1976,61 +1774,6 @@ async fn delete_workspace( Ok(format!("Deleted workspace {}", &w_id)) } -pub async fn invite_user_to_all_auto_invite_worspaces( - db: &DB, - email: &str, - rsmq: Option, - authorable: &impl AuditAuthorable, -) -> Result<()> { - let mut tx = db.begin().await?; - let domain = email.split('@').last().unwrap(); - let workspaces = sqlx::query!( - "SELECT workspace_id, auto_invite_operator, auto_add FROM workspace_settings ws WHERE (auto_invite_domain = $1 OR auto_invite_domain = '*') AND NOT EXISTS (SELECT 1 FROM usr WHERE workspace_id = ws.workspace_id AND email = $2)", - domain, - email - ) - .fetch_all(&mut *tx) - .await?; - let mut auto_added_workspace_usernames: Vec<(String, String)> = vec![]; - for r in workspaces { - if r.auto_add.is_some() && r.auto_add.unwrap() { - let operator = r.auto_invite_operator.unwrap_or(false); - let username = - auto_add_user(email, &r.workspace_id, &operator, &mut tx, authorable).await?; - auto_added_workspace_usernames.push((r.workspace_id, username)); - } else { - sqlx::query!( - "INSERT INTO workspace_invite - (workspace_id, email, is_admin, operator) - VALUES ($1, $2, false, $3) - ON CONFLICT DO NOTHING", - r.workspace_id, - email, - r.auto_invite_operator - ) - .execute(&mut *tx) - .await?; - } - } - tx.commit().await?; - - for workspace_username_tuple in auto_added_workspace_usernames { - let (w_id, username) = workspace_username_tuple; - handle_deployment_metadata( - &email, - &username, - db, - &w_id, - windmill_git_sync::DeployedObject::User { email: email.to_string() }, - Some(format!("Auto-added user '{}' to workspace", email)), - rsmq.clone(), - true, - ) - .await?; - } - Ok(()) -} - async fn invite_user( ApiAuthed { username, is_admin, .. }: ApiAuthed, Extension(db): Extension, diff --git a/backend/windmill-api/src/workspaces_ee.rs b/backend/windmill-api/src/workspaces_ee.rs new file mode 100644 index 0000000000..565a53b174 --- /dev/null +++ b/backend/windmill-api/src/workspaces_ee.rs @@ -0,0 +1,16 @@ +use crate::{ + db::{ApiAuthed, DB}, + workspaces::EditAutoInvite, +}; + +pub async fn edit_auto_invite( + _authed: ApiAuthed, + _db: DB, + _rsmq: Option, + _w_id: String, + _ea: EditAutoInvite, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::InternalErr( + "Not implemented on OSS".to_string(), + )) +} diff --git a/backend/windmill-autoscaling/Cargo.toml b/backend/windmill-autoscaling/Cargo.toml new file mode 100644 index 0000000000..7aada6f904 --- /dev/null +++ b/backend/windmill-autoscaling/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "windmill-autoscaling" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_autoscaling" +path = "./src/lib.rs" + +[features] +enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise"] +default = [] + +[dependencies] +uuid.workspace = true +serde.workspace = true +sqlx.workspace = true +serde_json.workspace = true +tracing.workspace = true +windmill-common = { workspace = true, default-features = false } +windmill-queue.workspace = true +rsmq_async.workspace = true +anyhow.workspace = true \ No newline at end of file diff --git a/backend/windmill-autoscaling/src/autoscaling_ee.rs b/backend/windmill-autoscaling/src/autoscaling_ee.rs new file mode 100644 index 0000000000..1c9defbede --- /dev/null +++ b/backend/windmill-autoscaling/src/autoscaling_ee.rs @@ -0,0 +1,6 @@ +use windmill_common::DB; + +pub async fn apply_all_autoscaling(_db: &DB) -> anyhow::Result<()> { + // Autoscaling is an ee feature + Ok(()) +} diff --git a/backend/windmill-autoscaling/src/lib.rs b/backend/windmill-autoscaling/src/lib.rs new file mode 100644 index 0000000000..28b9319244 --- /dev/null +++ b/backend/windmill-autoscaling/src/lib.rs @@ -0,0 +1,2 @@ +mod autoscaling_ee; +pub use autoscaling_ee::*; diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 09acf22157..f3adbfde84 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -56,6 +56,8 @@ mail-send.workspace = true futures-core.workspace = true async-stream.workspace = true const_format.workspace = true +crc.workspace = true +windmill-macros.workspace = true [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemalloc-ctl = { optional = true, workspace = true } diff --git a/backend/windmill-common/src/ee.rs b/backend/windmill-common/src/ee.rs index 2f32756e45..e25d36bee1 100644 --- a/backend/windmill-common/src/ee.rs +++ b/backend/windmill-common/src/ee.rs @@ -48,8 +48,20 @@ pub async fn send_critical_alert( } #[cfg(feature = "enterprise")] -pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () { +pub async fn maybe_renew_license_key_on_start( + _http_client: &reqwest::Client, + _db: &crate::db::DB, + force_renew_now: bool, +) -> bool { // Implementation is not open source + force_renew_now +} + +#[cfg(feature = "enterprise")] +pub enum RenewReason { + Manual, + Schedule, + OnStart, } #[cfg(feature = "enterprise")] @@ -57,7 +69,7 @@ pub async fn renew_license_key( _http_client: &reqwest::Client, _db: &crate::db::DB, _key: Option, - _manual: bool, + _reason: RenewReason, ) -> String { // Implementation is not open source "".to_string() @@ -74,3 +86,6 @@ pub async fn create_customer_portal_session( #[cfg(feature = "enterprise")] pub async fn worker_groups_alerts(_db: &DB) {} + +#[cfg(feature = "enterprise")] +pub async fn jobs_waiting_alerts(_db: &DB) {} diff --git a/backend/windmill-common/src/ee.rs~main b/backend/windmill-common/src/ee.rs~main deleted file mode 100644 index 482b61a0fe..0000000000 --- a/backend/windmill-common/src/ee.rs~main +++ /dev/null @@ -1,73 +0,0 @@ -#[cfg(feature = "enterprise")] -use crate::db::DB; -use crate::ee::LicensePlan::Community; -#[cfg(feature = "enterprise")] -use crate::error; -use serde::Deserialize; -use std::sync::Arc; -use tokio::sync::RwLock; - -lazy_static::lazy_static! { - pub static ref LICENSE_KEY_VALID: Arc> = Arc::new(RwLock::new(true)); - pub static ref LICENSE_KEY_ID: Arc> = Arc::new(RwLock::new("".to_string())); - pub static ref LICENSE_KEY: Arc> = Arc::new(RwLock::new("".to_string())); -} - -pub enum LicensePlan { - Community, - Pro, - Enterprise, -} - -pub async fn get_license_plan() -> LicensePlan { - // Implementation is not open source - return Community; -} - -#[derive(Deserialize)] -#[serde(untagged)] -pub enum CriticalErrorChannel {} - -pub enum CriticalAlertKind { - #[cfg(feature = "enterprise")] - CriticalError, - #[cfg(feature = "enterprise")] - RecoveredCriticalError, -} - -#[cfg(feature = "enterprise")] -pub async fn send_critical_alert( - _error_message: String, - _db: &DB, - _kind: CriticalAlertKind, - _channels: Option>, -) { -} - -#[cfg(feature = "enterprise")] -pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () { - // Implementation is not open source -} - -#[cfg(feature = "enterprise")] -pub async fn renew_license_key( - _http_client: &reqwest::Client, - _db: &crate::db::DB, - _key: Option, - _manual: bool, -) -> String { - // Implementation is not open source - "".to_string() -} - -#[cfg(feature = "enterprise")] -pub async fn create_customer_portal_session( - _http_client: &reqwest::Client, - _key: Option, -) -> error::Result { - // Implementation is not open source - Ok("".to_string()) -} - -#[cfg(feature = "enterprise")] -pub async fn worker_groups_alerts(_db: &DB) {} diff --git a/backend/windmill-common/src/ee.rs~main_0 b/backend/windmill-common/src/ee.rs~main_0 deleted file mode 100644 index 2f32756e45..0000000000 --- a/backend/windmill-common/src/ee.rs~main_0 +++ /dev/null @@ -1,76 +0,0 @@ -#[cfg(feature = "enterprise")] -use crate::db::DB; -use crate::ee::LicensePlan::Community; -#[cfg(feature = "enterprise")] -use crate::error; -use serde::Deserialize; -use std::sync::Arc; -use tokio::sync::RwLock; - -lazy_static::lazy_static! { - pub static ref LICENSE_KEY_VALID: Arc> = Arc::new(RwLock::new(true)); - pub static ref LICENSE_KEY_ID: Arc> = Arc::new(RwLock::new("".to_string())); - pub static ref LICENSE_KEY: Arc> = Arc::new(RwLock::new("".to_string())); -} - -pub enum LicensePlan { - Community, - Pro, - Enterprise, -} - -pub async fn get_license_plan() -> LicensePlan { - // Implementation is not open source - return Community; -} - -#[derive(Deserialize)] -#[serde(untagged)] -pub enum CriticalErrorChannel { - Email { email: String }, - Slack { slack_channel: String }, -} - -pub enum CriticalAlertKind { - #[cfg(feature = "enterprise")] - CriticalError, - #[cfg(feature = "enterprise")] - RecoveredCriticalError, -} - -#[cfg(feature = "enterprise")] -pub async fn send_critical_alert( - _error_message: String, - _db: &DB, - _kind: CriticalAlertKind, - _channels: Option>, -) { -} - -#[cfg(feature = "enterprise")] -pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () { - // Implementation is not open source -} - -#[cfg(feature = "enterprise")] -pub async fn renew_license_key( - _http_client: &reqwest::Client, - _db: &crate::db::DB, - _key: Option, - _manual: bool, -) -> String { - // Implementation is not open source - "".to_string() -} - -#[cfg(feature = "enterprise")] -pub async fn create_customer_portal_session( - _http_client: &reqwest::Client, - _key: Option, -) -> error::Result { - // Implementation is not open source - Ok("".to_string()) -} - -#[cfg(feature = "enterprise")] -pub async fn worker_groups_alerts(_db: &DB) {} diff --git a/backend/windmill-common/src/email_ee.rs b/backend/windmill-common/src/email_ee.rs new file mode 100644 index 0000000000..42aebbeec3 --- /dev/null +++ b/backend/windmill-common/src/email_ee.rs @@ -0,0 +1,11 @@ +use crate::server::Smtp; + +pub async fn send_email( + _subject: &str, + _content: &str, + _to: Vec, + _smtp: Smtp, + _client_timeout: Option, +) -> crate::error::Result<()> { + Ok(()) +} diff --git a/backend/windmill-common/src/flow_status.rs b/backend/windmill-common/src/flow_status.rs index 035cadecd8..a5c509d6ef 100644 --- a/backend/windmill-common/src/flow_status.rs +++ b/backend/windmill-common/src/flow_status.rs @@ -128,6 +128,7 @@ struct UntaggedFlowStatusModule { while_loop: Option, approvers: Option>, failed_retries: Option>, + skipped: Option, } #[derive(Serialize, Debug, Clone)] @@ -179,6 +180,7 @@ pub enum FlowStatusModule { approvers: Vec, #[serde(skip_serializing_if = "Vec::is_empty")] failed_retries: Vec, + skipped: bool, }, Failure { id: String, @@ -255,6 +257,7 @@ impl<'de> Deserialize<'de> for FlowStatusModule { branch_chosen: untagged.branch_chosen, approvers: untagged.approvers.unwrap_or_default(), failed_retries: untagged.failed_retries.unwrap_or_default(), + skipped: untagged.skipped.unwrap_or(false), }), "Failure" => Ok(FlowStatusModule::Failure { id: untagged diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index a5040a9da3..9b66cc65ac 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -101,6 +101,7 @@ pub struct FlowValue { #[serde(skip_serializing_if = "Option::is_none")] #[serde(default)] pub failure_module: Option>, + #[serde(skip_serializing_if = "Option::is_none")] #[serde(default)] pub preprocessor_module: Option>, #[serde(default)] @@ -269,6 +270,13 @@ pub struct FlowModule { pub delete_after_use: Option, #[serde(skip_serializing_if = "Option::is_none")] pub continue_on_error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub skip_if: Option, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct SkipIf { + pub expr: String, } #[derive(Deserialize)] @@ -416,6 +424,7 @@ pub enum FlowModuleValue { path: String, #[serde(skip_serializing_if = "Option::is_none")] hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] tag_override: Option, }, Flow { @@ -631,6 +640,7 @@ pub fn add_virtual_items_if_necessary(modules: &mut Vec) { priority: None, delete_after_use: None, continue_on_error: None, + skip_if: None, }); } } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 36b80c9fd7..b0420ca248 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -27,6 +27,7 @@ pub const OBJECT_STORE_CACHE_CONFIG_SETTING: &str = "object_store_cache_config"; pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; +pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const DEV_INSTANCE_SETTING: &str = "dev_instance"; pub const JWT_SECRET_SETTING: &str = "jwt_secret"; diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index f7cfe1c9ee..6feceb8d51 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -640,9 +640,7 @@ pub async fn get_logs_from_store( if log_offset > 0 { if let Some(file_index) = log_file_index.clone() { - tracing::debug!("Getting logs from store: {file_index:?}"); if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { - tracing::debug!("object store client present, streaming from there"); let logs = logs.to_string(); let stream = async_stream::stream! { diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index cd8059251e..ad96594ac8 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -17,10 +17,12 @@ use scripts::ScriptLang; use sqlx::{Pool, Postgres}; pub mod apps; +pub mod auth; #[cfg(feature = "benchmark")] pub mod bench; pub mod db; pub mod ee; +pub mod email_ee; pub mod error; pub mod external_ip; pub mod flow_status; @@ -32,9 +34,8 @@ pub mod job_s3_helpers_ee; pub mod jobs; pub mod more_serde; pub mod oauth2; +pub mod queue; pub mod s3_helpers; - -pub mod auth; pub mod schedule; pub mod scripts; pub mod server; @@ -95,6 +96,8 @@ lazy_static::lazy_static! { pub static ref JOB_RETENTION_SECS: Arc> = Arc::new(RwLock::new(0)); + pub static ref INSTANCE_NAME: String = rd_string(5); + } pub async fn shutdown_signal( @@ -141,6 +144,7 @@ pub async fn shutdown_signal( use tokio::sync::RwLock; #[cfg(feature = "prometheus")] use tokio::task::JoinHandle; +use utils::rd_string; #[cfg(feature = "prometheus")] pub async fn serve_metrics( diff --git a/backend/windmill-common/src/queue.rs b/backend/windmill-common/src/queue.rs new file mode 100644 index 0000000000..6cbb5611c5 --- /dev/null +++ b/backend/windmill-common/src/queue.rs @@ -0,0 +1,16 @@ +use std::collections::HashMap; + +use sqlx::{Pool, Postgres}; + +pub async fn get_queue_counts(db: &Pool) -> HashMap { + sqlx::query_as::<_, (String, i64)>( + "SELECT tag, count(*) as count FROM queue WHERE + scheduled_for <= now() - ('3 seconds')::interval AND running = false + GROUP BY tag", + ) + .fetch_all(db) + .await + .ok() + .map(|v| v.into_iter().map(|(k, v)| (k, v as u32)).collect()) + .unwrap_or_else(|| HashMap::new()) +} diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 198283db2d..4547c29d51 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -117,10 +117,10 @@ pub struct S3Object { #[cfg(feature = "parquet")] pub async fn get_etag_or_empty( - object_store_resource: &ObjectStoreResource, + object_store_resource: &mut ObjectStoreResource, s3_object: S3Object, ) -> Option { - let object_store_client = build_object_store_client(object_store_resource); + let object_store_client = build_object_store_client(object_store_resource).await; if object_store_client.is_err() { return None; } @@ -166,11 +166,11 @@ pub fn render_endpoint( } #[cfg(feature = "parquet")] -pub fn build_object_store_client( +pub async fn build_object_store_client( resource_ref: &ObjectStoreResource, ) -> error::Result> { match resource_ref { - ObjectStoreResource::S3(s3_resource_ref) => build_s3_client(&s3_resource_ref, None), + ObjectStoreResource::S3(s3_resource_ref) => build_s3_client(&s3_resource_ref).await, ObjectStoreResource::Azure(azure_blob_resource_ref) => { build_azure_blob_client(&azure_blob_resource_ref) } @@ -225,10 +225,21 @@ use aws_config::{default_provider::credentials::DefaultCredentialsChain, Region} use object_store::CredentialProvider; #[cfg(feature = "parquet")] -pub fn build_s3_client( - s3_resource_ref: &S3Resource, - credential_providers: Option, -) -> error::Result> { +pub async fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result> { + let static_creds = s3_resource_ref.access_key.as_ref().is_some_and(|x| x != "") + || s3_resource_ref.secret_key.as_ref().is_some_and(|x| x != ""); + + let credentials_provider = if !static_creds { + Some( + DefaultCredentialsChain::builder() + .region(Region::new(s3_resource_ref.region.clone())) + .build() + .await, + ) + } else { + None + }; + let s3_resource = s3_resource_ref.clone(); let endpoint = render_endpoint( s3_resource.endpoint, @@ -244,7 +255,7 @@ pub fn build_s3_client( .with_bucket_name(s3_resource.bucket) .with_endpoint(endpoint); - if let Some(credentials_provider) = credential_providers { + if let Some(credentials_provider) = credentials_provider { store_builder = store_builder.with_credentials(Arc::new(AwsCredentialAdapter { inner: credentials_provider, })); @@ -399,18 +410,7 @@ pub async fn build_s3_client_from_settings( ) -> error::Result> { let region = none_if_empty(settings.region) .unwrap_or_else(|| std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string())); - let access_key = none_if_empty(settings.access_key); - let secret_key = none_if_empty(settings.secret_key); - let credentials_provider = if access_key.is_none() && secret_key.is_none() { - Some( - DefaultCredentialsChain::builder() - .region(Region::new(region.clone())) - .build() - .await, - ) - } else { - None - }; + let s3_resource = S3Resource { endpoint: none_if_empty(settings.endpoint).unwrap_or_else(|| { std::env::var("S3_ENDPOINT").unwrap_or_else(|_| format!("s3.{region}.amazonaws.com")) @@ -419,15 +419,15 @@ pub async fn build_s3_client_from_settings( std::env::var("S3_CACHE_BUCKET").unwrap_or_else(|_| "missingbucket".to_string()) }), region, - access_key, - secret_key, + access_key: settings.access_key, + secret_key: settings.secret_key, use_ssl: !settings.allow_http.unwrap_or(true), path_style: settings.path_style, port: settings.port, token: None, }; - build_s3_client(&s3_resource, credentials_provider) + build_s3_client(&s3_resource).await } #[cfg(feature = "parquet")] diff --git a/backend/windmill-common/src/stats_ee.rs b/backend/windmill-common/src/stats_ee.rs index 48845d5f51..5d2dc82b82 100644 --- a/backend/windmill-common/src/stats_ee.rs +++ b/backend/windmill-common/src/stats_ee.rs @@ -8,11 +8,7 @@ pub async fn get_disable_stats_setting(_db: &DB) -> bool { false } -pub async fn schedule_stats( - _instance_name: String, - _db: &DB, - _http_client: &reqwest::Client, -) -> () { +pub async fn schedule_stats(_db: &DB, _http_client: &reqwest::Client) -> () { // stats details are closed source } @@ -23,10 +19,16 @@ struct JobsUsage { count: i64, } +pub enum SendStatsReason { + Manual, + Schedule, + OnStart, +} + pub async fn send_stats( - _instance_name: &String, _http_client: &reqwest::Client, _db: &DB, + _reason: SendStatsReason, ) -> Result<()> { // stats details are closed source Ok(()) diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 8676878352..04f95a3406 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -11,13 +11,11 @@ use crate::ee::LICENSE_KEY_ID; use crate::ee::{send_critical_alert, CriticalAlertKind}; use crate::error::{to_anyhow, Error, Result}; use crate::global_settings::UNIQUE_ID_SETTING; -use crate::server::Smtp; use crate::DB; use anyhow::Context; use gethostname::gethostname; use git_version::git_version; -use mail_send::mail_builder::MessageBuilder; -use mail_send::SmtpClientBuilder; + use rand::{distributions::Alphanumeric, thread_rng, Rng}; use reqwest::Client; use serde::{Deserialize, Serialize}; @@ -189,7 +187,7 @@ pub fn calculate_hash(s: &str) -> String { format!("{:x}", hasher.finalize()) } -pub async fn get_uid(db: &DB) -> Result { +pub async fn get_uid<'c, E: sqlx::Executor<'c, Database = Postgres>>(db: E) -> Result { let mut uid = LICENSE_KEY_ID.read().await.clone(); if uid == "" { @@ -206,6 +204,14 @@ pub async fn get_uid(db: &DB) -> Result { Ok(uid) } +pub fn map_string_to_number(s: &str, max_number: u64) -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + let mut hasher = DefaultHasher::new(); + s.hash(&mut hasher); + hasher.finish() % (max_number + 1) +} + #[derive(Serialize, Deserialize, PartialEq, Clone)] #[serde(rename_all = "lowercase")] pub enum Mode { @@ -228,56 +234,11 @@ impl std::fmt::Display for Mode { } } -pub async fn send_email( - subject: &str, - content: &str, - to: Vec, - smtp: Smtp, - client_timeout: Option, -) -> Result<()> { - let mut client = SmtpClientBuilder::new(smtp.host, smtp.port) - .implicit_tls(smtp.tls_implicit.unwrap_or(false)); - if std::env::var("ACCEPT_INVALID_CERTS").is_ok() { - client = client.allow_invalid_certs(); - } - let client = if let (Some(username), Some(password)) = (smtp.username, smtp.password) { - if !username.is_empty() { - client.credentials((username, password)) - } else { - client - } - } else { - client - }; - let message = MessageBuilder::new() - .from(("Windmill", smtp.from.as_str())) - .to(to.clone()) - .subject(subject) - .text_body(content); - - match client_timeout { - Some(timeout) => { - tokio::time::timeout(timeout, client.connect()) - .await - .map_err(to_anyhow)? - .map_err(to_anyhow)? - .send(message) - .await - .map_err(to_anyhow)?; - } - None => { - client - .connect() - .await - .map_err(to_anyhow)? - .send(message) - .await - .map_err(to_anyhow)?; - } - } - tracing::info!("Sent email to {:#?}: {subject}", to); - - return Ok(()); +// inspired from rails: https://github.com/rails/rails/blob/6e49cc77ab3d16c06e12f93158eaf3e507d4120e/activerecord/lib/active_record/migration.rb#L1308 +pub fn generate_lock_id(database_name: &str) -> i64 { + const CRC_IEEE: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); + // 0x3d32ad9e chosen by fair dice roll + 0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64) } pub async fn report_critical_error(error_message: String, _db: DB) -> () { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 9464eccdcc..37ae61c8d6 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -13,6 +13,7 @@ use std::{ sync::{atomic::AtomicBool, Arc}, }; use tokio::sync::RwLock; +use windmill_macros::annotations; use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::Smtp, DB}; @@ -303,46 +304,25 @@ fn parse_file(path: &str) -> Option { .flatten() } -pub struct Annotations { - pub npm_mode: bool, - pub nodejs_mode: bool, - pub native_mode: bool, +#[annotations("#")] +pub struct PythonAnnotations { + pub no_cache: bool, + pub no_uv: bool, +} + +#[annotations("//")] +pub struct TypeScriptAnnotations { + pub npm: bool, + pub nodejs: bool, + pub native: bool, pub nobundling: bool, } -pub fn get_annotation(inner_content: &str) -> Annotations { - let annotations = inner_content - .lines() - .take_while(|x| x.starts_with("//")) - .map(|x| x.to_string().replace("//", "").trim().to_string()) - .collect_vec(); - let nodejs_mode: bool = annotations.contains(&"nodejs".to_string()); - let npm_mode: bool = annotations.contains(&"npm".to_string()); - let native_mode: bool = annotations.contains(&"native".to_string()); - - //TODO: remove || npm_mode when bun build is more powerful - let nobundling: bool = - annotations.contains(&"nobundling".to_string()) || nodejs_mode || *DISABLE_BUNDLING; - - Annotations { npm_mode, nodejs_mode, native_mode, nobundling } -} - +#[annotations("--")] pub struct SqlAnnotations { pub return_last_result: bool, } -pub fn get_sql_annotations(inner_content: &str) -> SqlAnnotations { - let annotations = inner_content - .lines() - .take_while(|x| x.starts_with("--")) - .map(|x| x.to_string().replace("--", "").trim().to_string()) - .collect_vec(); - - let return_last_result: bool = annotations.contains(&"return_last_result".to_string()); - - SqlAnnotations { return_last_result } -} - pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) { if tokio::fs::metadata(&bin_path).await.is_ok() { (true, format!("loaded from local cache: {}\n", bin_path)) @@ -449,10 +429,13 @@ pub async fn save_cache( fn write_binary_file(main_path: &str, byts: &mut bytes::Bytes) -> error::Result<()> { use std::fs::{File, Permissions}; use std::io::Write; + + #[cfg(unix)] use std::os::unix::fs::PermissionsExt; let mut file = File::create(main_path)?; file.write_all(byts)?; + #[cfg(unix)] file.set_permissions(Permissions::from_mode(0o755))?; file.flush()?; Ok(()) diff --git a/backend/windmill-indexer/Cargo.toml b/backend/windmill-indexer/Cargo.toml index fa88503511..b345e02917 100644 --- a/backend/windmill-indexer/Cargo.toml +++ b/backend/windmill-indexer/Cargo.toml @@ -29,3 +29,4 @@ tempfile.workspace = true bytes.workspace = true object_store = { workspace = true, optional = true} tokio-tar.workspace = true +lazy_static.workspace = true diff --git a/backend/windmill-indexer/src/indexer_ee.rs b/backend/windmill-indexer/src/indexer_ee.rs index 79bcbcff77..da92ff0ef8 100644 --- a/backend/windmill-indexer/src/indexer_ee.rs +++ b/backend/windmill-indexer/src/indexer_ee.rs @@ -1,6 +1,6 @@ +use anyhow::anyhow; use sqlx::{Pool, Postgres}; use windmill_common::error::Error; -use anyhow::anyhow; #[derive(Clone)] pub struct IndexReader; diff --git a/backend/windmill-macros/Cargo.toml b/backend/windmill-macros/Cargo.toml new file mode 100644 index 0000000000..100b4678d3 --- /dev/null +++ b/backend/windmill-macros/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "windmill-macros" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +proc-macro = true + +[dependencies] +proc-macro2.workspace = true +quote.workspace = true +syn.workspace = true + +# Dependencies for tests +[dev-dependencies] +# tests/annotation.rs +lazy_static.workspace = true +itertools.workspace = true +regex.workspace = true diff --git a/backend/windmill-macros/src/lib.rs b/backend/windmill-macros/src/lib.rs new file mode 100644 index 0000000000..25ff8cbaa2 --- /dev/null +++ b/backend/windmill-macros/src/lib.rs @@ -0,0 +1,95 @@ +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, Ident, ItemStruct, Lit}; + +#[proc_macro_attribute] +pub fn annotations(attr: TokenStream, item: TokenStream) -> TokenStream { + let input = parse_macro_input!(item as ItemStruct); + let name = input.ident.clone(); + let fields = input + .fields + .iter() + .map(|f| f.ident.clone().unwrap()) + .collect::>(); + + // Match on the literal to extract the string value + let comm_lit = match parse_macro_input!(attr as Lit) { + Lit::Str(lit_str) => lit_str.value(), // This will give "#" without quotes + _ => panic!("Expected a string literal"), + }; + + // Generate regex + let mut reg = format!("^{}|", &comm_lit); + { + for field in fields.iter() { + reg.push_str(&(field.to_string())); + reg.push_str("\\b"); + } + + reg.push_str(r#"|\w+"#); + } + // Example of generated regex: + // ^# + // |ann1\b|ann2\b|ann3\b|ann4\b + // |\w+ + + TokenStream::from(quote! { + #[derive(Default, Debug)] + #input + + impl std::ops::BitOrAssign for #name{ + fn bitor_assign(&mut self, rhs: Self) { + // Unfold fields + // Read more: https://docs.rs/quote/latest/quote/macro.quote.html#interpolation + #( self.#fields |= rhs.#fields; )* + } + } + + impl #name { + /// Autogenerated by windmill-macros + pub fn parse(inner_content: &str) -> Self{ + let mut res = Self::default(); + lazy_static::lazy_static! { + static ref RE: regex::Regex = regex::Regex::new(#reg).unwrap(); + } + // Create lines stream + let mut lines = inner_content.lines(); + 'outer: while let Some(line) = lines.next() { + // If comment sign(s) on the right place + let mut comms = false; + // New instance + // We will apply it if in line only annotations + let mut new = Self::default(); + + 'inner: for (i, mat) in RE.find_iter(line).enumerate() { + + match mat.as_str(){ + #comm_lit if i == 0 => { + comms = true; + continue 'inner; + }, + + // Will expand into something like: + // "ann1" => new.ann1 = true, + // "ann2" => new.ann2 = true, + // "ann3" => new.ann3 = true, + #( stringify!(#fields) => new.#fields = true, )* + // Non annotations + _ => continue 'outer, + }; + } + + if !comms { + // We dont want to continue if line does not start with # + return res; + } + + // Apply changes + res |= new; + } + + res + } + } + }) +} diff --git a/backend/windmill-macros/tests/annotations.rs b/backend/windmill-macros/tests/annotations.rs new file mode 100644 index 0000000000..fd430dfaeb --- /dev/null +++ b/backend/windmill-macros/tests/annotations.rs @@ -0,0 +1,169 @@ +#[cfg(test)] +mod annotations_tests { + + extern crate windmill_macros; + use itertools::Itertools; + use windmill_macros::annotations; + + // Previous implementation. + // We have to make sure that new one works the same as old one + fn old(inner_content: &str) -> Annotations { + let annotations = inner_content + .lines() + .take_while(|x| x.starts_with("#")) + .map(|x| x.to_string().replace("#", "").trim().to_string()) + .collect_vec(); + + let ann1: bool = annotations.contains(&"ann1".to_string()); + let ann2: bool = annotations.contains(&"ann2".to_string()); + let ann3: bool = annotations.contains(&"ann3".to_string()); + let ann4: bool = annotations.contains(&"ann4".to_string()); + let ann5: bool = annotations.contains(&"ann5".to_string()); + + Annotations { ann1, ann2, ann3, ann4, ann5 } + } + + #[annotations("#")] + #[derive(Eq, PartialEq, Copy, Clone)] + pub struct Annotations { + pub ann1: bool, + pub ann2: bool, + pub ann3: bool, + pub ann4: bool, + pub ann5: bool, + } + + #[annotations("//")] + #[derive(Eq, PartialEq, Copy, Clone)] + pub struct SlashedAnnotations { + pub ann1: bool, + pub ann2: bool, + pub ann3: bool, + pub ann4: bool, + } + + #[annotations("--")] + #[derive(Eq, PartialEq, Copy, Clone)] + pub struct MinusedAnnotations { + pub ann1: bool, + pub ann2: bool, + } + + // e.g. rust, TS and JS + #[test] + fn slashed_annotations() { + let cont = "// ann1 +// ann2 +//ann3"; + assert_eq!( + SlashedAnnotations { ann1: true, ann2: true, ann3: true, ann4: false }, + SlashedAnnotations::parse(cont) + ); + } + + // e.g. Haskell, SQL + #[test] + fn minused_annotations() { + let cont = "-- ann1 +-- ann2"; + assert_eq!( + MinusedAnnotations { ann1: true, ann2: true }, + MinusedAnnotations::parse(cont) + ); + } + + #[test] + fn simple_integration() { + let cont = "# ann1"; + let expected = Annotations { ann1: true, ..Default::default() }; + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + + #[test] + fn multiline_integration() { + let cont = "# ann2 +# ann3 +# ann4 +# ann5 + "; + let expected = Annotations { + ann1: false, + ann2: true, + ann3: true, + ann4: true, + ann5: true, + // + }; + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + + #[test] + fn spacing_integration() { + // First line is ignored and not used + { + let cont = " +# ann2"; + let expected = Annotations { ..Default::default() }; + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + // Wrong spacing for ann3 + { + let cont = "# ann2 + # ann3"; + + let expected = Annotations { ann2: true, ..Default::default() }; + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + + // Drunk but valid spacing + { + let cont = "#ann1 +# ann2"; + let expected = Annotations { ann2: true, ann1: true, ..Default::default() }; + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + } + + #[test] + fn comments_inbetween_integration() { + let cont = "# ann2 +# Just comment, has nothing to do with annotations +# Another comment: ann1 ann2 ann3 +# ann4 is not valid annotation +# Actual annotation next line: +# ann5 + +# Should be ignored +# ann3 + "; + let expected = Annotations { ann2: true, ann5: true, ..Default::default() }; + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + + #[test] + fn hash_collision() { + // TODO + } + #[test] + fn non_matching_integration() { + { + let cont = r#" "ann1", ann2 "#; + let expected = Annotations::default(); + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + // Empty + { + let cont = ""; + let expected = Annotations::default(); + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + } +} diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index eca874aa63..c0235c9b26 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -482,6 +482,7 @@ where } } +#[cfg(feature = "enterprise")] #[derive(Deserialize)] struct RawFlowFailureModule { #[cfg(feature = "enterprise")] @@ -3535,6 +3536,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( priority: None, delete_after_use: None, continue_on_error: None, + skip_if: None, }], same_worker: false, failure_module: None, diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index c05bc26fc3..e5b47b7a81 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -18,6 +18,7 @@ parquet = ["windmill-common/parquet", "dep:object_store"] flow_testing = [] cloud = [] sqlx = [] +deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", "dep:deno_ast", "dep:deno_tls"] [dependencies] windmill-queue.workspace = true @@ -59,15 +60,15 @@ once_cell.workspace = true rsmq_async.workspace = true tokio-postgres.workspace = true bit-vec.workspace = true -deno_fetch.workspace = true -deno_webidl.workspace = true -deno_web.workspace = true -deno_net.workspace = true -deno_console.workspace = true -deno_url.workspace = true -deno_core.workspace = true -deno_ast.workspace = true -deno_tls.workspace = true +deno_fetch = { workspace = true, optional = true } +deno_webidl = { workspace = true, optional = true } +deno_web = { workspace = true, optional = true } +deno_net = { workspace = true, optional = true } +deno_console = { workspace = true, optional = true } +deno_url = { workspace = true, optional = true } +deno_core = { workspace = true, optional = true } +deno_ast = { workspace = true, optional = true } +deno_tls = { workspace = true, optional = true } postgres-native-tls.workspace = true native-tls.workspace = true mysql_async.workspace = true @@ -89,13 +90,17 @@ tar.workspace = true object_store = { workspace = true, optional = true} convert_case.workspace = true yaml-rust.workspace = true +swc_ecma_parser.workspace = true + [build-dependencies] -deno_fetch.workspace = true -deno_webidl.workspace = true -deno_web.workspace = true -deno_console.workspace = true -deno_url.workspace = true -deno_core.workspace = true -deno_net.workspace = true +deno_fetch = { workspace = true, optional = true } +deno_webidl = { workspace = true, optional = true } +deno_web = { workspace = true, optional = true } +deno_net = { workspace = true, optional = true } +deno_console = { workspace = true, optional = true } +deno_url = { workspace = true, optional = true } +deno_core = { workspace = true, optional = true } +deno_ast = { workspace = true, optional = true } +deno_tls = { workspace = true, optional = true } zstd.workspace = true diff --git a/backend/windmill-worker/build.rs b/backend/windmill-worker/build.rs index 8fa581fd5c..6bbc3fd75f 100644 --- a/backend/windmill-worker/build.rs +++ b/backend/windmill-worker/build.rs @@ -1,13 +1,24 @@ +#[cfg(feature = "deno_core")] use deno_fetch::FetchPermissions; +#[cfg(feature = "deno_core")] use deno_net::NetPermissions; +#[cfg(feature = "deno_core")] use deno_web::{BlobStore, TimersPermission}; +#[cfg(feature = "deno_core")] +use std::borrow::Cow; +#[cfg(feature = "deno_core")] use std::env; +#[cfg(feature = "deno_core")] use std::io::Write; -use std::path::PathBuf; +#[cfg(feature = "deno_core")] +use std::path::{Path, PathBuf}; +#[cfg(feature = "deno_core")] use std::sync::Arc; +// #[cfg(feature = "deno_core")] pub struct PermissionsContainer; +#[cfg(feature = "deno_core")] impl FetchPermissions for PermissionsContainer { #[inline(always)] fn check_net_url( @@ -15,19 +26,20 @@ impl FetchPermissions for PermissionsContainer { _url: &deno_core::url::Url, _api_name: &str, ) -> Result<(), deno_core::error::AnyError> { - Ok(()) + unreachable!("snapshotting") } #[inline(always)] - fn check_read( + fn check_read<'a>( &mut self, - _p: &std::path::Path, + _p: &'a std::path::Path, _api_name: &str, - ) -> Result<(), deno_core::error::AnyError> { - Ok(()) + ) -> Result, deno_core::error::AnyError> { + unreachable!("snapshotting") } } +#[cfg(feature = "deno_core")] impl TimersPermission for PermissionsContainer { #[inline(always)] fn allow_hrtime(&mut self) -> bool { @@ -35,21 +47,22 @@ impl TimersPermission for PermissionsContainer { } } +#[cfg(feature = "deno_core")] impl NetPermissions for PermissionsContainer { - fn check_read( + fn check_read<'a>( &mut self, - _p: &std::path::Path, + _p: &'a str, _api_name: &str, - ) -> Result<(), deno_core::error::AnyError> { - Ok(()) + ) -> Result { + unreachable!("snapshotting") } - fn check_write( + fn check_write<'a>( &mut self, - _p: &std::path::Path, + _p: &'a str, _api_name: &str, - ) -> Result<(), deno_core::error::AnyError> { - Ok(()) + ) -> Result { + unreachable!("snapshotting") } fn check_net>( @@ -57,16 +70,26 @@ impl NetPermissions for PermissionsContainer { _host: &(T, Option), _api_name: &str, ) -> Result<(), deno_core::error::AnyError> { - Ok(()) + unreachable!("snapshotting") + } + + fn check_write_path<'a>( + &mut self, + _: &'a Path, + _: &str, + ) -> Result, deno_core::anyhow::Error> { + todo!() } } +#[cfg(feature = "deno_core")] deno_core::extension!( fetch, esm_entry_point = "ext:fetch/src/runtime.js", esm = ["src/runtime.js"], ); +#[cfg(feature = "deno_core")] fn main() { println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); @@ -122,3 +145,6 @@ fn main() { println!("cargo:rerun-if-changed={}", path.display()); } } + +#[cfg(not(feature = "deno_core"))] +fn main() {} diff --git a/backend/windmill-worker/loader.bun.js b/backend/windmill-worker/loader.bun.js index b6efb68774..ce75cbfc15 100644 --- a/backend/windmill-worker/loader.bun.js +++ b/backend/windmill-worker/loader.bun.js @@ -8,29 +8,65 @@ const p = { "localhost", "127.0.0.1" ); + const w_id = "W_ID"; const current_path = "CURRENT_PATH"; + const token = "TOKEN"; + + const cdir = resolve("./"); + const cdirNoPrivate = cdir.replace(/^\/private/, ""); // for macos + const filterResolve = new RegExp( + `^(?!\\.\/main\\.ts)(?!${cdir}\/main\\.ts)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.mjs).*\\.ts$` + ); + + let cdirNodeModules = `${cdir}/node_modules/`; + + const filterLoad = new RegExp(`^${cdir}\/main\\.ts$`); + const transpiler = new Bun.Transpiler({ + loader: "tsx", + }); + + function replaceRelativeImports(code) { + const imports = transpiler.scanImports(code); + for (const imp of imports) { + if (imp.kind == "import-statement") { + if (imp.path.startsWith(".") && !imp.path.endsWith(".ts")) { + code = code.replaceAll(imp.path, imp.path + ".ts"); + } + } + } + return { + contents: code, + }; + } + + build.onLoad({ filter: filterLoad }, async (args) => { + const code = readFileSync(args.path, "utf8"); + return replaceRelativeImports(code); + }); build.onLoad({ filter: /.*\.url$/ }, async (args) => { const url = readFileSync(args.path, "utf8"); - const contents = await ( - await fetch(url, { - method: "GET", - headers: { Authorization: "Bearer TOKEN" }, - }) - ).text(); + const req = await fetch(url, { + method: "GET", + headers: { + Authorization: "Bearer " + token, + }, + }); + if (!req.ok) { + throw new Error( + `Failed to find relative import at ${url}`, + req.statusText + ); + } + const contents = await req.text(); return { - contents, + contents: replaceRelativeImports(contents).contents, loader: "tsx", }; }); - const cdir = resolve("./"); - const cdirNoPrivate = cdir.replace(/^\/private/, ""); // for macos - const filter = new RegExp( - `^(?!\\.\/main\\.ts)(?!${cdir}\/main\\.ts)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.mjs).*\\.ts$` - ); - let cdirNodeModules = `${cdir}/node_modules/`; - build.onResolve({ filter }, (args) => { + + build.onResolve({ filter: filterResolve }, (args) => { if (args.importer?.startsWith(cdirNodeModules)) { return undefined; } @@ -41,9 +77,10 @@ const p = { const isRelative = !args.path.startsWith("/"); + let endExt = args.path.endsWith(".ts") ? "" : ".ts"; const url = isRelative - ? `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${file_path}/../${args.path}` - : `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${args.path}`; + ? `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${file_path}/../${args.path}${endExt}` + : `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${args.path}${endExt}`; const file = isRelative ? resolve("./" + file_path + "/../" + args.path + ".url") : resolve("./" + args.path + ".url"); diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto index bbdf5ed306..06ced731c3 100644 --- a/backend/windmill-worker/nsjail/run.python3.config.proto +++ b/backend/windmill-worker/nsjail/run.python3.config.proto @@ -97,6 +97,13 @@ mount { is_bind: true } +mount { + dst: "/dev/shm" + fstype: "tmpfs" + rw: true + is_bind: false +} + mount { src: "/dev/random" dst: "/dev/random" diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 6639189f61..abd9b68415 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -1,3 +1,4 @@ +#[cfg(unix)] use std::{ collections::HashMap, os::unix::fs::PermissionsExt, @@ -5,6 +6,13 @@ use std::{ process::Stdio, }; +#[cfg(windows)] +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + process::Stdio, +}; + use anyhow::anyhow; use itertools::Itertools; use serde_json::value::RawValue; @@ -15,7 +23,7 @@ use windmill_common::{ jobs::QueuedJob, worker::{to_raw_value, write_file, write_file_at_user_defined_location, WORKER_CONFIG}, }; -use windmill_parser_yaml::AnsibleRequirements; +use windmill_parser_yaml::{AnsibleRequirements, ResourceOrVariablePath}; use windmill_queue::{append_logs, CanceledBy}; use crate::{ @@ -25,9 +33,9 @@ use crate::{ OccupancyMetrics, }, handle_child::handle_child, - python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}, + python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - TZ_ENV, + PROXY_ENVS, TZ_ENV, }; lazy_static::lazy_static! { @@ -72,7 +80,7 @@ async fn handle_ansible_python_deps( if requirements.is_empty() { "".to_string() } else { - pip_compile( + uv_pip_compile( job_id, &requirements, mem_peak, @@ -82,6 +90,8 @@ async fn handle_ansible_python_deps( worker_name, w_id, &mut Some(occupancy_metrics), + false, + false, ) .await .map_err(|e| { @@ -137,6 +147,7 @@ async fn install_galaxy_collections( galaxy_command .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) // .env("BASE_INTERNAL_URL", base_internal_url) @@ -175,7 +186,7 @@ async fn install_galaxy_collections( #[cfg(not(feature = "enterprise"))] fn check_ansible_exists() -> Result<(), error::Error> { if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() { - let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run rust jobs.", ANSIBLE_PLAYBOOK_PATH.as_str()); + let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run Ansible jobs.", ANSIBLE_PLAYBOOK_PATH.as_str()); return Err(error::Error::NotFound(msg)); } Ok(()) @@ -184,7 +195,7 @@ fn check_ansible_exists() -> Result<(), error::Error> { #[cfg(feature = "enterprise")] fn check_ansible_exists() -> Result<(), error::Error> { if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() { - let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run rust jobs.", ANSIBLE_PLAYBOOK_PATH.as_str()); + let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-ee-full` for your instance in order to run Ansible jobs.", ANSIBLE_PLAYBOOK_PATH.as_str()); return Err(error::Error::NotFound(msg)); } Ok(()) @@ -378,12 +389,14 @@ fi let file = write_file(job_dir, "wrapper.sh", &wrapper)?; + #[cfg(unix)] file.metadata()?.permissions().set_mode(0o777); // let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) // inject PYTHONPATH here - for some reason I had to do it in nsjail conf .envs(reserved_variables) .env("PATH", PATH_ENV.as_str()) @@ -545,22 +558,12 @@ async fn create_file_resources( } for file_res in &r.file_resources { - let r = client - .get_resource_value_interpolated::( - &file_res.resource_path, - Some(job_id.to_string()), - ) - .await?; + let r = + get_resource_or_variable_content(client, &file_res.resource_path, job_id.to_string()) + .await?; let path = file_res.target_path.clone(); - let validated_path = write_file_at_user_defined_location( - job_dir, - path.as_str(), - r.get("content").and_then(|v| v.as_str()).ok_or(anyhow!( - "Invalid text file resource {}, `content` field absent or invalid", - &file_res.resource_path - ))?, - ) - .map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?; + let validated_path = write_file_at_user_defined_location(job_dir, path.as_str(), &r) + .map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?; nsjail_mounts.push( define_nsjail_mount(job_dir, &validated_path) @@ -568,7 +571,7 @@ async fn create_file_resources( ); logs.push_str(&format!( - "\nCreated {} from {}", + "\nCreated {} from {:?}", file_res.target_path, file_res.resource_path )); } @@ -576,3 +579,26 @@ async fn create_file_resources( Ok(nsjail_mounts) } + +async fn get_resource_or_variable_content( + client: &crate::AuthedClient, + path: &ResourceOrVariablePath, + job_id: String, +) -> anyhow::Result { + Ok(match path { + ResourceOrVariablePath::Resource(p) => { + let r = client + .get_resource_value_interpolated::(&p, Some(job_id)) + .await?; + + r.get("content") + .and_then(|v| v.as_str()) + .ok_or(anyhow!( + "Invalid text file resource {}, `content` field absent or invalid", + p + ))? + .to_string() + } + ResourceOrVariablePath::Variable(p) => client.get_variable_value(&p).await?, + }) +} diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index e9b135d277..2c4c46ab7d 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -29,9 +29,12 @@ use crate::{ }, handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - POWERSHELL_CACHE_DIR, POWERSHELL_PATH, TZ_ENV, + POWERSHELL_CACHE_DIR, POWERSHELL_PATH, PROXY_ENVS, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + lazy_static::lazy_static! { pub static ref ANSI_ESCAPE_RE: Regex = Regex::new(r"\x1b\[[0-9;]*m").unwrap(); @@ -56,11 +59,42 @@ pub async fn handle_bash_job( append_logs(&job.id, &job.workspace_id, logs1, db).await; write_file(job_dir, "main.sh", &format!("set -e\n{content}"))?; - write_file( - job_dir, - "wrapper.sh", - &format!("set -o pipefail\nset -e\nmkfifo bp\ncat bp | tail -1 > ./result2.out &\n {bash} ./main.sh \"$@\" 2>&1 | tee bp\nwait $!", bash = BIN_BASH.as_str()), - )?; + let script = format!( + r#" +set -o pipefail +set -e + +# Function to kill child processes +cleanup() {{ + echo "Terminating child processes..." + + # Ignore SIGTERM and SIGINT + trap '' SIGTERM SIGINT + + # Kill the process group of the script (negative PID value) + pkill -P $$ + exit +}} + + +# Trap SIGTERM (or other signals) and call cleanup function +trap cleanup SIGTERM SIGINT + +# Create a named pipe +mkfifo bp + +# Start background processes +cat bp | tail -1 >> ./result2.out & + +# Run main.sh in the same process group +{bash} ./main.sh "$@" 2>&1 | tee bp & + +# Wait for all background processes to finish +wait +"#, + bash = BIN_BASH.as_str(), + ); + write_file(job_dir, "wrapper.sh", &script)?; let token = client.get_token().await; let mut reserved_variables = get_reserved_variables(job, &token, db).await?; @@ -109,6 +143,7 @@ pub async fn handle_bash_job( .current_dir(job_dir) .env_clear() .envs(reserved_variables) + .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .args(cmd_args) @@ -226,13 +261,19 @@ pub async fn handle_powershell_job( .collect::>() }; + #[cfg(windows)] + let split_char = '\\'; + + #[cfg(unix)] + let split_char = '/'; + let installed_modules = fs::read_dir(POWERSHELL_CACHE_DIR)? .filter_map(|x| { x.ok().map(|x| { x.path() .display() .to_string() - .split('/') + .split(split_char) .last() .unwrap_or_default() .to_lowercase() @@ -289,34 +330,83 @@ pub async fn handle_powershell_job( append_logs(&job.id, &job.workspace_id, logs2, db).await; // make sure default (only allhostsallusers) modules are loaded, disable autoload (cache can be large to explore especially on cloud) and add /tmp/windmill/cache to PSModulePath + #[cfg(unix)] let profile = format!( "$PSModuleAutoloadingPreference = 'None' $PSModulePathBackup = $env:PSModulePath -$env:PSModulePath = ($Env:PSModulePath -split ':')[-1] +$env:PSModulePath = \"$PSHome/Modules\" Get-Module -ListAvailable | Import-Module $env:PSModulePath = \"{}:$PSModulePathBackup\"", POWERSHELL_CACHE_DIR ); + + #[cfg(windows)] + let profile = format!( + "$PSModuleAutoloadingPreference = 'None' +$PSModulePathBackup = $env:PSModulePath +$env:PSModulePath = \"C:\\Program Files\\PowerShell\\7\\Modules\" +Get-Module -ListAvailable | Import-Module +$env:PSModulePath = \"{};$PSModulePathBackup\"", + POWERSHELL_CACHE_DIR + ); + + // NOTE: powershell error handling / termination is quite tricky compared to bash + // here we're trying to catch terminating errors and propagate the exit code + // to the caller such that the job will be marked as failed. It's up to the user + // to catch specific errors in their script not caught by the below as there is no + // generic set -eu as in bash + let strict_termination_start = "$ErrorActionPreference = 'Stop'\n\ + Set-StrictMode -Version Latest\n\ + try {\n"; + + let strict_termination_end = "\n\ + } catch {\n\ + Write-Output \"An error occurred:\n\"\ + Write-Output $_ + exit 1\n\ + }\n"; + // make sure param() is first let param_match = windmill_parser_bash::RE_POWERSHELL_PARAM.find(&content); let content: String = if let Some(param_match) = param_match { let param_match = param_match.as_str(); format!( - "{}\n{}\n{}", + "{}\n{}\n{}\n{}\n{}", param_match, profile, - content.replace(param_match, "") + strict_termination_start, + content.replace(param_match, ""), + strict_termination_end ) } else { format!("{}\n{}", profile, content) }; write_file(job_dir, "main.ps1", content.as_str())?; + + #[cfg(unix)] write_file( job_dir, "wrapper.sh", &format!("set -o pipefail\nset -e\nmkfifo bp\ncat bp | tail -1 > ./result2.out &\n{} -F ./main.ps1 \"$@\" 2>&1 | tee bp\nwait $!", POWERSHELL_PATH.as_str()), )?; + + #[cfg(windows)] + write_file( + job_dir, + "wrapper.ps1", + &format!( + "param([string[]]$args)\n\ + $ErrorActionPreference = 'Stop'\n\ + $pipe = New-TemporaryFile\n\ + & \"{}\" -File ./main.ps1 @args 2>&1 | Tee-Object -FilePath $pipe\n\ + Get-Content -Path $pipe | Select-Object -Last 1 | Set-Content -Path './result2.out'\n\ + Remove-Item $pipe\n\ + exit $LASTEXITCODE\n", + POWERSHELL_PATH.as_str() + ), + )?; + let token = client.get_token().await; let mut reserved_variables = get_reserved_variables(job, &token, db).await?; reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); @@ -346,6 +436,7 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", Command::new(NSJAIL_PATH.as_str()) .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) .envs(reserved_variables) .env("TZ", TZ_ENV.as_str()) .env("PATH", PATH_ENV.as_str()) @@ -355,10 +446,24 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", .stderr(Stdio::piped()) .spawn()? } else { - let mut cmd_args = vec!["wrapper.sh"]; - cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); - Command::new(BIN_BASH.as_str()) - .current_dir(job_dir) + let mut cmd; + let mut cmd_args; + + #[cfg(unix)] + { + cmd_args = vec!["wrapper.sh"]; + cmd_args.extend(pwsh_args.iter().map(|x| x.as_str())); + cmd = Command::new(BIN_BASH.as_str()); + } + + #[cfg(windows)] + { + cmd_args = vec![r".\wrapper.ps1".to_string()]; + cmd_args.extend(pwsh_args.iter().map(|x| x.replace("--", "-"))); + cmd = Command::new(POWERSHELL_PATH.as_str()); + } + + cmd.current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) @@ -366,11 +471,53 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"", .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .args(cmd_args) + .args(&cmd_args) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + cmd.env("SystemRoot", SYSTEM_ROOT.as_str()) + .env( + "LOCALAPPDATA", + std::env::var("LOCALAPPDATA") + .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), + ) + .env( + "ProgramData", + std::env::var("ProgramData") + .unwrap_or_else(|_| String::from("C:\\ProgramData")), + ) + .env( + "ProgramFiles", + std::env::var("ProgramFiles") + .unwrap_or_else(|_| String::from("C:\\Program Files")), + ) + .env( + "ProgramFiles(x86)", + std::env::var("ProgramFiles(x86)") + .unwrap_or_else(|_| String::from("C:\\Program Files (x86)")), + ) + .env( + "ProgramW6432", + std::env::var("ProgramW6432") + .unwrap_or_else(|_| String::from("C:\\Program Files")), + ) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ) + .env( + "PATHEXT", + std::env::var("PATHEXT").unwrap_or_else(|_| { + String::from(".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL") + }), + ); + } + + cmd.spawn()? }; + handle_child( &job.id, db, diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index 1552686b79..2438392680 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -5,7 +5,6 @@ use futures::{FutureExt, TryFutureExt}; use serde_json::{json, value::RawValue, Value}; use windmill_common::error::to_anyhow; use windmill_common::jobs::QueuedJob; -use windmill_common::worker::get_sql_annotations; use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::{ parse_bigquery_sig, parse_db_resource, parse_sql_blocks, parse_sql_statement_named_params, @@ -238,7 +237,7 @@ pub async fn do_bigquery( return Err(Error::BadRequest("Missing database argument".to_string())); }; - let annotations = get_sql_annotations(query); + let annotations = windmill_common::worker::SqlAnnotations::parse(query); let service_account = CustomServiceAccount::from_json(&database) .map_err(|e| Error::ExecutionErr(e.to_string()))?; diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index e17f140b32..5b7c2a06c0 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1,8 +1,14 @@ -use std::{collections::HashMap, fs, io, path::Path, process::Stdio, time::Instant}; +#[cfg(feature = "deno_core")] +use std::time::Instant; +use std::{collections::HashMap, fs, io, path::Path, process::Stdio}; use base64::Engine; use itertools::Itertools; + +#[cfg(not(feature = "deno_core"))] +use serde_json::value::to_raw_value; use serde_json::value::RawValue; + use sha2::Digest; use uuid::Uuid; use windmill_parser_ts::remove_pinned_imports; @@ -20,9 +26,12 @@ use crate::{ handle_child::handle_child, AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, - NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, TZ_ENV, + NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + use tokio::{fs::File, process::Command}; use tokio::io::AsyncReadExt; @@ -38,7 +47,7 @@ use windmill_common::{ get_latest_hash_for_path, jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, scripts::ScriptLang, - worker::{exists_in_cache, get_annotation, save_cache, write_file}, + worker::{exists_in_cache, save_cache, write_file}, DB, }; @@ -54,8 +63,26 @@ const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js"); const NSJAIL_CONFIG_RUN_BUN_CONTENT: &str = include_str!("../nsjail/run.bun.config.proto"); pub const BUN_LOCKB_SPLIT: &str = "\n//bun.lockb\n"; +pub const BUN_LOCKB_SPLIT_WINDOWS: &str = "\r\n//bun.lockb\r\n"; + pub const EMPTY_FILE: &str = ""; +fn split_lockfile(lockfile: &str) -> (&str, Option<&str>, bool) { + if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT) { + // Split using "\n//bun.lockb\n" + let (before, after_with_sep) = lockfile.split_at(index); + let after = &after_with_sep[BUN_LOCKB_SPLIT.len()..]; + (before, Some(after), after == EMPTY_FILE) + } else if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT_WINDOWS) { + // Split using "\r\n//bun.lockb\r\n" + let (before, after_with_sep) = lockfile.split_at(index); + let after = &after_with_sep[BUN_LOCKB_SPLIT_WINDOWS.len()..]; + (before, Some(after), after == EMPTY_FILE) + } else { + (lockfile, None, false) + } +} + pub async fn gen_bun_lockfile( mem_peak: &mut i32, canceled_by: &mut Option, @@ -112,6 +139,9 @@ pub async fn gen_bun_lockfile( .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + child_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + let mut child_process = start_child_process(child_cmd, &*BUN_PATH).await?; if let Some(db) = db { @@ -168,7 +198,12 @@ pub async fn gen_bun_lockfile( file.read_to_string(&mut content).await?; } if !npm_mode { + #[cfg(any(target_os = "linux", target_os = "macos"))] content.push_str(BUN_LOCKB_SPLIT); + + #[cfg(target_os = "windows")] + content.push_str(BUN_LOCKB_SPLIT_WINDOWS); + { let file = format!("{job_dir}/bun.lockb"); if !empty_deps && tokio::fs::metadata(&file).await.is_ok() { @@ -240,11 +275,15 @@ pub async fn install_bun_lockfile( child_cmd .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) .envs(common_bun_proc_envs) .args(vec!["install"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + child_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + let mut npm_logs = if npm_mode { "NPM mode\n".to_string() } else { @@ -453,6 +492,10 @@ pub async fn generate_wrapper_mjs( .args(vec!["run", "node_builder.ts"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + child.env("SystemRoot", SYSTEM_ROOT.as_str()); + let child_process = start_child_process(child, &*BUN_PATH).await?; handle_child( job_id, @@ -487,7 +530,7 @@ pub async fn generate_bun_bundle( mem_peak: &mut i32, canceled_by: &mut Option, common_bun_proc_envs: &HashMap, - occupancy_metrics: &mut OccupancyMetrics, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> Result<()> { let mut child = Command::new(&*BUN_PATH); child @@ -498,6 +541,10 @@ pub async fn generate_bun_bundle( .args(vec!["run", "node_builder.ts"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + child.env("SystemRoot", SYSTEM_ROOT.as_str()); + let mut child_process = start_child_process(child, &*BUN_PATH).await?; if let Some(db) = db { handle_child( @@ -512,7 +559,7 @@ pub async fn generate_bun_bundle( "bun build", timeout, false, - &mut Some(occupancy_metrics), + occupancy_metrics, ) .await?; } else { @@ -540,7 +587,11 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { if is_tar { extract_tar(fs::read(bun_cache_path)?.into(), job_dir).await?; } else { + #[cfg(unix)] tokio::fs::symlink(&bun_cache_path, dst).await?; + + #[cfg(windows)] + std::os::windows::fs::symlink_dir(&bun_cache_path, &dst)?; } } else if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS .read() @@ -553,7 +604,11 @@ pub async fn pull_codebase(w_id: &str, id: &str, job_dir: &str) -> Result<()> { if is_tar { extract_tar(bytes, job_dir).await?; } else { + #[cfg(unix)] tokio::fs::symlink(bun_cache_path, dst).await?; + + #[cfg(windows)] + std::os::windows::fs::symlink_dir(&bun_cache_path, &dst)?; } // extract_tar(bytes, job_dir).await?; @@ -619,7 +674,7 @@ pub async fn prebundle_bun_script( base_internal_url: &str, worker_name: &str, token: &str, - occupancy_metrics: &mut OccupancyMetrics, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> Result<()> { let (local_path, remote_path) = compute_bundle_local_and_remote_path( inner_content, @@ -632,7 +687,7 @@ pub async fn prebundle_bun_script( if exists_in_cache(&local_path, &remote_path).await { return Ok(()); } - let annotation = get_annotation(inner_content); + let annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); if annotation.nobundling { return Ok(()); } @@ -645,9 +700,9 @@ pub async fn prebundle_bun_script( &token, w_id, script_path, - if annotation.nodejs_mode { + if annotation.nodejs { LoaderMode::NodeBundle - } else if annotation.native_mode { + } else if annotation.native { LoaderMode::BrowserBundle } else { LoaderMode::BunBundle @@ -722,17 +777,24 @@ async fn compute_bundle_local_and_remote_path( let hash = windmill_common::utils::calculate_hash(&input_src); let local_path = format!("{BUN_BUNDLE_CACHE_DIR}/{hash}"); + + #[cfg(windows)] + let local_path = local_path.replace("/tmp", r"C:\tmp").replace("/", r"\"); + let remote_path = format!("{BUN_BUNDLE_OBJECT_STORE_PREFIX}{hash}"); (local_path, remote_path) } pub async fn prepare_job_dir(reqs: &str, job_dir: &str) -> Result<()> { - let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::>(); - let _ = write_file(job_dir, "package.json", &splitted[0])?; + let (pkg, lock, empty) = split_lockfile(reqs); + let _ = write_file(job_dir, "package.json", pkg)?; - if splitted[1] != EMPTY_FILE { - let _ = write_lockb(splitted[1], job_dir).await?; + if !empty { + if let Some(lock) = lock { + let _ = write_lockb(lock, job_dir).await?; + } } + Ok(()) } async fn write_lockb(splitted_lockb_2: &str, job_dir: &str) -> Result<()> { @@ -765,7 +827,7 @@ pub async fn handle_bun_job( new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { - let mut annotation = windmill_common::worker::get_annotation(inner_content); + let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if requirements_o.is_some() && !annotation.nobundling && codebase.is_none() { @@ -787,7 +849,7 @@ pub async fn handle_bun_job( if !codebase.is_some() && !has_bundle_cache { let _ = write_file(job_dir, "main.ts", inner_content)?; - } else if !annotation.native_mode && codebase.is_none() { + } else if !annotation.native && codebase.is_none() { let _ = write_file(job_dir, "package.json", r#"{ "type": "module" }"#)?; }; @@ -795,7 +857,7 @@ pub async fn handle_bun_job( get_common_bun_proc_envs(Some(&base_internal_url)).await; if codebase.is_some() { - annotation.nodejs_mode = true + annotation.nodejs = true } let (main_override, apply_preprocessor) = match get_main_override(job.args.as_ref()) { Some(main_override) => { @@ -809,16 +871,29 @@ pub async fn handle_bun_job( }; #[cfg(not(feature = "enterprise"))] - if annotation.nodejs_mode || annotation.npm_mode { + if annotation.nodejs || annotation.npm { return Err(error::Error::ExecutionErr( "Nodejs / npm mode is an EE feature".to_string(), )); } - let mut gbuntar_name = None; + let mut gbuntar_name: Option = None; if has_bundle_cache { - let target = format!("{job_dir}/main.js"); - std::os::unix::fs::symlink(&local_path, &target).map_err(|e| { + let target; + let symlink; + + #[cfg(unix)] + { + target = format!("{job_dir}/main.js"); + symlink = std::os::unix::fs::symlink(&local_path, &target); + } + #[cfg(windows)] + { + target = format!("{job_dir}\\main.js"); + symlink = std::os::windows::fs::symlink_dir(&local_path, &target); + } + + symlink.map_err(|e| { error::Error::ExecutionErr(format!( "could not copy cached binary from {local_path} to {job_dir}/main: {e:?}" )) @@ -826,22 +901,23 @@ pub async fn handle_bun_job( } else if let Some(codebase) = codebase.as_ref() { pull_codebase(&job.workspace_id, codebase, job_dir).await?; } else if let Some(reqs) = requirements_o.as_ref() { - let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::>(); - if splitted.len() != 2 && !annotation.npm_mode { + let (pkg, lock, empty) = split_lockfile(reqs); + + if lock.is_none() && !annotation.npm { return Err(error::Error::ExecutionErr( format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|") )); } - let _ = write_file(job_dir, "package.json", &splitted[0])?; - let lockb = if annotation.npm_mode { "" } else { splitted[1] }; - if lockb != EMPTY_FILE { + let _ = write_file(job_dir, "package.json", pkg)?; + let lockb = if annotation.npm { "" } else { lock.unwrap() }; + if !empty { let mut skip_install = false; let mut create_buntar = false; let mut buntar_path = "".to_string(); - if !annotation.npm_mode { - let _ = write_lockb(&splitted[1], job_dir).await?; + if !annotation.npm { + let _ = write_lockb(lockb, job_dir).await?; let mut sha_path = sha2::Sha256::new(); sha_path.update(lockb.as_bytes()); @@ -873,7 +949,7 @@ pub async fn handle_bun_job( job_dir, worker_name, common_bun_proc_envs.clone(), - annotation.npm_mode, + annotation.npm, &mut Some(occupancy_metrics), ) .await?; @@ -915,7 +991,7 @@ pub async fn handle_bun_job( worker_name, false, None, - annotation.npm_mode, + annotation.npm, &mut Some(occupancy_metrics), ) .await?; @@ -923,19 +999,19 @@ pub async fn handle_bun_job( // } } - let mut init_logs = if annotation.native_mode { + let mut init_logs = if annotation.native { "\n\n--- NATIVE CODE EXECUTION ---\n".to_string() } else if has_bundle_cache { - if annotation.nodejs_mode { + if annotation.nodejs { "\n\n--- NODE BUNDLE SNAPSHOT EXECUTION ---\n".to_string() } else { "\n\n--- BUN BUNDLE SNAPSHOT EXECUTION ---\n".to_string() } } else if codebase.is_some() { "\n\n--- NODE CODEBASE SNAPSHOT EXECUTION ---\n".to_string() - } else if annotation.native_mode { + } else if annotation.native { "\n\n--- NATIVE CODE EXECUTION ---\n".to_string() - } else if annotation.nodejs_mode { + } else if annotation.nodejs { write_file(job_dir, "main.ts", &remove_pinned_imports(inner_content)?)?; "\n\n--- NODE CODE EXECUTION ---\n".to_string() } else { @@ -955,7 +1031,7 @@ pub async fn handle_bun_job( } let write_wrapper_f = async { - if !has_bundle_cache && annotation.native_mode { + if !has_bundle_cache && annotation.native { return Ok(()) as error::Result<()>; } // let mut start = Instant::now(); @@ -1057,6 +1133,16 @@ try {{ if (step_id) {{ err["step_id"] = step_id; }} + const extra = {{}}; + Object.getOwnPropertyNames(e).forEach((key) => {{ + if (['line', 'name', 'stack', 'column', 'message', 'sourceURL', 'originalLine', 'originalColumn'].includes(key)) {{ + return; + }} + extra[key] = e[key]; + }}); + if (Object.keys(extra).length > 0) {{ + err["extra"] = extra; + }} await fs.writeFile("result.json", JSON.stringify(err)); process.exit(1); }} @@ -1068,7 +1154,7 @@ try {{ let reserved_variables_args_out_f = async { let args_and_out_f = async { - if !annotation.native_mode { + if !annotation.native { create_args_and_out_file(&client, job, job_dir, db).await?; } Ok(()) as Result<()> @@ -1085,7 +1171,7 @@ try {{ let build_cache = !has_bundle_cache && !annotation.nobundling && !codebase.is_some() - && (requirements_o.is_some() || annotation.native_mode); + && (requirements_o.is_some() || annotation.native); let write_loader_f = async { if build_cache { @@ -1095,9 +1181,9 @@ try {{ &client.get_token().await, &job.workspace_id, &job.script_path(), - if annotation.nodejs_mode { + if annotation.nodejs { LoaderMode::NodeBundle - } else if annotation.native_mode { + } else if annotation.native { LoaderMode::BrowserBundle } else { LoaderMode::BunBundle @@ -1113,7 +1199,7 @@ try {{ &client.get_token().await, &job.workspace_id, &job.script_path(), - if annotation.nodejs_mode { + if annotation.nodejs { LoaderMode::Node } else { LoaderMode::Bun @@ -1142,7 +1228,7 @@ try {{ mem_peak, canceled_by, &common_bun_proc_envs, - occupancy_metrics, + &mut Some(occupancy_metrics), ) .await?; if !local_path.is_empty() { @@ -1159,7 +1245,7 @@ try {{ } } } - if !annotation.native_mode { + if !annotation.native { let ex_wrapper = read_file_content(&format!("{job_dir}/wrapper.mjs")).await?; write_file( job_dir, @@ -1173,7 +1259,7 @@ try {{ } fs::remove_file(format!("{job_dir}/main.ts"))?; has_bundle_cache = true; - } else if annotation.nodejs_mode { + } else if annotation.nodejs { generate_wrapper_mjs( job_dir, &job.workspace_id, @@ -1189,52 +1275,64 @@ try {{ .await?; } } - if annotation.native_mode { - let env_code = format!( + if annotation.native { + #[cfg(not(feature = "deno_core"))] + { + tracing::error!( + r#""deno_core" feature is not activated, but "//native" annotation used. Returning empty value..."# + ); + return Ok(to_raw_value("").unwrap()); + } + + #[cfg(feature = "deno_core")] + { + let env_code = format!( "const process = {{ env: {{}} }};\nconst BASE_URL = '{base_internal_url}';\nconst BASE_INTERNAL_URL = '{base_internal_url}';\nprocess.env['BASE_URL'] = BASE_URL;process.env['BASE_INTERNAL_URL'] = BASE_INTERNAL_URL;\n{}", reserved_variables .iter() .map(|(k, v)| format!("process.env['{}'] = '{}';\n", k, v)) .collect::>() .join("\n")); - let js_code = read_file_content(&format!("{job_dir}/main.js")).await?; - let started_at = Instant::now(); - let args = crate::common::build_args_map(job, client, db) - .await? - .map(sqlx::types::Json); - let job_args = if args.is_some() { - args.as_ref() - } else { - job.args.as_ref() - }; - let result = crate::js_eval::eval_fetch_timeout( - env_code, - inner_content.clone(), - js_code, - job_args, - job.id, - job.timeout, - db, - mem_peak, - canceled_by, - worker_name, - &job.workspace_id, - false, - occupancy_metrics, - ) - .await?; - tracing::info!( - "Executed native code in {}ms", - started_at.elapsed().as_millis() - ); - append_logs( - &job.id, - &job.workspace_id, - format!("{}\n{}", init_logs, result.1), - db, - ) - .await; - return Ok(result.0); + let js_code = read_file_content(&format!("{job_dir}/main.js")).await?; + let started_at = Instant::now(); + let args = crate::common::build_args_map(job, client, db) + .await? + .map(sqlx::types::Json); + let job_args = if args.is_some() { + args.as_ref() + } else { + job.args.as_ref() + }; + + let result = crate::js_eval::eval_fetch_timeout( + env_code, + inner_content.clone(), + js_code, + job_args, + job.id, + job.timeout, + db, + mem_peak, + canceled_by, + worker_name, + &job.workspace_id, + false, + occupancy_metrics, + ) + .await?; + tracing::info!( + "Executed native code in {}ms", + started_at.elapsed().as_millis() + ); + append_logs( + &job.id, + &job.workspace_id, + format!("{}\n{}", init_logs, result.1), + db, + ) + .await; + return Ok(result.0); + } } append_logs(&job.id, &job.workspace_id, init_logs, db).await; @@ -1244,14 +1342,7 @@ try {{ job_dir, "run.config.proto", &NSJAIL_CONFIG_RUN_BUN_CONTENT - .replace( - "{LANG}", - if annotation.nodejs_mode { - "nodejs" - } else { - "bun" - }, - ) + .replace("{LANG}", if annotation.nodejs { "nodejs" } else { "bun" }) .replace("{JOB_DIR}", job_dir) .replace("{CACHE_DIR}", BUN_CACHE_DIR) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) @@ -1259,7 +1350,7 @@ try {{ "{SHARED_MOUNT}", &shared_mount.replace( "/tmp/shared", - if annotation.nodejs_mode { + if annotation.nodejs { "/tmp/nodejs/shared" } else { "/tmp/bun/shared" @@ -1269,7 +1360,7 @@ try {{ )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); - let args = if annotation.nodejs_mode { + let args = if annotation.nodejs { vec![ "--config", "run.config.proto", @@ -1313,7 +1404,7 @@ try {{ .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { - let cmd = if annotation.nodejs_mode { + let cmd = if annotation.nodejs { let script_path = format!("{job_dir}/wrapper.mjs"); let mut bun_cmd = Command::new(&*NODE_BIN_PATH); @@ -1326,6 +1417,10 @@ try {{ .args(vec!["--preserve-symlinks", &script_path]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + bun_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + bun_cmd } else { let script_path = format!("{job_dir}/wrapper.mjs"); @@ -1352,11 +1447,16 @@ try {{ .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + bun_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + bun_cmd }; + start_child_process( cmd, - if annotation.nodejs_mode { + if annotation.nodejs { &*NODE_BIN_PATH } else { &*BUN_PATH @@ -1371,7 +1471,7 @@ try {{ mem_peak, canceled_by, child, - false, + !*DISABLE_NSJAIL, worker_name, &job.workspace_id, "bun run", @@ -1461,10 +1561,10 @@ pub async fn start_worker( let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(Some(&base_internal_url)).await; - let mut annotation = windmill_common::worker::get_annotation(inner_content); + let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); //TODO: remove this when bun dedicated workers work without issues - annotation.nodejs_mode = true; + annotation.nodejs = true; let context = variables::get_reserved_variables( db, @@ -1489,20 +1589,20 @@ pub async fn start_worker( if let Some(codebase) = codebase.as_ref() { pull_codebase(w_id, codebase, job_dir).await?; } else if let Some(reqs) = requirements_o { - let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::>(); - if splitted.len() != 2 { + let (pkg, lock, empty) = split_lockfile(&reqs); + if lock.is_none() { return Err(error::Error::ExecutionErr( format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|") )); } - let _ = write_file(job_dir, "package.json", &splitted[0])?; - let lockb = splitted[1]; - if lockb != EMPTY_FILE { + let _ = write_file(job_dir, "package.json", pkg)?; + let lockb = lock.unwrap(); + if !empty { let _ = write_file_binary( job_dir, "bun.lockb", &base64::engine::general_purpose::STANDARD - .decode(&splitted[1]) + .decode(lockb) .map_err(|_| { error::Error::InternalErr("Could not decode bun.lockb".to_string()) })?, @@ -1518,7 +1618,7 @@ pub async fn start_worker( job_dir, worker_name, common_bun_proc_envs.clone(), - annotation.npm_mode, + annotation.npm, &mut None, ) .await?; @@ -1539,7 +1639,7 @@ pub async fn start_worker( worker_name, false, None, - annotation.npm_mode, + annotation.npm, &mut None, ) .await?; @@ -1617,7 +1717,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) { token, w_id, script_path, - if annotation.nodejs_mode { + if annotation.nodejs { LoaderMode::Node } else { LoaderMode::Bun @@ -1626,7 +1726,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) { .await?; } - if annotation.nodejs_mode && !codebase.is_some() { + if annotation.nodejs && !codebase.is_some() { generate_wrapper_mjs( job_dir, w_id, @@ -1642,7 +1742,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) { .await?; } - if annotation.nodejs_mode { + if annotation.nodejs { let script_path = format!("{job_dir}/wrapper.mjs"); handle_dedicated_process( diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 406f128917..e22f9be54c 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -736,8 +736,8 @@ async fn arg_value_hash_additions( .await; storage = s3_object.storage.clone(); - if let Some(s3_resource) = s3_resource_opt.ok().flatten() { - let etag = get_etag_or_empty(&s3_resource, s3_object.clone()).await; + if let Some(mut s3_resource) = s3_resource_opt.ok().flatten() { + let etag = get_etag_or_empty(&mut s3_resource, s3_object.clone()).await; tracing::warn!("Enriching s3 arg value with etag: {:?}", etag); result.insert(s3_object.s3.clone(), etag.unwrap_or_default()); // TODO: maybe inject a random value to invalidate the cache? } @@ -793,9 +793,9 @@ pub async fn get_cached_resource_value_if_valid( return None; } for (s3_file_key, s3_file_etag) in s3_etags { - if let Some(object_store_resource) = object_store_resource_opt.clone() { + if let Some(mut object_store_resource) = object_store_resource_opt.clone() { let etag = get_etag_or_empty( - &object_store_resource, + &mut object_store_resource, S3Object { s3: s3_file_key.clone(), storage: cached_resource.storage.clone(), diff --git a/backend/windmill-worker/src/dedicated_worker.rs b/backend/windmill-worker/src/dedicated_worker.rs index da3e25a699..1a332e76ef 100644 --- a/backend/windmill-worker/src/dedicated_worker.rs +++ b/backend/windmill-worker/src/dedicated_worker.rs @@ -77,7 +77,7 @@ pub async fn handle_dedicated_process( ) -> std::result::Result<(), error::Error> { //do not cache local dependencies - use crate::handle_child::process_status; + use crate::{handle_child::process_status, PROXY_ENVS}; let mut child = { let mut cmd = Command::new(command_path); @@ -85,6 +85,7 @@ pub async fn handle_dedicated_process( .env_clear() .envs(context_envs) .envs(envs) + .envs(PROXY_ENVS.clone()) .envs( reserved_variables .iter() diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index fbb875a012..2a33bba2b5 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -125,7 +125,8 @@ pub async fn generate_deno_lock( "--unstable-worker-options", "--unstable-http", "--lock=lock.json", - "--lock-write", + "--frozen=false", + "--allow-import", "--import-map", &import_map_path, "main.ts", @@ -156,10 +157,13 @@ pub async fn generate_deno_lock( } let path_lock = format!("{job_dir}/lock.json"); - let mut file = File::open(path_lock).await?; - let mut req_content = "".to_string(); - file.read_to_string(&mut req_content).await?; - Ok(req_content) + if let Ok(mut file) = File::open(path_lock).await { + let mut req_content = "".to_string(); + file.read_to_string(&mut req_content).await?; + Ok(req_content) + } else { + Ok("".to_string()) + } } #[tracing::instrument(level = "trace", skip_all)] @@ -366,10 +370,10 @@ try {{ } else if !*DISABLE_NSJAIL { args.push("--allow-net"); args.push("--allow-sys"); - args.push("--allow-hrtime"); args.push(allow_read.as_str()); args.push("--allow-write=./"); args.push("--allow-env"); + args.push("--allow-import"); args.push("--allow-run=git,/usr/bin/chromium"); } else { args.push("-A"); diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index b139859005..470e647383 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -1,3 +1,4 @@ +use crate::PROXY_ENVS; use std::{collections::HashMap, fs::DirBuilder, process::Stdio}; use itertools::Itertools; @@ -189,6 +190,7 @@ func Run(req Req) (interface{{}}, error){{ .env("BASE_INTERNAL_URL", base_internal_url) .env("GOPATH", GO_CACHE_DIR) .env("HOME", HOME_ENV.as_str()) + .envs(PROXY_ENVS.clone()) .args(vec!["build", "main.go"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -225,7 +227,12 @@ func Run(req Req) (interface{{}}, error){{ } } else { let target = format!("{job_dir}/main"); - std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| { + #[cfg(unix)] + let symlink = std::os::unix::fs::symlink(&bin_path, &target); + #[cfg(windows)] + let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target); + + symlink.map_err(|e| { Error::ExecutionErr(format!( "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" )) diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index d588ea8fc4..6e7520daf8 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -6,7 +6,11 @@ use nix::sys::signal::{self, Signal}; use nix::unistd::Pid; use sqlx::{Pool, Postgres}; +#[cfg(windows)] +use std::process::Stdio; use tokio::fs::File; +#[cfg(windows)] +use tokio::process::Command; use windmill_common::error::to_anyhow; use windmill_common::error::{self, Error}; @@ -49,6 +53,33 @@ use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM}; lazy_static::lazy_static! { pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true"); } + +// - kill windows process along with all child processes +#[cfg(windows)] +async fn kill_process_tree(pid: Option) -> Result<(), String> { + let pid = match pid { + Some(pid) => pid, + None => return Err("No PID provided to kill.".to_string()), + }; + + let output = Command::new("cmd") + .args(&["/C", "taskkill", "/PID", &pid.to_string(), "/T", "/F"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|e| format!("Failed to execute taskkill: {}", e))?; + + if output.status.success() { + Ok(()) + } else { + Err(format!( + "Failed to kill process tree. Error: {}", + String::from_utf8_lossy(&output.stderr) + )) + } +} + /// - wait until child exits and return with exit status /// - read lines from stdout and stderr and append them to the "queue"."logs" /// quitting early if output exceedes MAX_LOG_SIZE characters (not bytes) @@ -196,9 +227,26 @@ pub async fn handle_child( } } } - /* send SIGKILL and reap child process */ - let (_, kill) = future::join(set_reason, child.kill()).await; - kill.map(|()| Err(kill_reason)) + #[cfg(windows)] + { + let pid_to_kill = child.id(); + match kill_process_tree(pid_to_kill).await { + Ok(_) => tracing::debug!( + "successfully killed process tree with PID: {:?}", + pid_to_kill + ), + Err(e) => tracing::error!("failed to kill process tree: {:?}", e), + }; + set_reason.await; + return Ok(Err(kill_reason)); + } + + #[cfg(unix)] + { + /* send SIGKILL and reap child process */ + let (_, kill) = future::join(set_reason, child.kill()).await; + kill.map(|()| Err(kill_reason)) + } }; /* a future that reads output from the child and appends to the database */ @@ -366,9 +414,30 @@ async fn get_mem_peak(pid: Option, nsjail: bool) -> i32 { return -1; } let pid = if nsjail { - // This is a bit hacky, but the process id of the nsjail process is the pid of nsjail + 1. - // Ideally, we would get the number from fork() itself. This works in MOST cases. - pid.unwrap() + 1 + // Read /proc//task//children and extract pid + let nsjail_pid = pid.unwrap(); + let children_path = format!("/proc/{}/task/{}/children", nsjail_pid, nsjail_pid); + if let Ok(mut file) = File::open(children_path).await { + let mut contents = String::new(); + if tokio::io::AsyncReadExt::read_to_string(&mut file, &mut contents) + .await + .is_ok() + { + if let Some(child_pid) = contents.split_whitespace().next() { + if let Ok(child_pid) = child_pid.parse::() { + child_pid + } else { + return -1; + } + } else { + return -1; + } + } else { + return -1; + } + } else { + return -1; + } } else { pid.unwrap() }; diff --git a/backend/windmill-worker/src/job_logger.rs b/backend/windmill-worker/src/job_logger.rs index 5193fe79aa..05b9e5812b 100644 --- a/backend/windmill-worker/src/job_logger.rs +++ b/backend/windmill-worker/src/job_logger.rs @@ -1,5 +1,5 @@ -use deno_ast::swc::parser::lexer::util::CharExt; use itertools::Itertools; +use swc_ecma_parser::lexer::util::CharExt; #[cfg(all(feature = "enterprise", feature = "parquet"))] use object_store::path::Path; diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index 24f83c86af..da4eda39ef 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -6,56 +6,74 @@ * LICENSE-AGPL for a copy of the license. */ +#[cfg(feature = "deno_core")] use std::{ + borrow::Cow, cell::RefCell, - collections::HashMap, env, io::{self, BufReader}, + path::PathBuf, rc::Rc, - sync::Arc, }; +use std::{collections::HashMap, sync::Arc}; + +#[cfg(feature = "deno_core")] use deno_ast::ParseParams; +#[cfg(feature = "deno_core")] use deno_core::{ error::AnyError, op2, serde_v8, url, v8::{self, IsolateHandle}, Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions, }; +#[cfg(feature = "deno_core")] use deno_fetch::FetchPermissions; +#[cfg(feature = "deno_core")] use deno_net::NetPermissions; +#[cfg(feature = "deno_core")] use deno_tls::{rustls::RootCertStore, rustls_pemfile}; +#[cfg(feature = "deno_core")] use deno_web::{BlobStore, TimersPermission}; +#[cfg(feature = "deno_core")] use itertools::Itertools; use lazy_static::lazy_static; use regex::Regex; use serde_json::value::RawValue; use sqlx::types::Json; + +#[cfg(feature = "deno_core")] use tokio::{ sync::{mpsc, oneshot}, time::timeout, }; use uuid::Uuid; -use windmill_common::{error::Error, flow_status::JobResult, DB}; + +#[cfg(feature = "deno_core")] +use windmill_common::error::Error; + +use windmill_common::{flow_status::JobResult, DB}; use windmill_queue::CanceledBy; -use crate::{ - common::{unsafe_raw, OccupancyMetrics}, - handle_child::run_future_with_polling_update_job_poller, - AuthedClient, -}; +use crate::{common::OccupancyMetrics, AuthedClient}; + +#[cfg(feature = "deno_core")] +use crate::{common::unsafe_raw, handle_child::run_future_with_polling_update_job_poller}; #[derive(Debug, Clone)] pub struct IdContext { pub flow_job: Uuid, + #[allow(dead_code)] pub steps_results: HashMap, pub previous_id: String, } +#[cfg(feature = "deno_core")] pub struct ContainerRootCertStoreProvider { root_cert_store: RootCertStore, } +#[cfg(feature = "deno_core")] impl ContainerRootCertStoreProvider { fn new() -> ContainerRootCertStoreProvider { return ContainerRootCertStoreProvider { @@ -73,14 +91,17 @@ impl ContainerRootCertStoreProvider { } } +#[cfg(feature = "deno_core")] impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider { fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> { Ok(&self.root_cert_store) } } +#[cfg(feature = "deno_core")] pub struct PermissionsContainer; +#[cfg(feature = "deno_core")] impl FetchPermissions for PermissionsContainer { #[inline(always)] fn check_net_url( @@ -92,15 +113,16 @@ impl FetchPermissions for PermissionsContainer { } #[inline(always)] - fn check_read( + fn check_read<'a>( &mut self, - _p: &std::path::Path, + p: &'a std::path::Path, _api_name: &str, - ) -> Result<(), deno_core::error::AnyError> { - Ok(()) + ) -> Result, anyhow::Error> { + Ok(Cow::Borrowed(p)) } } +#[cfg(feature = "deno_core")] impl TimersPermission for PermissionsContainer { #[inline(always)] fn allow_hrtime(&mut self) -> bool { @@ -108,21 +130,22 @@ impl TimersPermission for PermissionsContainer { } } +#[cfg(feature = "deno_core")] impl NetPermissions for PermissionsContainer { - fn check_read( + fn check_read<'a>( &mut self, - _p: &std::path::Path, + p: &'a str, _api_name: &str, - ) -> Result<(), deno_core::error::AnyError> { - Ok(()) + ) -> Result { + Ok(PathBuf::from(p)) } - fn check_write( + fn check_write<'a>( &mut self, - _p: &std::path::Path, + p: &'a str, _api_name: &str, - ) -> Result<(), deno_core::error::AnyError> { - Ok(()) + ) -> Result { + Ok(PathBuf::from(p)) } fn check_net>( @@ -132,8 +155,17 @@ impl NetPermissions for PermissionsContainer { ) -> Result<(), deno_core::error::AnyError> { Ok(()) } + + fn check_write_path<'a>( + &mut self, + p: &'a std::path::Path, + _api_name: &str, + ) -> Result, AnyError> { + Ok(Cow::Borrowed(p)) + } } +#[cfg(feature = "deno_core")] pub struct OptAuthedClient(Option); pub async fn eval_timeout( @@ -142,7 +174,7 @@ pub async fn eval_timeout( flow_input: Option>>>, authed_client: Option<&AuthedClient>, by_id: Option, - ctx: Option>, + #[allow(unused_variables)] ctx: Option>, ) -> anyhow::Result> { let expr = expr.trim().to_string(); @@ -212,121 +244,133 @@ pub async fn eval_timeout( } } - let expr2 = expr.clone(); - let (sender, mut receiver) = oneshot::channel::(); - let has_client = authed_client.is_some(); - let authed_client = authed_client.cloned(); - timeout( - std::time::Duration::from_millis(10000), - tokio::task::spawn_blocking(move || { - let mut ops = vec![op_get_context()]; + #[cfg(not(feature = "deno_core"))] + { + #[allow(unreachable_code)] + return Err(anyhow::anyhow!("Deno core is not enabled".to_string()).into()); + } - if authed_client.is_some() { - ops.extend([ - // An op for summing an array of numbers - // The op-layer automatically deserializes inputs - // and serializes the returned Result & value - op_variable(), - op_resource(), - ]) - } + #[cfg(feature = "deno_core")] + { + let expr2 = expr.clone(); + let (sender, mut receiver) = oneshot::channel::(); + let has_client = authed_client.is_some(); + let authed_client = authed_client.cloned(); + return timeout( + std::time::Duration::from_millis(10000), + tokio::task::spawn_blocking(move || { + let mut ops = vec![op_get_context()]; - if by_id.is_some() && authed_client.is_some() { - ops.push(op_get_result()); - ops.push(op_get_id()); - } - - let ext = Extension { name: "js_eval", ops: ops.into(), ..Default::default() }; - let exts = vec![ext]; - // Use our snapshot to provision our new runtime - let options = RuntimeOptions { - extensions: exts, - // startup_snapshot: Some(Snapshot::Static(buffer)), - ..Default::default() - }; - - let mut context_keys = transform_context - .keys() - .filter(|x| expr.contains(&x.to_string())) - .map(|x| x.clone()) - .collect_vec(); - - if !context_keys.contains(&"previous_result".to_string()) - && (p_ids.is_some() && p_ids.as_ref().unwrap().iter().any(|x| expr.contains(x))) - || expr.contains("error") - { - // tracing::error!("PREVIOUS_RESULT"); - context_keys.push("previous_result".to_string()); - } - let has_flow_input = expr.contains("flow_input"); - if has_flow_input { - context_keys.push("flow_input".to_string()) - } - - let mut js_runtime = JsRuntime::new(options); - { - let op_state = js_runtime.op_state(); - let mut op_state = op_state.borrow_mut(); - let mut client = authed_client.clone(); - if let Some(client) = client.as_mut() { - client.force_client = Some( - reqwest::ClientBuilder::new() - .user_agent("windmill/beta") - .danger_accept_invalid_certs( - std::env::var("ACCEPT_INVALID_CERTS").is_ok(), - ) - .build() - .unwrap(), - ); + if authed_client.is_some() { + ops.extend([ + // An op for summing an array of numbers + // The op-layer automatically deserializes inputs + // and serializes the returned Result & value + op_variable(), + op_resource(), + ]) } - op_state.put(OptAuthedClient(client)); - op_state.put(TransformContext { - flow_input: if has_flow_input { flow_input } else { None }, - envs: transform_context - .into_iter() - .filter(|(a, _)| context_keys.contains(a)) - .collect(), - }) - } - sender - .send(js_runtime.v8_isolate().thread_safe_handle()) - .map_err(|_| Error::ExecutionErr("impossible to send v8 isolate".to_string()))?; + if by_id.is_some() && authed_client.is_some() { + ops.push(op_get_result()); + ops.push(op_get_id()); + } - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; + let ext = Extension { name: "js_eval", ops: ops.into(), ..Default::default() }; + let exts = vec![ext]; + // Use our snapshot to provision our new runtime + let options = RuntimeOptions { + extensions: exts, + // startup_snapshot: Some(Snapshot::Static(buffer)), + ..Default::default() + }; - // pretty frail but this it to make the expr more user friendly and not require the user to write await - let expr = ["variable", "resource"] - .into_iter() - .fold(expr, replace_with_await); + let mut context_keys = transform_context + .keys() + .filter(|x| expr.contains(&x.to_string())) + .map(|x| x.clone()) + .collect_vec(); - let expr = replace_with_await_result(expr); + if !context_keys.contains(&"previous_result".to_string()) + && (p_ids.is_some() && p_ids.as_ref().unwrap().iter().any(|x| expr.contains(x))) + || expr.contains("error") + { + // tracing::error!("PREVIOUS_RESULT"); + context_keys.push("previous_result".to_string()); + } + let has_flow_input = expr.contains("flow_input"); + if has_flow_input { + context_keys.push("flow_input".to_string()) + } - let r = runtime.block_on(eval( - &mut js_runtime, - &expr, - context_keys, - by_id, - has_client, - ctx, - ))?; + let mut js_runtime = JsRuntime::new(options); + { + let op_state = js_runtime.op_state(); + let mut op_state = op_state.borrow_mut(); + let mut client = authed_client.clone(); + if let Some(client) = client.as_mut() { + client.force_client = Some( + reqwest::ClientBuilder::new() + .user_agent("windmill/beta") + .danger_accept_invalid_certs( + std::env::var("ACCEPT_INVALID_CERTS").is_ok(), + ) + .build() + .unwrap(), + ); + } + op_state.put(OptAuthedClient(client)); + op_state.put(TransformContext { + flow_input: if has_flow_input { flow_input } else { None }, + envs: transform_context + .into_iter() + .filter(|(a, _)| context_keys.contains(a)) + .collect(), + }) + } - Ok(r) as anyhow::Result> - }), - ) - .await - .map_err(|_| { - if let Ok(isolate) = receiver.try_recv() { - isolate.terminate_execution(); - }; - Error::ExecutionErr(format!( - "The expression of evaluation `{expr2}` took too long to execute (>10000ms)" - )) - })?? + sender + .send(js_runtime.v8_isolate().thread_safe_handle()) + .map_err(|_| { + Error::ExecutionErr("impossible to send v8 isolate".to_string()) + })?; + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + // pretty frail but this it to make the expr more user friendly and not require the user to write await + let expr = ["variable", "resource"] + .into_iter() + .fold(expr, replace_with_await); + + let expr = replace_with_await_result(expr); + + let r = runtime.block_on(eval( + &mut js_runtime, + &expr, + context_keys, + by_id, + has_client, + ctx, + ))?; + + Ok(r) as anyhow::Result> + }), + ) + .await + .map_err(|_| { + if let Ok(isolate) = receiver.try_recv() { + isolate.terminate_execution(); + }; + Error::ExecutionErr(format!( + "The expression of evaluation `{expr2}` took too long to execute (>10000ms)" + )) + })??; + } } +#[cfg(feature = "deno_core")] fn replace_with_await(expr: String, fn_name: &str) -> String { let sep = format!("{}(", fn_name); let mut split = expr.split(&sep); @@ -345,10 +389,12 @@ lazy_static! { Regex::new(r"^(https?)://(([^:@\s]+):([^:@\s]+)@)?([^:@\s]+)(:(\d+))?$").unwrap(); } +#[cfg(feature = "deno_core")] fn replace_with_await_result(expr: String) -> String { RE.replace_all(&expr, "(await $r)").to_string() } +#[cfg(feature = "deno_core")] fn add_closing_bracket(s: &str) -> String { let mut s = s.to_string(); let mut level = 1; @@ -368,6 +414,7 @@ fn add_closing_bracket(s: &str) -> String { s } +#[cfg(feature = "deno_core")] async fn eval( context: &mut JsRuntime, expr: &str, @@ -508,6 +555,7 @@ function get_from_env(name) {{ // } // TODO: Can we a) share the api configuration here somehow or b) just implement this natively in deno, via the deno client? +#[cfg(feature = "deno_core")] #[op2(async)] #[string] async fn op_variable( @@ -522,6 +570,7 @@ async fn op_variable( } } +#[cfg(feature = "deno_core")] #[op2(async)] #[string] async fn op_get_result( @@ -540,6 +589,7 @@ async fn op_get_result( } } +#[cfg(feature = "deno_core")] #[op2(async)] #[string] async fn op_get_id( @@ -563,6 +613,7 @@ async fn op_get_id( } } +#[cfg(feature = "deno_core")] #[op2(async)] #[string] async fn op_resource( @@ -580,11 +631,13 @@ async fn op_resource( } } +#[cfg(feature = "deno_core")] pub struct TransformContext { pub envs: HashMap>>, pub flow_input: Option>>>, } +#[cfg(feature = "deno_core")] #[op2] #[string] fn op_get_context(op_state: Rc>, #[string] id: &str) -> String { @@ -605,6 +658,7 @@ fn op_get_context(op_state: Rc>, #[string] id: &str) -> String } } +#[cfg(feature = "deno_core")] pub fn transpile_ts(expr: String) -> anyhow::Result { let parsed = deno_ast::parse_module(ParseParams { specifier: url::Url::parse("file:///eval.ts")?, @@ -621,21 +675,30 @@ pub fn transpile_ts(expr: String) -> anyhow::Result { .text) } +#[cfg(not(feature = "deno_core"))] +pub fn transpile_ts(_expr: String) -> anyhow::Result { + Ok("require deno".to_string()) +} + +#[cfg(feature = "deno_core")] static RUNTIME_SNAPSHOT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/FETCH_SNAPSHOT.bin")); +#[cfg(feature = "deno_core")] pub struct MainArgs { args: Vec>>, } +#[cfg(feature = "deno_core")] pub struct LogString { pub s: String, } +#[cfg(feature = "deno_core")] pub struct NativeAnnotation { pub useragent: Option, pub proxy: Option<(String, Option<(String, String)>)>, } - +#[cfg(feature = "deno_core")] pub fn get_annotation(inner_content: &str) -> NativeAnnotation { let mut res = NativeAnnotation { useragent: None, proxy: None }; @@ -655,6 +718,7 @@ pub fn get_annotation(inner_content: &str) -> NativeAnnotation { res } +#[cfg(feature = "deno_core")] fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> { RE_PROXY.captures(s).map(|x| { ( @@ -675,7 +739,27 @@ fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> { ) }) } +#[cfg(not(feature = "deno_core"))] +pub async fn eval_fetch_timeout( + _env_code: String, + _ts_expr: String, + _js_expr: String, + _args: Option<&Json>>>, + _job_id: Uuid, + _job_timeout: Option, + _db: &DB, + _mem_peak: &mut i32, + _canceled_by: &mut Option, + _worker_name: &str, + _w_id: &str, + _load_client: bool, + _occupation_metrics: &mut OccupancyMetrics, +) -> anyhow::Result<(Box, String)> { + use serde_json::value::to_raw_value; + Ok((to_raw_value("require deno_core").unwrap(), "".to_string())) +} +#[cfg(feature = "deno_core")] pub async fn eval_fetch_timeout( env_code: String, ts_expr: String, @@ -851,8 +935,10 @@ pub async fn eval_fetch_timeout( Ok((res, format!("{extra_logs}{logs}"))) } +#[cfg(feature = "deno_core")] const WINDMILL_CLIENT: &str = include_str!("./windmill-client.js"); +#[cfg(feature = "deno_core")] async fn eval_fetch( js_runtime: &mut JsRuntime, expr: &str, @@ -898,6 +984,7 @@ import("file:///eval.ts").then((module) => module.main(...args)).then(JSON.strin Ok(unsafe_raw(r.unwrap_or_else(|| "null".to_string()))) } +#[cfg(feature = "deno_core")] #[op2] #[serde] fn op_get_static_args(op_state: Rc>) -> Vec> { @@ -910,6 +997,7 @@ fn op_get_static_args(op_state: Rc>) -> Vec> { .collect_vec() } +#[cfg(feature = "deno_core")] #[op2(fast)] fn op_log(op_state: Rc>, #[string] log: &str) { // tracing::error!("log: |{}|", log); @@ -920,6 +1008,7 @@ fn op_log(op_state: Rc>, #[string] log: &str) { .push_str(log); } +#[cfg(feature = "deno_core")] #[cfg(test)] mod tests { diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index f1c5bc3926..b790eb3faf 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -34,5 +34,7 @@ pub use worker::*; pub use result_processor::handle_job_error; -pub use bun_executor::{get_common_bun_proc_envs, install_bun_lockfile, prepare_job_dir}; +pub use bun_executor::{ + get_common_bun_proc_envs, install_bun_lockfile, prebundle_bun_script, prepare_job_dir, +}; pub use deno_executor::generate_deno_lock; diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index f50bad4619..7c4724418c 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -9,7 +9,7 @@ use tokio::net::TcpStream; use tokio_util::compat::TokioAsyncWriteCompatExt; use uuid::Uuid; use windmill_common::error::{self, Error}; -use windmill_common::worker::{get_sql_annotations, to_raw_value}; +use windmill_common::worker::to_raw_value; use windmill_common::{error::to_anyhow, jobs::QueuedJob}; use windmill_parser_sql::{parse_db_resource, parse_mssql_sig}; use windmill_queue::{append_logs, CanceledBy}; @@ -68,7 +68,7 @@ pub async fn do_mssql( return Err(Error::BadRequest("Missing database argument".to_string())); }; - let annotations = get_sql_annotations(query); + let annotations = windmill_common::worker::SqlAnnotations::parse(query); let mut config = Config::new(); diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index 03cde1d508..3ed6a9bf34 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -13,7 +13,7 @@ use tokio::sync::Mutex; use windmill_common::{ error::{to_anyhow, Error}, jobs::QueuedJob, - worker::{get_sql_annotations, to_raw_value}, + worker::to_raw_value, }; use windmill_parser_sql::{ parse_db_resource, parse_mysql_sig, parse_sql_blocks, parse_sql_statement_named_params, @@ -148,7 +148,7 @@ pub async fn do_mysql( return Err(Error::BadRequest("Missing database argument".to_string())); }; - let annotations = get_sql_annotations(query); + let annotations = windmill_common::worker::SqlAnnotations::parse(query); let opts = OptsBuilder::default() .db_name(Some(database.database)) diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 7c4192cf66..55f5abea31 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -18,19 +18,15 @@ use serde_json::value::RawValue; use serde_json::Map; use serde_json::Value; use tokio::sync::Mutex; -use tokio_postgres::types::IsNull; use tokio_postgres::Client; -use tokio_postgres::{ - types::{to_sql_checked, ToSql}, - NoTls, Row, -}; +use tokio_postgres::{types::ToSql, NoTls, Row}; use tokio_postgres::{ types::{FromSql, Type}, Column, }; use uuid::Uuid; use windmill_common::error::{self, Error}; -use windmill_common::worker::{get_sql_annotations, to_raw_value, CLOUD_HOSTED}; +use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; use windmill_common::{error::to_anyhow, jobs::QueuedJob}; use windmill_parser::{Arg, Typ}; use windmill_parser_sql::{ @@ -41,7 +37,7 @@ use windmill_queue::CanceledBy; use crate::common::{build_args_values, sizeof_val, OccupancyMetrics}; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::{AuthedClientBackgroundTask, MAX_RESULT_SIZE}; -use bytes::{Buf, BytesMut}; +use bytes::Buf; use lazy_static::lazy_static; use urlencoding::encode; @@ -85,7 +81,7 @@ fn do_postgresql_inner<'a>( let arg_t = arg .otyp .as_ref() - .ok_or_else(|| anyhow::anyhow!("Missing otyp for pg arg"))?; + .ok_or_else(|| anyhow::anyhow!("Missing otzyp for pg arg"))?; let typ = &arg.typ; let param = convert_val(value, arg_t, typ)?; query_params.push(param); @@ -98,6 +94,11 @@ fn do_postgresql_inner<'a>( let mut res: Vec = vec![]; + let query_params = query_params + .iter() + .map(|p| &**p as &(dyn ToSql + Sync)) + .collect_vec(); + if skip_collect { client .execute_raw(&query, query_params) @@ -191,7 +192,7 @@ pub async fn do_postgresql( return Err(Error::BadRequest("Missing database argument".to_string())); }; - let annotations = get_sql_annotations(query); + let annotations = windmill_common::worker::SqlAnnotations::parse(query); let sslmode = match database.sslmode.as_deref() { Some("allow") => "prefer".to_string(), @@ -414,133 +415,176 @@ pub async fn do_postgresql( return Ok(raw_result); } -#[derive(Debug)] -enum PgType { - String(String), - Bool(bool), - I8(i8), - I16(i16), - I32(i32), - I64(i64), - U32(u32), - F32(f32), - F64(f64), - Uuid(Uuid), - Decimal(Decimal), - Date(chrono::NaiveDate), - Time(chrono::NaiveTime), - Timestamp(chrono::NaiveDateTime), - None(Option), - Array(Vec), - Json(serde_json::Value), - Bytea(Vec), +fn map_as_single_type( + vec: &Vec, + f: impl Fn(&Value) -> Option, +) -> anyhow::Result>> { + vec.into_iter() + .map(|v| { + // allow nulls in arrays + if matches!(v, Value::Null) { + Some(None) + } else { + f(v).map(Some) + } + }) + .collect::>>>() + .ok_or_else(|| anyhow::anyhow!("Mixed types in array")) } -impl ToSql for PgType { - fn to_sql( - &self, - ty: &Type, - out: &mut BytesMut, - ) -> Result> { - match *self { - PgType::String(ref val) => val.to_sql(ty, out), - PgType::Bool(ref val) => val.to_sql(ty, out), - PgType::I8(ref val) => val.to_sql(ty, out), - PgType::I16(ref val) => val.to_sql(ty, out), - PgType::I32(ref val) => val.to_sql(ty, out), - PgType::I64(ref val) => val.to_sql(ty, out), - PgType::U32(ref val) => val.to_sql(ty, out), - PgType::F32(ref val) => val.to_sql(ty, out), - PgType::F64(ref val) => val.to_sql(ty, out), - PgType::Uuid(ref val) => val.to_sql(ty, out), - PgType::Decimal(ref val) => val.to_sql(ty, out), - PgType::Date(ref val) => val.to_sql(ty, out), - PgType::Time(ref val) => val.to_sql(ty, out), - PgType::Timestamp(ref val) => val.to_sql(ty, out), - PgType::None(ref val) => val.to_sql(ty, out), - PgType::Array(ref val) => val.to_sql(ty, out), - PgType::Json(ref val) => val.to_sql(ty, out), - PgType::Bytea(ref val) => val.to_sql(ty, out), +fn convert_vec_val( + vec: &Vec, + arg_t: &String, +) -> windmill_common::error::Result> { + match arg_t.as_str() { + "bool" | "boolean" => Ok(Box::new(map_as_single_type(vec, |v| v.as_bool())?)), + "char" | "character" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_i64().map(|x| x as i8) + })?)), + "smallint" | "smallserial" | "int2" | "serial2" => { + Ok(Box::new(map_as_single_type(vec, |v| { + v.as_i64().map(|x| x as i16) + })?)) } + "int" | "integer" | "int4" | "serial" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_i64().map(|x| x as i32) + })?)), + "numeric" | "decimal" => Ok(Box::new(map_as_single_type(vec, |v| { + if v.is_i64() { + Decimal::from_i64(v.as_i64().unwrap()) + } else if v.is_f64() { + Decimal::from_f64(v.as_f64().unwrap()) + } else { + None + } + })?)), + "oid" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_u64().map(|x| x as u32) + })?)), + "bigint" | "bigserial" | "int8" | "serial8" => { + Ok(Box::new(map_as_single_type(vec, |v| { + v.as_u64().map(|x| x as i64) + })?)) + } + "real" | "float4" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_f64().map(|x| x as f32) + })?)), + "double" | "float8" => Ok(Box::new(map_as_single_type(vec, |v| v.as_f64())?)), + "uuid" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| Uuid::parse_str(x).ok()).flatten() + })?)), + "date" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| { + chrono::NaiveDate::parse_from_str(x, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default() + }) + })?)), + "time" | "timetz" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| { + chrono::NaiveTime::parse_from_str(x, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default() + }) + })?)), + "timestamp" | "timestamptz" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| { + chrono::NaiveDateTime::parse_from_str(x, "%Y-%m-%dT%H:%M:%S.%3fZ") + .unwrap_or_default() + }) + })?)), + "jsonb" | "json" => Ok(Box::new(vec.clone().into_iter().map(Some).collect_vec())), + "bytea" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| { + engine::general_purpose::STANDARD + .decode(x) + .unwrap_or(vec![]) + }) + })?)), + "text" | "varchar" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| x.to_string()) + })?)), + _ => Err(anyhow::anyhow!("Unsupported JSON array type"))?, } - - fn accepts(_: &Type) -> bool { - true - } - - to_sql_checked!(); } -fn convert_val(value: &Value, arg_t: &String, typ: &Typ) -> windmill_common::error::Result { +fn convert_val( + value: &Value, + arg_t: &String, + typ: &Typ, +) -> windmill_common::error::Result> { match value { Value::Array(vec) if arg_t.ends_with("[]") => { let arg_t = arg_t.trim_end_matches("[]").to_string(); - let mut result = vec![]; - for val in vec { - result.push(convert_val(val, &arg_t, typ)?); - } - Ok(PgType::Array(result)) + convert_vec_val(vec, &arg_t) } - Value::Null => Ok(PgType::None(None::)), - Value::Bool(b) => Ok(PgType::Bool(b.clone())), - Value::Number(n) if matches!(typ, Typ::Str(_)) => Ok(PgType::String(n.to_string())), - Value::Number(n) if n.is_i64() && arg_t == "char" => { - Ok(PgType::I8(n.as_i64().unwrap() as i8)) - } - Value::Number(n) if n.is_i64() && (arg_t == "smallint" || arg_t == "smallserial") => { - Ok(PgType::I16(n.as_i64().unwrap() as i16)) + Value::Null => Ok(Box::new(None::)), + Value::Bool(b) => Ok(Box::new(b.clone())), + Value::Number(n) if matches!(typ, Typ::Str(_)) => Ok(Box::new(n.to_string())), + Value::Number(n) if arg_t == "char" && n.is_i64() => { + Ok(Box::new(n.as_i64().unwrap() as i8)) } Value::Number(n) - if n.is_i64() - && (arg_t == "int" - || arg_t == "integer" - || arg_t == "int4" - || arg_t == "serial") => + if (arg_t == "smallint" + || arg_t == "smallserial" + || arg_t == "int2" + || arg_t == "serial2") + && n.is_i64() => { - Ok(PgType::I32(n.as_i64().unwrap() as i32)) + Ok(Box::new(n.as_i64().unwrap() as i16)) } - Value::Number(n) if n.is_i64() && (arg_t == "numeric" || arg_t == "decimal") => Ok( - PgType::Decimal(Decimal::from_i64(n.as_i64().unwrap()).unwrap()), + Value::Number(n) + if (arg_t == "int" || arg_t == "integer" || arg_t == "int4" || arg_t == "serial") + && n.is_i64() => + { + Ok(Box::new(n.as_i64().unwrap() as i32)) + } + Value::Number(n) if (arg_t == "real" || arg_t == "float4") && n.as_f64().is_some() => { + Ok(Box::new(n.as_f64().unwrap() as f32)) + } + Value::Number(n) if (arg_t == "double" || arg_t == "float8") && n.as_f64().is_some() => { + Ok(Box::new(n.as_f64().unwrap())) + } + Value::Number(n) if (arg_t == "numeric" || arg_t == "decimal") && n.is_i64() => Ok( + Box::new(Decimal::from_i64(n.as_i64().unwrap()).unwrap_or_default()), ), - Value::Number(n) if n.is_i64() => Ok(PgType::I64(n.as_i64().unwrap())), - Value::Number(n) if n.is_u64() && arg_t == "oid" => { - Ok(PgType::U32(n.as_u64().unwrap() as u32)) - } - Value::Number(n) if n.is_u64() && (arg_t == "bigint" || arg_t == "bigserial") => { - Ok(PgType::I64(n.as_u64().unwrap() as i64)) - } - Value::Number(n) if n.is_f64() && arg_t == "real" => { - Ok(PgType::F32(n.as_f64().unwrap() as f32)) - } - Value::Number(n) if n.is_f64() && arg_t == "double" => Ok(PgType::F64(n.as_f64().unwrap())), - Value::Number(n) if n.is_f64() && (arg_t == "numeric" || arg_t == "decimal") => Ok( - PgType::Decimal(Decimal::from_f64(n.as_f64().unwrap()).unwrap()), + Value::Number(n) if (arg_t == "numeric" || arg_t == "decimal") && n.is_f64() => Ok( + Box::new(Decimal::from_f64(n.as_f64().unwrap()).unwrap_or_default()), ), - Value::Number(n) => Ok(PgType::F64(n.as_f64().unwrap())), - Value::String(s) if arg_t == "uuid" => Ok(PgType::Uuid(Uuid::parse_str(s)?)), + Value::Number(n) if arg_t == "oid" && n.is_u64() => { + Ok(Box::new(n.as_u64().unwrap() as u32)) + } + Value::Number(n) + if (arg_t == "bigint" + || arg_t == "bigserial" + || arg_t == "int8" + || arg_t == "serial8") + && n.is_u64() => + { + Ok(Box::new(n.as_u64().unwrap() as i64)) + } + Value::Number(n) if n.is_i64() => Ok(Box::new(n.as_i64().unwrap())), + Value::Number(n) => Ok(Box::new(n.as_f64().unwrap())), + Value::String(s) if arg_t == "uuid" => Ok(Box::new(Uuid::parse_str(s)?)), Value::String(s) if arg_t == "date" => { let date = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default(); - Ok(PgType::Date(date)) + Ok(Box::new(date)) } Value::String(s) if arg_t == "time" || arg_t == "timetz" => { let time = chrono::NaiveTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default(); - Ok(PgType::Time(time)) + Ok(Box::new(time)) } Value::String(s) if arg_t == "timestamp" || arg_t == "timestamptz" => { let datetime = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ") .unwrap_or_default(); - Ok(PgType::Timestamp(datetime)) + Ok(Box::new(datetime)) } Value::String(s) if arg_t == "bytea" => { let bytes = engine::general_purpose::STANDARD .decode(s) .unwrap_or(vec![]); - Ok(PgType::Bytea(bytes)) + Ok(Box::new(bytes)) } - Value::Object(_) => Ok(PgType::Json(value.clone())), - Value::String(s) => Ok(PgType::String(s.clone())), + Value::Object(_) => Ok(Box::new(value.clone())), + Value::String(s) => Ok(Box::new(s.clone())), _ => Err(Error::ExecutionErr(format!( "Unsupported type in query: {:?} and signature {arg_t:?}", value @@ -624,6 +668,9 @@ pub fn pg_cell_to_json_value( Type::TS_VECTOR => get_basic(row, column, column_i, |a: StringCollector| { Ok(JSONValue::String(a.0)) })?, + Type::OID => get_basic(row, column, column_i, |a: u32| { + Ok(JSONValue::Number(serde_json::Number::from(a))) + })?, // array types Type::BOOL_ARRAY => get_array(row, column, column_i, |a: bool| Ok(JSONValue::Bool(a)))?, Type::BIT_ARRAY => get_array(row, column, column_i, |a: bit_vec::BitVec| match a.len() { @@ -655,6 +702,10 @@ pub fn pg_cell_to_json_value( Type::FLOAT8_ARRAY => { get_array(row, column, column_i, |a: f64| Ok(f64_to_json_number(a)?))? } + Type::NUMERIC_ARRAY => get_array(row, column, column_i, |a: Decimal| { + Ok(serde_json::to_value(a) + .map_err(|_| anyhow::anyhow!("Cannot convert decimal to json"))?) + })?, // these types require a custom StringCollector struct as an intermediary (see struct at bottom) Type::TS_VECTOR_ARRAY => get_array(row, column, column_i, |a: StringCollector| { Ok(JSONValue::String(a.0)) @@ -766,7 +817,7 @@ fn get_array<'a, T: FromSql<'a>>( val_to_json_val: impl Fn(T) -> Result, ) -> Result { let raw_val_array = row - .try_get::<_, Option>>(column_i) + .try_get::<_, Option>>>(column_i) .with_context(|| { format!( "conversion issue for array at column_name `{}`", @@ -777,7 +828,11 @@ fn get_array<'a, T: FromSql<'a>>( Some(val_array) => { let mut result = vec![]; for val in val_array { - result.push(val_to_json_val(val)?); + result.push( + val.map(|v| val_to_json_val(v)) + .transpose()? + .unwrap_or(Value::Null), + ); } JSONValue::Array(result) } diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index e62d96b3fd..194e621fe1 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -143,7 +143,7 @@ fn check_php_exists() -> error::Result<()> { #[cfg(feature = "enterprise")] fn check_php_exists() -> error::Result<()> { if !Path::new(PHP_PATH.as_str()).exists() { - let msg = format!("Couldn't find php at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run php jobs.", PHP_PATH.as_str()); + let msg = format!("Couldn't find php at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-ee-full` for your instance in order to run php jobs.", PHP_PATH.as_str()); return Err(error::Error::NotFound(msg)); } Ok(()) @@ -345,7 +345,7 @@ try {{ mem_peak, canceled_by, child, - false, + !*DISABLE_NSJAIL, worker_name, &job.workspace_id, "php run", diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 87c1c829ae..112b3c84ca 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -29,6 +29,9 @@ lazy_static::lazy_static! { static ref PYTHON_PATH: String = std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string()); + static ref UV_PATH: String = + std::env::var("UV_PATH").unwrap_or_else(|_| "/usr/local/bin/uv".to_string()); + static ref FLOCK_PATH: String = std::env::var("FLOCK_PATH").unwrap_or_else(|_| "/usr/bin/flock".to_string()); static ref NON_ALPHANUM_CHAR: Regex = regex::Regex::new(r"[^0-9A-Za-z=.-]").unwrap(); @@ -36,6 +39,9 @@ lazy_static::lazy_static! { static ref PIP_TRUSTED_HOST: Option = std::env::var("PIP_TRUSTED_HOST").ok(); static ref PIP_INDEX_CERT: Option = std::env::var("PIP_INDEX_CERT").ok(); + static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE") + .ok().map(|flag| flag == "true").unwrap_or(false); + static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap(); @@ -59,11 +65,14 @@ use crate::{ read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, - AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY, - LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, - PIP_INDEX_URL, TZ_ENV, + AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, LOCK_CACHE_DIR, + NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, TZ_ENV, + UV_CACHE_DIR, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + pub async fn create_dependencies_dir(job_dir: &str) { DirBuilder::new() .recursive(true) @@ -93,7 +102,7 @@ pub fn handle_ephemeral_token(x: String) -> String { x } -pub async fn pip_compile( +pub async fn uv_pip_compile( job_id: &Uuid, requirements: &str, mem_peak: &mut i32, @@ -103,6 +112,10 @@ pub async fn pip_compile( worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + // Fallback to pip-compile. Will be removed in future + mut no_uv: bool, + // Debug-only flag + no_cache: bool, ) -> error::Result { let mut logs = String::new(); logs.push_str(&format!("\nresolving dependencies...")); @@ -139,83 +152,184 @@ pub async fn pip_compile( #[cfg(feature = "enterprise")] let requirements = replace_pip_secret(db, w_id, &requirements, worker_name, job_id).await?; - let req_hash = format!("py-{}", calculate_hash(&requirements)); - if let Some(cached) = sqlx::query_scalar!( - "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", - req_hash - ) - .fetch_optional(db) - .await? - { - logs.push_str(&format!("\nfound cached resolution: {req_hash}")); - return Ok(cached); + let mut req_hash = format!("py-{}", calculate_hash(&requirements)); + + if no_uv || *USE_PIP_COMPILE { + logs.push_str(&format!("\nFallback to pip-compile (Deprecated!)")); + // Set no_uv if not setted + no_uv = true; + // Make sure that if we put #no_uv (switch to pip-compile) to python code or used `USE_PIP_COMPILE=true` variable. + // Windmill will recalculate lockfile using pip-compile and dont take potentially broken lockfile (generated by uv) from cache (our db). + // It will recalculate lockfile even if inputs have not been changed. + req_hash.push_str("-no_uv"); + // Will be in format: + // py-000..000-no_uv + } + if !no_cache { + if let Some(cached) = sqlx::query_scalar!( + "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", + req_hash + ) + .fetch_optional(db) + .await? + { + logs.push_str(&format!("\nfound cached resolution: {req_hash}")); + return Ok(cached); + } } let file = "requirements.in"; write_file(job_dir, file, &requirements)?; - let mut args = vec![ - "-q", - "--no-header", - file, - "--resolver=backtracking", - "--strip-extras", - ]; - let mut pip_args = vec![]; - let pip_extra_index_url = PIP_EXTRA_INDEX_URL - .read() - .await - .clone() - .map(handle_ephemeral_token); - if let Some(url) = pip_extra_index_url.as_ref() { - args.extend(["--extra-index-url", url, "--no-emit-index-url"]); - pip_args.push(format!("--extra-index-url {}", url)); - } - let pip_index_url = PIP_INDEX_URL - .read() - .await - .clone() - .map(handle_ephemeral_token); - if let Some(url) = pip_index_url.as_ref() { - args.extend(["--index-url", url, "--no-emit-index-url"]); - pip_args.push(format!("--index-url {}", url)); - } - if let Some(host) = PIP_TRUSTED_HOST.as_ref() { - args.extend(["--trusted-host", host]); - } - if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { - args.extend(["--cert", cert_path]); - } - let pip_args_str = pip_args.join(" "); - if pip_args.len() > 0 { - args.extend(["--pip-args", &pip_args_str]); - } - tracing::debug!("pip-compile args: {:?}", args); + // Fallback pip-compile. Will be removed in future + if no_uv { + tracing::debug!("Fallback to pip-compile"); + + let mut args = vec![ + "-q", + "--no-header", + file, + "--resolver=backtracking", + "--strip-extras", + ]; + let mut pip_args = vec![]; + let pip_extra_index_url = PIP_EXTRA_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); + if let Some(url) = pip_extra_index_url.as_ref() { + args.extend(["--extra-index-url", url, "--no-emit-index-url"]); + pip_args.push(format!("--extra-index-url {}", url)); + } + let pip_index_url = PIP_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); + if let Some(url) = pip_index_url.as_ref() { + args.extend(["--index-url", url, "--no-emit-index-url"]); + pip_args.push(format!("--index-url {}", url)); + } + if let Some(host) = PIP_TRUSTED_HOST.as_ref() { + args.extend(["--trusted-host", host]); + } + if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { + args.extend(["--cert", cert_path]); + } + let pip_args_str = pip_args.join(" "); + if pip_args.len() > 0 { + args.extend(["--pip-args", &pip_args_str]); + } + tracing::debug!("pip-compile args: {:?}", args); + + let mut child_cmd = Command::new("pip-compile"); + child_cmd + .current_dir(job_dir) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let child_process = start_child_process(child_cmd, "pip-compile").await?; + append_logs(&job_id, &w_id, logs, db).await; + handle_child( + job_id, + db, + mem_peak, + canceled_by, + child_process, + false, + worker_name, + &w_id, + "pip-compile", + None, + false, + occupancy_metrics, + ) + .await + .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; + } else { + let mut args = vec![ + "pip", + "compile", + "-q", + "--no-header", + file, + "--strip-extras", + "-o", + "requirements.txt", + // Prefer main index over extra + // https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes + // TODO: Use env variable that can be toggled from UI + "--index-strategy", + "unsafe-best-match", + // Target to /tmp/windmill/cache/uv + "--cache-dir", + UV_CACHE_DIR, + // We dont want UV to manage python installations + "--python-preference", + "only-system", + "--no-python-downloads", + ]; + if no_cache { + args.extend(["--no-cache"]); + } + let pip_extra_index_url = PIP_EXTRA_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); + if let Some(url) = pip_extra_index_url.as_ref() { + args.extend(["--extra-index-url", url]); + } + let pip_index_url = PIP_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); + if let Some(url) = pip_index_url.as_ref() { + args.extend(["--index-url", url]); + } + if let Some(host) = PIP_TRUSTED_HOST.as_ref() { + args.extend(["--trusted-host", host]); + } + if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { + args.extend(["--cert", cert_path]); + } + tracing::debug!("uv args: {:?}", args); + + #[cfg(windows)] + let uv_cmd = "uv"; + + #[cfg(unix)] + let uv_cmd = UV_PATH.as_str(); + + let mut child_cmd = Command::new(uv_cmd); + child_cmd + .current_dir(job_dir) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let child_process = start_child_process(child_cmd, "/usr/local/bin/uv").await?; + append_logs(&job_id, &w_id, logs, db).await; + handle_child( + job_id, + db, + mem_peak, + canceled_by, + child_process, + false, + worker_name, + &w_id, + // TODO: Rename to uv-pip-compile? + "uv", + None, + false, + occupancy_metrics, + ) + .await + .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; + } - let mut child_cmd = Command::new("pip-compile"); - child_cmd - .current_dir(job_dir) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let child_process = start_child_process(child_cmd, "pip-compile").await?; - append_logs(&job_id, &w_id, logs, db).await; - handle_child( - job_id, - db, - mem_peak, - canceled_by, - child_process, - false, - worker_name, - &w_id, - "pip-compile", - None, - false, - occupancy_metrics, - ) - .await - .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; let path_lock = format!("{job_dir}/requirements.txt"); let mut file = File::open(path_lock).await?; let mut req_content = "".to_string(); @@ -384,7 +498,10 @@ except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.format_tb(exc_traceback) with open(result_json, 'w') as f: - err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }} + err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }} + extra = e.__dict__ + if extra and len(extra) > 0: + err['extra'] = extra flow_node_id = os.environ.get('WM_FLOW_STEP_ID') if flow_node_id: err['step_id'] = flow_node_id @@ -399,6 +516,9 @@ except BaseException as e: let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?; let additional_python_paths_folders = additional_python_paths.iter().join(":"); + #[cfg(windows)] + let additional_python_paths_folders = additional_python_paths_folders.replace(":", ";"); + if !*DISABLE_NSJAIL { let shared_deps = additional_python_paths .into_iter() @@ -445,6 +565,7 @@ mount {{ .env_clear() // inject PYTHONPATH here - for some reason I had to do it in nsjail conf .envs(reserved_variables) + .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -475,6 +596,10 @@ mount {{ .args(vec!["-u", "-m", "wrapper"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + python_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + start_child_process(python_cmd, PYTHON_PATH.as_str()).await? }; @@ -774,6 +899,7 @@ async fn handle_python_deps( let requirements = match requirements_o { Some(r) => r, None => { + let annotation = windmill_common::worker::PythonAnnotations::parse(inner_content); let mut already_visited = vec![]; let requirements = windmill_parser_py_imports::parse_python_imports( @@ -788,7 +914,7 @@ async fn handle_python_deps( if requirements.is_empty() { "".to_string() } else { - pip_compile( + uv_pip_compile( job_id, &requirements, mem_peak, @@ -798,6 +924,8 @@ async fn handle_python_deps( worker_name, w_id, occupancy_metrics, + annotation.no_uv, + annotation.no_cache, ) .await .map_err(|e| { @@ -876,15 +1004,6 @@ pub async fn handle_python_reqs( if let Some(host) = PIP_TRUSTED_HOST.as_ref() { vars.push(("TRUSTED_HOST", host)); } - if let Some(http_proxy) = HTTP_PROXY.as_ref() { - vars.push(("HTTP_PROXY", http_proxy)); - } - if let Some(https_proxy) = HTTPS_PROXY.as_ref() { - vars.push(("HTTPS_PROXY", https_proxy)); - } - if let Some(no_proxy) = NO_PROXY.as_ref() { - vars.push(("NO_PROXY", no_proxy)); - } let _ = write_file( job_dir, @@ -899,6 +1018,9 @@ pub async fn handle_python_reqs( let mut req_with_penv: Vec<(String, String)> = vec![]; for req in requirements { + if req.starts_with('#') { + continue; + } let venv_p = format!( "{PIP_CACHE_DIR}/{}", req.replace(' ', "").replace('/', "").replace(':', "") @@ -1010,13 +1132,19 @@ pub async fn handle_python_reqs( .current_dir(job_dir) .env_clear() .envs(vars) + .envs(PROXY_ENVS.clone()) .args(vec!["--config", "download.config.proto"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { let fssafe_req = NON_ALPHANUM_CHAR.replace_all(&req, "_").to_string(); + #[cfg(unix)] let req = format!("'{}'", req); + + #[cfg(windows)] + let req = format!("{}", req); + let mut command_args = vec![ PYTHON_PATH.as_str(), "-m", @@ -1058,33 +1186,42 @@ pub async fn handle_python_reqs( } let mut envs = vec![("PATH", PATH_ENV.as_str())]; - if let Some(http_proxy) = HTTP_PROXY.as_ref() { - envs.push(("HTTP_PROXY", http_proxy)); - } - if let Some(https_proxy) = HTTPS_PROXY.as_ref() { - envs.push(("HTTPS_PROXY", https_proxy)); - } - if let Some(no_proxy) = NO_PROXY.as_ref() { - envs.push(("NO_PROXY", no_proxy)); - } envs.push(("HOME", HOME_ENV.as_str())); tracing::debug!("pip install command: {:?}", command_args); - let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); - flock_cmd - .env_clear() - .envs(envs) - .args([ - "-x", - &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req), - "--command", - &command_args.join(" "), - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - start_child_process(flock_cmd, FLOCK_PATH.as_str()).await? + #[cfg(unix)] + { + let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); + flock_cmd + .env_clear() + .envs(PROXY_ENVS.clone()) + .envs(envs) + .args([ + "-x", + &format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req), + "--command", + &command_args.join(" "), + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + start_child_process(flock_cmd, FLOCK_PATH.as_str()).await? + } + + #[cfg(windows)] + { + let mut pip_cmd = Command::new(PYTHON_PATH.as_str()); + pip_cmd + .env_clear() + .envs(envs) + .envs(PROXY_ENVS.clone()) + .env("SystemRoot", SYSTEM_ROOT.as_str()) + .args(&command_args[1..]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + start_child_process(pip_cmd, PYTHON_PATH.as_str()).await? + } }; let child = handle_child( diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index ab6fdd247f..9391fd9a69 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -20,15 +20,31 @@ use crate::{ }, handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - RUST_CACHE_DIR, TZ_ENV, + PROXY_ENVS, RUST_CACHE_DIR, TZ_ENV, }; +#[cfg(windows)] +use crate::SYSTEM_ROOT; + const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.config.proto"); lazy_static::lazy_static! { - static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| "/usr/local/cargo".to_string()); - static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| "/usr/local/rustup".to_string()); - static ref CARGO_PATH: String = format!("{}/bin/cargo", std::env::var("CARGO_HOME").unwrap_or("/usr/local/cargo/bin/cargo".to_string())); + static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable"); + static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() }); + static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() }); + static ref CARGO_PATH: String = format!("{}/bin/cargo", CARGO_HOME.as_str()); +} + +#[cfg(windows)] +lazy_static::lazy_static! { + static ref CARGO_HOME_DEFAULT: String = format!("{}\\.cargo", *HOME_DIR); + static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR); +} + +#[cfg(unix)] +lazy_static::lazy_static! { + static ref CARGO_HOME_DEFAULT: String = "/usr/local/cargo".to_string(); + static ref RUSTUP_HOME_DEFAULT: String = "/usr/local/rustup".to_string(); } const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/"; @@ -126,6 +142,14 @@ pub async fn generate_cargo_lockfile( .args(vec!["generate-lockfile"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + { + gen_lockfile_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + gen_lockfile_cmd.env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), + ); + } let gen_lockfile_process = start_child_process(gen_lockfile_cmd, CARGO_PATH.as_str()).await?; handle_child( job_id, @@ -168,6 +192,7 @@ pub async fn build_rust_crate( build_rust_cmd .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) @@ -176,6 +201,16 @@ pub async fn build_rust_crate( .args(vec!["build", "--release"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + { + build_rust_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + build_rust_cmd.env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), + ); + } + let build_rust_process = start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await?; handle_child( job_id, @@ -247,7 +282,7 @@ fn check_cargo_exists() -> Result<(), Error> { #[cfg(feature = "enterprise")] fn check_cargo_exists() -> Result<(), Error> { if !Path::new(CARGO_PATH.as_str()).exists() { - let msg = format!("Couldn't find cargo at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run rust jobs.", CARGO_PATH.as_str()); + let msg = format!("Couldn't find cargo at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-ee-full` for your instance in order to run rust jobs.", CARGO_PATH.as_str()); return Err(Error::NotFound(msg)); } Ok(()) @@ -279,7 +314,13 @@ pub async fn handle_rust_job( let cache_logs = if cache { let target = format!("{job_dir}/main"); - std::os::unix::fs::symlink(&bin_path, &target).map_err(|e| { + + #[cfg(unix)] + let symlink = std::os::unix::fs::symlink(&bin_path, &target); + #[cfg(windows)] + let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target); + + symlink.map_err(|e| { Error::ExecutionErr(format!( "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" )) @@ -360,6 +401,9 @@ pub async fn handle_rust_job( .stdout(Stdio::piped()) .stderr(Stdio::piped()); + #[cfg(windows)] + run_rust.env("SystemRoot", SYSTEM_ROOT.as_str()); + start_child_process(run_rust, compiled_executable_name).await? }; handle_child( diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index bf9dae41ff..f89b29832a 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -9,7 +9,6 @@ use serde_json::{json, value::RawValue, Value}; use sha2::{Digest, Sha256}; use std::collections::HashMap; use windmill_common::error::to_anyhow; -use windmill_common::worker::get_sql_annotations; use windmill_common::jobs::QueuedJob; use windmill_common::{error::Error, worker::to_raw_value}; @@ -266,7 +265,7 @@ pub async fn do_snowflake( return Err(Error::BadRequest("Missing database argument".to_string())); }; - let annotations = get_sql_annotations(query); + let annotations = windmill_common::worker::SqlAnnotations::parse(query); let qualified_username = format!( "{}.{}", diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 6c0fce3d3b..f4d78bf0ef 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -237,6 +237,7 @@ pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock"); pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip"); +pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip"); pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); pub const DENO_CACHE_DIR_DEPS: &str = concatcp!(ROOT_CACHE_DIR, "deno/deps"); @@ -323,6 +324,20 @@ lazy_static::lazy_static! { pub static ref NO_PROXY: Option = std::env::var("no_proxy").ok().or(std::env::var("NO_PROXY").ok()); pub static ref HTTP_PROXY: Option = std::env::var("http_proxy").ok().or(std::env::var("HTTP_PROXY").ok()); pub static ref HTTPS_PROXY: Option = std::env::var("https_proxy").ok().or(std::env::var("HTTPS_PROXY").ok()); + + pub static ref PROXY_ENVS: Vec<(&'static str, String)> = { + let mut proxy_env = Vec::new(); + if let Some(no_proxy) = NO_PROXY.as_ref() { + proxy_env.push(("NO_PROXY", no_proxy.to_string())); + } + if let Some(http_proxy) = HTTP_PROXY.as_ref() { + proxy_env.push(("HTTP_PROXY", http_proxy.to_string())); + } + if let Some(https_proxy) = HTTPS_PROXY.as_ref() { + proxy_env.push(("HTTPS_PROXY", https_proxy.to_string())); + } + proxy_env + }; pub static ref DENO_PATH: String = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string()); pub static ref BUN_PATH: String = std::env::var("BUN_PATH").unwrap_or_else(|_| "/usr/bin/bun".to_string()); pub static ref NPM_PATH: String = std::env::var("NPM_PATH").unwrap_or_else(|_| "/usr/bin/npm".to_string()); @@ -348,7 +363,6 @@ lazy_static::lazy_static! { pub static ref PIP_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); pub static ref JOB_DEFAULT_TIMEOUT: Arc>> = Arc::new(RwLock::new(None)); - static ref MAX_TIMEOUT: u64 = std::env::var("TIMEOUT") .ok() .and_then(|x| x.parse::().ok()) @@ -389,6 +403,12 @@ lazy_static::lazy_static! { } + +#[cfg(windows)] +lazy_static::lazy_static! { + pub static ref SYSTEM_ROOT: String = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string()); +} + //only matter if CLOUD_HOSTED pub const MAX_RESULT_SIZE: usize = 1024 * 1024 * 2; // 2MB @@ -1267,16 +1287,16 @@ pub async fn run_worker { + #[cfg(feature = "prometheus")] + if let Some(wb) = worker_busy.as_ref() { + wb.set(1); + tracing::debug!("set worker busy to 1"); + } + + occupancy_metrics.running_job_started_at = Some(Instant::now()); + last_executed_job = None; jobs_executed += 1; @@ -2061,15 +2081,10 @@ pub fn build_envs( hm }; - if let Some(ref env) = *HTTPS_PROXY { - envs.insert("HTTPS_PROXY".to_string(), env.to_string()); - } - if let Some(ref env) = *HTTP_PROXY { - envs.insert("HTTP_PROXY".to_string(), env.to_string()); - } - if let Some(ref env) = *NO_PROXY { - envs.insert("NO_PROXY".to_string(), env.to_string()); + for (k, v) in PROXY_ENVS.iter() { + envs.insert(k.to_string(), v.to_string()); } + Ok(envs) } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 6474af6d46..a1a4fd9904 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -561,6 +561,7 @@ pub async fn update_flow_status_after_job_completion_internal< branch_chosen: None, approvers: vec![], failed_retries: vec![], + skipped: false, } } else { success = false; @@ -698,6 +699,20 @@ pub async fn update_flow_status_after_job_completion_internal< } } if success || (flow_jobs.is_some() && (skip_loop_failures || skip_branch_failure)) { + let is_skipped = if current_module.as_ref().is_some_and(|m| m.skip_if.is_some()) { + sqlx::query_scalar!( + "SELECT job_kind = 'identity' FROM completed_job WHERE id = $1", + job_id_for_status + ) + .fetch_one(db) + .await + .map_err(|e| { + Error::InternalErr(format!("error during skip check: {e:#}")) + })? + .unwrap_or(false) + } else { + false + }; success = true; ( true, @@ -709,6 +724,7 @@ pub async fn update_flow_status_after_job_completion_internal< branch_chosen, approvers: vec![], failed_retries: old_status.retry.failed_jobs.clone(), + skipped: is_skipped, }), ) } else { @@ -2210,6 +2226,23 @@ async fn push_next_flow_job drop(resume_messages); + let is_skipped = if let Some(skip_if) = &module.skip_if { + let idcontext = get_transform_context(&flow_job, previous_id.as_str(), &status).await?; + compute_bool_from_expr( + skip_if.expr.to_string(), + arc_flow_job_args.clone(), + arc_last_job_result.clone(), + None, + Some(idcontext.clone()), + Some(client), + Some((resumes.clone(), resume.clone(), approvers.clone())), + None, + ) + .await? + } else { + false + }; + let args: windmill_common::error::Result<_> = if module.mock.is_some() && module.mock.as_ref().unwrap().enabled { let mut hm = HashMap::new(); @@ -2249,7 +2282,16 @@ async fn push_next_flow_job ); Ok(Marc::new(hm)) } else { - match &module.get_value() { + let value = module.get_value(); + match &value { + Ok(_) if matches!(value, Ok(FlowModuleValue::Identity)) || is_skipped => serde_json::from_str( + &serde_json::to_string(&PreviousResult { + previous_result: Some(&arc_last_job_result), + }) + .unwrap(), + ) + .map(Marc::new) + .map_err(|e| error::Error::InternalErr(format!("identity: {e:#}"))), Ok( FlowModuleValue::Script { input_transforms, .. } | FlowModuleValue::RawScript { input_transforms, .. } @@ -2270,16 +2312,7 @@ async fn push_next_flow_job ) .await .map(Marc::new) - } - Ok(FlowModuleValue::Identity) => serde_json::from_str( - &serde_json::to_string(&PreviousResult { - previous_result: Some(&arc_last_job_result), - }) - .unwrap(), - ) - .map(Marc::new) - .map_err(|e| error::Error::InternalErr(format!("identity: {e:#}"))), - + }, Ok(_) => Ok(arc_flow_job_args.clone()), Err(e) => { return Err(error::Error::InternalErr(format!( @@ -2305,6 +2338,7 @@ async fn push_next_flow_job resumes.clone(), resume.clone(), approvers.clone(), + is_skipped, ) .await?; tracing::info!(id = %flow_job.id, root_id = %job_root, "next flow transform computed"); @@ -2326,6 +2360,7 @@ async fn push_next_flow_job branch_chosen: None, approvers: vec![], failed_retries: vec![], + skipped: false, })) .bind(flow_job.id) .execute(db) @@ -2368,15 +2403,15 @@ async fn push_next_flow_job } tracing::debug!(id = %flow_job.id, root_id = %job_root, "pushing job {i} of {len}"); let payload_tag = match &job_payloads { - ContinuePayload::SingleJob(payload) => payload.clone(), - ContinuePayload::BranchAllJobs(payloads) => payloads[i].clone(), + ContinuePayload::SingleJob(payload) => payload, + ContinuePayload::BranchAllJobs(payloads) => &payloads[i], ContinuePayload::ForloopJobs { flow_value, delete_after_use, .. } => { let mut fv = flow_value.clone(); if let Some(failure_module) = fv.failure_module.as_mut() { failure_module.id_append(&format!("{}-{i}", &status.step.to_string())); } - JobPayloadWithTag { + &JobPayloadWithTag { payload: JobPayload::RawFlow { value: fv, path: Some(format!("{}/forloop-{i}", flow_job.script_path())), @@ -2555,13 +2590,17 @@ async fn push_next_flow_job }; tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed perms for job {i} of {len}"); - + let tag = if flow_job.tag == "flow" || flow_job.tag == format!("flow-{}", flow_job.workspace_id) { + payload_tag.tag.clone() + } else { + Some(flow_job.tag.clone()) + }; let tx2 = PushIsolationLevel::Transaction(tx); let (uuid, mut inner_tx) = push( &db, tx2, &flow_job.workspace_id, - payload_tag.payload, + payload_tag.payload.clone(), push_args, &flow_job.created_by, &flow_job.email, @@ -2575,11 +2614,7 @@ async fn push_next_flow_job continue_on_same_worker, err, flow_job.visible_to_owner, - if flow_job.tag == "flow" || flow_job.tag == format!("flow-{}", flow_job.workspace_id) { - payload_tag.tag - } else { - Some(flow_job.tag.clone()) - }, + tag, payload_tag.timeout, Some(module.id.clone()), new_job_priority_override, @@ -2971,6 +3006,7 @@ async fn compute_next_flow_transform( resumes: Arc>, resume: Arc>, approvers: Arc>, + is_skipped: bool, ) -> error::Result { if module.mock.is_some() && module.mock.as_ref().unwrap().enabled { return Ok(NextFlowTransform::Continue( @@ -2997,6 +3033,9 @@ async fn compute_next_flow_transform( let delete_after_use = module.delete_after_use.unwrap_or(false); tracing::debug!(id = %flow_job.id, "computing next flow transform for {:?}", &module.value); + if is_skipped { + return trivial_next_job(JobPayload::Identity); + } match &module.get_value()? { FlowModuleValue::Identity => trivial_next_job(JobPayload::Identity), FlowModuleValue::Flow { path, .. } => { @@ -3490,6 +3529,7 @@ fn is_simple_modules(modules: &Vec, flow: &FlowValue) -> bool { && modules[0].retry.is_none() && modules[0].stop_after_if.is_none() && modules[0].stop_after_all_iters_if.is_none() + && modules[0].skip_if.is_none() && (modules[0].mock.is_none() || modules[0].mock.as_ref().is_some_and(|m| !m.enabled)) && flow.failure_module.is_none(); is_simple @@ -3721,6 +3761,11 @@ async fn script_to_payload( module: &FlowModule, tag_override: &Option, ) -> Result { + let tag_override = if tag_override.as_ref().is_some_and(|x| x.trim().is_empty()) { + None + } else { + tag_override.clone() + }; let (payload, tag, delete_after_use, script_timeout) = if script_hash.is_none() { let (jp, tag, delete_after_use, script_timeout) = script_path_to_payload(script_path, db, &flow_job.workspace_id, Some(true)).await?; diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 6f25ec606b..f71c73f618 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -12,7 +12,7 @@ use windmill_common::flows::{FlowModule, FlowModuleValue}; use windmill_common::get_latest_deployed_hash_for_path; use windmill_common::jobs::JobPayload; use windmill_common::scripts::ScriptHash; -use windmill_common::worker::{get_annotation, to_raw_value, to_raw_value_owned, write_file}; +use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file}; use windmill_common::{ error::{self, to_anyhow}, flows::FlowValue, @@ -26,7 +26,7 @@ use windmill_parser_ts::parse_expr_for_imports; use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel}; use crate::common::OccupancyMetrics; -use crate::python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}; +use crate::python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}; use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile}; use crate::{ bun_executor::gen_bun_lockfile, @@ -291,7 +291,7 @@ pub async fn handle_dependency_job( } if language == ScriptLang::Bun || language == ScriptLang::Bunnative { - let anns = get_annotation(&content); - if anns.native_mode && language == ScriptLang::Bun { + let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content); + if anns.native && language == ScriptLang::Bun { language = ScriptLang::Bunnative; - } else if !anns.native_mode && language == ScriptLang::Bunnative { + } else if !anns.native && language == ScriptLang::Bunnative { language = ScriptLang::Bun; }; } @@ -1003,10 +1003,10 @@ async fn lock_modules<'c>( fn skip_creating_new_lock(language: &ScriptLang, content: &str) -> bool { if language == &ScriptLang::Bun || language == &ScriptLang::Bunnative { - let anns = get_annotation(&content); - if anns.native_mode && language == &ScriptLang::Bun { + let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content); + if anns.native && language == &ScriptLang::Bun { return false; - } else if !anns.native_mode && language == &ScriptLang::Bunnative { + } else if !anns.native && language == &ScriptLang::Bunnative { return false; }; } @@ -1077,11 +1077,13 @@ async fn lock_modules_app( match new_lock { Ok(new_lock) => { append_logs(&job.id, &job.workspace_id, logs, db).await; - let anns = get_annotation(&content); - let nlang = if anns.native_mode && language == ScriptLang::Bun { + let anns = + windmill_common::worker::TypeScriptAnnotations::parse( + &content, + ); + let nlang = if anns.native && language == ScriptLang::Bun { Some(ScriptLang::Bunnative) - } else if !anns.native_mode && language == ScriptLang::Bunnative - { + } else if !anns.native && language == ScriptLang::Bunnative { Some(ScriptLang::Bun) } else { None @@ -1280,7 +1282,7 @@ async fn python_dep( occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> std::result::Result { create_dependencies_dir(job_dir).await; - let req: std::result::Result = pip_compile( + let req: std::result::Result = uv_pip_compile( job_id, &reqs, mem_peak, @@ -1290,6 +1292,8 @@ async fn python_dep( worker_name, w_id, occupancy_metrics, + false, + false, ) .await; // install the dependencies to pre-fill the cache @@ -1434,8 +1438,9 @@ async fn capture_dependency_job( .await } ScriptLang::Bun | ScriptLang::Bunnative => { - let npm_mode = npm_mode - .unwrap_or_else(|| windmill_common::worker::get_annotation(job_raw_code).npm_mode); + let npm_mode = npm_mode.unwrap_or_else(|| { + windmill_common::worker::TypeScriptAnnotations::parse(job_raw_code).npm + }); if !raw_deps { let _ = write_file(job_dir, "main.ts", job_raw_code)?; } @@ -1472,7 +1477,7 @@ async fn capture_dependency_job( base_internal_url, worker_name, &token, - occupancy_metrics, + &mut Some(occupancy_metrics), ) .await?; } diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 2d88e6abed..3a229dc1e8 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.402.3"; +export const VERSION = "v1.416.2"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/apps.ts b/cli/apps.ts index 1330c3ba80..ca966e2112 100644 --- a/cli/apps.ts +++ b/cli/apps.ts @@ -1,6 +1,6 @@ // deno-lint-ignore-file no-explicit-any import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; -import { colors, Command, log, SEP, Table, yamlParse } from "./deps.ts"; +import { colors, Command, log, SEP, Table, yamlParseFile } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; import { ListableApp, Policy } from "./gen/types.gen.ts"; @@ -25,7 +25,7 @@ export async function pushApp( return; } alreadySynced.push(localPath); - remotePath.replaceAll(SEP, "/"); + remotePath = remotePath.replaceAll(SEP, "/"); let app: any = undefined; // deleting old app if it exists in raw mode try { @@ -40,8 +40,8 @@ export async function pushApp( if (!localPath.endsWith(SEP)) { localPath += SEP; } - const localAppRaw = await Deno.readTextFile(localPath + "app.yaml"); - const localApp = yamlParse(localAppRaw) as AppFile; + const path = localPath + "app.yaml"; + const localApp = (await yamlParseFile(path)) as AppFile; function replaceInlineScripts(rec: any) { if (!rec) { diff --git a/cli/conf.ts b/cli/conf.ts index fa18de880b..eab154fec9 100644 --- a/cli/conf.ts +++ b/cli/conf.ts @@ -1,4 +1,4 @@ -import { log, yamlParse } from "./deps.ts"; +import { log, yamlParseFile } from "./deps.ts"; export interface SyncOptions { stateful?: boolean; @@ -40,9 +40,7 @@ export interface Codebase { export async function readConfigFile(): Promise { try { - const conf = yamlParse( - await Deno.readTextFile("wmill.yaml") - ) as SyncOptions; + const conf = (await yamlParseFile("wmill.yaml")) as SyncOptions; if (conf?.defaultTs == undefined) { log.warn( "No defaultTs defined in your wmill.yaml. Using 'bun' as default." diff --git a/cli/deps.ts b/cli/deps.ts index a8e14f0d3d..53e96a4f9c 100644 --- a/cli/deps.ts +++ b/cli/deps.ts @@ -21,7 +21,29 @@ export { copy } from "jsr:@std/io/copy"; export { readAll } from "jsr:@std/io/read-all"; export * as log from "jsr:@std/log"; -export { stringify as yamlStringify, parse as yamlParse } from "jsr:@std/yaml"; +export { stringify as yamlStringify } from "jsr:@std/yaml"; + +import { parse as yamlParse, ParseOptions } from "jsr:@std/yaml"; + +export async function yamlParseFile(path: string, options: ParseOptions = {}) { + try { + return yamlParse(await Deno.readTextFile(path), options); + } catch (e) { + throw new Error(`Error parsing yaml ${path}`, { cause: e }); + } +} + +export function yamlParseContent( + path: string, + content: string, + options: ParseOptions = {} +) { + try { + return yamlParse(content, options); + } catch (e) { + throw new Error(`Error parsing yaml ${path}`, { cause: e }); + } +} // other diff --git a/cli/flow.ts b/cli/flow.ts index 601f0be0bc..edca024f66 100644 --- a/cli/flow.ts +++ b/cli/flow.ts @@ -1,7 +1,7 @@ // deno-lint-ignore-file no-explicit-any import { GlobalOptions, isSuperset } from "./types.ts"; import { Confirm, SEP, log, yamlStringify } from "./deps.ts"; -import { colors, Command, Table, yamlParse } from "./deps.ts"; +import { colors, Command, Table, yamlParseFile } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; @@ -29,21 +29,23 @@ export function replaceInlineScripts( ) { modules.forEach((m) => { if (m.value.type == "rawscript") { - const path = m.value.content.split(" ")[1]; - m.value.content = Deno.readTextFileSync(localPath + path); - const lock = m.value.lock; - if (removeLocks && removeLocks.includes(path)) { - m.value.lock = undefined; - } else if ( - lock && - typeof lock == "string" && - lock.trimStart().startsWith("!inline ") - ) { - const path = lock.split(" ")[1]; - try { - m.value.lock = readInlinePathSync(localPath + path); - } catch { - log.error(`Lock file ${path} not found`); + if (m.value.content.startsWith("!inline")) { + const path = m.value.content.split(" ")[1]; + m.value.content = Deno.readTextFileSync(localPath + path); + const lock = m.value.lock; + if (removeLocks && removeLocks.includes(path)) { + m.value.lock = undefined; + } else if ( + lock && + typeof lock == "string" && + lock.trimStart().startsWith("!inline ") + ) { + const path = lock.split(" ")[1]; + try { + m.value.lock = readInlinePathSync(localPath + path); + } catch { + log.error(`Lock file ${path} not found`); + } } } } else if (m.value.type == "forloopflow") { @@ -87,8 +89,7 @@ export async function pushFlow( if (!localPath.endsWith(SEP)) { localPath += SEP; } - const localFlowRaw = await Deno.readTextFile(localPath + "flow.yaml"); - const localFlow = yamlParse(localFlowRaw) as FlowFile; + const localFlow = (await yamlParseFile(localPath + "flow.yaml")) as FlowFile; replaceInlineScripts(localFlow.value.modules, localPath, undefined); @@ -120,6 +121,7 @@ export async function pushFlow( }); } catch (e) { throw new Error( + //@ts-ignore `Failed to create flow ${remotePath}: ${e.body ?? e.message}` ); } diff --git a/cli/folder.ts b/cli/folder.ts index 0820cc0c9d..5cf2812670 100644 --- a/cli/folder.ts +++ b/cli/folder.ts @@ -73,6 +73,7 @@ export async function pushFolder( }, }); } catch (e) { + //@ts-ignore console.error(e.body); throw e; } @@ -87,6 +88,7 @@ export async function pushFolder( }, }); } catch (e) { + //@ts-ignore throw Error(`Failed to create folder ${name}: ${e.body ?? e.message}`); } } diff --git a/cli/gen/core/OpenAPI.ts b/cli/gen/core/OpenAPI.ts index 6af5246be2..cddc3f88de 100644 --- a/cli/gen/core/OpenAPI.ts +++ b/cli/gen/core/OpenAPI.ts @@ -54,7 +54,7 @@ export const OpenAPI: OpenAPIConfig = { PASSWORD: undefined, TOKEN: getEnv("WM_TOKEN"), USERNAME: undefined, - VERSION: '1.401.0', + VERSION: '1.407.2', WITH_CREDENTIALS: true, interceptors: { request: new Interceptors(), diff --git a/cli/gen/types.gen.ts b/cli/gen/types.gen.ts index 92bfe933d9..fe297f8bde 100644 --- a/cli/gen/types.gen.ts +++ b/cli/gen/types.gen.ts @@ -1015,6 +1015,9 @@ export type FlowModule = { skip_if_stopped?: boolean; expr: string; }; + skip_if?: { + expr: string; + }; sleep?: InputTransform; cache_ttl?: number; timeout?: number; @@ -1176,6 +1179,7 @@ export type FlowStatusModule = { approver: string; }>; failed_retries?: Array<(string)>; + skipped?: boolean; }; export type type4 = 'WaitingForPriorSteps' | 'WaitingForEvents' | 'WaitingForExecutor' | 'InProgress' | 'Success' | 'Failure'; diff --git a/cli/instance.ts b/cli/instance.ts index cf2d5f5b02..1e86282d1a 100644 --- a/cli/instance.ts +++ b/cli/instance.ts @@ -3,7 +3,7 @@ import { path, Confirm, yamlStringify, - yamlParse, + yamlParseFile, Command, setClient, Table, @@ -25,8 +25,8 @@ import { import { add as workspaceSetup, addWorkspace, - allWorkspaces, removeWorkspace, + setActiveWorkspace, } from "./workspace.ts"; import { pushInstanceSettings, @@ -35,8 +35,9 @@ import { pushInstanceConfigs, type SimplifiedSettings, } from "./settings.ts"; -import { sleep, deepEqual } from "./utils.ts"; +import { deepEqual } from "./utils.ts"; import { GlobalOptions } from "./types.ts"; +import { getActiveWorkspace } from "./workspace.ts"; export interface Instance { remote: string; @@ -180,18 +181,44 @@ export type InstanceSyncOptions = { instance?: string; baseUrl?: string; token?: string; + folderPerInstance?: boolean; yes?: boolean; + prefix?: string; }; -export async function pickInstance(opts: InstanceSyncOptions, allowNew: boolean) { +export async function pickInstance( + opts: InstanceSyncOptions, + allowNew: boolean +) { const instances = await allInstances(); + if (opts.baseUrl && opts.token && opts.instance) { + log.info("Using instance defined by --instance, --base-url and --token"); + + setClient( + opts.token, + opts.baseUrl.endsWith("/") ? opts.baseUrl.slice(0, -1) : opts.baseUrl + ); + + return { + name: opts.instance, + remote: opts.baseUrl, + token: opts.token, + prefix: opts.prefix ?? opts.instance, + }; + } if (opts.baseUrl && opts.token) { - log.info("Using instance fully defined by --base-url and --token") + log.info("Using instance fully defined by --base-url and --token"); + + setClient( + opts.token, + opts.baseUrl.endsWith("/") ? opts.baseUrl.slice(0, -1) : opts.baseUrl + ); + return { name: "custom", remote: opts.baseUrl, token: opts.token, - prefix: "custom", + prefix: opts.prefix ?? "custom", }; } if (!allowNew && instances.length < 1) { @@ -284,23 +311,25 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { log.info("No instance-level changes to apply"); } - sleep(1000); - if (opts.includeWorkspaces) { log.info("\nPulling all workspaces"); + const rootDir = Deno.cwd(); + const localWorkspaces = await getLocalWorkspaces( + rootDir, + instance.prefix, + opts.folderPerInstance + ); + + const previousActiveWorkspace = await getActiveWorkspace(undefined); const remoteWorkspaces = await wmill.listWorkspacesAsSuperAdmin({ page: 1, perPage: 1000, }); - let localWorkspaces = await allWorkspaces(); - localWorkspaces = localWorkspaces.filter((w) => - w.name.startsWith(instance.prefix + "_") - ); - const rootDir = Deno.cwd(); for (const remoteWorkspace of remoteWorkspaces) { log.info("\nPulling workspace " + remoteWorkspace.id); - sleep(1000); - const workspaceName = instance.prefix + "_" + remoteWorkspace.id; + const workspaceName = opts?.folderPerInstance + ? instance.prefix + "/" + remoteWorkspace.id + : instance.prefix + "_" + remoteWorkspace.id; await Deno.mkdir(path.join(rootDir, workspaceName), { recursive: true, }); @@ -327,31 +356,37 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { includeSettings: true, includeUsers: true, includeKey: true, + yes: opts.yes, }); } const localWorkspacesToDelete = localWorkspaces.filter( - (w) => !remoteWorkspaces.find((r) => r.id === w.workspaceId) + (w) => !remoteWorkspaces.find((r) => r.id === w.id) ); if (localWorkspacesToDelete.length > 0) { - const confirmDelete = await Confirm.prompt({ - message: - "Do you want to delete the local copy of workspaces that don't exist anymore on the instance?\n" + - localWorkspacesToDelete.map((w) => w.workspaceId).join(", "), - default: true, - }); + const confirmDelete = + opts.yes || + (await Confirm.prompt({ + message: + "Do you want to delete the local copy of workspaces that don't exist anymore on the instance?\n" + + localWorkspacesToDelete.map((w) => w).join(", "), + default: true, + })); if (confirmDelete) { for (const workspace of localWorkspacesToDelete) { - await removeWorkspace(workspace.name, false, {}); - await Deno.remove(path.join(rootDir, workspace.name), { + await removeWorkspace(workspace.id, false, {}); + await Deno.remove(path.join(rootDir, workspace.dir), { recursive: true, }); } } } + if (previousActiveWorkspace) { + await setActiveWorkspace(previousActiveWorkspace?.name); + } log.info(colors.green.underline.bold("All workspaces pulled")); } } @@ -409,37 +444,46 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { log.info("No instance-level changes to apply"); } - sleep(1000); - if (opts.includeWorkspaces) { instances = await allInstances(); - const localPrefix = (await Select.prompt({ - message: "What is the prefix of the local workspaces you want to sync?", - options: [ - ...instances.map((i) => ({ - name: `${i.prefix} (${i.name} - ${i.remote})`, - value: i.prefix, - })), - ], - default: instance.prefix as unknown, - })) as unknown as string; + const rootDir = Deno.cwd(); + + let localPrefix; + if (opts.prefix) { + localPrefix = opts.prefix; + } else { + localPrefix = (await Select.prompt({ + message: "What is the prefix of the local workspaces you want to sync?", + options: [ + ...instances.map((i) => ({ + name: `${i.prefix} (${i.name} - ${i.remote})`, + value: i.prefix, + })), + ], + default: instance.prefix as unknown, + })) as unknown as string; + } const remoteWorkspaces = await wmill.listWorkspacesAsSuperAdmin({ page: 1, perPage: 1000, }); - let localWorkspaces = await allWorkspaces(); - localWorkspaces = localWorkspaces.filter((w) => - w.name.startsWith(localPrefix + "_") + + const previousActiveWorkspace = await getActiveWorkspace(undefined); + + const localWorkspaces = await getLocalWorkspaces( + rootDir, + localPrefix, + opts.folderPerInstance ); - log.info("\nPushing all workspaces"); - const rootDir = Deno.cwd(); + log.info( + `\nPushing all workspaces: ${localWorkspaces.map((x) => x.id).join(", ")}` + ); for (const localWorkspace of localWorkspaces) { - log.info("\nPushing workspace " + localWorkspace.workspaceId); - sleep(1000); + log.info("\nPushing workspace " + localWorkspace.id); try { - await Deno.chdir(path.join(rootDir, localWorkspace.name)); + await Deno.chdir(path.join(rootDir, localWorkspace.dir)); } catch (_) { throw new Error( "Workspace folder not found, are you in the right directory?" @@ -447,9 +491,9 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { } try { - const workspaceSettings = yamlParse( - await Deno.readTextFile("settings.yaml") - ) as SimplifiedSettings; + const workspaceSettings = (await yamlParseFile( + "settings.yaml" + )) as SimplifiedSettings; await workspaceSetup( { token: instance.token, @@ -459,8 +503,8 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { createWorkspaceName: workspaceSettings.name, createUsername: undefined, }, - localWorkspace.name, - localWorkspace.workspaceId, + localWorkspace.dir, + localWorkspace.id, instance.remote ); } catch (_) { @@ -470,7 +514,7 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { continue; } await push({ - workspace: localWorkspace.name, + workspace: localWorkspace.dir, token: undefined, baseUrl: undefined, includeGroups: true, @@ -478,19 +522,22 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { includeSettings: true, includeUsers: true, includeKey: true, + yes: opts.yes, }); } const workspacesToDelete = remoteWorkspaces.filter( - (w) => !localWorkspaces.find((l) => l.workspaceId === w.id) + (w) => !localWorkspaces.find((l) => l.id === w.id) ); if (workspacesToDelete.length > 0) { - const confirmDelete = await Confirm.prompt({ - message: - "Do you want to delete the following remote workspaces that don't exist locally?\n" + - workspacesToDelete.map((w) => w.id).join(", "), - default: true, - }); + const confirmDelete = + opts.yes || + (await Confirm.prompt({ + message: + "Do you want to delete the following remote workspaces that don't exist locally?\n" + + workspacesToDelete.map((w) => w.id).join(", "), + default: true, + })); if (confirmDelete) { for (const workspace of workspacesToDelete) { @@ -499,10 +546,46 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { } } } + if (previousActiveWorkspace) { + await setActiveWorkspace(previousActiveWorkspace?.name); + } log.info(colors.green.underline.bold("All workspaces pushed")); } } +async function getLocalWorkspaces( + rootDir: string, + localPrefix: string, + folderPerInstance?: boolean +) { + const localWorkspaces: { dir: string; id: string }[] = []; + + if (!(await Deno.stat(localPrefix).catch(() => null))) { + await Deno.mkdir(localPrefix); + } + if (folderPerInstance) { + for await (const dir of Deno.readDir(rootDir + "/" + localPrefix)) { + const dirName = dir.name; + localWorkspaces.push({ + dir: localPrefix + "/" + dirName, + id: dirName, + }); + } + log.info(localWorkspaces); + } else { + for await (const dir of Deno.readDir(rootDir)) { + const dirName = dir.name; + if (dirName.startsWith(localPrefix + "_")) { + localWorkspaces.push({ + dir: dirName, + id: dirName.substring(localPrefix.length + 1), + }); + } + } + } + return localWorkspaces; +} + async function switchI(opts: {}, instanceName: string) { const all = await allInstances(); if (all.findIndex((x) => x.name === instanceName) === -1) { @@ -544,7 +627,10 @@ async function whoami(opts: {}) { log.info(colors.green.underline(`global whoami infos:`)); log.info(JSON.stringify(whoamiInfo, null, 2)); } catch (error) { - log.error(colors.red(`Failed to retrieve whoami information: ${error.message}`)); + log.error( + //@ts-ignore + colors.red(`Failed to retrieve whoami information: ${error.message}`) + ); } } @@ -585,7 +671,7 @@ const command = new Command() .description("Remove an instance") .complete("instance", async () => (await allInstances()).map((x) => x.name)) .arguments("") - .action(async (instance) => { + .action(async (instance: any) => { const instances = await allInstances(); const choice = (await Select.prompt({ @@ -614,6 +700,15 @@ const command = new Command() .option("--skip-configs", "Skip pulling configs (worker groups and SMTP)") .option("--skip-groups", "Skip pulling instance groups") .option("--include-workspaces", "Also pull workspaces") + .option("--folder-per-instance", "Create a folder per instance") + .option( + "--instance ", + "Name of the instance to pull from, override the active instance" + ) + .option( + "--prefix ", + "Prefix of the local workspaces to pull, used to create the folders when using --include-workspaces" + ) .action(instancePull as any) .command("push") @@ -626,13 +721,14 @@ const command = new Command() .option("--skip-configs", "Skip pushing configs (worker groups and SMTP)") .option("--skip-groups", "Skip pushing instance groups") .option("--include-workspaces", "Also push workspaces") + .option("--folder-per-instance", "Create a folder per instance") .option( - "--instance", + "--instance ", "Name of the instance to push to, override the active instance" ) .option( - "--base-url", - "If used with --token, will be used as the base url for the instance" + "--prefix ", + "Prefix of the local workspaces folders to push" ) .action(instancePush as any) .command("whoami") diff --git a/cli/local_encryption.ts b/cli/local_encryption.ts new file mode 100644 index 0000000000..4277093f8f --- /dev/null +++ b/cli/local_encryption.ts @@ -0,0 +1,101 @@ +import crypto from "node:crypto"; + +// Helper function to convert strings to Uint8Array (binary) +function encode(input: string): Uint8Array { + return new TextEncoder().encode(input); +} + +// Helper function to convert Uint8Array (binary) to base64 +function toBase64(arr: Uint8Array): string { + return btoa(String.fromCharCode(...arr)); +} + +// Helper function to convert base64 to Uint8Array (binary) +function fromBase64(base64: string): Uint8Array { + return new Uint8Array( + atob(base64) + .split("") + .map((char) => char.charCodeAt(0)) + ); +} + +// Function to derive a 256-bit key from any input string using SHA-256 +async function deriveKey( + keyString: string +): Promise { + const keyMaterial = encode(keyString); + const keyHash = await crypto.subtle.digest("SHA-256", keyMaterial); // Generate SHA-256 hash + // Import the hash as a CryptoKey for AES-GCM + return crypto.subtle.importKey("raw", keyHash, { name: "AES-GCM" }, false, [ + "encrypt", + "decrypt", + ]); +} + +// Encrypt function +export async function encrypt( + plaintext: string, + keyString: string +): Promise { + const key = await deriveKey(keyString); // Derive a 256-bit AES key from any input string + const iv = crypto.getRandomValues(new Uint8Array(12)); // AES-GCM needs a 12-byte IV + const encrypted = await crypto.subtle.encrypt( + { + name: "AES-GCM", + iv, + tagLength: 128, + }, + key, + encode(plaintext) // convert plaintext to binary + ); + + // Concatenate IV and encrypted data + const combined = new Uint8Array(iv.length + encrypted.byteLength); + combined.set(iv, 0); // first part is the IV + combined.set(new Uint8Array(encrypted), iv.length); // second part is the ciphertext + + // Convert to base64 for storage/transmission + return toBase64(combined); +} + +// Decrypt function +export async function decrypt( + combinedCiphertext: string, + keyString: string +): Promise { + const key = await deriveKey(keyString); // Derive the same 256-bit AES key from the input string + const combined = fromBase64(combinedCiphertext); // decode base64 to binary + + // Split the IV and the ciphertext + const iv = combined.slice(0, 12); // First 12 bytes are the IV + const ciphertext = combined.slice(12); // The rest is the encrypted data + console.log(); + + // log.info({keyString, key, ciphertext}) + // Perform decryption + const decrypted = await crypto.subtle.decrypt( + { + name: "AES-GCM", + iv, + tagLength: 128, + }, + key, + ciphertext + ); + + // Convert decrypted data from binary to string + return new TextDecoder().decode(decrypted); +} + +// // Example usage: +// const key = "any-length-key-you-want"; // Now can be any length +// const message = "This is a secret message."; + +// encrypt(message, key).then((combinedCiphertext) => { +// console.log("Encrypted message:", combinedCiphertext); + +// // Now decrypt it +// decrypt(combinedCiphertext, key).then((decryptedMessage) => { +// console.log("Decrypted message:", decryptedMessage); +// }); +// }); diff --git a/cli/main.ts b/cli/main.ts index c1aa86cf3e..562c3fde45 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.402.3"; +export const VERSION = "1.416.2"; const command = new Command() .name("wmill") @@ -101,9 +101,13 @@ const command = new Command() "wmill.yaml", yamlStringify({ defaultTs: "bun", - includes: ["**"], + includes: ["f/**"], excludes: [], codebases: [], + skipVariables: true, + skipResources: true, + skipSecrets: true, + includeSchedules: false, }) ); log.info(colors.green("wmill.yaml created")); @@ -213,9 +217,11 @@ function isMain() { const isMain = import.meta.main; if (isMain) { if (!Deno.args.includes("completions")) { - log.warn( - "Using the deno runtime for the Windmill CLI is deprecated, you can now use node: deno uninstall wmill && npm install -g windmill-cli" - ); + if (Deno.env.get("SKIP_DENO_DEPRECATION_WARNING") !== "true") { + log.warn( + "Using the deno runtime for the Windmill CLI is deprecated, you can now use node: deno uninstall wmill && npm install -g windmill-cli. To skip this warning set SKIP_DENO_DEPRECATION_WARNING=true" + ); + } } } return isMain; diff --git a/cli/metadata.ts b/cli/metadata.ts index a3b4738cb7..92ac7dfc4b 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -1,6 +1,13 @@ // deno-lint-ignore-file no-explicit-any import { GlobalOptions } from "./types.ts"; -import { SEP, colors, log, path, yamlParse, yamlStringify } from "./deps.ts"; +import { + SEP, + colors, + log, + path, + yamlParseFile, + yamlStringify, +} from "./deps.ts"; import { ScriptMetadata, defaultScriptMetadata, @@ -119,9 +126,9 @@ export async function generateFlowLockInternal( return remote_path; } - const flowValue = yamlParse( - await Deno.readTextFile(folder! + SEP + "flow.yaml") - ) as FlowFile; + const flowValue = (await yamlParseFile( + folder! + SEP + "flow.yaml" + )) as FlowFile; if (!justUpdateMetadataLock) { const changedScripts = []; @@ -798,7 +805,7 @@ export async function parseMetadataFile( try { metadataFilePath = scriptPath + ".script.yaml"; await Deno.stat(metadataFilePath); - const payload: any = yamlParse(await Deno.readTextFile(metadataFilePath)); + const payload: any = await yamlParseFile(metadataFilePath); replaceLock(payload); return { @@ -840,9 +847,9 @@ export async function parseMetadataFile( codebases, false ); - scriptInitialMetadata = yamlParse( - await Deno.readTextFile(metadataFilePath) - ) as ScriptMetadata; + scriptInitialMetadata = (await yamlParseFile( + metadataFilePath + )) as ScriptMetadata; replaceLock(scriptInitialMetadata); } catch (e) { log.info( @@ -868,8 +875,7 @@ interface Lock { const WMILL_LOCKFILE = "wmill-lock.yaml"; export async function readLockfile(): Promise { try { - const lockfile = await Deno.readTextFile(WMILL_LOCKFILE); - const read = yamlParse(lockfile); + const read = await yamlParseFile(WMILL_LOCKFILE); if (typeof read == "object") { return read as Lock; } else { diff --git a/cli/schedule.ts b/cli/schedule.ts index fe2db1af19..25ffdd5b49 100644 --- a/cli/schedule.ts +++ b/cli/schedule.ts @@ -1,5 +1,5 @@ // deno-lint-ignore-file no-explicit-any -import { colors, Command, log, Table } from "./deps.ts"; +import { colors, Command, log, SEP, Table } from "./deps.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; import * as wmill from "./gen/services.gen.ts"; @@ -42,8 +42,7 @@ export async function pushSchedule( schedule: Schedule | ScheduleFile | undefined, localSchedule: ScheduleFile ): Promise { - path = removeType(path, "schedule"); - + path = removeType(path, "schedule").replaceAll(SEP, "/"); log.debug(`Processing local schedule ${path}`); // deleting old app if it exists in raw mode diff --git a/cli/settings.ts b/cli/settings.ts index b3a1b6a272..f9dbac1fa6 100644 --- a/cli/settings.ts +++ b/cli/settings.ts @@ -1,7 +1,7 @@ import { yamlStringify } from "./deps.ts"; import { Confirm } from "./deps.ts"; import { colors } from "./deps.ts"; -import { yamlParse } from "./deps.ts"; +import { yamlParseFile } from "./deps.ts"; import { log } from "./deps.ts"; import { compareInstanceObjects } from "./instance.ts"; import { isSuperset } from "./types.ts"; @@ -9,6 +9,7 @@ import { deepEqual } from "./utils.ts"; import * as wmill from "./gen/services.gen.ts"; import { Config, GlobalSetting } from "./gen/types.gen.ts"; import { removeWorkerPrefix } from "./worker_groups.ts"; +import process from "node:process"; export interface SimplifiedSettings { // slack_team_id?: string; @@ -112,10 +113,10 @@ export async function pushWorkspaceSettings( workspace, requestBody: localSettings.auto_invite_enabled ? { - operator: localSettings.auto_invite_as === "operator", - invite_all: true, - auto_add: localSettings.auto_invite_mode === "add", - } + operator: localSettings.auto_invite_as === "operator", + invite_all: true, + auto_add: localSettings.auto_invite_mode === "add", + } : {}, }); } catch (_) { @@ -127,10 +128,10 @@ export async function pushWorkspaceSettings( workspace, requestBody: localSettings.auto_invite_enabled ? { - operator: localSettings.auto_invite_as === "operator", - invite_all: false, - auto_add: localSettings.auto_invite_mode === "add", - } + operator: localSettings.auto_invite_as === "operator", + invite_all: false, + auto_add: localSettings.auto_invite_mode === "add", + } : {}, }); } @@ -155,7 +156,7 @@ export async function pushWorkspaceSettings( settings.error_handler_extra_args ) || localSettings.error_handler_muted_on_cancel !== - settings.error_handler_muted_on_cancel + settings.error_handler_muted_on_cancel ) { log.debug(`Updating error handler...`); await wmill.editErrorHandler({ @@ -259,20 +260,69 @@ export async function pushWorkspaceKey( } } +const INSTANCE_SETTINGS_PATH = "instance_settings.yaml"; + +export async function readInstanceSettings() { + let localSettings: GlobalSetting[] = []; + + try { + localSettings = (await yamlParseFile(INSTANCE_SETTINGS_PATH)) as GlobalSetting[]; + } catch { + log.warn(`No ${INSTANCE_SETTINGS_PATH} found`); + } + return localSettings; +} + + +import { decrypt, encrypt } from "./local_encryption.ts"; + +const SENSITIVE_FIELD: string[] = ["license_key", "jwt_secret"] + +async function processInstanceSettings(settings: GlobalSetting[], mode: "encode" | "decode"): Promise { + const encKey = process.env.WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY; + if (encKey) { + const res: GlobalSetting[] = [] + + for (const s of settings) { + if (SENSITIVE_FIELD.includes(s.name) && typeof s.value === "string") { + res.push(await processField(s, "value", encKey, mode) as GlobalSetting); + } else if (s.name == "oauths") { + if (typeof s.value === "object") { + const oauths = s.value as { [key: string]: any }; + for (const [k, v] of Object.entries(oauths)) { + oauths[k] = await processField(v, "secret", encKey, mode); + } + res.push(s); + } else { + log.warn(`Unexpected oauths value type: ${typeof s.value}`); + res.push(s); + } + } else { + res.push(s); + } + } + return res; + } else { + log.warn("No encryption key found, skipping encryption. Recommend setting WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY"); + } + return settings; +} + +async function processField(obj: { [key: string]: any }, field: string, encKey: string, mode: "encode" | "decode"): Promise<{ [key: string]: any }> { + return { + ...obj, + [field]: mode === "encode" ? await encrypt(obj[field], encKey) : await decrypt(obj[field], encKey) as any, + } +} + export async function pullInstanceSettings(preview = false) { const remoteSettings = await wmill.listGlobalSettings(); if (preview) { - let localSettings: GlobalSetting[] = []; - - try { - localSettings = yamlParse( - await Deno.readTextFile("instance_settings.yaml") - ) as GlobalSetting[]; - } catch {} - + const localSettings: GlobalSetting[] = await readInstanceSettings(); + const processedSettings = await processInstanceSettings(remoteSettings, "encode"); return compareInstanceObjects( - remoteSettings, + processedSettings, localSettings, "name", "setting" @@ -280,12 +330,13 @@ export async function pullInstanceSettings(preview = false) { } else { log.info("Pulling settings from instance"); + const processedSettings = await processInstanceSettings(remoteSettings, "encode"); await Deno.writeTextFile( - "instance_settings.yaml", - yamlStringify(remoteSettings as any) + INSTANCE_SETTINGS_PATH, + yamlStringify(processedSettings) ); - log.info(colors.green("Settings written to instance_settings.yaml")); + log.info(colors.green(`Settings written to ${INSTANCE_SETTINGS_PATH}`)); } } @@ -294,9 +345,8 @@ export async function pushInstanceSettings( baseUrl?: string ) { const remoteSettings = await wmill.listGlobalSettings(); - let localSettings = (await Deno.readTextFile("instance_settings.yaml") - .then((raw) => yamlParse(raw)) - .catch(() => [])) as GlobalSetting[]; + let localSettings: GlobalSetting[] = await readInstanceSettings(); + localSettings = await processInstanceSettings(localSettings, "decode"); if (baseUrl) { localSettings = localSettings.filter((s) => s.name !== "base_url"); @@ -354,6 +404,17 @@ export async function pushInstanceSettings( } } +export async function readLocalConfigs() { + let localConfigs: Config[] = []; + + try { + localConfigs = (await yamlParseFile("instance_configs.yaml")) as Config[]; + } catch { + log.warn("No instance_configs.yaml found"); + } + return localConfigs; +} + export async function pullInstanceConfigs(preview = false) { const remoteConfigs = (await wmill.listConfigs()).map((x) => { return { @@ -363,12 +424,7 @@ export async function pullInstanceConfigs(preview = false) { }); if (preview) { - let localConfigs: Config[] = []; - try { - localConfigs = yamlParse( - await Deno.readTextFile("instance_configs.yaml") - ) as Config[]; - } catch {} + const localConfigs: Config[] = await readLocalConfigs(); return compareInstanceObjects( remoteConfigs, @@ -395,9 +451,7 @@ export async function pushInstanceConfigs(preview: boolean = false) { name: removeWorkerPrefix(x.name), }; }); - const localConfigs = (await Deno.readTextFile("instance_configs.yaml") - .then((raw) => yamlParse(raw)) - .catch(() => [])) as Config[]; + const localConfigs = await readLocalConfigs(); if (preview) { return compareInstanceObjects( @@ -415,7 +469,9 @@ export async function pushInstanceConfigs(preview: boolean = false) { } try { await wmill.updateConfig({ - name: config.name.startsWith('worker__') ? config.name : `worker__${config.name}`, + name: config.name.startsWith("worker__") + ? config.name + : `worker__${config.name}`, requestBody: config.config, }); } catch (err) { diff --git a/cli/sync.ts b/cli/sync.ts index 088571362d..342dda3162 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -9,7 +9,7 @@ import { path, log, yamlStringify, - yamlParse, + yamlParseContent, SEP, } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; @@ -112,7 +112,7 @@ async function addCodebaseDigestIfRelevant( if (isTs) { const c = findCodebase(path, codebases); if (c) { - const parsed: any = yamlParse(content); + const parsed: any = yamlParseContent(path, content); if (parsed && typeof parsed == "object") { parsed["codebase"] = c.digest; parsed["lock"] = undefined; @@ -504,7 +504,7 @@ function ZipFSElement( if (formatExtension) { const fileContent: string = parsed["value"]["content"]; - if (typeof(fileContent) === "string") { + if (typeof fileContent === "string") { r.push({ isDirectory: false, path: @@ -618,8 +618,8 @@ export async function elementsToMap( for await (const entry of readDirRecursiveWithIgnore(ignore, els)) { if (entry.isDirectory || entry.ignored) continue; const path = entry.path; - if (json && path.endsWith(".yaml")) continue; - if (!json && path.endsWith(".json")) continue; + if (json && path.endsWith(".yaml") && !isFileResource(path)) continue; + if (!json && path.endsWith(".json") && !isFileResource(path)) continue; const ext = json ? ".json" : ".yaml"; if (!skips.includeSchedules && path.endsWith(".schedule" + ext)) continue; if (!skips.includeUsers && path.endsWith(".user" + ext)) continue; @@ -660,7 +660,7 @@ export async function elementsToMap( if (json) { o = JSON.parse(content); } else { - o = yamlParse(content); + o = yamlParseContent(path, content); } if (o["is_secret"]) { continue; @@ -704,7 +704,7 @@ async function compareDynFSElement( function parseYaml(k: string, v: string) { if (k.endsWith(".script.yaml")) { - const o: any = yamlParse(v); + const o: any = yamlParseContent(k, v); if (typeof o == "object") { if (Array.isArray(o?.["lock"])) { o["lock"] = o["lock"].join("\n"); @@ -715,7 +715,7 @@ async function compareDynFSElement( } return o; } else if (k.endsWith(".app.yaml")) { - const o: any = yamlParse(v); + const o: any = yamlParseContent(k, v); const o2 = o["policy"]; if (typeof o2 == "object") { @@ -728,7 +728,7 @@ async function compareDynFSElement( } return o; } else { - return yamlParse(v); + return yamlParseContent(k, v); } } for (const [k, v] of Object.entries(m1)) { @@ -948,7 +948,7 @@ export async function pull(opts: GlobalOptions & SyncOptions) { if ( !opts.yes && !(await Confirm.prompt({ - message: `Do you want to apply these ${changes.length} changes?`, + message: `Do you want to apply these ${changes.length} changes to your local files?`, default: true, })) ) { @@ -1251,7 +1251,7 @@ export async function push(opts: GlobalOptions & SyncOptions) { if ( !opts.yes && !(await Confirm.prompt({ - message: `Do you want to apply these ${changes.length} changes?`, + message: `Do you want to apply these ${changes.length} changes to the remote?`, default: true, })) ) { diff --git a/cli/types.ts b/cli/types.ts index 84691a37e2..e264521cb7 100644 --- a/cli/types.ts +++ b/cli/types.ts @@ -6,7 +6,7 @@ import { colors, log, path, - yamlParse, + yamlParseContent, yamlStringify, } from "./deps.ts"; import { pushApp } from "./apps.ts"; @@ -115,7 +115,7 @@ export async function pushObj( newObj: any, plainSecrets: boolean, alreadySynced: string[], - message?: string, + message?: string ) { const typeEnding = getTypeStrFromPath(p); @@ -155,7 +155,7 @@ export async function pushObj( export function parseFromPath(p: string, content: string): any { return p.endsWith(".yaml") - ? yamlParse(content) + ? yamlParseContent(p, content) : p.endsWith(".json") ? JSON.parse(content) : content; @@ -164,7 +164,7 @@ export function parseFromFile(p: string): any { if (p.endsWith(".json")) { return JSON.parse(Deno.readTextFileSync(p)); } else if (p.endsWith(".yaml") || p.endsWith(".yml")) { - return yamlParse(Deno.readTextFileSync(p)); + return yamlParseContent(p, Deno.readTextFileSync(p)); } else { throw new Error("Could not read file " + p); } @@ -227,7 +227,7 @@ export function getTypeStrFromPath( return typeEnding; } else { if (isFileResource(p)) { - return "resource" + return "resource"; } throw new Error("Could not infer type of path " + JSON.stringify(parsed)); } diff --git a/cli/user.ts b/cli/user.ts index f95747306a..2ddeb997c9 100644 --- a/cli/user.ts +++ b/cli/user.ts @@ -13,7 +13,7 @@ import { log, Table, yamlStringify, - yamlParse, + yamlParseFile, } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; import { @@ -170,6 +170,7 @@ export async function pushWorkspaceUser( }, }); } catch (e) { + //@ts-ignore console.error(e.body); throw e; } @@ -189,6 +190,7 @@ export async function pushWorkspaceUser( }, }); } catch (e) { + //@ts-ignore console.error(e.body); throw e; } @@ -251,6 +253,7 @@ export async function pushGroup( }, }); } catch (e) { + //@ts-ignore console.error(e.body); throw e; } @@ -301,6 +304,7 @@ export async function pushGroup( }); } } catch (e) { + //@ts-ignore console.error(e.body); throw e; } @@ -329,6 +333,7 @@ export async function pushGroup( }, }); } catch (e) { + //@ts-ignore console.error(e.body); throw e; } @@ -368,11 +373,13 @@ export async function pushGroup( }); } } catch (e) { + //@ts-ignore console.error(e.body); throw e; } } } catch (e) { + //@ts-ignore console.error(e.body); throw e; } @@ -383,11 +390,7 @@ export async function pullInstanceUsers(preview: boolean = false) { const remoteUsers = await wmill.globalUsersExport(); if (preview) { - let localUsers: ExportedUser[] = []; - try { - const raw = await Deno.readTextFile("instance_users.yaml"); - localUsers = yamlParse(raw) as ExportedUser[]; - } catch {} + const localUsers: ExportedUser[] = await readInstanceUsers(); return compareInstanceObjects(remoteUsers, localUsers, "email", "user"); } else { log.info("Pulling users from instance..."); @@ -399,11 +402,31 @@ export async function pullInstanceUsers(preview: boolean = false) { } } +export async function readInstanceUsers() { + let localUsers: ExportedUser[] = []; + try { + localUsers = (await yamlParseFile("instance_users.yaml")) as ExportedUser[]; + } catch { + log.warn("No instance_users.yaml file found"); + } + return localUsers; +} + +export async function readInstanceGroups() { + let localGroups: InstanceGroup[] = []; + try { + localGroups = (await yamlParseFile( + "instance_groups.yaml" + )) as ExportedInstanceGroup[]; + } catch { + log.warn("No instance_groups.yaml file found"); + } + return localGroups; +} + export async function pushInstanceUsers(preview: boolean = false) { const remoteUsers = await wmill.globalUsersExport(); - const localUsers = (await Deno.readTextFile("instance_users.yaml") - .then((raw) => yamlParse(raw)) - .catch(() => [])) as ExportedUser[]; + const localUsers: ExportedUser[] = await readInstanceUsers(); if (preview) { return compareInstanceObjects(localUsers, remoteUsers, "email", "user"); @@ -421,11 +444,7 @@ export async function pullInstanceGroups(preview = false) { const remoteGroups = await wmill.exportInstanceGroups(); if (preview) { - let localGroups: InstanceGroup[] = []; - try { - const raw = await Deno.readTextFile("instance_groups.yaml"); - localGroups = yamlParse(raw) as InstanceGroup[]; - } catch {} + const localGroups = await readInstanceGroups(); return compareInstanceObjects(remoteGroups, localGroups, "name", "group"); } else { log.info("Pulling groups from instance..."); @@ -441,9 +460,7 @@ export async function pullInstanceGroups(preview = false) { export async function pushInstanceGroups(preview: boolean = false) { const remoteGroups = await wmill.exportInstanceGroups(); - const localGroups = (await Deno.readTextFile("instance_groups.yaml") - .then((raw) => yamlParse(raw)) - .catch(() => [])) as ExportedInstanceGroup[]; + const localGroups = await readInstanceGroups(); if (preview) { return compareInstanceObjects(localGroups, remoteGroups, "name", "group"); diff --git a/cli/utils.ts b/cli/utils.ts index 5f2de77f6b..fcb07bea9e 100644 --- a/cli/utils.ts +++ b/cli/utils.ts @@ -55,7 +55,10 @@ export function deepEqual(a: T, b: T): boolean { if (a.valueOf !== Object.prototype.valueOf) { return a.valueOf() === b.valueOf(); } - if (a.toString !== Object.prototype.toString) { + if ( + a.toString !== Object.prototype.toString && + typeof a.toString == "function" + ) { return a.toString() === b.toString(); } diff --git a/cli/workspace.ts b/cli/workspace.ts index face504183..4338e21cfe 100644 --- a/cli/workspace.ts +++ b/cli/workspace.ts @@ -34,10 +34,10 @@ export async function allWorkspaces(): Promise { } async function getActiveWorkspaceName( - opts: GlobalOptions + opts: GlobalOptions | undefined ): Promise { - if (opts.workspace) { - return opts.workspace; + if (opts?.workspace) { + return opts?.workspace; } try { return await Deno.readTextFile((await getRootStore()) + "/activeWorkspace"); @@ -47,7 +47,7 @@ async function getActiveWorkspaceName( } export async function getActiveWorkspace( - opts: GlobalOptions + opts: GlobalOptions | undefined ): Promise { const name = await getActiveWorkspaceName(opts); if (!name) { @@ -115,7 +115,12 @@ async function switchC(opts: GlobalOptions, workspaceName: string) { return; } - return await Deno.writeTextFile( + await setActiveWorkspace(workspaceName); + return; +} + +export async function setActiveWorkspace(workspaceName: string) { + await Deno.writeTextFile( (await getRootStore()) + "/activeWorkspace", workspaceName ); @@ -241,10 +246,8 @@ export async function add( }, opts ); - await Deno.writeTextFile( - (await getRootStore()) + "/activeWorkspace", - workspaceName - ); + await setActiveWorkspace(workspaceName); + log.info( colors.green.underline( `Added workspace ${workspaceName} for ${workspaceId} on ${remote}!` diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index b94bdec85e..16c2361a0d 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -11,13 +11,16 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - -RUN apt-get -y update && apt-get install -y curl nodejs awscli +RUN apt-get -y update && apt-get install -y curl procps nodejs awscli ENV TZ=Etc/UTC RUN /usr/local/bin/python3 -m pip install pip-tools -COPY --from=oven/bun:1.1.27 /usr/local/bin/bun /usr/bin/bun +# Install UV +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv + +COPY --from=oven/bun:1.1.32 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index b94bdec85e..65c6b55a19 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -11,13 +11,15 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - -RUN apt-get -y update && apt-get install -y curl nodejs awscli +RUN apt-get -y update && apt-get install -y curl procps nodejs awscli ENV TZ=Etc/UTC RUN /usr/local/bin/python3 -m pip install pip-tools +# Install UV +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.1.27 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.32 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/docker/RHEL9/Dockerfile b/docker/RHEL9/Dockerfile index 3a9ed24bb6..10e0f01096 100644 --- a/docker/RHEL9/Dockerfile +++ b/docker/RHEL9/Dockerfile @@ -4,11 +4,13 @@ ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm FROM ${RUST_IMAGE} AS rust_base -RUN yum install -y rust-toolset - RUN yum update -y && \ yum install -y git openssl-devel npm nodejs rustfmt +# Install rust manually +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo install cargo-chef --version ^0.1 WORKDIR /windmill @@ -35,7 +37,7 @@ COPY /backend/parsers/windmill-parser-wasm/pkg/ /backend/parsers/windmill-parser COPY /typescript-client/docs/ /frontend/static/tsdocs/ RUN npm run generate-backend-client -ENV NODE_OPTIONS "--max-old-space-size=8192" +ENV NODE_OPTIONS "--max-old-space-size=10240" RUN npm run build diff --git a/docker/RHEL9/README.md b/docker/RHEL9/README.md index af858d8be5..17d39b86ca 100644 --- a/docker/RHEL9/README.md +++ b/docker/RHEL9/README.md @@ -3,17 +3,16 @@ This directory contains the Dockerfiles for building Windmill binaries for Red Hat Linux 9. We build Windmill on the Red Hat Universal Base Image 9. Windmill requires the xmlsec1-devel package which is not available in the default UBI9 repositories. It is however included in the CodeReady Builder for RHEL9 repository which requires a RedHat subscription. -Moreover, only rust v1.75 is supported on Red Hat Linux 9. To make Windmill compatible with Rust v1.75, you need to pin the following libraries: -``` -aws-config = "=1.4.0" -aws-sdk-sts = "=1.25.0" -aws-sdk-ssooidc = "=1.25.0" -aws-sdk-sso = "=1.25.0" -``` - -Make sure to include `aws-sdk-ssooidc` and `aws-sdk-sso` in the Cargo.toml of windmill-common as well to enforce the correct versions of the nested dependencies. Make them optional and include them in the `parquet` feature. -It's also possible that you need to add `#[async_recursion]` to the `lock_modules` function in the `backend/windmill-worker/src/worker_lockfiles.rs` file for it to compile. Once the image is built, you can simply copy the binary on any Red Hat Linux 9 machine and run it. You will just need to install the xmlsec1 package which can be installed directly using `yum/dnf install xmlsec1`. - +## Notes + - you will need to register on Red Hat and have an individual developer subscription and pass the username and password to docker build: + ``` + docker build \ + -f docker/RHEL9/Dockerfile \ + --build-arg features="$features" \ + --secret id=rh_username,src=/path/to/rh_username \ + --secret id=rh_password,src=/path/to/rh_password \ + . + ``` \ No newline at end of file diff --git a/frontend/openapi-ts-error-1726231138297.log b/frontend/openapi-ts-error-1726231138297.log deleted file mode 100644 index f78fbf5b1c..0000000000 --- a/frontend/openapi-ts-error-1726231138297.log +++ /dev/null @@ -1,28 +0,0 @@ -Error parsing /git/windmill/backend/windmill-api/openapi.yaml: duplicated mapping key (8350:3) - - 8347 | application/json: - 8348 | schema: {} - 8349 | - 8350 | /w/{workspace}/job_helpers/loa ... -----------^ - 8351 | get: - 8352 | summary: Load a preview of ... -ParserError: Error parsing /git/windmill/backend/windmill-api/openapi.yaml: duplicated mapping key (8350:3) - - 8347 | application/json: - 8348 | schema: {} - 8349 | - 8350 | /w/{workspace}/job_helpers/loa ... -----------^ - 8351 | get: - 8352 | summary: Load a preview of ... - at Object.parse (/git/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/parsers/yaml.js:44:23) - at getResult (/git/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/util/plugins.js:116:22) - at runNextPlugin (/git/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/util/plugins.js:64:32) - at /git/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/util/plugins.js:55:9 - at new Promise () - at Object.run (/git/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/util/plugins.js:54:12) - at parseFile (/git/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/parse.js:130:38) - at parse (/git/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/parse.js:56:30) - at async $RefParser.parse (/git/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/index.js:115:28) - at async $RefParser.resolve (/git/windmill/frontend/node_modules/@apidevtools/json-schema-ref-parser/dist/lib/index.js:145:13) \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json index fbfe82be4e..028ac3e356 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.402.3", + "version": "1.416.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.402.3", + "version": "1.416.2", "license": "AGPL-3.0", "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", diff --git a/frontend/package.json b/frontend/package.json index b85fc1caba..9c4f3a2a7d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.402.3", + "version": "1.416.2", "scripts": { "dev": "vite dev", "build": "vite build", @@ -177,6 +177,11 @@ "svelte": "./package/components/icons/WindmillIcon.svelte", "default": "./package/components/icons/WindmillIcon.svelte" }, + "./components/icons/WindmillIcon2.svelte": { + "types": "./package/components/icons/WindmillIcon2.d.ts", + "svelte": "./package/components/icons/WindmillIcon2.svelte", + "default": "./package/components/icons/WindmillIcon2.svelte" + }, "./components/IconedResourceType.svelte": { "types": "./package/components/IconedResourceType.svelte.d.ts", "svelte": "./package/components/IconedResourceType.svelte", @@ -295,6 +300,14 @@ "types": "./package/utils.d.ts", "default": "./package/utils.js" }, + "./components/icons/store": { + "types": "./package/components/icons/store.d.ts", + "default": "./package/components/icons/store.js" + }, + "./script_helpers": { + "types": "./package/script_helpers.d.ts", + "default": "./package/script_helpers.js" + }, "./infer": { "types": "./package/infer.d.ts", "default": "./package/infer.js" @@ -336,6 +349,11 @@ "types": "./package/components/DropdownV2.svelte.d.ts", "svelte": "./package/components/DropdownV2.svelte", "default": "./package/components/DropdownV2.svelte" + }, + "./components/flows/FlowHistoryInner.svelte": { + "types": "./package/components/flows/FlowHistoryInner.svelte.d.ts", + "svelte": "./package/components/flows/FlowHistoryInner.svelte", + "default": "./package/components/flows/FlowHistoryInner.svelte" } }, "files": [ @@ -358,6 +376,9 @@ "components/icons/WindmillIcon.svelte": [ "./package/components/icons/WindmillIcon.svelte.d.ts" ], + "components/icons/WindmillIcon2.svelte": [ + "./package/components/icons/WindmillIcon2.svelte.d.ts" + ], "components/scriptEditor/LogPanel.svelte": [ "./package/components/scriptEditor/LogPanel.svelte.d.ts" ], @@ -418,6 +439,9 @@ "components/EditableSchemaWrapper.svelte": [ "./package/components/schema/EditableSchemaWrapper.svelte.d.ts" ], + "components/flows/FlowHistoryInner.svelte": [ + "./package/components/flows/FlowHistoryInner.svelte.d.ts" + ], "utils": [ "./package/utils.d.ts" ], @@ -453,6 +477,12 @@ ], "components/DropdownV2.svelte": [ "./package/components/DropdownV2.svelte.d.ts" + ], + "script_helpers": [ + "./package/script_helpers.d.ts" + ], + "components/icons/store": [ + "./package/components/icons/store.d.ts" ] } }, diff --git a/frontend/src/lib/assets/app.css b/frontend/src/lib/assets/app.css index 99bc17ae8e..6a386c559d 100644 --- a/frontend/src/lib/assets/app.css +++ b/frontend/src/lib/assets/app.css @@ -145,3 +145,23 @@ svelte-virtual-list-contents > * + * { rgba(255, 69, 58, 0.2) 20px ); } + +.bg-draggedover { + background-image: repeating-linear-gradient( + -45deg, + rgba(0, 0, 128, 0.2), + rgba(0, 0, 192, 0.2) 10px, + rgba(0, 0, 128, 0.2) 10px, + rgba(0, 0, 192, 0.2) 20px + ); +} + +.bg-draggedover-dark { + background-image: repeating-linear-gradient( + -45deg, + rgba(0, 0, 128, 0.6), + rgba(0, 0, 192, 0.6) 10px, + rgba(0, 0, 128, 0.6) 10px, + rgba(0, 0, 192, 0.6) 20px + ); +} diff --git a/frontend/src/lib/ata/index.ts b/frontend/src/lib/ata/index.ts index 3af147b45c..5414df8b18 100644 --- a/frontend/src/lib/ata/index.ts +++ b/frontend/src/lib/ata/index.ts @@ -120,11 +120,16 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => { ? f.raw : '/' + config.scriptPath + (f.raw.startsWith('../') ? '/../' : '/.') + f.raw let url = config.root + path - // console.log('FOO', config.scriptPath, path, f.raw) - console.log('fetching local file', url, f.raw) + let localPath = f.raw + if (f.raw.startsWith('.') && !f.raw.endsWith('.ts')) { + url += '.ts' + localPath += '.ts' + } + + console.log('fetching local file', url, f.raw, localPath) const res = await fetch(url) if (res.ok) { - config.delegate.localFile?.(await res.text(), f.raw) + config.delegate.localFile?.(await res.text(), localPath) } }) } diff --git a/frontend/src/lib/components/AppConnectDrawer.svelte b/frontend/src/lib/components/AppConnectDrawer.svelte index 28cb801616..4f30737736 100644 --- a/frontend/src/lib/components/AppConnectDrawer.svelte +++ b/frontend/src/lib/components/AppConnectDrawer.svelte @@ -6,7 +6,7 @@ import AppConnectInner from './AppConnectInner.svelte' import DarkModeObserver from './DarkModeObserver.svelte' - let expressOAuthSetup = false + export let expressOAuthSetup = false let drawer: Drawer let resourceType = '' @@ -18,8 +18,7 @@ let appConnectInner: AppConnectInner | undefined = undefined let rtToLoad: string | undefined = '' - export async function open(rt?: string, express?: boolean) { - expressOAuthSetup = express ?? false + export async function open(rt?: string) { rtToLoad = rt drawer.openDrawer?.() } @@ -27,7 +26,7 @@ $: appConnectInner && onRtToLoadChange(rtToLoad) function onRtToLoadChange(rtToLoad: string | undefined) { - appConnectInner?.open(rtToLoad, expressOAuthSetup) + appConnectInner?.open(rtToLoad) } const dispatch = createEventDispatcher() @@ -60,6 +59,7 @@ bind:manual on:close={drawer?.closeDrawer} on:refresh + express={expressOAuthSetup} />
{#if step > 1} diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index dbe162fd64..e568589b49 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -31,6 +31,7 @@ export let isGoogleSignin = false export let disabled = false export let manual = true + export let express = false let isValid = true @@ -81,10 +82,7 @@ let pathError = '' - let expressOAuthSetup = false - - export async function open(rt?: string, express?: boolean) { - expressOAuthSetup = express ?? false + export async function open(rt?: string) { if (!rt) { loadResourceTypes() } @@ -95,12 +93,12 @@ valueToken = undefined await loadConnects() manual = !connects?.includes(resourceType) - if (manual && expressOAuthSetup) { + if (manual && express) { dispatch('error', 'Express OAuth setup is not available for non OAuth resource types') return } if (rt) { - if (!manual && expressOAuthSetup) { + if (!manual && express) { await getScopesAndParams() step = 2 } @@ -171,7 +169,6 @@ } function popupListener(event) { - console.log('popupListener', event.data, event.origin, window.location.origin) let data = event.data if (event.origin == null || event.origin !== window.location.origin) { return @@ -187,7 +184,7 @@ value = data.res.access_token! valueToken = data.res step = 4 - if (expressOAuthSetup) { + if (express) { path = `u/${$userStore?.username}/${resourceType}_${new Date().getTime()}` next() } @@ -329,252 +326,254 @@ let editScopes = false - a.localeCompare(b)) - .map((key) => ({ - key - })) - : undefined} - bind:filteredItems={filteredConnects} - f={(x) => x.key} -/> - a[0].localeCompare(b[0]))} - bind:filteredItems={filteredConnectsManual} - f={(x) => x[0]} -/> -{#if step == 1} -
- -
- -

OAuth APIs

-
- {#if filteredConnects} - {#each filteredConnects as { key }} - - {/each} - {:else} - {#each new Array(3) as _} - - {/each} - {/if} -
- {#if connects && connects.length == 0} -
No OAuth APIs has been setup on the instance. To add oauth APIs, first sync the resource - types with the hub, then add oauth configuration. See documentation -
- {/if} - -

Others

- - {#if connectsManual && connectsManual?.length < 10} -
- Resource Types have not been synced with the hub. Go to the admins workspace to sync them (and - add a schedule to do daily): -

1. Go to the "admins" workspaces: - sync resource types -

-

- 2: Run the synchronization script: - sync resource types -

-
- {/if} - -
- {#if filteredConnectsManual} - {#each filteredConnectsManual as [key, _]} - {#if nativeLanguagesCategory.includes(key)} - - {/if} - {/each} - {/if} -
- -
-
- {#if filteredConnectsManual} - {#each filteredConnectsManual as [key, _]} - {#if !nativeLanguagesCategory.includes(key)} - - - {/if} - {/each} - {:else} - {#each new Array(9) as _} - - {/each} - {/if} -
-{:else if step == 2 && manual} - a.localeCompare(b)) + .map((key) => ({ + key + })) + : undefined} + bind:filteredItems={filteredConnects} + f={(x) => x.key} /> - - {#if apiTokenApps[resourceType]} -

Instructions

-
-
    - {#each apiTokenApps[resourceType].instructions as step} -
  1. - {@html step} -
  2. - {/each} -
-
- {#if apiTokenApps[resourceType].img} -
- connect -
- {/if} - {:else if !emptyString(resourceTypeInfo?.description)} -

{resourceTypeInfo?.name} description

-
- -
- {/if} - {#if resourceType == 'postgresql' || resourceType == 'mysql' || resourceType == 'mongodb'} - - {/if} - -

Resource description -
- - -
-

- {#if renderDescription} -
-
GH Markdown
-