diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 5792e5753b..e20d023bdc 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -40,11 +40,11 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER RUN /usr/local/bin/python3 -m pip install pip-tools -COPY --from=oven/bun:1.1.30 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.31 /usr/local/bin/bun /usr/bin/bun ARG TARGETPLATFORM -RUN curl -Lsf https://github.com/denoland/deno/releases/download/v2.0.0/deno-x86_64-unknown-linux-gnu.zip -o deno.zip +RUN curl -Lsf https://github.com/denoland/deno/releases/download/v2.0.2/deno-x86_64-unknown-linux-gnu.zip -o deno.zip # RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v2.0.0/deno-aarch64-unknown-linux-gnu.zip -o deno.zip || true RUN unzip deno.zip && rm deno.zip && mv deno /usr/bin/deno diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index f0928cbd33..f1f3cae9f8 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -26,6 +26,7 @@ permissions: write-all jobs: build: runs-on: ubicloud + if: (github.event_name != 'issue_comment') || (contains(github.event.comment.body, '/buildimage_all') || contains(github.event.comment.body, '/buildimage_base')) steps: - uses: actions/checkout@v4 with: @@ -87,6 +88,7 @@ jobs: build_ee: runs-on: ubicloud + if: (github.event_name != 'issue_comment') || (contains(github.event.comment.body, '/buildimage_ee') || contains(github.event.comment.body, '/buildimage_nsjail')) || contains(github.event.comment.body, '/buildimage_all') steps: - uses: actions/checkout@v4 with: @@ -148,7 +150,7 @@ jobs: build_ee_312: runs-on: ubicloud - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} steps: - uses: actions/checkout@v4 with: @@ -244,7 +246,7 @@ jobs: attach_amd64_binary_to_release: needs: [build, build_ee] runs-on: ubicloud - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} env: ARCH: amd64 steps: @@ -337,10 +339,10 @@ jobs: with: fetch-depth: 0 - name: Prepare test run - if: ${{ ! startsWith(github.ref, 'refs/tags/') }} + if: ${{ ! startsWith(github.ref, 'refs/tags/v') }} run: cd integration_tests && ./build.sh - name: Test run - if: ${{ ! startsWith(github.ref, 'refs/tags/') }} + if: ${{ ! startsWith(github.ref, 'refs/tags/v') }} timeout-minutes: 15 env: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} @@ -356,7 +358,7 @@ jobs: tag_latest: runs-on: ubicloud needs: [run_integration_test, build] - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) steps: - uses: actions/checkout@v4 with: @@ -375,7 +377,7 @@ jobs: tag_latest_ee: runs-on: ubicloud needs: [run_integration_test, build_ee] - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) steps: - uses: actions/checkout@v4 with: @@ -394,7 +396,7 @@ jobs: verify_ee_image_vulnerabilities: runs-on: ubicloud needs: [tag_latest_ee] - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -436,7 +438,7 @@ jobs: build_ee_nsjail: needs: [build_ee] runs-on: ubicloud - if: github.event_name != 'pull_request' + if: (github.event_name != 'issue_comment') || (github.event_name != 'pull_request') || (contains(github.event.comment.body, '/buildimage_nsjail') || contains(github.event.comment.body, '/buildimage_all')) steps: - uses: actions/checkout@v4 with: @@ -459,6 +461,8 @@ jobs: type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=sha,enable=true,priority=100,prefix=,suffix=,format=short + type=ref,event=branch + type=ref,event=pr - name: Login to registry uses: docker/login-action@v3 @@ -476,7 +480,6 @@ jobs: file: "./docker/DockerfileNsjail" tags: | ${{ steps.meta-ee-public.outputs.tags }} - ghcr.io/windmill-labs/windmill-ee-nsjail:main labels: | ${{ steps.meta-ee-public.outputs.labels }} org.opencontainers.image.licenses=Windmill-Enterprise-License @@ -570,7 +573,7 @@ jobs: bucket-region: us-east-1 build_ee_cuda: - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build_ee] runs-on: ubicloud steps: @@ -614,7 +617,7 @@ jobs: org.opencontainers.image.licenses=Windmill-Enterprise-License build_slim: - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build] runs-on: ubicloud steps: @@ -657,7 +660,7 @@ jobs: ${{ steps.meta-ee-public.outputs.labels }} build_ee_slim: - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build_ee] runs-on: ubicloud steps: @@ -701,7 +704,7 @@ jobs: org.opencontainers.image.licenses=Windmill-Enterprise-License build_full: - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build] runs-on: ubicloud steps: @@ -744,7 +747,7 @@ jobs: ${{ steps.meta-public.outputs.labels }} build_ee_full: - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/v') }} needs: [build_ee] runs-on: ubicloud steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 31ef1b08a2..111225c3a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,144 @@ # Changelog +## [1.416.2](https://github.com/windmill-labs/windmill/compare/v1.416.1...v1.416.2) (2024-11-02) + + +### Bug Fixes + +* apply NO_PROXY and HTTP_PROXY, HTTPS_PROXY more consistently ([567d621](https://github.com/windmill-labs/windmill/commit/567d6216d2631a90fbe59ec6142c38b3b352eea7)) + +## [1.416.1](https://github.com/windmill-labs/windmill/compare/v1.416.0...v1.416.1) (2024-11-01) + + +### Bug Fixes + +* **prometheus:** fix incorrect worker_busy set to 1 ([53f9136](https://github.com/windmill-labs/windmill/commit/53f9136658b9fc1795793d82408c1f1f04adcf06)) + +## [1.416.0](https://github.com/windmill-labs/windmill/compare/v1.415.2...v1.416.0) (2024-11-01) + + +### Features + +* private hub user accessible url setting ([#4617](https://github.com/windmill-labs/windmill/issues/4617)) ([79edf89](https://github.com/windmill-labs/windmill/commit/79edf89bd17827d5f1d946739385327b6c0520bf)) + + +### Bug Fixes + +* **frontend:** improve tag selector for workspace script drawer ([66f6985](https://github.com/windmill-labs/windmill/commit/66f69859ad2de51aef5a133df4ab4397d0f61ccf)) + +## [1.415.2](https://github.com/windmill-labs/windmill/compare/v1.415.1...v1.415.2) (2024-11-01) + + +### Bug Fixes + +* **s3:** align s3 handler additional creds providers ([984c6dd](https://github.com/windmill-labs/windmill/commit/984c6dd10c63097eb195883c4d8a9681ab1b49e0)) + +## [1.415.1](https://github.com/windmill-labs/windmill/compare/v1.415.0...v1.415.1) (2024-10-31) + + +### Bug Fixes + +* **cli:** improve --instance handling wmill instance push ([cb005a1](https://github.com/windmill-labs/windmill/commit/cb005a15baef4272bc58c7e80a43e44723556d31)) + +## [1.415.0](https://github.com/windmill-labs/windmill/compare/v1.414.2...v1.415.0) (2024-10-31) + + +### Features + +* **cli:** opts.instance as instace name and prefix ([#4609](https://github.com/windmill-labs/windmill/issues/4609)) ([a07f57e](https://github.com/windmill-labs/windmill/commit/a07f57e698107056d045d8d5c2458e04c809fcc8)) + + +### Bug Fixes + +* improve express oauth setup ([ba4aed5](https://github.com/windmill-labs/windmill/commit/ba4aed5bf51c65204332cfc158d0ffd9c7095ec7)) +* improve user resource input ([8c7f53b](https://github.com/windmill-labs/windmill/commit/8c7f53b2ebe0990cd93879258d004ac89dc8b24c)) + +## [1.414.2](https://github.com/windmill-labs/windmill/compare/v1.414.1...v1.414.2) (2024-10-29) + + +### Bug Fixes + +* **cli:** improve instance sync for CI/CD + --folder-per-instance ([212579a](https://github.com/windmill-labs/windmill/commit/212579a514d070355fe0d9e0215593bacfa05e1f)) + +## [1.414.1](https://github.com/windmill-labs/windmill/compare/v1.414.0...v1.414.1) (2024-10-29) + + +### Bug Fixes + +* **apps:** enable text selection on aggrid tables by default ([b0b9180](https://github.com/windmill-labs/windmill/commit/b0b9180fb907c92b95a48ff286eb1dae59bb4981)) +* **apps:** public apps can take full height ([703db7d](https://github.com/windmill-labs/windmill/commit/703db7d4412795b4323a2eefcc39ae3cf43bc748)) +* **bun:** handle bun lockfile created with windows ([#4602](https://github.com/windmill-labs/windmill/issues/4602)) ([dcf5e2f](https://github.com/windmill-labs/windmill/commit/dcf5e2f03f977e241f6785530dad61a70c5bdd79)) +* **frontend:** make script and schema scrollable on script detail page ([6e222b3](https://github.com/windmill-labs/windmill/commit/6e222b3b1a419e5543fdf35420a7413699688b41)) +* **frontend:** new approval steps default to timeout 1800 ([b86de62](https://github.com/windmill-labs/windmill/commit/b86de6280e03e014e8ddf85b2b5f8fd030d0467a)) + +## [1.414.0](https://github.com/windmill-labs/windmill/compare/v1.413.2...v1.414.0) (2024-10-29) + +* Issue with previous release, re-releasing + +## [1.413.2](https://github.com/windmill-labs/windmill/compare/v1.413.1...v1.413.2) (2024-10-29) + + +### Bug Fixes + +* **backend:** in flows, workspace scripts should use their set tags instead of the default one ([5b7c6d7](https://github.com/windmill-labs/windmill/commit/5b7c6d7d62dcfd09fec374e781bdf5c5bafe4a9d)) +* **cli:** fix wmill instance pull --instance ([3c62f5e](https://github.com/windmill-labs/windmill/commit/3c62f5ea83d1da8bd3705468969d56e2fe680751)) +* **frontend:** fix script and flow renaming ([d743e00](https://github.com/windmill-labs/windmill/commit/d743e0056353a4fca445a7089e3afc1fd4e8c219)) + +## [1.413.1](https://github.com/windmill-labs/windmill/compare/v1.413.0...v1.413.1) (2024-10-28) + + +### Bug Fixes + +* **cli:** fix wmill instance push --base-url and --instance ([8298710](https://github.com/windmill-labs/windmill/commit/82987105a6fd6ec272c170fb094453a0267143be)) + +## [1.413.0](https://github.com/windmill-labs/windmill/compare/v1.412.0...v1.413.0) (2024-10-28) + + +### Features + +* autoscaling v0 ([#4593](https://github.com/windmill-labs/windmill/issues/4593)) ([fe7d044](https://github.com/windmill-labs/windmill/commit/fe7d044a66e8ec223a337cb704d3e58942dd1502)) + + +### Bug Fixes + +* add run immediately popover to run again ([e54d253](https://github.com/windmill-labs/windmill/commit/e54d25368541dc6109a0f99022a120d28455f9bd)) +* **docs:** smtp setup documentation link ([#4590](https://github.com/windmill-labs/windmill/issues/4590)) ([bac3205](https://github.com/windmill-labs/windmill/commit/bac32057259d893140d649c9dfec2ca75e395ad4)) + +## [1.412.0](https://github.com/windmill-labs/windmill/compare/v1.411.1...v1.412.0) (2024-10-25) + + +### Features + +* add Spotify oauth provider ([#4581](https://github.com/windmill-labs/windmill/issues/4581)) ([a46aa64](https://github.com/windmill-labs/windmill/commit/a46aa644b096e71e75b59507224ed92f7d2f99ba)) + + +### Bug Fixes + +* **app builder:** date input default value improvements ([9f43d5d](https://github.com/windmill-labs/windmill/commit/9f43d5dcd92ddcd0c0baeaa5271779520af652f4)) +* **bash:** correctly propagate sigterm for cancelled bash scripts ([134cfdb](https://github.com/windmill-labs/windmill/commit/134cfdb30eb8d29c2ecb1b78b8fabae2b6e10700)) +* do not update created_at of scripts on lockfile generation ([d1a28eb](https://github.com/windmill-labs/windmill/commit/d1a28eb7cac5f465e2b07f870a507b0cc5cc722a)) +* initialize empty smtp settings correctly ([84e0524](https://github.com/windmill-labs/windmill/commit/84e05249505c3d2f7eb7a4917f6bca503df81017)) + +## [1.411.1](https://github.com/windmill-labs/windmill/compare/v1.411.0...v1.411.1) (2024-10-22) + + +### Bug Fixes + +* update bun to 1.1.32 ([#4568](https://github.com/windmill-labs/windmill/issues/4568)) ([0586446](https://github.com/windmill-labs/windmill/commit/058644667129f0d79ec147aacdda449142ae0ab9)) + +## [1.411.0](https://github.com/windmill-labs/windmill/compare/v1.410.3...v1.411.0) (2024-10-21) + + +### Features + +* **cli:** encrypt sensitive instance settings ([#4561](https://github.com/windmill-labs/windmill/issues/4561)) ([b8a6a11](https://github.com/windmill-labs/windmill/commit/b8a6a116354b10f5977e54edb365d6711e160538)) + + +### Bug Fixes + +* Do not ignore file resources with json file ext ([#4562](https://github.com/windmill-labs/windmill/issues/4562)) ([2079b2e](https://github.com/windmill-labs/windmill/commit/2079b2e7e19aa2fe327f2ae66d1b5eba988b9b0a)) +* update bun to 1.1.31 and deno to 2.0.2 ([0d90396](https://github.com/windmill-labs/windmill/commit/0d9039641b3348e75599937188c35dd89a000584)) + ## [1.410.3](https://github.com/windmill-labs/windmill/compare/v1.410.2...v1.410.3) (2024-10-20) diff --git a/Dockerfile b/Dockerfile index 505cf61d50..8683aa9738 100644 --- a/Dockerfile +++ b/Dockerfile @@ -162,7 +162,7 @@ ENV GO_PATH=/usr/local/go/bin/go RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - -RUN apt-get -y update && apt-get install -y curl nodejs awscli && apt-get clean \ +RUN apt-get -y update && apt-get install -y curl procps nodejs awscli && apt-get clean \ && rm -rf /var/lib/apt/lists/* # go build is slower the first time it is ran, so we prewarm it in the build @@ -175,9 +175,9 @@ RUN /usr/local/bin/python3 -m pip install pip-tools COPY --from=builder /frontend/build /static_frontend COPY --from=builder /windmill/target/release/windmill ${APP}/windmill -COPY --from=denoland/deno:2.0.0 --chmod=755 /usr/bin/deno /usr/bin/deno +COPY --from=denoland/deno:2.0.2 --chmod=755 /usr/bin/deno /usr/bin/deno -COPY --from=oven/bun:1.1.30 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.32 /usr/local/bin/bun /usr/bin/bun COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer diff --git a/backend/.sqlx/query-02b516dac764662194db1bc33e365c01f40bae70af3683f1f09748f6020f0d49.json b/backend/.sqlx/query-02b516dac764662194db1bc33e365c01f40bae70af3683f1f09748f6020f0d49.json deleted file mode 100644 index 99f3606bcd..0000000000 --- a/backend/.sqlx/query-02b516dac764662194db1bc33e365c01f40bae70af3683f1f09748f6020f0d49.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT tag, count(*) as count FROM queue WHERE\n scheduled_for <= now() - ('3 seconds')::interval AND running = false\n GROUP BY tag", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - null - ] - }, - "hash": "02b516dac764662194db1bc33e365c01f40bae70af3683f1f09748f6020f0d49" -} diff --git a/backend/.sqlx/query-19f0abd79372698f378cb6deea3ee6d098a2758d16ede000809bd9a09660b604.json b/backend/.sqlx/query-19f0abd79372698f378cb6deea3ee6d098a2758d16ede000809bd9a09660b604.json deleted file mode 100644 index 8796b86902..0000000000 --- a/backend/.sqlx/query-19f0abd79372698f378cb6deea3ee6d098a2758d16ede000809bd9a09660b604.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT created_at FROM metrics WHERE id = 'telemetry' ORDER BY created_at DESC LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "created_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "19f0abd79372698f378cb6deea3ee6d098a2758d16ede000809bd9a09660b604" -} diff --git a/backend/.sqlx/query-c2060e8cacef6c3b5ce51ed203a2dbafc18d66f2924d1fe518c6728997647db2.json b/backend/.sqlx/query-1a4d291c2f239f7b50c116594cebb031862e1a18ad9204e02a0194817db26d6a.json similarity index 75% rename from backend/.sqlx/query-c2060e8cacef6c3b5ce51ed203a2dbafc18d66f2924d1fe518c6728997647db2.json rename to backend/.sqlx/query-1a4d291c2f239f7b50c116594cebb031862e1a18ad9204e02a0194817db26d6a.json index 6a08bbeab3..b8c5b89128 100644 --- a/backend/.sqlx/query-c2060e8cacef6c3b5ce51ed203a2dbafc18d66f2924d1fe518c6728997647db2.json +++ b/backend/.sqlx/query-1a4d291c2f239f7b50c116594cebb031862e1a18ad9204e02a0194817db26d6a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT email, login_type::text, verified, super_admin, name, company, username from password ORDER BY super_admin DESC, email LIMIT $1 OFFSET $2", + "query": "SELECT email, login_type::text, verified, super_admin, name, company, username, NULL::bool as operator_only FROM password ORDER BY super_admin DESC, email LIMIT $1 OFFSET $2", "describe": { "columns": [ { @@ -37,6 +37,11 @@ "ordinal": 6, "name": "username", "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "operator_only", + "type_info": "Bool" } ], "parameters": { @@ -52,8 +57,9 @@ false, true, true, - true + true, + null ] }, - "hash": "c2060e8cacef6c3b5ce51ed203a2dbafc18d66f2924d1fe518c6728997647db2" + "hash": "1a4d291c2f239f7b50c116594cebb031862e1a18ad9204e02a0194817db26d6a" } diff --git a/backend/.sqlx/query-1f5f0858909eb5bac63c4e3b1add95226bd94ca3facb92a620ffa59dacad6705.json b/backend/.sqlx/query-1f5f0858909eb5bac63c4e3b1add95226bd94ca3facb92a620ffa59dacad6705.json new file mode 100644 index 0000000000..0adff2fe2c --- /dev/null +++ b/backend/.sqlx/query-1f5f0858909eb5bac63c4e3b1add95226bd94ca3facb92a620ffa59dacad6705.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO autoscaling_event (worker_group, event_type, desired_workers, reason) VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "autoscaling_event_type", + "kind": { + "Enum": [ + "full_scaleout", + "scalein", + "scaleout" + ] + } + } + }, + "Int4", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1f5f0858909eb5bac63c4e3b1add95226bd94ca3facb92a620ffa59dacad6705" +} diff --git a/backend/.sqlx/query-8813665f8adfcab0daefbac2cc6b50e427dfd1c12d12895451affd307dc59c37.json b/backend/.sqlx/query-2c14d3a88193f16ad3b8cd590749cb5537995f2499f6cb8f0f316fb62902d542.json similarity index 77% rename from backend/.sqlx/query-8813665f8adfcab0daefbac2cc6b50e427dfd1c12d12895451affd307dc59c37.json rename to backend/.sqlx/query-2c14d3a88193f16ad3b8cd590749cb5537995f2499f6cb8f0f316fb62902d542.json index ce8adcd5b7..93017f5aec 100644 --- a/backend/.sqlx/query-8813665f8adfcab0daefbac2cc6b50e427dfd1c12d12895451affd307dc59c37.json +++ b/backend/.sqlx/query-2c14d3a88193f16ad3b8cd590749cb5537995f2499f6cb8f0f316fb62902d542.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT email, login_type::TEXT, super_admin, verified, name, company, username FROM password WHERE email = $1", + "query": "SELECT email, login_type::TEXT, super_admin, verified, name, company, username, NULL::bool as operator_only FROM password WHERE email = $1", "describe": { "columns": [ { @@ -37,6 +37,11 @@ "ordinal": 6, "name": "username", "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "operator_only", + "type_info": "Bool" } ], "parameters": { @@ -51,8 +56,9 @@ false, true, true, - true + true, + null ] }, - "hash": "8813665f8adfcab0daefbac2cc6b50e427dfd1c12d12895451affd307dc59c37" + "hash": "2c14d3a88193f16ad3b8cd590749cb5537995f2499f6cb8f0f316fb62902d542" } diff --git a/backend/.sqlx/query-2eec077cc9e27d7ccd160cbaac118c321c422705f79e69550bb60f377083bcef.json b/backend/.sqlx/query-2eec077cc9e27d7ccd160cbaac118c321c422705f79e69550bb60f377083bcef.json new file mode 100644 index 0000000000..f3eecb9006 --- /dev/null +++ b/backend/.sqlx/query-2eec077cc9e27d7ccd160cbaac118c321c422705f79e69550bb60f377083bcef.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, name, company, username\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC\n LIMIT $1 OFFSET $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "operator_only", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "login_type", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "verified", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "super_admin", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "company", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "username", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + null, + null, + false, + false, + true, + true, + true + ] + }, + "hash": "2eec077cc9e27d7ccd160cbaac118c321c422705f79e69550bb60f377083bcef" +} diff --git a/backend/.sqlx/query-4eca060026a0cb19c5794cd56ace89fc04765191f251945d14bbe78718714f6e.json b/backend/.sqlx/query-4eca060026a0cb19c5794cd56ace89fc04765191f251945d14bbe78718714f6e.json new file mode 100644 index 0000000000..3b812a53db --- /dev/null +++ b/backend/.sqlx/query-4eca060026a0cb19c5794cd56ace89fc04765191f251945d14bbe78718714f6e.json @@ -0,0 +1,39 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT event_type::AUTOSCALING_EVENT_TYPE AS \"event_type: _\", EXTRACT(EPOCH FROM (NOW() - applied_at))::int as seconds_ago FROM autoscaling_event WHERE worker_group = $1 ORDER BY applied_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "event_type: _", + "type_info": { + "Custom": { + "name": "autoscaling_event_type", + "kind": { + "Enum": [ + "full_scaleout", + "scalein", + "scaleout" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "seconds_ago", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "4eca060026a0cb19c5794cd56ace89fc04765191f251945d14bbe78718714f6e" +} diff --git a/backend/.sqlx/query-6776dc50f184188756ad7fe263b0304333536768527525a43bdd45aedffa3c4f.json b/backend/.sqlx/query-6776dc50f184188756ad7fe263b0304333536768527525a43bdd45aedffa3c4f.json deleted file mode 100644 index 4187b628b7..0000000000 --- a/backend/.sqlx/query-6776dc50f184188756ad7fe263b0304333536768527525a43bdd45aedffa3c4f.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT pg_try_advisory_xact_lock($1)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "pg_try_advisory_xact_lock", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - null - ] - }, - "hash": "6776dc50f184188756ad7fe263b0304333536768527525a43bdd45aedffa3c4f" -} diff --git a/backend/.sqlx/query-6d134b137ae81534e145fc5b6474cf963ee26a3ad3a0a3d8dc064cb14c8fd9a6.json b/backend/.sqlx/query-6d134b137ae81534e145fc5b6474cf963ee26a3ad3a0a3d8dc064cb14c8fd9a6.json new file mode 100644 index 0000000000..03351cdeab --- /dev/null +++ b/backend/.sqlx/query-6d134b137ae81534e145fc5b6474cf963ee26a3ad3a0a3d8dc064cb14c8fd9a6.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, worker_group, event_type::text, desired_workers, reason, applied_at FROM autoscaling_event WHERE worker_group = $1 ORDER BY applied_at DESC LIMIT 5", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "worker_group", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "event_type", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "desired_workers", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "reason", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "applied_at", + "type_info": "Timestamp" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + null, + false, + true, + false + ] + }, + "hash": "6d134b137ae81534e145fc5b6474cf963ee26a3ad3a0a3d8dc064cb14c8fd9a6" +} diff --git a/backend/.sqlx/query-9ed77e78e6295c62745ac3ac3b7e5f544f654d7cddc13acb5d3f4fdc12a8875f.json b/backend/.sqlx/query-9ed77e78e6295c62745ac3ac3b7e5f544f654d7cddc13acb5d3f4fdc12a8875f.json deleted file mode 100644 index 9c7af05c42..0000000000 --- a/backend/.sqlx/query-9ed77e78e6295c62745ac3ac3b7e5f544f654d7cddc13acb5d3f4fdc12a8875f.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT created_at FROM metrics WHERE id = 'license_key_renewal' ORDER BY created_at DESC LIMIT 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "created_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "9ed77e78e6295c62745ac3ac3b7e5f544f654d7cddc13acb5d3f4fdc12a8875f" -} diff --git a/backend/.sqlx/query-ec47955683d811b12e82ff3e6aeafa0df46a320c436bad5eb1acc127df138a61.json b/backend/.sqlx/query-d697b7311430e7bd5375ec5494179f4071e3bfe123d9600249cfa80c1103edd8.json similarity index 51% rename from backend/.sqlx/query-ec47955683d811b12e82ff3e6aeafa0df46a320c436bad5eb1acc127df138a61.json rename to backend/.sqlx/query-d697b7311430e7bd5375ec5494179f4071e3bfe123d9600249cfa80c1103edd8.json index 2ed5e047db..e35c4e2730 100644 --- a/backend/.sqlx/query-ec47955683d811b12e82ff3e6aeafa0df46a320c436bad5eb1acc127df138a61.json +++ b/backend/.sqlx/query-d697b7311430e7bd5375ec5494179f4071e3bfe123d9600249cfa80c1103edd8.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE script SET lock = $1, created_at = now() WHERE hash = $2 AND workspace_id = $3", + "query": "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "ec47955683d811b12e82ff3e6aeafa0df46a320c436bad5eb1acc127df138a61" + "hash": "d697b7311430e7bd5375ec5494179f4071e3bfe123d9600249cfa80c1103edd8" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 89bed47aa1..d12b1b2698 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -117,9 +117,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.15" +version = "0.6.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +checksum = "23a1e53f0f5d86382dafe1cf314783b2044280f406e7e1506368220ad11b1338" dependencies = [ "anstyle", "anstyle-parse", @@ -132,43 +132,43 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" dependencies = [ "anstyle", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.90" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95" +checksum = "74f37166d7d48a0284b99dd824694c26119c700b53bf0d1540cdb147dbdaaf13" [[package]] name = "arc-swap" @@ -441,7 +441,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -486,9 +486,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.16" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "103db485efc3e41214fe4fda9f3dbeae2eb9082f48fd236e6095627a9422066e" +checksum = "0cb8f1d480b0ea3783ab015936d2a55c87e219676f0c0b7dec61494043f21857" dependencies = [ "brotli 7.0.0", "bzip2", @@ -512,7 +512,7 @@ dependencies = [ "async-task", "concurrent-queue", "fastrand 2.1.1", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "slab", ] @@ -527,7 +527,7 @@ dependencies = [ "async-io", "async-lock", "blocking", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "once_cell", ] @@ -541,7 +541,7 @@ dependencies = [ "cfg-if", "concurrent-queue", "futures-io", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "parking", "polling", "rustix", @@ -571,7 +571,7 @@ dependencies = [ "bytes", "http 1.1.0", "rand 0.8.5", - "reqwest 0.12.8", + "reqwest 0.12.9", "serde", "serde-aux", "serde_json", @@ -588,7 +588,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -605,7 +605,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-io", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "gloo-timers", "kv-log-macro", "log", @@ -636,14 +636,14 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] name = "async-stripe" -version = "0.34.1" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109605d984dd71a9a278e2d43a2831a20e80f25c3bcc25174096d12352cfc469" +checksum = "58d670cf4d47a1b8ffef54286a5625382e360a34ee76902fd93ad8c7032a0c30" dependencies = [ "chrono", "futures-util", @@ -678,7 +678,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -732,9 +732,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.8" +version = "1.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7198e6f03240fdceba36656d8be440297b6b82270325908c7381f37d826a74f6" +checksum = "2d6448cfb224dd6a9b9ac734f58622dd0d4751f3589f3b777345745f46b2eb14" dependencies = [ "aws-credential-types", "aws-runtime", @@ -799,9 +799,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.46.0" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc2faec3205d496c7e57eff685dd944203df7ce16a4116d0281c44021788a7b" +checksum = "ded855583fa1d22e88fe39fd6062b062376e50a8211989e07cf5e38d52eb3453" dependencies = [ "aws-credential-types", "aws-runtime", @@ -821,9 +821,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.47.0" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c93c241f52bc5e0476e259c953234dab7e2a35ee207ee202e86c0095ec4951dc" +checksum = "9177ea1192e6601ae16c7273385690d88a7ed386a00b74a6bc894d12103cd933" dependencies = [ "aws-credential-types", "aws-runtime", @@ -843,9 +843,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.46.0" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b259429be94a3459fa1b00c5684faee118d74f9577cc50aebadc36e507c63b5f" +checksum = "823ef553cf36713c97453e2ddff1eb8f62be7f4523544e2a5db64caf80100f0a" dependencies = [ "aws-credential-types", "aws-runtime", @@ -866,9 +866,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.4" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc8db6904450bafe7473c6ca9123f88cc11089e41a025408f992db4e22d3be68" +checksum = "5619742a0d8f253be760bfbb8e8e8368c69e3587e4637af5754e488a611499b1" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -939,9 +939,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.2" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a065c0fe6fdbdf9f11817eb68582b2ab4aff9e9c39e986ae48f7ec576c6322db" +checksum = "be28bd063fa91fd871d131fc8b68d7cd4c5fa0869bea68daca50dcb1cbd76be2" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -983,9 +983,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.7" +version = "1.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147100a7bea70fa20ef224a6bad700358305f5dc0f84649c53769761395b355b" +checksum = "07c9cdc179e6afbf5d391ab08c85eac817b51c87e1892a5edb5f7bbdc64314b4" dependencies = [ "base64-simd 0.8.0", "bytes", @@ -1179,9 +1179,9 @@ dependencies = [ [[package]] name = "bigdecimal" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d712318a27c7150326677b321a5fa91b55f6d9034ffd67f20319e147d40cee" +checksum = "8f850665a0385e070b64c38d2354e6c104c8479c59868d1e48a0c13ee2c7a1c1" dependencies = [ "autocfg", "libm", @@ -1212,13 +1212,13 @@ dependencies = [ "lazy_static", "lazycell", "log", - "prettyplease 0.2.24", + "prettyplease 0.2.25", "proc-macro2", "quote", "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.81", + "syn 2.0.86", "which 4.4.2", ] @@ -1237,7 +1237,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -1357,7 +1357,7 @@ dependencies = [ "async-channel 2.3.1", "async-task", "futures-io", - "futures-lite 2.3.0", + "futures-lite 2.4.0", "piper", ] @@ -1381,7 +1381,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", "syn_derive", ] @@ -1484,7 +1484,7 @@ checksum = "bcfcc3cd946cb52f0bbfdbbcfa2f4e24f75ebb6c0e1002f7c25904fada18b9ec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -1495,9 +1495,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "bytes-utils" @@ -1599,9 +1599,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.31" +version = "1.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "e3788d6ac30243803df38a3e9991cf37e41210232916d41a8222ae378f912624" dependencies = [ "jobserver", "libc", @@ -1744,7 +1744,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -1764,9 +1764,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "combine" @@ -2124,7 +2124,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -2172,7 +2172,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -2194,7 +2194,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -2233,7 +2233,7 @@ dependencies = [ "arrow-array", "arrow-ipc", "arrow-schema", - "async-compression 0.4.16", + "async-compression 0.4.17", "async-trait", "bytes", "bzip2", @@ -2715,7 +2715,7 @@ dependencies = [ "quote", "strum 0.25.0", "strum_macros 0.25.3", - "syn 2.0.81", + "syn 2.0.86", "thiserror", ] @@ -2777,7 +2777,7 @@ checksum = "22a1abc6bd8af41aa2496ceceef94f4277092c781a9495c437327a17659553a2" dependencies = [ "deno_core", "deno_native_certs", - "rustls 0.23.15", + "rustls 0.23.16", "rustls-pemfile 2.2.0", "rustls-tokio-stream", "rustls-webpki 0.102.8", @@ -2907,7 +2907,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -3004,7 +3004,7 @@ checksum = "f2b99bf03862d7f545ebc28ddd33a665b50865f4dfd84031a393823879bd4c54" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -3160,7 +3160,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -3180,7 +3180,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -3260,9 +3260,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastdivide" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59668941c55e5c186b8b58c391629af56774ec768f73c08bbcd56f09348eb00b" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" [[package]] name = "fastrand" @@ -3395,7 +3395,7 @@ checksum = "32016f1242eb82af5474752d00fd8ebcd9004bd69b462b1c91de833972d08ed4" dependencies = [ "proc-macro2", "swc_macros_common", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -3427,7 +3427,7 @@ checksum = "e99b8b3c28ae0e84b604c75f721c21dc77afb3706076af5e8216d15fd1deaae3" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -3439,7 +3439,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -3451,7 +3451,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -3556,9 +3556,9 @@ dependencies = [ [[package]] name = "futures-lite" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" +checksum = "3f1fa2f9765705486b33fd2acf1577f8ec449c2ba1f318ae5447697b7c08d210" dependencies = [ "fastrand 2.1.1", "futures-core", @@ -3575,7 +3575,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -3828,7 +3828,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -4265,7 +4265,7 @@ dependencies = [ "http 1.1.0", "hyper 1.5.0", "hyper-util", - "rustls 0.23.15", + "rustls 0.23.16", "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", @@ -4463,7 +4463,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -4700,9 +4700,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "libredox" @@ -4889,7 +4889,7 @@ dependencies = [ "base64 0.22.1", "gethostname", "mail-builder", - "rustls 0.23.15", + "rustls 0.23.16", "rustls-pki-types", "smtp-proto", "tokio", @@ -5113,7 +5113,7 @@ checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -5152,7 +5152,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", "termcolor", "thiserror", ] @@ -5445,7 +5445,7 @@ dependencies = [ "percent-encoding", "quick-xml 0.36.2", "rand 0.8.5", - "reqwest 0.12.8", + "reqwest 0.12.9", "ring 0.17.8", "serde", "serde_json", @@ -5562,7 +5562,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -5839,7 +5839,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -5865,29 +5865,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -6057,12 +6057,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.24" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "910d41a655dac3b764f1ade94821093d3610248694320cd072303a8eedcf221d" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -6115,7 +6115,7 @@ checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" dependencies = [ "proc-macro-rules-macros", "proc-macro2", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -6127,14 +6127,14 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] name = "proc-macro2" -version = "1.0.88" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3a7fc5db1e57d5a779a352c8cdb57b29aa4c40cc69c3a68a7fedc815fbf2f9" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -6188,7 +6188,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "syn 2.0.81", + "syn 2.0.86", "thiserror", "typify", "unicode-ident", @@ -6208,7 +6208,7 @@ dependencies = [ "serde_json", "serde_tokenstream", "serde_yaml", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -6251,7 +6251,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -6365,7 +6365,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.0.0", - "rustls 0.23.15", + "rustls 0.23.16", "socket2 0.5.7", "thiserror", "tokio", @@ -6382,7 +6382,7 @@ dependencies = [ "rand 0.8.5", "ring 0.17.8", "rustc-hash 2.0.0", - "rustls 0.23.15", + "rustls 0.23.16", "slab", "thiserror", "tinyvec", @@ -6391,10 +6391,11 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +checksum = "e346e016eacfff12233c243718197ca12f148c84e1e84268a896699b41c71780" dependencies = [ + "cfg_aliases", "libc", "once_cell", "socket2 0.5.7", @@ -6603,9 +6604,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -6724,11 +6725,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.8" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ - "async-compression 0.4.16", + "async-compression 0.4.17", "base64 0.22.1", "bytes", "encoding_rs", @@ -6751,7 +6752,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.15", + "rustls 0.23.16", "rustls-native-certs 0.8.0", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -6934,7 +6935,7 @@ dependencies = [ "proc-macro2", "quote", "rust-embed-utils", - "syn 2.0.81", + "syn 2.0.86", "walkdir", ] @@ -7026,9 +7027,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.37" +version = "0.38.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" dependencies = [ "bitflags 2.6.0", "errno", @@ -7051,9 +7052,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.15" +version = "0.23.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993" +checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" dependencies = [ "log", "once_cell", @@ -7133,7 +7134,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22557157d7395bc30727745b365d923f1ecc230c4c80b176545f3f4f08c46e33" dependencies = [ "futures", - "rustls 0.23.15", + "rustls 0.23.16", "socket2 0.5.7", "tokio", ] @@ -7314,7 +7315,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -7414,9 +7415,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" dependencies = [ "serde_derive", ] @@ -7455,13 +7456,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -7472,7 +7473,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -7547,7 +7548,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -7602,7 +7603,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -7971,7 +7972,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -8016,7 +8017,7 @@ dependencies = [ "once_cell", "paste", "percent-encoding", - "rustls 0.23.15", + "rustls 0.23.16", "rustls-pemfile 2.2.0", "serde", "serde_json", @@ -8042,7 +8043,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -8065,7 +8066,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.81", + "syn 2.0.86", "tempfile", "tokio", "url", @@ -8217,7 +8218,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -8271,7 +8272,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -8284,7 +8285,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -8392,7 +8393,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -8441,7 +8442,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -8526,7 +8527,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -8633,7 +8634,7 @@ checksum = "63db0adcff29d220c3d151c5b25c0eabe7e32dd936212b84cdaa1392e3130497" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -8644,7 +8645,7 @@ checksum = "f486687bfb7b5c560868f69ed2d458b880cebc9babebcb67e49f31b55c5bf847" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -8667,7 +8668,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -8683,9 +8684,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.81" +version = "2.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "198514704ca887dd5a1e408c6c6cdcba43672f9b4062e1b24aa34e74e6d7faae" +checksum = "e89275301d38033efb81a6e60e3497e734dfcc62571f2854bf4b16690398824c" dependencies = [ "proc-macro2", "quote", @@ -8701,7 +8702,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -8727,7 +8728,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -8935,9 +8936,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ff6c40d3aedb5e06b57c6f669ad17ab063dd1e63d977c6a88e7f4dfa4f04020" +checksum = "c65998313f8e17d0d553d28f91a0df93e4dbbbf770279c7bc21ca0f09ea1a1f6" dependencies = [ "filetime", "libc", @@ -8977,22 +8978,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "5d171f59dbaa811dbbb1aee1e73db92ec2b122911a48e1390dfe327a821ddede" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "b08be0f17bd307950653ce45db00cd31200d82b624b36e181337d9c7d92765b5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -9217,7 +9218,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -9272,7 +9273,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.15", + "rustls 0.23.16", "rustls-pki-types", "tokio", ] @@ -9486,7 +9487,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" dependencies = [ - "async-compression 0.4.16", + "async-compression 0.4.17", "bitflags 2.6.0", "bytes", "futures-core", @@ -9544,7 +9545,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -9607,7 +9608,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea6023f9fe4b69267ccd3ed7d203d931c43c5f82dbaa0f07202bc17193a5f43" dependencies = [ "loki-api", - "reqwest 0.12.8", + "reqwest 0.12.9", "serde", "serde_json", "snap", @@ -9781,7 +9782,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.81", + "syn 2.0.86", "thiserror", "unicode-ident", ] @@ -9798,7 +9799,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.81", + "syn 2.0.86", "typify-impl", ] @@ -9898,9 +9899,9 @@ checksum = "10103c57044730945224467c09f71a4db0071c123a0648cc3e818913bde6b561" [[package]] name = "unicode-id-start" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97e2a3c5fc9de285c0e805d98eba666adb4b2d9e1049ce44821ff7707cc34e91" +checksum = "2f322b60f6b9736017344fa0635d64be2f458fbc04eef65f6be22976dd1ffd5b" [[package]] name = "unicode-ident" @@ -10007,7 +10008,7 @@ dependencies = [ "log", "native-tls", "once_cell", - "rustls 0.23.15", + "rustls 0.23.16", "rustls-pki-types", "serde", "serde_json", @@ -10107,9 +10108,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a84c137d37ab0142f0f2ddfe332651fdbf252e7b7dbb4e67b6c1f1b2e925101" +checksum = "3ef4c4aa54d5d05a279399bfa921ec387b7aba77caf7a682ae8d86785b8fdad2" [[package]] name = "vcpkg" @@ -10193,7 +10194,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", "wasm-bindgen-shared", ] @@ -10227,7 +10228,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -10260,7 +10261,7 @@ checksum = "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -10394,7 +10395,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "axum", @@ -10411,7 +10412,7 @@ dependencies = [ "prometheus", "quote", "rand 0.8.5", - "reqwest 0.12.8", + "reqwest 0.12.9", "rsmq_async", "serde", "serde_json", @@ -10426,6 +10427,7 @@ dependencies = [ "uuid 1.11.0", "windmill-api", "windmill-api-client", + "windmill-autoscaling", "windmill-common", "windmill-git-sync", "windmill-indexer", @@ -10435,7 +10437,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "argon2", @@ -10479,7 +10481,7 @@ dependencies = [ "quick_cache", "rand 0.8.5", "regex", - "reqwest 0.12.8", + "reqwest 0.12.9", "rsa 0.7.2", "rsmq_async", "rust-embed", @@ -10520,7 +10522,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.410.3" +version = "1.416.2" dependencies = [ "base64 0.21.7", "chrono", @@ -10538,7 +10540,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.410.3" +version = "1.416.2" dependencies = [ "chrono", "serde", @@ -10549,9 +10551,24 @@ dependencies = [ "windmill-common", ] +[[package]] +name = "windmill-autoscaling" +version = "1.416.2" +dependencies = [ + "anyhow", + "rsmq_async", + "serde", + "serde_json", + "sqlx", + "tracing", + "uuid 1.11.0", + "windmill-common", + "windmill-queue", +] + [[package]] name = "windmill-common" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "async-stream", @@ -10578,7 +10595,7 @@ dependencies = [ "prometheus", "rand 0.8.5", "regex", - "reqwest 0.12.8", + "reqwest 0.12.9", "serde", "serde_json", "sha2 0.10.8", @@ -10597,7 +10614,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.410.3" +version = "1.416.2" dependencies = [ "regex", "rsmq_async", @@ -10612,7 +10629,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "bytes", @@ -10634,19 +10651,19 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.410.3" +version = "1.416.2" dependencies = [ "itertools 0.13.0", "lazy_static", "proc-macro2", "quote", "regex", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] name = "windmill-parser" -version = "1.410.3" +version = "1.416.2" dependencies = [ "convert_case 0.6.0", "serde", @@ -10655,7 +10672,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "lazy_static", @@ -10667,7 +10684,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "gosyn", @@ -10679,7 +10696,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "lazy_static", @@ -10691,7 +10708,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10702,7 +10719,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10713,7 +10730,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "async-recursion", @@ -10731,7 +10748,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10741,14 +10758,14 @@ dependencies = [ "quote", "regex", "serde_json", - "syn 2.0.81", + "syn 2.0.86", "toml 0.7.8", "windmill-parser", ] [[package]] name = "windmill-parser-sql" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "lazy_static", @@ -10760,7 +10777,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "lazy_static", @@ -10778,7 +10795,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10799,7 +10816,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "serde_json", @@ -10809,7 +10826,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "async-recursion", @@ -10825,7 +10842,7 @@ dependencies = [ "lazy_static", "prometheus", "regex", - "reqwest 0.12.8", + "reqwest 0.12.9", "rsmq_async", "serde", "serde_json", @@ -10842,7 +10859,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.410.3" +version = "1.416.2" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10852,7 +10869,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.410.3" +version = "1.416.2" dependencies = [ "anyhow", "async-recursion", @@ -10892,7 +10909,7 @@ dependencies = [ "prometheus", "rand 0.8.5", "regex", - "reqwest 0.12.8", + "reqwest 0.12.9", "rsmq_async", "rust_decimal", "serde", @@ -11218,7 +11235,7 @@ checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", "synstructure", ] @@ -11240,7 +11257,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", ] [[package]] @@ -11260,7 +11277,7 @@ checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.81", + "syn 2.0.86", "synstructure", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index b0ced81436..1c0abc299c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.410.3" +version = "1.416.2" authors.workspace = true edition.workspace = true @@ -13,6 +13,7 @@ members = [ "./windmill-common", "./windmill-audit", "./windmill-git-sync", + "./windmill-autoscaling", "./windmill-indexer", "./windmill-macros", "./parsers/windmill-parser", @@ -28,7 +29,7 @@ members = [ ] [workspace.package] -version = "1.410.3" +version = "1.416.2" authors = ["Ruben Fiszel "] edition = "2021" @@ -45,7 +46,7 @@ lto = "thin" [features] default = [] -enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"] +enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"] enterprise_saml = ["windmill-api/enterprise_saml"] stripe = ["windmill-api/stripe"] benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"] @@ -72,6 +73,7 @@ windmill-git-sync.workspace = true windmill-api = { workspace = true, default-features = false } windmill-worker.workspace = true windmill-indexer = { workspace = true, optional = true } +windmill-autoscaling = { workspace = true, optional = true } futures.workspace = true tracing.workspace = true sqlx.workspace = true @@ -116,6 +118,7 @@ windmill-worker = { path = "./windmill-worker" } windmill-common = { path = "./windmill-common", default-features = false } windmill-audit = { path = "./windmill-audit" } windmill-git-sync = { path = "./windmill-git-sync" } +windmill-autoscaling = { path = "./windmill-autoscaling" } windmill-indexer = {path = "./windmill-indexer"} windmill-macros = {path = "./windmill-macros"} windmill-parser = { path = "./parsers/windmill-parser" } @@ -157,7 +160,7 @@ hex = "^0" sql-builder = "^3" argon2 = "^0" quick_cache = "^0" -rand = "0.8.5" +rand = "^0" rand_core = { version = "^0", features = ["std"] } magic-crypt = "^3" git-version = "^0" @@ -169,7 +172,7 @@ urlencoding = "^2" url = "^2" async-oauth2 = "^0" reqwest = { version = "^0.12", features = ["json", "stream", "gzip"] } -time = "0.3.16" +time = "^0" serde_urlencoded = "^0" tokio-tar = "^0" tempfile = "^3" @@ -195,7 +198,7 @@ swc_ecma_visit = "=0.104.8" async-recursion = "^1" -base64 = "0.21.0" +base64 = "^0" base32 = "^0" hmac = "0.12.1" sha2 = "0.10.6" @@ -219,7 +222,7 @@ serde_derive = "1.0.147" const_format = { version = "0.2", features = ["rust_1_64", "rust_1_51"] } dyn-iter = "0.2.0" rsa = "0.7.2" -async-stripe = { version = "0.34.1", features = [ +async-stripe = { version = "0.39.1", features = [ "runtime-tokio-hyper", "checkout", "billing", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 2ada4955ca..e7dd9365ec 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ce38a7f85ca257c48471c46e3811a1281d9e2f27 +f136a2f499e0fe7c10c54c79488851980d796eb2 \ No newline at end of file diff --git a/backend/migrations/20241024125924_autoscaling.down.sql b/backend/migrations/20241024125924_autoscaling.down.sql new file mode 100644 index 0000000000..8f536a7553 --- /dev/null +++ b/backend/migrations/20241024125924_autoscaling.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +DROP TABLE autoscaling_event; +DROP TYPE autoscaling_event_type; diff --git a/backend/migrations/20241024125924_autoscaling.up.sql b/backend/migrations/20241024125924_autoscaling.up.sql new file mode 100644 index 0000000000..fa3b161d56 --- /dev/null +++ b/backend/migrations/20241024125924_autoscaling.up.sql @@ -0,0 +1,13 @@ +-- Add up migration script here +CREATE TYPE AUTOSCALING_EVENT_TYPE AS ENUM ('full_scaleout', 'scalein', 'scaleout'); + +CREATE TABLE autoscaling_event ( + id SERIAL PRIMARY KEY, + worker_group TEXT NOT NULL, + event_type AUTOSCALING_EVENT_TYPE NOT NULL, + desired_workers INTEGER NOT NULL, + applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + reason TEXT +); + +CREATE INDEX autoscaling_event_worker_group_idx ON autoscaling_event (worker_group, applied_at); \ No newline at end of file diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index 034aece2f4..994d9999fd 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -152,5 +152,24 @@ "vismanet_erp_interactive_api:read", "vismanet_erp_interactive_api:update" ] + }, + "spotify": { + "auth_url": "https://accounts.spotify.com/authorize", + "token_url": "https://accounts.spotify.com/api/token", + "scopes": [ + "user-read-playback-state", + "user-modify-playback-state", + "user-read-currently-playing", + "playlist-read-private", + "playlist-read-collaborative", + "playlist-modify-private", + "playlist-modify-public", + "user-follow-read", + "user-read-playback-position", + "user-read-recently-played", + "user-top-read", + "user-library-modify", + "user-library-read" + ] } } \ No newline at end of file diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 34487e04ef..841a763ec2 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -61,6 +61,7 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma "opensearchpy" => "opensearch-py", "lokalise" => "python-lokalise-api", "msgraph" => "msgraph-sdk", + "pythonjsonlogger" => "python-json-logger", }; fn replace_import(x: String) -> String { diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index f008f2f34a..8c29c97b7e 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -188,9 +188,15 @@ pub struct AnsiblePlaybookOptions { pub force_handlers: Option<()>, } +#[derive(Debug, Clone)] +pub enum ResourceOrVariablePath { + Resource(String), + Variable(String), +} + #[derive(Debug, Clone)] pub struct FileResource { - pub resource_path: String, + pub resource_path: ResourceOrVariablePath, pub target_path: String, } @@ -309,7 +315,7 @@ pub fn parse_ansible_reqs( } } } - Yaml::String(key) if key == "file_resources" => { + Yaml::String(key) if key == "files" || key == "file_resources" => { if let Yaml::Array(file_resources) = value { let resources: anyhow::Result> = file_resources.iter().map(parse_file_resource).collect(); @@ -440,7 +446,24 @@ fn parse_file_resource(yaml: &Yaml) -> anyhow::Result { "No `target` provided for file resource {}. Please input a target relative path for the ansible playbook to see this file.", resource_path ))?; - return Ok(FileResource { resource_path: resource_path.clone(), target_path }); + return Ok(FileResource { + resource_path: ResourceOrVariablePath::Resource(resource_path.clone()), + target_path, + }); + } + if let Some(Yaml::String(resource_path)) = f.get(&Yaml::String("variable".to_string())) { + let target_path = f + .get(&Yaml::String("target".to_string())) + .and_then(|x| x.as_str()) + .map(|x| x.to_string()) + .ok_or(anyhow!( + "No `target` provided for file resource {}. Please input a target relative path for the ansible playbook to see this file.", + resource_path + ))?; + return Ok(FileResource { + resource_path: ResourceOrVariablePath::Variable(resource_path.clone()), + target_path, + }); } return Err(anyhow!( "File resource should have a `resource` field, linking to a text file resource" diff --git a/backend/src/ee.rs b/backend/src/ee.rs index ef944b984b..91816cd1ba 100644 --- a/backend/src/ee.rs +++ b/backend/src/ee.rs @@ -1,16 +1,8 @@ -use anyhow::anyhow; -#[cfg(feature = "enterprise")] -use windmill_common::error::{Error, Result}; - -pub async fn set_license_key(_license_key: String) -> anyhow::Result<()> { +pub async fn set_license_key(_license_key: String) -> () { // Implementation is not open source - Err(anyhow!("License cannot be set in Windmill CE")) } #[cfg(feature = "enterprise")] -pub async fn verify_license_key() -> Result<()> { +pub async fn verify_license_key() -> () { // Implementation is not open source - Err(Error::InternalErr( - "License always invalid in Windmill CE".to_string(), - )) } diff --git a/backend/src/main.rs b/backend/src/main.rs index 84deb4e6b5..11d12f9dab 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -27,7 +27,7 @@ use uuid::Uuid; use windmill_api::HTTP_CLIENT; #[cfg(feature = "enterprise")] -use windmill_common::ee::schedule_key_renewal; +use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICENSE_KEY_VALID}; use windmill_common::{ global_settings::{ @@ -284,7 +284,8 @@ async fn windmill_main() -> anyhow::Result<()> { tracing::info!("Binary is in 'indexer' mode"); #[cfg(not(feature = "tantivy"))] { - panic!("Indexer mode requires the tantivy feature flag"); + tracing::error!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features."); + panic!("Indexer mode requires compiling with the tantivy feature flag."); } #[cfg(feature = "tantivy")] Mode::Indexer @@ -307,7 +308,8 @@ async fn windmill_main() -> anyhow::Result<()> { Mode::Standalone }); - let num_workers = if mode == Mode::Server || mode == Mode::Indexer { + #[allow(unused_mut)] + let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer { 0 } else { std::env::var("NUM_WORKERS") @@ -422,6 +424,50 @@ Windmill Community Edition {GIT_VERSION} display_config(&ENV_SETTINGS); + if let Err(e) = reload_base_url_setting(&db).await { + tracing::error!("Error loading base url: {:?}", e) + } + + if let Err(e) = reload_critical_error_channels_setting(&db).await { + tracing::error!("Could loading critical error emails setting: {:?}", e); + } + + #[cfg(feature = "enterprise")] + { + // load the license key and check if it's valid + // if not valid and not server mode just quit + // if not expired and server mode then force renewal + // if key still invalid and num_workers > 0, set to 0 + if let Err(err) = reload_license_key(&db).await { + tracing::error!("Failed to reload license key: {err:#}"); + } + let valid_key = *LICENSE_KEY_VALID.read().await; + if !valid_key && !server_mode { + panic!("Invalid license key, workers require a valid license key"); + } + if server_mode { + // only force renewal if invalid but not empty (= expired) + let renewed_now = maybe_renew_license_key_on_start( + &HTTP_CLIENT, + &db, + !valid_key && !LICENSE_KEY_ID.read().await.is_empty(), + ) + .await; + if renewed_now { + if let Err(err) = reload_license_key(&db).await { + tracing::error!("Failed to reload license key: {err:#}"); + } + } + if num_workers > 0 { + let valid_key = *LICENSE_KEY_VALID.read().await; + if !valid_key { + tracing::warn!("License key invalid, setting num_workers to 0"); + num_workers = 0; + } + } + } + } + let worker_mode = num_workers > 0; if server_mode || worker_mode || indexer_mode { @@ -448,7 +494,16 @@ Windmill Community Edition {GIT_VERSION} initial_load(&db, killpill_tx.clone(), worker_mode, server_mode, is_agent).await; - monitor_db(&db, &base_internal_url, rsmq.clone(), server_mode, true).await; + monitor_db( + &db, + &base_internal_url, + rsmq.clone(), + server_mode, + worker_mode, + true, + killpill_tx.clone(), + ) + .await; monitor_pool(&db).await; @@ -548,8 +603,11 @@ Windmill Community Edition {GIT_VERSION} rx.recv().await?; } } - tracing::info!("Starting phase 2 of shutdown"); - killpill_phase2_tx.send(())?; + if killpill_phase2_tx.receiver_count() > 0 { + tracing::info!("Starting phase 2 of shutdown"); + killpill_phase2_tx.send(())?; + tracing::info!("Phase 2 of shutdown completed"); + } Ok(()) as anyhow::Result<()> }; @@ -575,7 +633,9 @@ Windmill Community Edition {GIT_VERSION} &base_internal_url, rsmq.clone(), server_mode, - false + worker_mode, + false, + tx.clone(), ) .await; }, @@ -618,7 +678,15 @@ Windmill Community Edition {GIT_VERSION} }, LICENSE_KEY_SETTING => { if let Err(e) = reload_license_key(&db).await { - tracing::error!(error = %e, "Could not reload license key setting"); + tracing::error!("Failed to reload license key: {e:#}"); + } + #[cfg(feature = "enterprise")] + if worker_mode { + let valid_key = *LICENSE_KEY_VALID.read().await; + if !valid_key { + tracing::error!("Invalid license key, exiting..."); + tx.send(()).expect("send"); + } } }, DEFAULT_TAGS_PER_WORKSPACE_SETTING => { @@ -764,15 +832,10 @@ Windmill Community Edition {GIT_VERSION} Ok(()) as anyhow::Result<()> }; - if mode == Mode::Server || mode == Mode::Standalone { + if server_mode { schedule_stats(&db, &HTTP_CLIENT).await; } - #[cfg(feature = "enterprise")] - if mode == Mode::Server || mode == Mode::Standalone { - schedule_key_renewal(&HTTP_CLIENT, &db).await; - } - futures::try_join!( shutdown_signal, workers_f, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 576a38de72..0618cfd8d8 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -27,7 +27,7 @@ use windmill_api::{ DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT, SAML_METADATA, SCIM_TOKEN, }; #[cfg(feature = "enterprise")] -use windmill_common::ee::{worker_groups_alerts, jobs_waiting_alerts}; +use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts, LICENSE_KEY_VALID}; use windmill_common::{ auth::JWT_SECRET, ee::CriticalErrorChannel, @@ -76,9 +76,6 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING; #[cfg(feature = "enterprise")] use crate::ee::verify_license_key; -#[cfg(feature = "enterprise")] -use windmill_common::ee::LICENSE_KEY_VALID; - use crate::ee::set_license_key; lazy_static::lazy_static! { @@ -152,18 +149,10 @@ pub async fn initial_load( tracing::error!("Error reloading custom tags: {:?}", e) } - if let Err(e) = reload_base_url_setting(db).await { - tracing::error!("Error reloading base url: {:?}", e) - } - if let Err(e) = reload_hub_base_url_setting(db, server_mode).await { tracing::error!("Error reloading hub base url: {:?}", e) } - if let Err(e) = reload_critical_error_channels_setting(&db).await { - tracing::error!("Could not reload critical error emails setting: {:?}", e); - } - if let Err(e) = reload_jwt_secret_setting(&db).await { tracing::error!("Could not reload jwt secret setting: {:?}", e); } @@ -182,11 +171,6 @@ pub async fn initial_load( reload_scim_token_setting(&db).await; } - #[cfg(feature = "enterprise")] - if let Err(e) = reload_license_key(&db).await { - tracing::error!("Error reloading license key: {:?}", e) - } - if worker_mode { reload_extra_pip_index_url_setting(&db).await; reload_pip_index_url_setting(&db).await; @@ -854,8 +838,10 @@ pub async fn reload_request_size(db: &DB) { } } -pub async fn reload_license_key(db: &DB) -> error::Result<()> { - let q = load_value_from_global_settings(db, LICENSE_KEY_SETTING).await?; +pub async fn reload_license_key(db: &DB) -> anyhow::Result<()> { + let q = load_value_from_global_settings(db, LICENSE_KEY_SETTING) + .await + .map_err(|err| anyhow::anyhow!("Error reloading license key: {}", err.to_string()))?; let mut value = std::env::var("LICENSE_KEY") .ok() @@ -873,9 +859,7 @@ pub async fn reload_license_key(db: &DB) -> error::Result<()> { tracing::error!("Could not parse LICENSE_KEY found: {:#?}", &q); } }; - - set_license_key(value).await?; - + set_license_key(value).await; Ok(()) } @@ -1014,7 +998,9 @@ pub async fn monitor_db( base_internal_url: &str, rsmq: Option, server_mode: bool, + _worker_mode: bool, initial_load: bool, + _killpill_tx: tokio::sync::broadcast::Sender<()>, ) { let zombie_jobs_f = async { if server_mode && !initial_load { @@ -1035,15 +1021,14 @@ pub async fn monitor_db( let verify_license_key_f = async { #[cfg(feature = "enterprise")] - if let Err(e) = verify_license_key().await { - tracing::error!("Error verifying license key: {:?}", e); - let mut l = LICENSE_KEY_VALID.write().await; - *l = false; - } else { - let is_valid = LICENSE_KEY_VALID.read().await.clone(); - if !is_valid { - let mut l = LICENSE_KEY_VALID.write().await; - *l = true; + if !initial_load { + verify_license_key().await; + if _worker_mode { + let valid_key = *LICENSE_KEY_VALID.read().await; + if !valid_key { + tracing::error!("Invalid license key, exiting..."); + _killpill_tx.send(()).expect("send"); + } } } }; @@ -1068,6 +1053,15 @@ pub async fn monitor_db( } }; + let apply_autoscaling_f = async { + #[cfg(feature = "enterprise")] + if server_mode && !initial_load { + if let Err(e) = windmill_autoscaling::apply_all_autoscaling(db).await { + tracing::error!("Error applying autoscaling: {:?}", e); + } + } + }; + join!( expired_items_f, zombie_jobs_f, @@ -1075,6 +1069,7 @@ pub async fn monitor_db( verify_license_key_f, worker_groups_alerts_f, jobs_waiting_alerts_f, + apply_autoscaling_f, ); } @@ -1092,19 +1087,11 @@ pub async fn expose_queue_metrics(db: &Pool) { .unwrap_or(true); if metrics_enabled || save_metrics { - let queue_counts = sqlx::query!( - "SELECT tag, count(*) as count FROM queue WHERE - scheduled_for <= now() - ('3 seconds')::interval AND running = false - GROUP BY tag" - ) - .fetch_all(db) - .await - .ok() - .unwrap_or_else(|| vec![]); + let queue_counts = windmill_common::queue::get_queue_counts(db).await; for q in queue_counts { - let count = q.count.unwrap_or(0); - let tag = q.tag; + let count = q.1; + let tag = q.0; if metrics_enabled { let metric = (*QUEUE_COUNT).with_label_values(&[&tag]); metric.set(count as i64); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 327dd04c7b..eebb8fd0a9 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.410.3 + version: 1.416.2 title: Windmill API contact: @@ -1467,6 +1467,11 @@ paths: parameters: - $ref: "#/components/parameters/Page" - $ref: "#/components/parameters/PerPage" + - name: active_only + in: query + description: filter only active users + schema: + type: boolean responses: "200": description: user @@ -8198,6 +8203,29 @@ paths: items: $ref: "#/components/schemas/Config" + /configs/list_autoscaling_events/{worker_group}: + get: + summary: List autoscaling events + operationId: listAutoscalingEvents + tags: + - config + parameters: + - name: worker_group + in: path + required: true + schema: + type: string + responses: + "200": + description: List of autoscaling events + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/AutoscalingEvent" + + /w/{workspace}/acls/get/{kind}/{path}: get: summary: get granular acls @@ -10735,6 +10763,7 @@ components: - "users.delete" - "users.update" - "users.login" + - "users.login_failure" - "users.logout" - "users.accept_invite" - "users.decline_invite" @@ -10746,6 +10775,7 @@ components: - "users.impersonate" - "users.leave_workspace" - "oauth.login" + - "oauth.login_failure" - "oauth.signup" - "variables.create" - "variables.delete" @@ -11766,6 +11796,8 @@ components: type: string username: type: string + operator_only: + type: boolean required: - email @@ -12467,3 +12499,21 @@ components: properties: dancer: type: string + + AutoscalingEvent: + type: object + properties: + id: + type: integer + format: int64 + worker_group: + type: string + event_type: + type: string + desired_workers: + type: integer + reason: + type: string + applied_at: + type: string + format: date-time diff --git a/backend/windmill-api/src/configs.rs b/backend/windmill-api/src/configs.rs index 64c8a1884f..81b65216e0 100644 --- a/backend/windmill-api/src/configs.rs +++ b/backend/windmill-api/src/configs.rs @@ -29,6 +29,10 @@ pub fn global_service() -> Router { .route("/update/:name", post(update_config).delete(delete_config)) .route("/get/:name", get(get_config)) .route("/list", get(list_configs)) + .route( + "/list_autoscaling_events/:worker_group", + get(list_autoscaling_events), + ) } #[derive(Serialize, Deserialize, FromRow)] @@ -177,6 +181,30 @@ async fn delete_config( Ok(format!("Deleted config {name}")) } +#[derive(Serialize, Deserialize, FromRow)] +struct AutoscalingEvent { + id: i64, + worker_group: String, + event_type: Option, + desired_workers: i32, + reason: Option, + applied_at: chrono::NaiveDateTime, +} + +async fn list_autoscaling_events( + Extension(db): Extension, + Path(worker_group): Path, +) -> error::JsonResult> { + let events = sqlx::query_as!( + AutoscalingEvent, + "SELECT id, worker_group, event_type::text, desired_workers, reason, applied_at FROM autoscaling_event WHERE worker_group = $1 ORDER BY applied_at DESC LIMIT 5", + worker_group + ) + .fetch_all(&db) + .await?; + Ok(Json(events)) +} + #[cfg(feature = "enterprise")] async fn list_configs( authed: ApiAuthed, diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index c44b8c18e6..6186b0ada1 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -173,17 +173,6 @@ pub async fn migrate(db: &DB) -> Result<(), Error> { let migrator = db.acquire().await?; let mut custom_migrator = CustomMigrator { inner: migrator }; - if let Err(err) = fix_flow_versioning_migration(&mut custom_migrator, db).await { - tracing::error!("Could not apply flow versioning fix migration: {err:#}"); - } - - let db2 = db.clone(); - let _ = tokio::task::spawn(async move { - if let Err(err) = fix_job_completed_index(&db2).await { - tracing::error!("Could not apply job completed index fix migration: {err:#}"); - } - }); - match sqlx::migrate!("../migrations") .run_direct(&mut custom_migrator) .await @@ -199,6 +188,17 @@ pub async fn migrate(db: &DB) -> Result<(), Error> { Err(err) => Err(err), }?; + if let Err(err) = fix_flow_versioning_migration(&mut custom_migrator, db).await { + tracing::error!("Could not apply flow versioning fix migration: {err:#}"); + } + + let db2 = db.clone(); + let _ = tokio::task::spawn(async move { + if let Err(err) = fix_job_completed_index(&db2).await { + tracing::error!("Could not apply job completed index fix migration: {err:#}"); + } + }); + Ok(()) } @@ -302,7 +302,7 @@ macro_rules! run_windmill_migration { .await?; tracing::info!("Finished applying {migration_job_name} migration"); } else { - tracing::info!("migration {migration_job_name} already done"); + tracing::debug!("migration {migration_job_name} already done"); } let _ = sqlx::query("SELECT pg_advisory_unlock(4242)") @@ -311,7 +311,7 @@ macro_rules! run_windmill_migration { tx.commit().await?; tracing::info!("released lock for {migration_job_name}"); } else { - tracing::info!("migration {migration_job_name} already done"); + tracing::debug!("migration {migration_job_name} already done"); } } diff --git a/backend/windmill-api/src/ee.rs b/backend/windmill-api/src/ee.rs index b20521e8b4..cddb639e95 100644 --- a/backend/windmill-api/src/ee.rs +++ b/backend/windmill-api/src/ee.rs @@ -4,7 +4,7 @@ use std::sync::Arc; #[cfg(feature = "enterprise")] use tokio::sync::RwLock; -pub async fn validate_license_key(_license_key: String) -> anyhow::Result { +pub async fn validate_license_key(_license_key: String) -> anyhow::Result<(String, bool)> { // Implementation is not open source Err(anyhow!("License can't be validated in Windmill CE")) } diff --git a/backend/windmill-api/src/embeddings.rs b/backend/windmill-api/src/embeddings.rs index 5cc7226753..fd222bdbfa 100644 --- a/backend/windmill-api/src/embeddings.rs +++ b/backend/windmill-api/src/embeddings.rs @@ -53,6 +53,7 @@ use crate::{resources::ResourceType, HTTP_CLIENT}; lazy_static::lazy_static! { pub static ref EMBEDDINGS_DB: Arc>> = Arc::new(RwLock::new(None)); pub static ref MODEL_INSTANCE: Arc>>> = Arc::new(RwLock::new(None)); + pub static ref HUB_EMBEDDINGS_PULLING_INTERVAL_SECS: u64 = std::env::var("HUB_EMBEDDINGS_PULLING_INTERVAL_SECS").ok().map(|x| x.parse::().ok()).flatten().unwrap_or(3600 * 24); } #[cfg(feature = "embedding")] @@ -607,7 +608,10 @@ pub fn load_embeddings_db(db: &Pool) -> () { drop(model_instance_lock); loop { update_embeddings_db(&db_clone).await; - tokio::time::sleep(std::time::Duration::from_secs(3600 * 24)).await; + tokio::time::sleep(std::time::Duration::from_secs( + *HUB_EMBEDDINGS_PULLING_INTERVAL_SECS, + )) + .await; } } else { tracing::error!( diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 3644e04d30..4fc9ce514f 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -544,7 +544,6 @@ async fn get_latest_version( Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> JsonResult> { - let path = path.to_path(); let mut tx = user_db.begin(&authed).await?; @@ -1333,7 +1332,6 @@ mod tests { }, "type": "script", "path": "test", - "tag_override": Option::::None, }, }, { @@ -1377,14 +1375,12 @@ mod tests { "input_transforms": {}, "type": "script", "path": "test", - "tag_override": Option::::None, }, "stop_after_if": { "expr": "previous.isEmpty()", "skip_if_stopped": false } }, - "preprocessor_module": Option::::None }); assert_eq!(dbg!(serde_json::json!(fv)), dbg!(expect)); } diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 53cd958c6f..7244b2be17 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -2815,7 +2815,6 @@ pub async fn run_flow_by_path_inner( let flow_path = flow_path.to_path(); check_scopes(&authed, || format!("run:flow/{flow_path}"))?; - let (tag, dedicated_worker, has_preprocessor) = sqlx::query!( "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor FROM flow @@ -2910,7 +2909,6 @@ pub async fn restart_flow( ) -> error::Result<(StatusCode, String)> { check_license_key_valid().await?; - let completed_job = sqlx::query_as::<_, CompletedJob>( "SELECT *, result->'wm_labels' as labels from completed_job WHERE id = $1 and workspace_id = $2", ) @@ -3010,7 +3008,6 @@ pub async fn run_script_by_path_inner( check_scopes(&authed, || format!("run:script/{script_path}"))?; - let (job_payload, tag, _delete_after_use, timeout) = script_path_to_payload(script_path, &db, &w_id, run_query.skip_preprocessor).await?; let scheduled_for = run_query.get_scheduled_for(&db).await?; @@ -3018,7 +3015,6 @@ pub async fn run_script_by_path_inner( let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag).await?; - let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into(), rsmq); let (uuid, tx) = push( @@ -3067,7 +3063,6 @@ pub async fn run_workflow_as_code( Query(wkflow_query): Query, Json(task): Json, ) -> error::Result<(StatusCode, String)> { - let mut i = 1; if *CLOUD_HOSTED { @@ -3079,23 +3074,18 @@ pub async fn run_workflow_as_code( check_license_key_valid().await?; check_tag_available_for_workspace(&w_id, &run_query.tag).await?; - if *CLOUD_HOSTED { tracing::info!("workflow_as_code_tracing id {i} "); i += 1; } - let job = get_queued_job(&job_id, &w_id, &db).await?; - - if *CLOUD_HOSTED { tracing::info!("workflow_as_code_tracing id {i} "); i += 1; } - let job = not_found_if_none(job, "Queued Job", &job_id.to_string())?; let (job_payload, tag, _delete_after_use, timeout) = match job.job_kind { JobKind::Preview => ( @@ -3118,18 +3108,12 @@ pub async fn run_workflow_as_code( run_query.timeout, ), JobKind::Script => { - script_path_to_payload( - job.script_path(), - &db, - &w_id, - run_query.skip_preprocessor, - ) - .await? + script_path_to_payload(job.script_path(), &db, &w_id, run_query.skip_preprocessor) + .await? } _ => return Err(anyhow::anyhow!("Not supported").into()), }; - if *CLOUD_HOSTED { tracing::info!("workflow_as_code_tracing id {i} "); i += 1; @@ -3143,22 +3127,18 @@ pub async fn run_workflow_as_code( let tag = run_query.tag.clone().or(tag).or(Some(job.tag)); - if *CLOUD_HOSTED { tracing::info!("workflow_as_code_tracing id {i} "); i += 1; } - let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into(), rsmq); - if *CLOUD_HOSTED { tracing::info!("workflow_as_code_tracing id {i} "); i += 1; } - let (uuid, mut tx) = push( &db, tx, @@ -3185,7 +3165,6 @@ pub async fn run_workflow_as_code( ) .await?; - if *CLOUD_HOSTED { tracing::info!("workflow_as_code_tracing id {i} "); i += 1; @@ -3203,16 +3182,13 @@ pub async fn run_workflow_as_code( tracing::info!("Skipping update of flow status for job {job_id} in workspace {w_id}"); } - if *CLOUD_HOSTED { tracing::info!("workflow_as_code_tracing id {i} "); i += 1; } - tx.commit().await?; - if *CLOUD_HOSTED { tracing::info!("workflow_as_code_tracing id {i} "); } @@ -3866,7 +3842,6 @@ pub async fn run_wait_result_flow_by_path_internal( let flow_path = flow_path.to_path(); check_scopes(&authed, || format!("run:flow/{flow_path}"))?; - let scheduled_for = run_query.get_scheduled_for(&db).await?; let (tag, dedicated_worker, early_return, has_preprocessor) = sqlx::query!( @@ -4362,7 +4337,6 @@ async fn add_batch_jobs( } } "flow" => { - let mut uuids: Vec = Vec::new(); let payload = if let Some(ref fv) = batch_info.flow_value { JobPayload::RawFlow { value: fv.clone(), path: None, restarted_from: None } @@ -4580,7 +4554,6 @@ pub async fn run_job_by_hash_inner( #[cfg(feature = "enterprise")] check_license_key_valid().await?; - let hash = script_hash.0; let ( path, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 45a0ea7196..3ea78ffd6d 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -84,12 +84,14 @@ mod stripe_ee; mod tracing_init; mod triggers; mod users; +mod users_ee; mod utils; mod variables; mod webhook_util; mod websocket_triggers; mod workers; mod workspaces; +mod workspaces_ee; pub const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); @@ -456,9 +458,13 @@ async fn ee_license() -> &'static str { #[cfg(feature = "enterprise")] async fn ee_license() -> String { - use windmill_common::ee::LICENSE_KEY_ID; + use windmill_common::ee::{LICENSE_KEY_ID, LICENSE_KEY_VALID}; - LICENSE_KEY_ID.read().await.clone() + if *LICENSE_KEY_VALID.read().await { + LICENSE_KEY_ID.read().await.clone() + } else { + "".to_string() + } } async fn openapi() -> &'static str { diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 8a72a5459a..0299145a38 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -28,13 +28,13 @@ use serde::Deserialize; #[cfg(feature = "enterprise")] use windmill_common::ee::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel}; use windmill_common::{ + email_ee::send_email, error::{self, JsonResult, Result}, global_settings::{ AUTOMATE_USERNAME_CREATION_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - HUB_BASE_URL_SETTING, + HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, }, server::Smtp, - utils::send_email, }; #[cfg(feature = "parquet")] @@ -163,8 +163,13 @@ pub async fn test_license_key( Json(TestKey { license_key }): Json, ) -> error::Result { require_super_admin(&db, &authed.email).await?; - validate_license_key(license_key).await?; - Ok("Sent test email".to_string()) + let (_, expired) = validate_license_key(license_key).await?; + + if expired { + Err(error::Error::BadRequest("Expired license key".to_string())) + } else { + Ok("Valid license key".to_string()) + } } pub async fn get_local_settings( @@ -257,6 +262,7 @@ pub async fn get_global_setting( && !key.starts_with("default_success_handler_") && key != AUTOMATE_USERNAME_CREATION_SETTING && key != HUB_BASE_URL_SETTING + && key != HUB_ACCESSIBLE_URL_SETTING && key != EMAIL_DOMAIN_SETTING { require_super_admin(&db, &authed.email).await?; @@ -301,7 +307,6 @@ pub async fn send_stats(Extension(db): Extension, authed: ApiAuthed) -> Resu windmill_common::stats_ee::send_stats( &HTTP_CLIENT, &db, - true, windmill_common::stats_ee::SendStatsReason::Manual, ) .await?; diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs new file mode 100644 index 0000000000..c9a812c4a0 --- /dev/null +++ b/backend/windmill-api/src/users.rs @@ -0,0 +1,3169 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +#![allow(non_snake_case)] + +use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; +use std::sync::Arc; + +use crate::db::ApiAuthed; + +#[cfg(feature = "enterprise")] +use crate::ee::ExternalJwks; +use crate::oauth2_ee::InstanceEvent; +use crate::utils::{ + generate_instance_wide_unique_username, get_instance_username_or_create_pending, +}; +use crate::{ + db::DB, utils::require_super_admin, webhook_util::WebhookShared, COOKIE_DOMAIN, IS_SECURE, +}; +use argon2::{Argon2, PasswordHash, PasswordVerifier}; +use axum::{ + async_trait, + extract::{Extension, FromRequestParts, OriginalUri, Path, Query}, + http::request::Parts, + response::{IntoResponse, Response}, + routing::{delete, get, post}, + Json, Router, +}; +use chrono::TimeZone; +use hyper::{header::LOCATION, StatusCode}; +use lazy_static::lazy_static; +use quick_cache::sync::Cache; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; +use time::OffsetDateTime; +#[cfg(feature = "enterprise")] +use tokio::sync::RwLock; +use tower_cookies::{Cookie, Cookies}; +use tracing::{Instrument, Span}; +use windmill_audit::audit_ee::{audit_log, AuditAuthor}; +use windmill_audit::ActionKind; +use windmill_common::auth::fetch_authed_from_permissioned_as; +use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING; +use windmill_common::users::{truncate_token, username_to_permissioned_as}; +use windmill_common::utils::paginate; +use windmill_common::worker::CLOUD_HOSTED; +use windmill_common::{ + auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, JWT_SECRET}, + db::UserDB, + error::{self, Error, JsonResult, Result}, + users::SUPERADMIN_SECRET_EMAIL, + utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath}, +}; +use windmill_git_sync::handle_deployment_metadata; + +pub const TTL_TOKEN_DB_H: u32 = 72; + +const COOKIE_NAME: &str = "token"; +const COOKIE_PATH: &str = "/"; + +pub fn workspaced_service() -> Router { + Router::new() + .route("/list", get(list_users)) + .route("/list_usage", get(list_user_usage)) + .route("/list_usernames", get(list_usernames)) + .route("/exists", post(exists_username)) + .route("/get/:user", get(get_workspace_user)) + .route("/update/:user", post(update_workspace_user)) + .route("/delete/:user", delete(delete_workspace_user)) + .route("/is_owner/*path", get(is_owner_of_path)) + .route("/whois/:username", get(whois)) + .route("/whoami", get(whoami)) + .route("/leave", post(leave_workspace)) + .route("/username_to_email/:username", get(username_to_email)) +} + +pub fn global_service() -> Router { + Router::new() + .route("/exists/:email", get(exists_email)) + .route("/email", get(get_email)) + .route("/whoami", get(global_whoami)) + .route("/list_invites", get(list_invites)) + .route("/decline_invite", post(decline_invite)) + .route("/accept_invite", post(accept_invite)) + .route("/list_as_super_admin", get(list_users_as_super_admin)) + .route("/setpassword", post(set_password)) + .route("/create", post(create_user)) + .route("/update/:user", post(update_user)) + .route("/delete/:user", delete(delete_user)) + .route("/username_info/:user", get(get_instance_username_info)) + .route("/rename/:user", post(rename_user)) + .route("/tokens/create", post(create_token)) + .route("/tokens/delete/:token_prefix", delete(delete_token)) + .route("/tokens/list", get(list_tokens)) + .route("/tokens/impersonate", post(impersonate)) + .route("/usage", get(get_usage)) + .route("/all_runnables", get(get_all_runnables)) + .route("/refresh_token", get(refresh_token)) + .route( + "/tutorial_progress", + post(update_tutorial_progress).get(get_tutorial_progress), + ) + .route("/leave_instance", post(leave_instance)) + .route("/export", get(export_global_users)) + .route("/overwrite", post(overwrite_global_users)) + + // .route("/list_invite_codes", get(list_invite_codes)) + // .route("/create_invite_code", post(create_invite_code)) + // .route("/signup", post(signup)) + // .route("/lost_password", post(lost_password)) + // .route("/use_magic_link", get(use_magic_link)) +} + +pub fn make_unauthed_service() -> Router { + Router::new() + .route("/login", post(login)) + .route("/logout", post(logout).get(logout)) + .route("/is_first_time_setup", get(is_first_time_setup)) +} + +fn username_override_from_label(label: Option) -> Option { + match label { + Some(label) + if label.starts_with("webhook-") + || label.starts_with("http-") + || label.starts_with("email-") => + { + Some(label) + } + Some(label) if label.starts_with("ephemeral-script-end-user-") => Some( + label + .trim_start_matches("ephemeral-script-end-user-") + .to_string(), + ), + Some(label) if label == "Ephemeral lsp token" => Some("lsp".to_string()), + Some(label) if label != "ephemeral-script" && label != "session" && !label.is_empty() => { + Some(format!("label-{label}")) + } + _ => None, + } +} + +#[derive(Clone)] +pub struct ExpiringAuthCache { + pub authed: ApiAuthed, + pub expiry: chrono::DateTime, +} + +pub struct AuthCache { + cache: Cache<(String, String), ExpiringAuthCache>, + db: DB, + superadmin_secret: Option, + #[cfg(feature = "enterprise")] + ext_jwks: Option>>, +} + +impl AuthCache { + pub fn new( + db: DB, + superadmin_secret: Option, + #[cfg(feature = "enterprise")] ext_jwks: Option>>, + ) -> Self { + AuthCache { + cache: Cache::new(300), + db, + superadmin_secret, + #[cfg(feature = "enterprise")] + ext_jwks, + } + } + + pub async fn invalidate(&self, w_id: &str, token: String) { + self.cache.remove(&(w_id.to_string(), token)); + } + + pub async fn get_authed(&self, w_id: Option, token: &str) -> Option { + let key = ( + w_id.as_ref().unwrap_or(&"".to_string()).to_string(), + token.to_string(), + ); + let s = self.cache.get(&key).map(|c| c.to_owned()); + match s { + Some(ExpiringAuthCache { authed, expiry }) if expiry > chrono::Utc::now() => { + Some(authed) + } + #[cfg(feature = "enterprise")] + _ if token.starts_with("jwt_ext_") => { + let authed_and_exp = match crate::ee::jwt_ext_auth( + w_id.as_ref(), + token.trim_start_matches("jwt_ext_"), + self.ext_jwks.clone(), + ) + .await + { + Ok(r) => Some(r), + Err(e) => { + tracing::error!("JWT_EXT auth error: {:?}", e); + None + } + }; + + if let Some((authed, exp)) = authed_and_exp.clone() { + self.cache.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc.timestamp_nanos(exp as i64 * 1_000_000_000), + }, + ); + + Some(authed) + } else { + None + } + } + _ if token.starts_with("jwt_") => { + let jwt_secret = JWT_SECRET.read().await; + if !jwt_secret.is_empty() { + let jwt_token = token.trim_start_matches("jwt_"); + + let jwt_result = jsonwebtoken::decode::( + jwt_token, + &jsonwebtoken::DecodingKey::from_secret(jwt_secret.as_bytes()), + &jsonwebtoken::Validation::new(jsonwebtoken::Algorithm::HS256), + ); + + match jwt_result { + Ok(payload) => { + if w_id.is_some_and(|w_id| w_id != payload.claims.workspace_id) { + tracing::error!("JWT auth error: workspace_id mismatch"); + return None; + } + + let username_override = + username_override_from_label(payload.claims.label); + let authed = crate::db::ApiAuthed { + email: payload.claims.email, + username: payload.claims.username, + is_admin: payload.claims.is_admin, + is_operator: payload.claims.is_operator, + groups: payload.claims.groups, + folders: payload.claims.folders, + scopes: None, + username_override, + }; + + self.cache.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc + .timestamp_nanos(payload.claims.exp as i64 * 1_000_000_000), + }, + ); + + Some(authed) + } + Err(err) => { + tracing::error!("JWT auth error: {:?}", err); + None + } + } + } else { + tracing::error!("JWT auth error: no jwt secret set"); + None + } + } + _ => { + let user_o = sqlx::query_as::<_, (Option, Option, bool, Option>, Option)>( + "UPDATE token SET last_used_at = now() WHERE token = $1 AND (expiration > NOW() \ + OR expiration IS NULL) AND (workspace_id IS NULL OR workspace_id = $2) RETURNING owner, email, super_admin, scopes, label", + ) + .bind(token) + .bind(w_id.as_ref()) + .fetch_optional(&self.db) + .await + .ok() + .flatten(); + + if let Some(user) = user_o { + let authed_o = { + match user { + (Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => { + let username_override = username_override_from_label(label); + if let Some((prefix, name)) = owner.split_once('/') { + if prefix == "u" { + let (is_admin, is_operator) = if super_admin { + (true, false) + } else { + let r = sqlx::query!( + "SELECT is_admin, operator FROM usr where username = $1 AND \ + workspace_id = $2 AND disabled = false", + name, + &w_id.as_ref().unwrap() + ) + .fetch_one(&self.db) + .await + .ok(); + if let Some(r) = r { + (r.is_admin, r.operator) + } else { + (false, true) + } + }; + + let w_id = &w_id.unwrap(); + let groups = + get_groups_for_user(w_id, &name, &email, &self.db) + .await + .ok() + .unwrap_or_default(); + + let folders = + get_folders_for_user(w_id, &name, &groups, &self.db) + .await + .ok() + .unwrap_or_default(); + + Some(ApiAuthed { + email: email, + username: name.to_string(), + is_admin, + is_operator, + groups, + folders, + scopes: None, + username_override, + }) + } else { + let groups = vec![name.to_string()]; + let folders = get_folders_for_user( + &w_id.unwrap(), + "", + &groups, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + Some(ApiAuthed { + email: email, + username: format!("group-{name}"), + is_admin: false, + groups, + is_operator: false, + folders, + scopes: None, + username_override, + }) + } + } else { + let groups = vec![]; + let folders = vec![]; + Some(ApiAuthed { + email: email, + username: owner, + is_admin: super_admin, + is_operator: true, + groups, + folders, + scopes: None, + username_override, + }) + } + } + (_, Some(email), super_admin, scopes, label) => { + let username_override = username_override_from_label(label); + if w_id.is_some() { + let row_o = sqlx::query_as::<_, (String, bool, bool)>( + "SELECT username, is_admin, operator FROM usr where email = $1 AND \ + workspace_id = $2 AND disabled = false", + ) + .bind(&email) + .bind(&w_id.as_ref().unwrap()) + .fetch_optional(&self.db) + .await + .unwrap_or(Some(("error".to_string(), false, false))); + + match row_o { + Some((username, is_admin, is_operator)) => { + let groups = get_groups_for_user( + &w_id.as_ref().unwrap(), + &username, + &email, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + + let folders = get_folders_for_user( + &w_id.unwrap(), + &username, + &groups, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + Some(ApiAuthed { + email, + username, + is_admin: is_admin || super_admin, + is_operator, + groups, + folders, + scopes, + username_override, + }) + } + None if super_admin => Some(ApiAuthed { + email: email.clone(), + username: email, + is_admin: super_admin, + is_operator: false, + groups: vec![], + folders: vec![], + scopes, + username_override, + }), + None => None, + } + } else { + Some(ApiAuthed { + email: email.to_string(), + username: email, + is_admin: super_admin, + is_operator: true, + groups: Vec::new(), + folders: Vec::new(), + scopes, + username_override, + }) + } + } + _ => None, + } + }; + if let Some(authed) = authed_o.as_ref() { + self.cache.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc::now() + + chrono::Duration::try_seconds(120).unwrap(), + }, + ); + } + authed_o + } else if self + .superadmin_secret + .as_ref() + .map(|x| x == token) + .unwrap_or(false) + { + Some(ApiAuthed { + email: SUPERADMIN_SECRET_EMAIL.to_string(), + username: "superadmin_secret".to_string(), + is_admin: true, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + }) + } else { + None + } + } + } + } +} + +async fn extract_token(parts: &mut Parts, state: &S) -> Option { + let auth_header = parts + .headers + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|s| s.strip_prefix("Bearer ")); + + let from_cookie = match auth_header { + Some(x) => Some(x.to_owned()), + None => Extension::::from_request_parts(parts, state) + .await + .ok() + .and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())), + }; + + #[derive(Deserialize)] + struct Token { + token: Option, + } + match from_cookie { + Some(token) => Some(token), + None => Query::::from_request_parts(parts, state) + .await + .ok() + .and_then(|token| token.token.clone()), + } +} + +#[derive(Clone, Debug)] +pub struct Tokened { + pub token: String, +} + +struct BruteForceCounter { + counter: AtomicU64, + last_reset: AtomicI64, +} + +lazy_static! { + static ref BRUTE_FORCE_COUNTER: BruteForceCounter = + BruteForceCounter { last_reset: AtomicI64::new(0), counter: AtomicU64::new(0) }; +} + +impl BruteForceCounter { + async fn increment(&self) { + let now = time::OffsetDateTime::now_utc().unix_timestamp(); + if self.counter.fetch_add(1, Ordering::Relaxed) > 10000 { + tracing::error!( + "Brute force attack to find valid token detected, sleeping unauthorized response for 2 seconds" + ); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + if now - self.last_reset.load(Ordering::Relaxed) > 60 { + self.counter.store(0, Ordering::Relaxed); + self.last_reset.store(now, Ordering::Relaxed); + } + } +} + +#[async_trait] +impl FromRequestParts for Tokened +where + S: Send + Sync, +{ + type Rejection = (StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method == http::Method::OPTIONS { + return Ok(Tokened { token: "".to_string() }); + }; + let already_tokened = parts.extensions.get::(); + if let Some(tokened) = already_tokened { + Ok(tokened.clone()) + } else { + let token_o = extract_token(parts, state).await; + if let Some(token) = token_o { + let tokened = Self { token }; + parts.extensions.insert(tokened.clone()); + Ok(tokened) + } else { + BRUTE_FORCE_COUNTER.increment().await; + Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) + } + } + } +} + +pub async fn maybe_refresh_folders( + path: &str, + w_id: &str, + authed: ApiAuthed, + db: &DB, +) -> ApiAuthed { + if authed.is_admin { + return authed; + } + let splitted = path.split('/').collect::>(); + if splitted.len() >= 2 + && splitted[0] == "f" + && !authed.folders.iter().any(|(f, _, _)| f == splitted[1]) + { + let name = &authed.username; + let groups = get_groups_for_user(w_id, name, &authed.email, db) + .await + .ok() + .unwrap_or_default(); + + let folders = get_folders_for_user(w_id, name, &groups, db) + .await + .ok() + .unwrap_or_default(); + ApiAuthed { folders, ..authed } + } else { + authed + } +} + +#[async_trait] +impl FromRequestParts for ApiAuthed +where + S: Send + Sync, +{ + type Rejection = (StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method == http::Method::OPTIONS { + return Ok(ApiAuthed { + email: "".to_owned(), + username: "".to_owned(), + is_admin: false, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + }); + }; + let already_authed = parts.extensions.get::(); + if let Some(authed) = already_authed { + Ok(authed.clone()) + } else { + let already_tokened = parts.extensions.get::(); + let token_o = if let Some(token) = already_tokened { + Some(token.token.clone()) + } else { + extract_token(parts, state).await + }; + let original_uri = OriginalUri::from_request_parts(parts, state) + .await + .ok() + .map(|x| x.0) + .unwrap_or_default(); + let path_vec: Vec<&str> = original_uri.path().split("/").collect(); + + let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" { + Some(path_vec[3].to_owned()) + } else { + if path_vec.len() >= 5 + && path_vec[0] == "" + && path_vec[2] == "srch" + && path_vec[3] == "w" + { + Some(path_vec[4].to_string()) + } else { + None + } + }; + if let Some(token) = token_o { + if let Ok(Extension(cache)) = + Extension::>::from_request_parts(parts, state).await + { + if let Some(authed) = cache.get_authed(workspace_id.clone(), &token).await { + parts.extensions.insert(authed.clone()); + if authed.scopes.as_ref().is_some_and(|scopes| { + scopes + .iter() + .any(|s| s.starts_with("jobs:") || s.starts_with("run:")) + }) && (path_vec.len() < 3 + || (path_vec[4] != "jobs" && path_vec[4] != "jobs_u")) + { + BRUTE_FORCE_COUNTER.increment().await; + return Err(( + StatusCode::UNAUTHORIZED, + format!("Unauthorized scoped token: {:?}", authed.scopes), + )); + } + Span::current().record("username", &authed.username.as_str()); + Span::current().record("email", &authed.email); + + if let Some(workspace_id) = workspace_id { + Span::current().record("workspace_id", &workspace_id); + } + return Ok(authed); + } + } + } + BRUTE_FORCE_COUNTER.increment().await; + Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) + } + } +} + +pub fn check_scopes(authed: &ApiAuthed, required: F) -> error::Result<()> +where + F: FnOnce() -> String, +{ + if authed.scopes.as_ref().is_some_and(|scopes| { + scopes + .iter() + .any(|s| s.starts_with("jobs:") || s.starts_with("run:")) + }) { + let req = &required(); + if !authed.scopes.as_ref().unwrap().contains(req) { + return Err(Error::BadRequest(format!("missing required scope: {req}"))); + } + } + Ok(()) +} + +pub fn get_scope_tags(authed: &ApiAuthed) -> Option> { + authed.scopes.as_ref()?.iter().find_map(|s| { + if s.starts_with("if_jobs:filter_tags:") { + Some( + s.trim_start_matches("if_jobs:filter_tags:") + .split(",") + .collect::>(), + ) + } else { + None + } + }) +} + +#[derive(Clone, Debug)] +pub struct OptAuthed(pub Option); + +#[async_trait] +impl FromRequestParts for OptAuthed +where + S: Send + Sync, +{ + type Rejection = (StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + ApiAuthed::from_request_parts(parts, state) + .await + .map(|authed| Self(Some(authed))) + .or_else(|_| Ok(Self(None))) + } +} + +pub async fn fetch_api_authed( + username: String, + email: String, + w_id: &str, + db: &DB, + username_override: String, +) -> error::Result { + let permissioned_as = username_to_permissioned_as(username.as_str()); + let authed = + fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?; + Ok(ApiAuthed { + username: username, + email: email, + is_admin: authed.is_admin, + is_operator: authed.is_operator, + groups: authed.groups, + folders: authed.folders, + scopes: authed.scopes, + username_override: Some(username_override), + }) +} + +#[derive(FromRow, Serialize)] +pub struct User { + pub workspace_id: String, + pub email: String, + pub username: String, + pub is_admin: bool, + pub created_at: chrono::DateTime, + pub operator: bool, + pub disabled: bool, + pub role: Option, +} + +#[derive(Serialize)] +pub struct UserWithUsage { + pub email: String, + pub executions: Option, +} + +#[derive(FromRow, Serialize, Debug)] +pub struct GlobalUserInfo { + email: String, + login_type: Option, + super_admin: bool, + verified: bool, + name: Option, + company: Option, + username: Option, + #[serde(skip_serializing_if = "Option::is_none")] + operator_only: Option, +} + +#[derive(Serialize, Debug)] +pub struct UserInfo { + pub workspace_id: String, + pub email: String, + pub username: String, + pub is_admin: bool, + pub is_super_admin: bool, + pub created_at: chrono::DateTime, + pub groups: Vec, + pub operator: bool, + pub disabled: bool, + pub role: Option, + pub folders_read: Vec, + pub folders: Vec, + pub folders_owners: Vec, +} + +#[derive(FromRow, Serialize)] +pub struct WorkspaceInvite { + pub workspace_id: String, + pub email: String, + pub is_admin: bool, + pub operator: bool, +} + +#[allow(dead_code)] +#[derive(Deserialize)] +pub struct NewUser { + pub email: String, + pub password: String, + pub super_admin: bool, + pub name: Option, + pub company: Option, +} + +#[derive(Deserialize)] +pub struct AcceptInvite { + pub workspace_id: String, + pub username: Option, +} + +#[derive(Deserialize)] +pub struct DeclineInvite { + pub workspace_id: String, +} + +#[derive(Deserialize)] +pub struct EditUser { + pub is_super_admin: Option, + pub name: Option, +} + +#[derive(Deserialize)] +pub struct EditWorkspaceUser { + pub is_admin: Option, + pub operator: Option, + pub disabled: Option, +} + +#[allow(dead_code)] +#[derive(Deserialize)] +pub struct EditPassword { + pub password: String, +} + +#[derive(FromRow, Serialize)] +pub struct TruncatedToken { + pub label: Option, + pub token_prefix: Option, + pub expiration: Option>, + pub created_at: chrono::DateTime, + pub last_used_at: chrono::DateTime, + pub scopes: Option>, +} + +#[derive(Deserialize)] +pub struct NewToken { + pub label: Option, + pub expiration: Option>, + pub impersonate_email: Option, + pub scopes: Option>, + pub workspace_id: Option, +} + +#[derive(Deserialize)] +pub struct Login { + pub email: String, + pub password: String, +} + +lazy_static::lazy_static! { + static ref FIRST_TIME_SETUP: Arc = Arc::new(AtomicBool::new(true)); +} + +pub async fn is_first_time_setup(Extension(db): Extension) -> JsonResult { + if !FIRST_TIME_SETUP.load(std::sync::atomic::Ordering::Relaxed) { + return Ok(Json(false)); + } + let single_user = sqlx::query_scalar!("SELECT 1 FROM password LIMIT 2") + .fetch_all(&db) + .await + .ok() + .unwrap_or_default() + .len() + == 1; + if single_user { + let user_is_admin_and_password_changeme = sqlx::query_scalar!( + "SELECT 1 FROM password WHERE email = 'admin@windmill.dev' AND password_hash = '$argon2id$v=19$m=4096,t=3,p=1$oLJo/lPn/gezXCuFOEyaNw$i0T2tCkw3xUFsrBIKZwr8jVNHlIfoxQe+HfDnLtd12I'" + ).fetch_all(&db) + .await + .ok() + .unwrap_or_default() + .len() == 1; + if user_is_admin_and_password_changeme { + let base_url_is_not_set = + sqlx::query_scalar!("SELECT COUNT(*) FROM global_settings WHERE name = 'base_url'") + .fetch_optional(&db) + .await + .ok() + .flatten() + .flatten() + .unwrap_or(0) + == 0; + if base_url_is_not_set { + return Ok(Json(true)); + } + } + } + FIRST_TIME_SETUP.store(false, std::sync::atomic::Ordering::Relaxed); + Ok(Json(false)) +} + +#[derive(Deserialize)] +struct WorkspaceUsername { + pub username: String, +} + +async fn exists_username( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(WorkspaceUsername { username }): Json, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)", + &w_id, + &username + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + tx.commit().await?; + Ok(Json(exists)) +} + +async fn list_users( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + if *CLOUD_HOSTED && w_id == "demo" { + require_admin(authed.is_admin, &authed.username)?; + } + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query_as!( + User, + " + SELECT * + FROM usr + WHERE workspace_id = $1 + ", + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +async fn list_user_usage( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + if *CLOUD_HOSTED && w_id == "demo" { + require_admin(authed.is_admin, &authed.username)?; + } + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query_as!( + UserWithUsage, + " + SELECT usr.email, usage.executions + FROM usr + , LATERAL ( + SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0)::BIGINT executions + FROM completed_job + WHERE workspace_id = $1 + AND job_kind NOT IN ('flow', 'flowpreview') + AND email = usr.email + AND now() - '1 week'::interval < created_at + ) usage + WHERE workspace_id = $1 + ", + w_id + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +#[derive(Deserialize)] +struct ActiveUsersOnly { + active_only: Option, +} + +async fn list_users_as_super_admin( + authed: ApiAuthed, + Extension(db): Extension, + Query(pagination): Query, + Query(ActiveUsersOnly { active_only }): Query, +) -> JsonResult> { + require_super_admin(&db, &authed.email).await?; + let per_page = pagination.per_page.unwrap_or(10000).max(1); + let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page; + + let rows = if active_only.is_some_and(|x| x) { + sqlx::query_as!( + GlobalUserInfo, + "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login')), + authors as (SELECT distinct email FROM usr WHERE usr.operator IS false) + SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, name, company, username + FROM password + WHERE email IN (SELECT email FROM active_users) + ORDER BY super_admin DESC + LIMIT $1 OFFSET $2", + per_page as i32, + offset as i32 + ) + .fetch_all(&db) + .await? + } else { + sqlx::query_as!( + GlobalUserInfo, + "SELECT email, login_type::text, verified, super_admin, name, company, username, NULL::bool as operator_only FROM password ORDER BY super_admin DESC, email LIMIT \ + $1 OFFSET $2", + per_page as i32, + offset as i32 + ) + .fetch_all(&db) + .await? + }; + + Ok(Json(rows)) +} + +#[derive(Serialize, Deserialize)] +struct Progress { + progress: u64, +} +async fn get_tutorial_progress( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult { + let res = sqlx::query_scalar!( + "SELECT progress::bigint FROM tutorial_progress WHERE email = $1", + authed.email + ) + .fetch_optional(&db) + .await? + .flatten() + .unwrap_or_default() as u64; + Ok(Json(Progress { progress: res })) +} + +async fn update_tutorial_progress( + authed: ApiAuthed, + Extension(db): Extension, + Json(progress): Json, +) -> Result { + sqlx::query_scalar!( + "INSERT INTO tutorial_progress VALUES ($2, $1::bigint::bit(64)) ON CONFLICT (email) DO UPDATE SET progress = $1::bigint::bit(64)", + progress.progress as i64, + authed.email + ) + .execute(&db) + .await?; + Ok("tutorial progress updated".to_string()) +} + +async fn list_usernames( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + if *CLOUD_HOSTED && w_id == "demo" { + return Ok(Json(vec![ + authed.username, + "other_usernames_redacted_in_demo_workspace".to_string(), + ])); + } + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query_scalar!("SELECT username from usr WHERE workspace_id = $1", &w_id) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +async fn list_invites( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + let mut tx = db.begin().await?; + let rows = sqlx::query_as!( + WorkspaceInvite, + "SELECT * from workspace_invite WHERE email = $1", + authed.email + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json(rows)) +} + +#[derive(Deserialize)] +struct LogoutQuery { + rd: Option, +} +async fn logout( + Tokened { token }: Tokened, + cookies: Cookies, + Extension(db): Extension, + Query(LogoutQuery { rd }): Query, +) -> Result { + let mut cookie = Cookie::new(COOKIE_NAME, ""); + cookie.set_path(COOKIE_PATH); + if COOKIE_DOMAIN.is_some() { + cookie.set_domain(COOKIE_DOMAIN.clone().unwrap()); + } + cookies.remove(cookie); + let mut tx = db.begin().await?; + let email = sqlx::query_scalar!("DELETE FROM token WHERE token = $1 RETURNING email", token) + .fetch_optional(&mut *tx) + .await?; + if let Some(email) = email { + let email = email.unwrap_or("noemail".to_string()); + audit_log( + &mut *tx, + &AuditAuthor { email: email.clone(), username: email, username_override: None }, + "users.logout", + ActionKind::Delete, + "global", + Some(&truncate_token(&token)), + None, + ) + .await?; + } + tx.commit().await?; + if let Some(rd) = rd { + Ok((StatusCode::TEMPORARY_REDIRECT, [(LOCATION, rd)]).into_response()) + } else { + Ok((StatusCode::OK, "logged out successfully".to_string()).into_response()) + } +} + +async fn whoami( + Extension(db): Extension, + Path(w_id): Path, + ApiAuthed { username, email, is_admin, groups, folders, .. }: ApiAuthed, +) -> JsonResult { + let user = get_user(&w_id, &username, &db).await?; + if let Some(user) = user { + Ok(Json(user)) + } else { + Ok(Json(UserInfo { + workspace_id: w_id, + email: email.clone(), + username: email, + is_admin, + is_super_admin: is_admin, + created_at: chrono::Utc::now(), + groups: groups, + operator: false, + disabled: false, + role: Some("superadmin".to_string()), + folders_read: folders.clone().into_iter().map(|x| x.0).collect(), + folders: folders + .clone() + .into_iter() + .filter_map(|x| if x.1 { Some(x.0) } else { None }) + .collect(), + folders_owners: folders + .into_iter() + .filter_map(|x| if x.2 { Some(x.0) } else { None }) + .collect(), + })) + } +} + +async fn global_whoami( + Extension(db): Extension, + ApiAuthed { email, .. }: ApiAuthed, + Tokened { token }: Tokened, +) -> JsonResult { + let user = sqlx::query_as!( + GlobalUserInfo, + "SELECT email, login_type::TEXT, super_admin, verified, name, company, username, NULL::bool as operator_only FROM password WHERE \ + email = $1", + email + ) + .fetch_one(&db) + .await + .map_err(|e| Error::InternalErr(format!("fetching global identity: {e:#}"))); + + if let Ok(user) = user { + Ok(Json(user)) + } else if std::env::var("SUPERADMIN_SECRET").ok() == Some(token) { + Ok(Json(GlobalUserInfo { + email: email.clone(), + login_type: Some("superadmin_secret".to_string()), + super_admin: true, + verified: true, + name: None, + company: None, + username: None, + operator_only: None, + })) + } else { + Err(user.unwrap_err()) + } +} + +async fn exists_email(Extension(db): Extension, Path(email): Path) -> JsonResult { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM password WHERE email = $1)", + email + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + Ok(Json(exists)) +} + +async fn get_email(ApiAuthed { email, .. }: ApiAuthed) -> Result { + Ok(email) +} + +async fn get_usage( + Extension(db): Extension, + ApiAuthed { email, .. }: ApiAuthed, +) -> Result { + let usage = sqlx::query_scalar!( + " + SELECT usage.usage FROM usage + WHERE is_workspace = false + AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date) + AND id = $1", + email + ) + .fetch_optional(&db) + .await? + .unwrap_or(0); + Ok(usage.to_string()) +} + +async fn get_user(w_id: &str, username: &str, db: &DB) -> Result> { + let user = sqlx::query_as!( + User, + "SELECT * FROM usr where username = $1 AND workspace_id = $2", + username, + w_id + ) + .fetch_optional(db) + .await?; + let is_super_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + user.as_ref().map(|x| &x.email) + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + let groups = get_groups_for_user( + &w_id, + username, + &user + .as_ref() + .map(|x| x.email.to_string()) + .unwrap_or_else(|| "".to_string()), + db, + ) + .await?; + let folders = get_folders_for_user(&w_id, username, &groups, db).await?; + + Ok(user.map(|usr| UserInfo { + groups, + workspace_id: usr.workspace_id, + email: usr.email, + username: usr.username, + is_admin: usr.is_admin, + is_super_admin, + created_at: usr.created_at, + operator: usr.operator, + disabled: usr.disabled, + role: usr.role, + folders_read: folders.clone().into_iter().map(|x| x.0).collect(), + folders: folders + .clone() + .into_iter() + .filter_map(|x| if x.1 { Some(x.0) } else { None }) + .collect(), + folders_owners: folders + .into_iter() + .filter_map(|x| if x.2 { Some(x.0) } else { None }) + .collect(), + })) +} + +pub async fn is_owner_of_path( + authed: ApiAuthed, + Path((_w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + if authed.is_admin { + Ok(Json(true)) + } else { + Ok(Json(require_owner_of_path(&authed, path).is_ok())) + } +} + +pub fn require_owner_of_path(authed: &ApiAuthed, path: &str) -> Result<()> { + if authed.is_admin { + return Ok(()); + } + if !path.is_empty() { + let splitted = path.split("/").collect::>(); + if splitted[0] == "u" { + if splitted[1] == authed.username { + Ok(()) + } else { + Err(Error::BadRequest(format!( + "only the owner {} is authorized to perform this operation", + splitted[1] + ))) + } + } else if splitted[0] == "f" { + crate::folders::require_is_owner(authed, splitted[1]) + } else { + Err(Error::BadRequest(format!( + "Not recognized path kind: {}", + path + ))) + } + } else { + Err(Error::BadRequest(format!( + "Cannot be owner of an empty path" + ))) + } +} + +pub fn get_perm_in_extra_perms_for_authed( + v: serde_json::Value, + authed: &ApiAuthed, +) -> Option { + match v { + serde_json::Value::Object(obj) => { + let mut keys = vec![format!("u/{}", authed.username)]; + for g in authed.groups.iter() { + keys.push(format!("g/{}", g)); + } + let mut res = None; + for k in keys { + if let Some(v) = obj.get(&k) { + if let Some(v) = v.as_bool() { + if v { + return Some(true); + } + res = Some(v); + } + } + } + res + } + _ => None, + } +} + +pub async fn require_is_writer( + authed: &ApiAuthed, + path: &str, + w_id: &str, + db: DB, + query: &str, + kind: &str, +) -> Result<()> { + if authed.is_admin { + return Ok(()); + } + if !path.is_empty() { + if require_owner_of_path(authed, path).is_ok() { + return Ok(()); + } + if path.starts_with("f/") && path.split('/').count() >= 2 { + let folder = path.split('/').nth(1).unwrap(); + let extra_perms = sqlx::query_scalar!( + "SELECT extra_perms FROM folder WHERE name = $1 AND workspace_id = $2", + folder, + w_id + ) + .fetch_optional(&db) + .await?; + if let Some(perms) = extra_perms { + let is_folder_writer = + get_perm_in_extra_perms_for_authed(perms, authed).unwrap_or(false); + if is_folder_writer { + return Ok(()); + } + } + } + let extra_perms = sqlx::query_scalar(query) + .bind(path) + .bind(w_id) + .fetch_optional(&db) + .await?; + if let Some(perms) = extra_perms { + let perm = get_perm_in_extra_perms_for_authed(perms, authed); + match perm { + Some(true) => Ok(()), + Some(false) => Err(Error::BadRequest(format!( + "User {} is not a writer of {kind} path {path}", + authed.username + ))), + None => Err(Error::BadRequest(format!( + "User {} has neither read or write permission on {kind} {path}", + authed.username + ))), + } + } else { + Err(Error::BadRequest(format!( + "{path} does not exist yet and user {} is not an owner of the parent folder", + authed.username + ))) + } + } else { + Err(Error::BadRequest(format!( + "Cannot be writer of an empty path" + ))) + } +} +async fn whois( + Extension(db): Extension, + Path((w_id, username)): Path<(String, String)>, +) -> JsonResult { + let user_o = get_user(&w_id, &username, &db).await?; + let user = not_found_if_none(user_o, "User", username)?; + Ok(Json(user)) +} + +// async fn create_invite_code( +// ApiAuthed { email, .. }: ApiAuthed, +// Extension(db): Extension, +// Json(nu): Json, +// ) -> Result<(StatusCode, String)> { + +// let mut tx = db.begin().await?; +// require_super_admin(&mut tx, email).await?; + +// sqlx::query!( +// "INSERT INTO invite_code +// (code, seats_left) +// VALUES ($1, $2)", +// nu.code, +// nu.seats +// ) +// .execute(&mut tx) +// .await?; + +// tx.commit().await?; + +// Ok(( +// StatusCode::CREATED, +// format!("new invite code {}", nu.code), +// )) +// } + +async fn decline_invite( + authed: ApiAuthed, + Extension(db): Extension, + Json(nu): Json, +) -> Result<(StatusCode, String)> { + let mut tx = db.begin().await?; + + let is_admin = sqlx::query_scalar!( + "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2 RETURNING is_admin", + nu.workspace_id, + authed.email, + ) + .fetch_optional(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.decline_invite", + ActionKind::Delete, + &nu.workspace_id, + Some(&authed.email), + None, + ) + .await?; + tx.commit().await?; + + if is_admin.is_some() { + Ok(( + StatusCode::OK, + format!( + "user {} declined invite to workspace {}", + &authed.email, nu.workspace_id + ), + )) + } else { + Err(Error::NotFound(format!( + "invite for {} not found", + authed.email + ))) + } +} + +lazy_static! { + pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z][a-zA-Z_0-9]*$"#).unwrap(); +} + +async fn accept_invite( + authed: ApiAuthed, + Extension(webhook): Extension, + Extension(db): Extension, + Extension(rsmq): Extension>, + Json(nu): Json, +) -> Result<(StatusCode, String)> { + let mut tx = db.begin().await?; + + let r = sqlx::query!( + "DELETE FROM workspace_invite WHERE workspace_id = $1 AND email = $2 RETURNING is_admin, operator", + nu.workspace_id, + authed.email, + ) + .fetch_optional(&mut *tx) + .await?; + + if let Some(r) = r { + let already_in_workspace = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)", + &nu.workspace_id, + &authed.email, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + if already_in_workspace { + tx.commit().await?; + return Ok(( + StatusCode::CREATED, + format!( + "user {} accepted invite to workspace {}", + &authed.email, nu.workspace_id + ), + )); + } + let username; + (tx, username) = join_workspace( + &nu.workspace_id, + &authed, + nu.username, + r.is_admin, + r.operator, + tx, + ) + .await?; + + audit_log( + &mut *tx, + &ApiAuthed { username: username.clone(), ..authed.clone() }, + "users.accept_invite", + ActionKind::Create, + &nu.workspace_id, + Some(&authed.email), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &username, + &db, + &nu.workspace_id, + windmill_git_sync::DeployedObject::User { email: authed.email.clone() }, + Some(format!("User '{}' accepted invite", &authed.email)), + rsmq, + true, + ) + .await?; + webhook.send_instance_event(InstanceEvent::UserJoinedWorkspace { + email: authed.email.clone(), + workspace: nu.workspace_id.clone(), + username: username, + }); + Ok(( + StatusCode::CREATED, + format!( + "user {} accepted invite to workspace {}", + &authed.email, nu.workspace_id + ), + )) + } else { + Err(Error::NotFound(format!( + "invite for {} not found", + authed.email + ))) + } +} + +async fn join_workspace<'c>( + w_id: &str, + authed: &ApiAuthed, + username: Option, + is_admin: bool, + operator: bool, + mut tx: sqlx::Transaction<'c, sqlx::Postgres>, +) -> error::Result<(sqlx::Transaction<'c, sqlx::Postgres>, String)> { + let automate_username_creation = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = $1", + AUTOMATE_USERNAME_CREATION_SETTING, + ) + .fetch_optional(&mut *tx) + .await? + .map(|v| v.as_bool()) + .flatten() + .unwrap_or(false); + + let username = if automate_username_creation { + if username.is_some() && username.unwrap().len() > 0 { + return Err(Error::BadRequest( + "username is not allowed when username creation is automated".to_string(), + )); + } + get_instance_username_or_create_pending(&mut tx, &authed.email).await? + } else { + let username = username.ok_or(Error::BadRequest("username is required".to_string()))?; + let already_exists_username = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND username = $2)", + &w_id, + username, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + if already_exists_username { + return Err(Error::BadRequest(format!( + "user with username {} already exists in workspace {}", + username, w_id + ))); + } + + if !VALID_USERNAME.is_match(&username) { + return Err(windmill_common::error::Error::BadRequest(format!( + "Usermame can only contain alphanumeric characters and underscores and must start with a letter" + ))); + } + username.to_string() + }; + + let already_exists_email = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE workspace_id = $1 AND email = $2)", + &w_id, + authed.email, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + if already_exists_email { + return Err(Error::BadRequest(format!( + "user with email {} already exists in workspace {}", + authed.email, w_id + ))); + } + + sqlx::query!( + "INSERT INTO usr + (workspace_id, email, username, is_admin, operator) + VALUES ($1, $2, $3, $4, $5)", + &w_id, + authed.email, + username, + is_admin, + operator + ) + .execute(&mut *tx) + .await?; + sqlx::query_as!( + Group, + "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + &w_id, + username, + "all", + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &AuditAuthor { username: username.clone(), ..authed.into() }, + "users.add_to_workspace", + ActionKind::Create, + &w_id, + Some(&authed.email), + None, + ) + .await?; + Ok((tx, username)) +} + +async fn leave_instance(Extension(db): Extension, authed: ApiAuthed) -> Result { + let mut tx = db.begin().await?; + sqlx::query!("DELETE FROM password WHERE email = $1", &authed.email) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.leave", + ActionKind::Delete, + "global", + Some(&authed.email), + None, + ) + .await?; + tx.commit().await?; + + Ok(format!("Left instance",)) +} + +async fn get_workspace_user( + ApiAuthed { username, is_admin, .. }: ApiAuthed, + Extension(db): Extension, + Path((w_id, username_to_update)): Path<(String, String)>, +) -> Result> { + require_admin(is_admin, &username)?; + + let user = sqlx::query_as!( + User, + "SELECT * FROM usr WHERE username = $1 AND workspace_id = $2", + &username_to_update, + &w_id + ) + .fetch_optional(&db) + .await?; + + let user = not_found_if_none(user, "User", username_to_update)?; + + Ok(Json(user)) +} + +async fn update_workspace_user( + authed: ApiAuthed, + Extension(db): Extension, + Extension(rsmq): Extension>, + Path((w_id, username_to_update)): Path<(String, String)>, + Json(eu): Json, +) -> Result { + let mut tx = db.begin().await?; + + require_admin(authed.is_admin, &authed.username)?; + + if let Some(a) = eu.is_admin { + sqlx::query_scalar!( + "UPDATE usr SET is_admin = $1 WHERE username = $2 AND workspace_id = $3", + a, + &username_to_update, + &w_id + ) + .execute(&mut *tx) + .await?; + } + + if let Some(a) = eu.operator { + sqlx::query_scalar!( + "UPDATE usr SET operator = $1 WHERE username = $2 AND workspace_id = $3", + a, + &username_to_update, + &w_id + ) + .execute(&mut *tx) + .await?; + } + + if let Some(a) = eu.disabled { + sqlx::query_scalar!( + "UPDATE usr SET disabled = $1 WHERE username = $2 AND workspace_id = $3", + a, + &username_to_update, + &w_id + ) + .execute(&mut *tx) + .await?; + } + + audit_log( + &mut *tx, + &authed, + "users.update", + ActionKind::Update, + &w_id, + Some(&username_to_update), + None, + ) + .await?; + + let user_email = sqlx::query_scalar!( + "SELECT email FROM usr WHERE username = $1 AND workspace_id = $2", + &username_to_update, + &w_id + ) + .fetch_one(&mut *tx) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::User { email: user_email.clone() }, + Some(format!("Updated user '{}'", &user_email)), + rsmq, + true, + ) + .await?; + + Ok(format!("user {} updated", user_email)) +} + +async fn update_user( + authed: ApiAuthed, + Path(email_to_update): Path, + Extension(db): Extension, + Json(eu): Json, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + + if let Some(sa) = eu.is_super_admin { + sqlx::query_scalar!( + "UPDATE password SET super_admin = $1 WHERE email = $2", + sa, + &email_to_update + ) + .execute(&mut *tx) + .await?; + } + + if let Some(n) = eu.name { + sqlx::query_scalar!( + "UPDATE password SET name = $1 WHERE email = $2", + n, + &email_to_update + ) + .execute(&mut *tx) + .await?; + } + + audit_log( + &mut *tx, + &authed, + "users.update", + ActionKind::Update, + "global", + Some(&email_to_update), + None, + ) + .await?; + tx.commit().await?; + Ok(format!("email {} updated", &email_to_update)) +} + +async fn delete_user( + authed: ApiAuthed, + Path(email_to_delete): Path, + Extension(db): Extension, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + + sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) + .execute(&mut *tx) + .await?; + + let usernames = sqlx::query_scalar!( + "DELETE FROM usr WHERE email = $1 RETURNING username", + &email_to_delete + ) + .fetch_all(&mut *tx) + .await?; + + for username in usernames { + sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) + .execute(&mut *tx) + .await?; + + sqlx::query!("DELETE FROM usr_to_group WHERE usr = $1", &username) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "DELETE FROM workspace_invite WHERE email = $1", + &email_to_delete + ) + .execute(&mut *tx) + .await?; + } + audit_log( + &mut *tx, + &authed, + "users.delete", + ActionKind::Delete, + "global", + Some(&email_to_delete), + None, + ) + .await?; + tx.commit().await?; + Ok(format!("email {} deleted", &email_to_delete)) +} + +lazy_static::lazy_static! { + pub static ref NEW_USER_WEBHOOK: Option = std::env::var("NEW_USER_WEBHOOK").ok(); + +} + +async fn create_user( + authed: ApiAuthed, + Extension(db): Extension, + Extension(webhook): Extension, + Extension(argon2): Extension>>, + Extension(rsmq): Extension>, + Json(nu): Json, +) -> Result<(StatusCode, String)> { + crate::users_ee::create_user(authed, db, webhook, argon2, rsmq, nu).await +} + +async fn delete_workspace_user( + authed: ApiAuthed, + Extension(db): Extension, + Extension(rsmq): Extension>, + Path((w_id, username_to_delete)): Path<(String, String)>, +) -> Result { + let mut tx = db.begin().await?; + + require_admin(authed.is_admin, &authed.username)?; + + let email_to_delete_o = sqlx::query_scalar!( + "SELECT email FROM usr where username = $1 AND workspace_id = $2", + username_to_delete, + &w_id, + ) + .fetch_optional(&db) + .await?; + + let email_to_delete = not_found_if_none(email_to_delete_o, "User", &username_to_delete)?; + + sqlx::query_scalar!( + "DELETE FROM usr WHERE email = $1 AND workspace_id = $2", + email_to_delete, + &w_id + ) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "DELETE FROM usr_to_group WHERE usr = $1 AND workspace_id = $2", + &username_to_delete, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.delete", + ActionKind::Delete, + &w_id, + Some(&username_to_delete), + None, + ) + .await?; + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::User { email: email_to_delete.clone() }, + Some(format!( + "Removed user '{}' from workspace", + &email_to_delete + )), + rsmq, + true, + ) + .await?; + + Ok(format!("username {} deleted", username_to_delete)) +} + +async fn set_password( + Extension(db): Extension, + Extension(argon2): Extension>>, + authed: ApiAuthed, + Json(ep): Json, +) -> Result { + crate::users_ee::set_password(db, argon2, authed, ep).await +} + +async fn login( + cookies: Cookies, + Extension(db): Extension, + Extension(argon2): Extension>>, + Json(Login { email, password }): Json, +) -> Result { + let mut tx = db.begin().await?; + let email = email.to_lowercase(); + let audit_author = + AuditAuthor { email: email.clone(), username: email.clone(), username_override: None }; + let email_w_h: Option<(String, String, bool, bool)> = sqlx::query_as( + "SELECT email, password_hash, super_admin, first_time_user FROM password WHERE email = $1 AND login_type = \ + 'password'", + ) + .bind(&email) + .fetch_optional(&mut *tx) + .await?; + + if let Some((email, hash, super_admin, first_time_user)) = email_w_h { + let parsed_hash = + PasswordHash::new(&hash).map_err(|e| Error::InternalErr(e.to_string()))?; + if argon2 + .verify_password(password.as_bytes(), &parsed_hash) + .is_err() + { + audit_log( + &mut *tx, + &audit_author, + "users.login_failure", + ActionKind::Create, + "global", + None, + None, + ) + .await?; + Err(Error::BadRequest("Invalid login".to_string())) + } else { + if first_time_user { + sqlx::query_scalar!( + "UPDATE password SET first_time_user = false WHERE email = $1", + &email + ) + .execute(&mut *tx) + .await?; + let mut c = Cookie::new("first_time", "1"); + if let Some(domain) = COOKIE_DOMAIN.as_ref() { + c.set_domain(domain); + } + c.set_secure(false); + c.set_expires(time::OffsetDateTime::now_utc() + time::Duration::minutes(15)); + c.set_http_only(false); + c.set_path("/"); + + cookies.add(c); + } + + let token = create_session_token(&email, super_admin, &mut tx, cookies).await?; + + audit_log( + &mut *tx, + &audit_author, + "users.login", + ActionKind::Create, + "global", + Some(&truncate_token(&token)), + None, + ) + .await?; + + tx.commit().await?; + Ok(token) + } + } else { + audit_log( + &mut *tx, + &audit_author, + "users.login_failure", + ActionKind::Create, + "global", + None, + None, + ) + .await?; + Err(Error::BadRequest("Invalid login".to_string())) + } +} + +async fn refresh_token( + Extension(db): Extension, + authed: ApiAuthed, + cookies: Cookies, +) -> Result { + let mut tx = db.begin().await?; + + let super_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + &authed.email + ) + .fetch_optional(&mut *tx) + .await? + .unwrap_or(false); + + let _ = create_session_token(&authed.email, super_admin, &mut tx, cookies).await?; + + tx.commit().await?; + Ok("token refreshed".to_string()) +} + +pub async fn create_session_token<'c>( + email: &str, + super_admin: bool, + tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, + cookies: Cookies, +) -> Result { + let token = rd_string(32); + sqlx::query!( + "INSERT INTO token + (token, email, label, expiration, super_admin) + VALUES ($1, $2, $3, now() + ($4 || ' hours')::interval, $5)", + token, + email, + "session", + TTL_TOKEN_DB_H.to_string(), + super_admin + ) + .execute(&mut **tx) + .await?; + let mut cookie = Cookie::new(COOKIE_NAME, token.clone()); + cookie.set_secure(IS_SECURE.read().await.clone()); + cookie.set_same_site(Some(tower_cookies::cookie::SameSite::Lax)); + cookie.set_http_only(true); + cookie.set_path(COOKIE_PATH); + if COOKIE_DOMAIN.is_some() { + cookie.set_domain(COOKIE_DOMAIN.clone().unwrap()); + } + let mut expire: OffsetDateTime = time::OffsetDateTime::now_utc(); + expire += time::Duration::days(3); + cookie.set_expires(expire); + cookies.add(cookie); + Ok(token) +} + +async fn create_token( + Extension(db): Extension, + authed: ApiAuthed, + Json(new_token): Json, +) -> Result<(StatusCode, String)> { + let token = rd_string(32); + let mut tx = db.begin().await?; + + let is_super_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + authed.email + ) + .fetch_optional(&mut *tx) + .await? + .unwrap_or(false); + sqlx::query!( + "INSERT INTO token + (token, email, label, expiration, super_admin, scopes, workspace_id) + VALUES ($1, $2, $3, $4, $5, $6, $7)", + token, + authed.email, + new_token.label, + new_token.expiration, + is_super_admin, + new_token.scopes.as_ref().map(|x| x.as_slice()), + new_token.workspace_id, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.token.create", + ActionKind::Create, + &"global", + Some(&token[0..10]), + None, + ) + .instrument(tracing::info_span!("token", email = &authed.email)) + .await?; + tx.commit().await?; + Ok((StatusCode::CREATED, token)) +} + +async fn impersonate( + Extension(db): Extension, + authed: ApiAuthed, + Json(new_token): Json, +) -> Result<(StatusCode, String)> { + let token = rd_string(32); + require_super_admin(&db, &authed.email).await?; + + if new_token.impersonate_email.is_none() { + return Err(Error::BadRequest( + "impersonate_username is required".to_string(), + )); + } + + let impersonated = new_token.impersonate_email.unwrap(); + + let is_super_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + impersonated + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + let mut tx = db.begin().await?; + + sqlx::query!( + "INSERT INTO token + (token, email, label, expiration, super_admin) + VALUES ($1, $2, $3, $4, $5)", + token, + impersonated, + new_token.label, + new_token.expiration, + is_super_admin + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.impersonate", + ActionKind::Delete, + &"global", + Some(&token[0..10]), + Some([("impersonated", &format!("{impersonated}")[..])].into()), + ) + .instrument(tracing::info_span!("token", email = &impersonated)) + .await?; + tx.commit().await?; + Ok((StatusCode::CREATED, token)) +} + +#[derive(Deserialize)] +struct ListTokenQuery { + exclude_ephemeral: Option, +} + +async fn list_tokens( + Extension(db): Extension, + ApiAuthed { email, .. }: ApiAuthed, + Query(query): Query, + Query(pagination): Query, +) -> JsonResult> { + let (per_page, offset) = paginate(pagination); + let rows = if query.exclude_ephemeral.unwrap_or(false) { + sqlx::query_as!( + TruncatedToken, + "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, \ + last_used_at, scopes FROM token WHERE email = $1 AND label != 'ephemeral-script' + ORDER BY created_at DESC LIMIT $2 OFFSET $3", + email, + per_page as i64, + offset as i64, + ) + .fetch_all(&db) + .await? + } else { + sqlx::query_as!( + TruncatedToken, + "SELECT label, concat(substring(token for 10)) as token_prefix, expiration, created_at, \ + last_used_at, scopes FROM token WHERE email = $1 + ORDER BY created_at DESC LIMIT $2 OFFSET $3", + email, + per_page as i64, + offset as i64, + ) + .fetch_all(&db) + .await? + }; + Ok(Json(rows)) +} + +async fn delete_token( + Extension(db): Extension, + authed: ApiAuthed, + Path(token_prefix): Path, +) -> Result { + let mut tx = db.begin().await?; + + let tokens_deleted: Vec = sqlx::query_scalar( + "DELETE FROM token + WHERE email = $1 + AND token LIKE concat($2::text, '%') + RETURNING concat(substring(token for 10), '*****')", + ) + .bind(&authed.email) + .bind(&token_prefix) + .fetch_all(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.token.delete", + ActionKind::Delete, + &"global", + Some(&token_prefix), + None, + ) + .await?; + tx.commit().await?; + + Ok(format!( + "deleted {} tokens {:?} with prefix {}", + tokens_deleted.len(), + tokens_deleted, + token_prefix + )) +} + +async fn leave_workspace( + Extension(db): Extension, + Path(w_id): Path, + authed: ApiAuthed, +) -> Result { + let mut tx = db.begin().await?; + sqlx::query!( + "DELETE FROM usr WHERE workspace_id = $1 AND username = $2", + &w_id, + authed.username + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.leave_workspace", + ActionKind::Delete, + &w_id, + None, + None, + ) + .await?; + tx.commit().await?; + + Ok(format!("left workspace {w_id}")) +} + +#[derive(Serialize)] +struct Runnable { + workspace: String, + endpoint_async: String, + endpoint_sync: String, + endpoint_openai_sync: String, + summary: String, + description: String, + schema: Option, + kind: String, + path: String, +} + +async fn get_all_runnables( + Extension(db): Extension, + authed: ApiAuthed, + Tokened { token }: Tokened, + Extension(cache): Extension>, +) -> JsonResult> { + let mut tx = db.clone().begin(&authed).await?; + let mut runnables = Vec::new(); + let workspaces = sqlx::query_scalar!( + "SELECT workspace.id as id FROM workspace, usr WHERE usr.workspace_id = workspace.id AND \ + usr.email = $1 AND deleted = false", + authed.email + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + for workspace in workspaces { + let nauthed = cache + .get_authed(Some(workspace.clone()), &token) + .await + .ok_or_else(|| { + Error::BadRequest(format!("not authorized to access workspace: {workspace}")) + })?; + let mut tx = db.clone().begin(&nauthed).await?; + let flows = sqlx::query!( + "SELECT flow.workspace_id as workspace, flow.path, summary, description, flow_version.schema + FROM flow + LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] + WHERE flow.workspace_id = $1", + workspace + ) + .fetch_all(&mut *tx) + .await?; + runnables.extend( + flows + .into_iter() + .map(|f| Runnable { + workspace: f.workspace.clone(), + endpoint_async: format!("/w/{}/jobs/run/f/{}", &f.workspace, &f.path), + endpoint_sync: format!( + "/w/{}/jobs/run_wait_result/f/{}", + &f.workspace, &f.path + ), + endpoint_openai_sync: format!( + "/w/{}/jobs/openai_sync/f/{}", + &f.workspace, &f.path + ), + summary: f.summary, + description: f.description, + schema: f.schema, + kind: "flow".to_string(), + path: f.path, + }) + .collect::>(), + ); + let scripts = sqlx::query!( + "SELECT workspace_id as workspace, path, summary, description, schema FROM script as o WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1) and workspace_id = $1", workspace + ) + .fetch_all(&mut *tx) + .await?; + runnables.extend( + scripts + .into_iter() + .map(|s| Runnable { + workspace: s.workspace.clone(), + endpoint_async: format!("/w/{}/jobs/run/p/{}", &s.workspace, &s.path), + endpoint_sync: format!( + "/w/{}/jobs/run_wait_result/p/{}", + &s.workspace, &s.path + ), + endpoint_openai_sync: format!( + "/w/{}/jobs/openai_sync/p/{}", + &s.workspace, &s.path + ), + summary: s.summary, + description: s.description, + schema: s.schema, + kind: "script".to_string(), + path: s.path, + }) + .collect::>(), + ); + tx.commit().await?; + } + Ok(Json(runnables)) +} + +//used by oauth +#[allow(dead_code)] +#[derive(Deserialize, Debug, Clone)] +pub struct LoginUserInfo { + pub email: Option, + pub name: Option, + pub company: Option, + + pub displayName: Option, +} + +#[derive(Serialize)] +struct InstanceUsernameInfo { + username: String, + workspace_usernames: Vec, +} + +#[derive(Serialize)] +struct WorkspaceUsernameInfo { + workspace_id: String, + username: String, +} +async fn get_instance_username_info( + ApiAuthed { email, .. }: ApiAuthed, + Path(user_email): Path, + Extension(db): Extension, +) -> JsonResult { + require_super_admin(&db, &email).await?; + let mut tx = db.begin().await?; + let instance_username = match sqlx::query_scalar!( + "SELECT username FROM password WHERE email = $1", + &user_email + ) + .fetch_one(&mut *tx) + .await? + { + Some(username) => username, + None => generate_instance_wide_unique_username(&mut tx, &user_email).await?, + }; + + let workspace_usernames = sqlx::query_as!( + WorkspaceUsernameInfo, + "SELECT workspace_id, username FROM usr WHERE email = $1", + &user_email + ) + .fetch_all(&mut *tx) + .await?; + + Ok(Json(InstanceUsernameInfo { + username: instance_username, + workspace_usernames: workspace_usernames, + })) +} + +async fn username_to_email( + Path((w_id, username)): Path<(String, String)>, + Extension(db): Extension, +) -> Result { + let email = sqlx::query_scalar!( + "SELECT email FROM usr WHERE username = $1 AND workspace_id = $2", + &username, + &w_id + ) + .fetch_optional(&db) + .await?; + + let email = not_found_if_none(email, "user", username)?; + + Ok(email) +} + +#[cfg(feature = "enterprise")] +#[derive(Serialize, Deserialize)] +struct ExportedGlobalUser { + email: String, + password_hash: Option, + login_type: String, + super_admin: bool, + verified: bool, + name: Option, + company: Option, + first_time_user: bool, + username: Option, +} + +#[cfg(feature = "enterprise")] +async fn export_global_users( + Extension(db): Extension, + authed: ApiAuthed, +) -> JsonResult> { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + let users = sqlx::query_as!( + ExportedGlobalUser, + "SELECT email, password_hash, login_type, super_admin, verified, name, company, first_time_user, username FROM password" + ) + .fetch_all(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.export_export", + ActionKind::Execute, + "global", + None, + None, + ) + .await?; + + tx.commit().await?; + + Ok(Json(users)) +} + +#[cfg(not(feature = "enterprise"))] +async fn export_global_users() -> JsonResult { + Err(Error::BadRequest( + "This feature is only available in the enterprise version".to_string(), + )) +} + +#[cfg(feature = "enterprise")] +async fn overwrite_global_users( + Extension(db): Extension, + authed: ApiAuthed, + Json(users): Json>, +) -> Result { + require_super_admin(&db, &authed.email).await?; + let mut tx = db.begin().await?; + sqlx::query!("DELETE FROM password") + .execute(&mut *tx) + .await?; + for user in users { + sqlx::query!( + "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, company, first_time_user, username) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + user.email, + user.password_hash, + user.login_type, + user.super_admin, + user.verified, + user.name, + user.company, + user.first_time_user, + user.username + ) + .execute(&mut *tx) + .await?; + } + audit_log( + &mut *tx, + &authed, + "users.import_global", + ActionKind::Create, + "global", + None, + None, + ) + .await?; + tx.commit().await?; + Ok("loaded global users".to_string()) +} + +#[cfg(not(feature = "enterprise"))] +async fn overwrite_global_users() -> JsonResult { + Err(Error::BadRequest( + "This feature is only available in the enterprise version".to_string(), + )) +} + +#[derive(Deserialize)] +struct RenameUser { + new_username: String, +} + +async fn rename_user( + authed: ApiAuthed, + Path(user_email): Path, + Extension(db): Extension, + Json(ru): Json, +) -> Result { + require_super_admin(&db, &authed.email).await?; + + let mut tx = db.begin().await?; + + let username_conflict = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 and email != $2 UNION SELECT 1 FROM password WHERE username = $1 UNION SELECT 1 FROM pending_user WHERE username = $1)", + &ru.new_username, + &user_email + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + if username_conflict { + return Err(Error::BadRequest(format!( + "username {} already used by another user", + &ru.new_username + ))); + } + + if !VALID_USERNAME.is_match(&ru.new_username) { + return Err(windmill_common::error::Error::BadRequest(format!( + "Usermame can only contain alphanumeric characters and underscores and must start with a letter" + ))); + } + + sqlx::query!( + "UPDATE password SET username = $1 WHERE email = $2", + ru.new_username, + user_email + ) + .execute(&mut *tx) + .await?; + + let workspace_usernames = sqlx::query!( + "SELECT workspace_id, username FROM usr WHERE email = $1", + &user_email + ) + .fetch_all(&mut *tx) + .await?; + + for w_u in workspace_usernames { + if ru.new_username == w_u.username { + continue; + } + update_username_in_workpsace( + &mut tx, + &user_email, + &w_u.username, + &ru.new_username, + &w_u.workspace_id, + ) + .await?; + } + + audit_log( + &mut *tx, + &authed, + "users.rename", + ActionKind::Update, + "global", + Some(&user_email), + None, + ) + .await?; + tx.commit().await?; + Ok(format!( + "updated username of user {} to {}", + &user_email, &ru.new_username + )) +} + +async fn update_username_in_workpsace<'c>( + tx: &mut sqlx::Transaction<'c, sqlx::Postgres>, + email: &str, + old_username: &str, + new_username: &str, + w_id: &str, +) -> error::Result<()> { + // ---- instance and workspace users ---- + sqlx::query!( + "UPDATE usr SET username = $1 WHERE email = $2", + new_username, + email + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE usr_to_group SET usr = $1 WHERE usr = $2", + new_username, + old_username + ) + .execute(&mut **tx) + .await?; + + // ---- queue ---- + sqlx::query!( + r#"UPDATE queue SET script_path = REGEXP_REPLACE(script_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE script_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE queue SET schedule_path = REGEXP_REPLACE(schedule_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE schedule_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE queue SET permissioned_as = ('u/' || $1) WHERE permissioned_as = ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE queue SET canceled_by = $1 WHERE canceled_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + sqlx::query!( + "UPDATE queue SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + // ---- completed_job ---- + sqlx::query!( + r#"UPDATE completed_job SET script_path = REGEXP_REPLACE(script_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE script_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE completed_job SET schedule_path = REGEXP_REPLACE(schedule_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE schedule_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE completed_job SET permissioned_as = ('u/' || $1) WHERE permissioned_as = ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE completed_job SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + sqlx::query!( + "UPDATE completed_job SET canceled_by = $1 WHERE canceled_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + // ---- resources---- + sqlx::query!( + r#"UPDATE resource SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE resource_type SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE resource SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE resource SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- variables ---- + + sqlx::query!( + r#"UPDATE variable SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE variable SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- scripts ---- + sqlx::query!( + r#"UPDATE script SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE script SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + sqlx::query!( + "UPDATE script SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- flows ---- + sqlx::query!( + r#"INSERT INTO flow + (workspace_id, path, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, concurrency_key, versions, value, schema, edited_by, edited_at) + SELECT workspace_id, REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1'), summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, concurrency_key, versions, value, schema, edited_by, edited_at + FROM flow + WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE flow_version SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "DELETE FROM flow WHERE path LIKE ('u/' || $1 || '/%') AND workspace_id = $2", + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE flow SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- draft ---- + sqlx::query!( + r#"UPDATE draft SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE draft SET value = to_json(jsonb_set(to_jsonb(value), ARRAY['path'], to_jsonb(REGEXP_REPLACE(value->>'path','u/' || $2 || '/(.*)','u/' || $1 || '/\1')))) WHERE value->>'path' LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ).execute(&mut **tx) + .await?; + + // ---- app ---- + sqlx::query!( + r#"UPDATE app SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE app SET policy = jsonb_set(policy, ARRAY['on_behalf_of'], to_jsonb('u/' || $1)) WHERE policy->>'on_behalf_of' = ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE app SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- app_version ---- + + sqlx::query!( + "UPDATE app_version SET created_by = $1 WHERE created_by = $2 AND EXISTS (SELECT 1 FROM app WHERE workspace_id = $3 AND app.id = app_version.app_id)", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- schedules ---- + + sqlx::query!( + r#"UPDATE schedule SET path = REGEXP_REPLACE(path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + r#"UPDATE schedule SET script_path = REGEXP_REPLACE(script_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\1') WHERE script_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE schedule SET edited_by = $1 WHERE edited_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + sqlx::query!( + "UPDATE schedule SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- group_ ---- + + sqlx::query!( + "UPDATE group_ SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- folders ---- + + sqlx::query!( + "UPDATE folder SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE folder SET owners = ARRAY_REPLACE(owners, 'u/' || $2, 'u/' || $1) WHERE ('u/' || $2) = ANY(owners) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "UPDATE folder SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- input ---- + + sqlx::query!( + "UPDATE input SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + // ---- favorite ---- + + sqlx::query!( + "UPDATE favorite SET usr = $1 WHERE usr = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + // ---- capture ---- + + sqlx::query!( + "UPDATE capture SET created_by = $1 WHERE created_by = $2 AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + // ---- token ---- + + sqlx::query!( + "UPDATE token SET owner = ('u/' || $1) WHERE owner = ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await + .unwrap(); + + sqlx::query!( + r#"UPDATE token SET scopes = array(select regexp_replace(unnest(scopes), 'run:([^/]+)/u/' || $2 || '/(.+)', 'run:\1/u/' || $1 || '/\2')) WHERE EXISTS (SELECT 1 FROM UNNEST(scopes) scope WHERE scope LIKE ('run:%/u/' || $2 || '/%')) AND workspace_id = $3"#, + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + // ---- raw_app ---- + + sqlx::query!( + "UPDATE raw_app SET extra_perms = extra_perms - ('u/' || $2) || jsonb_build_object(('u/' || $1), extra_perms->('u/' || $2)) WHERE extra_perms ? ('u/' || $2) AND workspace_id = $3", + new_username, + old_username, + w_id + ) + .execute(&mut **tx) + .await?; + + Ok(()) +} diff --git a/backend/windmill-api/src/users_ee.rs b/backend/windmill-api/src/users_ee.rs new file mode 100644 index 0000000000..e0f15a9317 --- /dev/null +++ b/backend/windmill-api/src/users_ee.rs @@ -0,0 +1,41 @@ +use std::sync::Arc; + +use crate::db::ApiAuthed; + +use crate::users::{EditPassword, NewUser}; +use crate::{db::DB, webhook_util::WebhookShared}; +use argon2::Argon2; + +use http::StatusCode; + +use windmill_common::error::{Error, Result}; + +pub async fn create_user( + _authed: ApiAuthed, + _db: DB, + _webhook: WebhookShared, + _argon2: Arc>, + _rsmq: Option, + mut _nu: NewUser, +) -> Result<(StatusCode, String)> { + Err(Error::InternalErr( + "Not implemented in Windmill's Open Source repository".to_string(), + )) +} + +pub async fn set_password( + _db: DB, + _argon2: Arc>, + _authed: ApiAuthed, + _ep: EditPassword, +) -> Result { + Err(Error::InternalErr( + "Not implemented in Windmill's Open Source repository".to_string(), + )) +} + +pub fn send_email_if_possible(_subject: &str, _content: &str, _to: &str) { + tracing::warn!( + "send_email_if_possible is not implemented in Windmill's Open Source repository" + ); +} diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index d8a81ab741..ea4fccc160 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -155,25 +155,6 @@ pub async fn get_instance_username_or_create_pending<'c>( } } -pub async fn get_and_delete_pending_username_or_generate<'c>( - tx: &mut Transaction<'c, Postgres>, - email: &str, -) -> error::Result { - let username = sqlx::query_scalar!("SELECT username FROM pending_user WHERE email = $1", email) - .fetch_optional(&mut **tx) - .await?; - - if let Some(username) = username { - sqlx::query!("DELETE FROM pending_user WHERE email = $1", email) - .execute(&mut **tx) - .await?; - Ok(username) - } else { - let username = generate_instance_wide_unique_username(&mut *tx, email).await?; - Ok(username) - } -} - pub fn content_plain(body: Body) -> Response { use axum::http::header; Response::builder() diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index cf48781331..018ac8de5f 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -9,14 +9,15 @@ use std::collections::HashMap; use crate::db::ApiAuthed; -use crate::utils::{get_instance_username_or_create_pending, INVALID_USERNAME_CHARS}; +use crate::users_ee::send_email_if_possible; +use crate::utils::get_instance_username_or_create_pending; use crate::BASE_URL; use crate::{ apps::AppWithLastVersion, db::DB, folders::Folder, resources::{Resource, ResourceType}, - users::{send_email_if_possible, WorkspaceInvite, VALID_USERNAME}, + users::{WorkspaceInvite, VALID_USERNAME}, utils::require_super_admin, webhook_util::WebhookShared, }; @@ -34,7 +35,7 @@ use itertools::Itertools; use regex::Regex; use uuid::Uuid; -use windmill_audit::audit_ee::{audit_log, AuditAuthor, AuditAuthorable}; +use windmill_audit::audit_ee::audit_log; use windmill_audit::ActionKind; use windmill_common::db::UserDB; use windmill_common::s3_helpers::LargeFileStorage; @@ -218,11 +219,12 @@ struct EditDeployTo { deploy_to: Option, } +#[allow(dead_code)] #[derive(Deserialize)] -struct EditAutoInvite { - operator: Option, - invite_all: Option, - auto_add: Option, +pub struct EditAutoInvite { + pub operator: Option, + pub invite_all: Option, + pub auto_add: Option, } #[derive(Deserialize)] @@ -573,250 +575,21 @@ async fn edit_deploy_to() -> Result { )); } -const BANNED_DOMAINS: &str = include_str!("../banned_domains.txt"); +pub const BANNED_DOMAINS: &str = include_str!("../banned_domains.txt"); async fn is_allowed_auto_domain(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult { let domain = email.split('@').last().unwrap(); return Ok(Json(!BANNED_DOMAINS.contains(domain))); } -async fn auto_add_user( - email: &str, - w_id: &str, - operator: &bool, - tx: &mut Transaction<'_, Postgres>, - authorable: &impl AuditAuthorable, -) -> Result { - let automate_username_creation = sqlx::query_scalar!( - "SELECT value FROM global_settings WHERE name = $1", - AUTOMATE_USERNAME_CREATION_SETTING, - ) - .fetch_optional(&mut **tx) - .await? - .map(|v| v.as_bool()) - .flatten() - .unwrap_or(false); - - let username = if automate_username_creation { - get_instance_username_or_create_pending(&mut *tx, &email).await? - } else { - let mut username = email - .split('@') - .next() - .unwrap() - .to_string() - .replace(".", ""); - - username = INVALID_USERNAME_CHARS - .replace_all(&mut username, "") - .to_string(); - - if username.is_empty() { - username = "user".to_string() - } - - let base_username = username.clone(); - let mut username_conflict = true; - let mut i = 1; - while username_conflict { - if i > 1000 { - return Err(Error::InternalErr(format!( - "too many username conflicts for {}", - email - ))); - } - if i > 1 { - username = format!("{}{}", base_username, i) - } - username_conflict = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 AND workspace_id = $2)", - &username, - &w_id - ) - .fetch_one(&mut **tx) - .await? - .unwrap_or(false); - i += 1; - } - username - }; - - sqlx::query!( - "INSERT INTO usr (workspace_id, username, email, is_admin, operator) VALUES ($1, $2, $3, false, $4) ON CONFLICT DO NOTHING", - &w_id, - &username, - &email, - &operator - ) - .execute(&mut **tx) - .await?; - - sqlx::query_as!( - Group, - "INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", - &w_id, - username, - "all", - ) - .execute(&mut **tx) - .await?; - let audit_author = if authorable.username() == authorable.email() && authorable.email() == email - { - // if the user is auto adding themselves (e.g. by joining the instance), we use their newly created workspace username for audit logs - AuditAuthor { - username: username.clone(), - email: email.to_string(), - username_override: None, - } - } else { - AuditAuthor { - username: authorable.username().to_string(), - email: authorable.email().to_string(), - username_override: authorable.username_override().map(|x| x.to_string()), - } - }; - audit_log( - &mut **tx, - &audit_author, - "users.auto_invite_add", - ActionKind::Create, - &w_id, - Some(email), - None, - ) - .await?; - Ok(username) -} - async fn edit_auto_invite( authed: ApiAuthed, Extension(db): Extension, Extension(rsmq): Extension>, Path(w_id): Path, - ApiAuthed { is_admin, email, username, .. }: ApiAuthed, Json(ea): Json, ) -> Result { - require_admin(is_admin, &username)?; - - // #[cfg(not(feature = "enterprise"))] - // { - // return Err(Error::BadRequest( - // "Auto-invite is only available on enterprise".to_string(), - // )); - // } - - let domain = if ea.invite_all.is_some_and(|x| x) { - if *CLOUD_HOSTED { - return Err(Error::BadRequest( - "invite_all is only available locally".to_string(), - )); - } else { - "*" - } - } else { - email.split('@').last().unwrap() - }; - - let mut tx = db.begin().await?; - - let mut users_to_auto_add = Option::None; - - if let (Some(operator), Some(auto_add)) = (ea.operator, ea.auto_add) { - if BANNED_DOMAINS.contains(domain) { - return Err(Error::BadRequest(format!( - "Domain {} is not allowed", - domain - ))); - } - - sqlx::query!( - "UPDATE workspace_settings SET auto_invite_domain = $1, auto_invite_operator = $2, auto_add = $4 WHERE workspace_id = $3", - domain, - operator, - &w_id, - auto_add, - ) - .execute(&mut *tx) - .await?; - - if auto_add { - users_to_auto_add = Some(sqlx::query!( - "SELECT email FROM password WHERE ($2::text = '*' OR email LIKE CONCAT('%', $2::text)) AND NOT EXISTS ( - SELECT 1 FROM usr WHERE workspace_id = $1::text AND email = password.email - )", - &w_id, - domain - ) - .fetch_all(&mut *tx).await?); - - for user in users_to_auto_add.as_ref().unwrap() { - auto_add_user(&user.email, &w_id, &operator, &mut tx, &authed).await?; - send_email_if_possible( - &format!("Added to Windmill's workspace: {w_id}"), - &format!( - "You have been granted access to Windmill's workspace {w_id} by {email}. - - Access the workspace at {}/?workspace={w_id}", - BASE_URL.read().await.clone() - ), - &user.email, - ); - } - } else { - sqlx::query!( - "INSERT INTO workspace_invite - (workspace_id, email, is_admin, operator) - SELECT $1::text, email, false, $3 FROM password WHERE ($2::text = '*' OR email LIKE CONCAT('%', $2::text)) AND NOT EXISTS ( - SELECT 1 FROM usr WHERE workspace_id = $1::text AND email = password.email - ) - ON CONFLICT DO NOTHING", - &w_id, - domain, - operator - ) - .execute(&mut *tx) - .await?; - } - } else { - sqlx::query!( - "UPDATE workspace_settings SET auto_invite_domain = NULL, auto_invite_operator = NULL, auto_add = NULL WHERE workspace_id = $1", - &w_id, - ) - .execute(&mut *tx) - .await?; - } - audit_log( - &mut *tx, - &authed, - "workspaces.edit_auto_invite_domain", - ActionKind::Update, - &w_id, - Some(&authed.email), - Some([("operator", &format!("{:?}", ea.operator)[..])].into()), - ) - .await?; - tx.commit().await?; - - if let Some(users) = users_to_auto_add { - for user in users { - handle_deployment_metadata( - &email, - &username, - &db, - &w_id, - windmill_git_sync::DeployedObject::User { email: user.email.clone() }, - Some(format!("Auto-added user '{}' to workspace", &user.email)), - rsmq.clone(), - true, - ) - .await?; - } - } - - Ok(format!( - "Edit auto-invite for workspace {} to {}", - &w_id, domain - )) + crate::workspaces_ee::edit_auto_invite(authed, db, rsmq, w_id, ea).await } async fn edit_webhook( @@ -2001,61 +1774,6 @@ async fn delete_workspace( Ok(format!("Deleted workspace {}", &w_id)) } -pub async fn invite_user_to_all_auto_invite_worspaces( - db: &DB, - email: &str, - rsmq: Option, - authorable: &impl AuditAuthorable, -) -> Result<()> { - let mut tx = db.begin().await?; - let domain = email.split('@').last().unwrap(); - let workspaces = sqlx::query!( - "SELECT workspace_id, auto_invite_operator, auto_add FROM workspace_settings ws WHERE (auto_invite_domain = $1 OR auto_invite_domain = '*') AND NOT EXISTS (SELECT 1 FROM usr WHERE workspace_id = ws.workspace_id AND email = $2)", - domain, - email - ) - .fetch_all(&mut *tx) - .await?; - let mut auto_added_workspace_usernames: Vec<(String, String)> = vec![]; - for r in workspaces { - if r.auto_add.is_some() && r.auto_add.unwrap() { - let operator = r.auto_invite_operator.unwrap_or(false); - let username = - auto_add_user(email, &r.workspace_id, &operator, &mut tx, authorable).await?; - auto_added_workspace_usernames.push((r.workspace_id, username)); - } else { - sqlx::query!( - "INSERT INTO workspace_invite - (workspace_id, email, is_admin, operator) - VALUES ($1, $2, false, $3) - ON CONFLICT DO NOTHING", - r.workspace_id, - email, - r.auto_invite_operator - ) - .execute(&mut *tx) - .await?; - } - } - tx.commit().await?; - - for workspace_username_tuple in auto_added_workspace_usernames { - let (w_id, username) = workspace_username_tuple; - handle_deployment_metadata( - &email, - &username, - db, - &w_id, - windmill_git_sync::DeployedObject::User { email: email.to_string() }, - Some(format!("Auto-added user '{}' to workspace", email)), - rsmq.clone(), - true, - ) - .await?; - } - Ok(()) -} - async fn invite_user( ApiAuthed { username, is_admin, .. }: ApiAuthed, Extension(db): Extension, diff --git a/backend/windmill-api/src/workspaces_ee.rs b/backend/windmill-api/src/workspaces_ee.rs new file mode 100644 index 0000000000..565a53b174 --- /dev/null +++ b/backend/windmill-api/src/workspaces_ee.rs @@ -0,0 +1,16 @@ +use crate::{ + db::{ApiAuthed, DB}, + workspaces::EditAutoInvite, +}; + +pub async fn edit_auto_invite( + _authed: ApiAuthed, + _db: DB, + _rsmq: Option, + _w_id: String, + _ea: EditAutoInvite, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::InternalErr( + "Not implemented on OSS".to_string(), + )) +} diff --git a/backend/windmill-autoscaling/Cargo.toml b/backend/windmill-autoscaling/Cargo.toml new file mode 100644 index 0000000000..7aada6f904 --- /dev/null +++ b/backend/windmill-autoscaling/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "windmill-autoscaling" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_autoscaling" +path = "./src/lib.rs" + +[features] +enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise"] +default = [] + +[dependencies] +uuid.workspace = true +serde.workspace = true +sqlx.workspace = true +serde_json.workspace = true +tracing.workspace = true +windmill-common = { workspace = true, default-features = false } +windmill-queue.workspace = true +rsmq_async.workspace = true +anyhow.workspace = true \ No newline at end of file diff --git a/backend/windmill-autoscaling/src/autoscaling_ee.rs b/backend/windmill-autoscaling/src/autoscaling_ee.rs new file mode 100644 index 0000000000..1c9defbede --- /dev/null +++ b/backend/windmill-autoscaling/src/autoscaling_ee.rs @@ -0,0 +1,6 @@ +use windmill_common::DB; + +pub async fn apply_all_autoscaling(_db: &DB) -> anyhow::Result<()> { + // Autoscaling is an ee feature + Ok(()) +} diff --git a/backend/windmill-autoscaling/src/lib.rs b/backend/windmill-autoscaling/src/lib.rs new file mode 100644 index 0000000000..28b9319244 --- /dev/null +++ b/backend/windmill-autoscaling/src/lib.rs @@ -0,0 +1,2 @@ +mod autoscaling_ee; +pub use autoscaling_ee::*; diff --git a/backend/windmill-common/src/ee.rs b/backend/windmill-common/src/ee.rs index 6df93aca78..e25d36bee1 100644 --- a/backend/windmill-common/src/ee.rs +++ b/backend/windmill-common/src/ee.rs @@ -48,8 +48,13 @@ pub async fn send_critical_alert( } #[cfg(feature = "enterprise")] -pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () { +pub async fn maybe_renew_license_key_on_start( + _http_client: &reqwest::Client, + _db: &crate::db::DB, + force_renew_now: bool, +) -> bool { // Implementation is not open source + force_renew_now } #[cfg(feature = "enterprise")] diff --git a/backend/windmill-common/src/ee.rs~main b/backend/windmill-common/src/ee.rs~main deleted file mode 100644 index 482b61a0fe..0000000000 --- a/backend/windmill-common/src/ee.rs~main +++ /dev/null @@ -1,73 +0,0 @@ -#[cfg(feature = "enterprise")] -use crate::db::DB; -use crate::ee::LicensePlan::Community; -#[cfg(feature = "enterprise")] -use crate::error; -use serde::Deserialize; -use std::sync::Arc; -use tokio::sync::RwLock; - -lazy_static::lazy_static! { - pub static ref LICENSE_KEY_VALID: Arc> = Arc::new(RwLock::new(true)); - pub static ref LICENSE_KEY_ID: Arc> = Arc::new(RwLock::new("".to_string())); - pub static ref LICENSE_KEY: Arc> = Arc::new(RwLock::new("".to_string())); -} - -pub enum LicensePlan { - Community, - Pro, - Enterprise, -} - -pub async fn get_license_plan() -> LicensePlan { - // Implementation is not open source - return Community; -} - -#[derive(Deserialize)] -#[serde(untagged)] -pub enum CriticalErrorChannel {} - -pub enum CriticalAlertKind { - #[cfg(feature = "enterprise")] - CriticalError, - #[cfg(feature = "enterprise")] - RecoveredCriticalError, -} - -#[cfg(feature = "enterprise")] -pub async fn send_critical_alert( - _error_message: String, - _db: &DB, - _kind: CriticalAlertKind, - _channels: Option>, -) { -} - -#[cfg(feature = "enterprise")] -pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () { - // Implementation is not open source -} - -#[cfg(feature = "enterprise")] -pub async fn renew_license_key( - _http_client: &reqwest::Client, - _db: &crate::db::DB, - _key: Option, - _manual: bool, -) -> String { - // Implementation is not open source - "".to_string() -} - -#[cfg(feature = "enterprise")] -pub async fn create_customer_portal_session( - _http_client: &reqwest::Client, - _key: Option, -) -> error::Result { - // Implementation is not open source - Ok("".to_string()) -} - -#[cfg(feature = "enterprise")] -pub async fn worker_groups_alerts(_db: &DB) {} diff --git a/backend/windmill-common/src/ee.rs~main_0 b/backend/windmill-common/src/ee.rs~main_0 deleted file mode 100644 index 2f32756e45..0000000000 --- a/backend/windmill-common/src/ee.rs~main_0 +++ /dev/null @@ -1,76 +0,0 @@ -#[cfg(feature = "enterprise")] -use crate::db::DB; -use crate::ee::LicensePlan::Community; -#[cfg(feature = "enterprise")] -use crate::error; -use serde::Deserialize; -use std::sync::Arc; -use tokio::sync::RwLock; - -lazy_static::lazy_static! { - pub static ref LICENSE_KEY_VALID: Arc> = Arc::new(RwLock::new(true)); - pub static ref LICENSE_KEY_ID: Arc> = Arc::new(RwLock::new("".to_string())); - pub static ref LICENSE_KEY: Arc> = Arc::new(RwLock::new("".to_string())); -} - -pub enum LicensePlan { - Community, - Pro, - Enterprise, -} - -pub async fn get_license_plan() -> LicensePlan { - // Implementation is not open source - return Community; -} - -#[derive(Deserialize)] -#[serde(untagged)] -pub enum CriticalErrorChannel { - Email { email: String }, - Slack { slack_channel: String }, -} - -pub enum CriticalAlertKind { - #[cfg(feature = "enterprise")] - CriticalError, - #[cfg(feature = "enterprise")] - RecoveredCriticalError, -} - -#[cfg(feature = "enterprise")] -pub async fn send_critical_alert( - _error_message: String, - _db: &DB, - _kind: CriticalAlertKind, - _channels: Option>, -) { -} - -#[cfg(feature = "enterprise")] -pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () { - // Implementation is not open source -} - -#[cfg(feature = "enterprise")] -pub async fn renew_license_key( - _http_client: &reqwest::Client, - _db: &crate::db::DB, - _key: Option, - _manual: bool, -) -> String { - // Implementation is not open source - "".to_string() -} - -#[cfg(feature = "enterprise")] -pub async fn create_customer_portal_session( - _http_client: &reqwest::Client, - _key: Option, -) -> error::Result { - // Implementation is not open source - Ok("".to_string()) -} - -#[cfg(feature = "enterprise")] -pub async fn worker_groups_alerts(_db: &DB) {} diff --git a/backend/windmill-common/src/email_ee.rs b/backend/windmill-common/src/email_ee.rs new file mode 100644 index 0000000000..42aebbeec3 --- /dev/null +++ b/backend/windmill-common/src/email_ee.rs @@ -0,0 +1,11 @@ +use crate::server::Smtp; + +pub async fn send_email( + _subject: &str, + _content: &str, + _to: Vec, + _smtp: Smtp, + _client_timeout: Option, +) -> crate::error::Result<()> { + Ok(()) +} diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 958e9f13ef..9b66cc65ac 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -101,6 +101,7 @@ pub struct FlowValue { #[serde(skip_serializing_if = "Option::is_none")] #[serde(default)] pub failure_module: Option>, + #[serde(skip_serializing_if = "Option::is_none")] #[serde(default)] pub preprocessor_module: Option>, #[serde(default)] @@ -423,6 +424,7 @@ pub enum FlowModuleValue { path: String, #[serde(skip_serializing_if = "Option::is_none")] hash: Option, + #[serde(skip_serializing_if = "Option::is_none")] tag_override: Option, }, Flow { diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 36b80c9fd7..b0420ca248 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -27,6 +27,7 @@ pub const OBJECT_STORE_CACHE_CONFIG_SETTING: &str = "object_store_cache_config"; pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; +pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const DEV_INSTANCE_SETTING: &str = "dev_instance"; pub const JWT_SECRET_SETTING: &str = "jwt_secret"; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 0f25e12e46..ad96594ac8 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -17,10 +17,12 @@ use scripts::ScriptLang; use sqlx::{Pool, Postgres}; pub mod apps; +pub mod auth; #[cfg(feature = "benchmark")] pub mod bench; pub mod db; pub mod ee; +pub mod email_ee; pub mod error; pub mod external_ip; pub mod flow_status; @@ -32,9 +34,8 @@ pub mod job_s3_helpers_ee; pub mod jobs; pub mod more_serde; pub mod oauth2; +pub mod queue; pub mod s3_helpers; - -pub mod auth; pub mod schedule; pub mod scripts; pub mod server; diff --git a/backend/windmill-common/src/queue.rs b/backend/windmill-common/src/queue.rs new file mode 100644 index 0000000000..6cbb5611c5 --- /dev/null +++ b/backend/windmill-common/src/queue.rs @@ -0,0 +1,16 @@ +use std::collections::HashMap; + +use sqlx::{Pool, Postgres}; + +pub async fn get_queue_counts(db: &Pool) -> HashMap { + sqlx::query_as::<_, (String, i64)>( + "SELECT tag, count(*) as count FROM queue WHERE + scheduled_for <= now() - ('3 seconds')::interval AND running = false + GROUP BY tag", + ) + .fetch_all(db) + .await + .ok() + .map(|v| v.into_iter().map(|(k, v)| (k, v as u32)).collect()) + .unwrap_or_else(|| HashMap::new()) +} diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 198283db2d..4547c29d51 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -117,10 +117,10 @@ pub struct S3Object { #[cfg(feature = "parquet")] pub async fn get_etag_or_empty( - object_store_resource: &ObjectStoreResource, + object_store_resource: &mut ObjectStoreResource, s3_object: S3Object, ) -> Option { - let object_store_client = build_object_store_client(object_store_resource); + let object_store_client = build_object_store_client(object_store_resource).await; if object_store_client.is_err() { return None; } @@ -166,11 +166,11 @@ pub fn render_endpoint( } #[cfg(feature = "parquet")] -pub fn build_object_store_client( +pub async fn build_object_store_client( resource_ref: &ObjectStoreResource, ) -> error::Result> { match resource_ref { - ObjectStoreResource::S3(s3_resource_ref) => build_s3_client(&s3_resource_ref, None), + ObjectStoreResource::S3(s3_resource_ref) => build_s3_client(&s3_resource_ref).await, ObjectStoreResource::Azure(azure_blob_resource_ref) => { build_azure_blob_client(&azure_blob_resource_ref) } @@ -225,10 +225,21 @@ use aws_config::{default_provider::credentials::DefaultCredentialsChain, Region} use object_store::CredentialProvider; #[cfg(feature = "parquet")] -pub fn build_s3_client( - s3_resource_ref: &S3Resource, - credential_providers: Option, -) -> error::Result> { +pub async fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result> { + let static_creds = s3_resource_ref.access_key.as_ref().is_some_and(|x| x != "") + || s3_resource_ref.secret_key.as_ref().is_some_and(|x| x != ""); + + let credentials_provider = if !static_creds { + Some( + DefaultCredentialsChain::builder() + .region(Region::new(s3_resource_ref.region.clone())) + .build() + .await, + ) + } else { + None + }; + let s3_resource = s3_resource_ref.clone(); let endpoint = render_endpoint( s3_resource.endpoint, @@ -244,7 +255,7 @@ pub fn build_s3_client( .with_bucket_name(s3_resource.bucket) .with_endpoint(endpoint); - if let Some(credentials_provider) = credential_providers { + if let Some(credentials_provider) = credentials_provider { store_builder = store_builder.with_credentials(Arc::new(AwsCredentialAdapter { inner: credentials_provider, })); @@ -399,18 +410,7 @@ pub async fn build_s3_client_from_settings( ) -> error::Result> { let region = none_if_empty(settings.region) .unwrap_or_else(|| std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string())); - let access_key = none_if_empty(settings.access_key); - let secret_key = none_if_empty(settings.secret_key); - let credentials_provider = if access_key.is_none() && secret_key.is_none() { - Some( - DefaultCredentialsChain::builder() - .region(Region::new(region.clone())) - .build() - .await, - ) - } else { - None - }; + let s3_resource = S3Resource { endpoint: none_if_empty(settings.endpoint).unwrap_or_else(|| { std::env::var("S3_ENDPOINT").unwrap_or_else(|_| format!("s3.{region}.amazonaws.com")) @@ -419,15 +419,15 @@ pub async fn build_s3_client_from_settings( std::env::var("S3_CACHE_BUCKET").unwrap_or_else(|_| "missingbucket".to_string()) }), region, - access_key, - secret_key, + access_key: settings.access_key, + secret_key: settings.secret_key, use_ssl: !settings.allow_http.unwrap_or(true), path_style: settings.path_style, port: settings.port, token: None, }; - build_s3_client(&s3_resource, credentials_provider) + build_s3_client(&s3_resource).await } #[cfg(feature = "parquet")] diff --git a/backend/windmill-common/src/stats_ee.rs b/backend/windmill-common/src/stats_ee.rs index c6dc9946bf..5d2dc82b82 100644 --- a/backend/windmill-common/src/stats_ee.rs +++ b/backend/windmill-common/src/stats_ee.rs @@ -28,7 +28,6 @@ pub enum SendStatsReason { pub async fn send_stats( _http_client: &reqwest::Client, _db: &DB, - _skip_job_usage: bool, _reason: SendStatsReason, ) -> Result<()> { // stats details are closed source diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index d5f9c5c099..04f95a3406 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -11,13 +11,11 @@ use crate::ee::LICENSE_KEY_ID; use crate::ee::{send_critical_alert, CriticalAlertKind}; use crate::error::{to_anyhow, Error, Result}; use crate::global_settings::UNIQUE_ID_SETTING; -use crate::server::Smtp; use crate::DB; use anyhow::Context; use gethostname::gethostname; use git_version::git_version; -use mail_send::mail_builder::MessageBuilder; -use mail_send::SmtpClientBuilder; + use rand::{distributions::Alphanumeric, thread_rng, Rng}; use reqwest::Client; use serde::{Deserialize, Serialize}; @@ -206,6 +204,14 @@ pub async fn get_uid<'c, E: sqlx::Executor<'c, Database = Postgres>>(db: E) -> R Ok(uid) } +pub fn map_string_to_number(s: &str, max_number: u64) -> u64 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + let mut hasher = DefaultHasher::new(); + s.hash(&mut hasher); + hasher.finish() % (max_number + 1) +} + #[derive(Serialize, Deserialize, PartialEq, Clone)] #[serde(rename_all = "lowercase")] pub enum Mode { @@ -235,58 +241,6 @@ pub fn generate_lock_id(database_name: &str) -> i64 { 0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64) } -pub async fn send_email( - subject: &str, - content: &str, - to: Vec, - smtp: Smtp, - client_timeout: Option, -) -> Result<()> { - let mut client = SmtpClientBuilder::new(smtp.host, smtp.port) - .implicit_tls(smtp.tls_implicit.unwrap_or(false)); - if std::env::var("ACCEPT_INVALID_CERTS").is_ok() { - client = client.allow_invalid_certs(); - } - let client = if let (Some(username), Some(password)) = (smtp.username, smtp.password) { - if !username.is_empty() { - client.credentials((username, password)) - } else { - client - } - } else { - client - }; - let message = MessageBuilder::new() - .from(("Windmill", smtp.from.as_str())) - .to(to.clone()) - .subject(subject) - .text_body(content); - - match client_timeout { - Some(timeout) => { - tokio::time::timeout(timeout, client.connect()) - .await - .map_err(to_anyhow)? - .map_err(to_anyhow)? - .send(message) - .await - .map_err(to_anyhow)?; - } - None => { - client - .connect() - .await - .map_err(to_anyhow)? - .send(message) - .await - .map_err(to_anyhow)?; - } - } - tracing::info!("Sent email to {:#?}: {subject}", to); - - return Ok(()); -} - pub async fn report_critical_error(error_message: String, _db: DB) -> () { tracing::error!("CRITICAL ERROR: {error_message}"); diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 9c2474d95f..abd9b68415 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -23,7 +23,7 @@ use windmill_common::{ jobs::QueuedJob, worker::{to_raw_value, write_file, write_file_at_user_defined_location, WORKER_CONFIG}, }; -use windmill_parser_yaml::AnsibleRequirements; +use windmill_parser_yaml::{AnsibleRequirements, ResourceOrVariablePath}; use windmill_queue::{append_logs, CanceledBy}; use crate::{ @@ -35,7 +35,7 @@ use crate::{ handle_child::handle_child, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - TZ_ENV, + PROXY_ENVS, TZ_ENV, }; lazy_static::lazy_static! { @@ -147,6 +147,7 @@ async fn install_galaxy_collections( galaxy_command .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) // .env("BASE_INTERNAL_URL", base_internal_url) @@ -185,7 +186,7 @@ async fn install_galaxy_collections( #[cfg(not(feature = "enterprise"))] fn check_ansible_exists() -> Result<(), error::Error> { if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() { - let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run rust jobs.", ANSIBLE_PLAYBOOK_PATH.as_str()); + let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run Ansible jobs.", ANSIBLE_PLAYBOOK_PATH.as_str()); return Err(error::Error::NotFound(msg)); } Ok(()) @@ -194,7 +195,7 @@ fn check_ansible_exists() -> Result<(), error::Error> { #[cfg(feature = "enterprise")] fn check_ansible_exists() -> Result<(), error::Error> { if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() { - let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run rust jobs.", ANSIBLE_PLAYBOOK_PATH.as_str()); + let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-ee-full` for your instance in order to run Ansible jobs.", ANSIBLE_PLAYBOOK_PATH.as_str()); return Err(error::Error::NotFound(msg)); } Ok(()) @@ -395,6 +396,7 @@ fi nsjail_cmd .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) // inject PYTHONPATH here - for some reason I had to do it in nsjail conf .envs(reserved_variables) .env("PATH", PATH_ENV.as_str()) @@ -556,22 +558,12 @@ async fn create_file_resources( } for file_res in &r.file_resources { - let r = client - .get_resource_value_interpolated::( - &file_res.resource_path, - Some(job_id.to_string()), - ) - .await?; + let r = + get_resource_or_variable_content(client, &file_res.resource_path, job_id.to_string()) + .await?; let path = file_res.target_path.clone(); - let validated_path = write_file_at_user_defined_location( - job_dir, - path.as_str(), - r.get("content").and_then(|v| v.as_str()).ok_or(anyhow!( - "Invalid text file resource {}, `content` field absent or invalid", - &file_res.resource_path - ))?, - ) - .map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?; + let validated_path = write_file_at_user_defined_location(job_dir, path.as_str(), &r) + .map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?; nsjail_mounts.push( define_nsjail_mount(job_dir, &validated_path) @@ -579,7 +571,7 @@ async fn create_file_resources( ); logs.push_str(&format!( - "\nCreated {} from {}", + "\nCreated {} from {:?}", file_res.target_path, file_res.resource_path )); } @@ -587,3 +579,26 @@ async fn create_file_resources( Ok(nsjail_mounts) } + +async fn get_resource_or_variable_content( + client: &crate::AuthedClient, + path: &ResourceOrVariablePath, + job_id: String, +) -> anyhow::Result { + Ok(match path { + ResourceOrVariablePath::Resource(p) => { + let r = client + .get_resource_value_interpolated::(&p, Some(job_id)) + .await?; + + r.get("content") + .and_then(|v| v.as_str()) + .ok_or(anyhow!( + "Invalid text file resource {}, `content` field absent or invalid", + p + ))? + .to_string() + } + ResourceOrVariablePath::Variable(p) => client.get_variable_value(&p).await?, + }) +} diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 128eb7d25f..2c4c46ab7d 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -29,7 +29,7 @@ use crate::{ }, handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - POWERSHELL_CACHE_DIR, POWERSHELL_PATH, TZ_ENV, + POWERSHELL_CACHE_DIR, POWERSHELL_PATH, PROXY_ENVS, TZ_ENV, }; #[cfg(windows)] @@ -59,11 +59,42 @@ pub async fn handle_bash_job( append_logs(&job.id, &job.workspace_id, logs1, db).await; write_file(job_dir, "main.sh", &format!("set -e\n{content}"))?; - write_file( - job_dir, - "wrapper.sh", - &format!("set -o pipefail\nset -e\nmkfifo bp\ncat bp | tail -1 > ./result2.out &\n {bash} ./main.sh \"$@\" 2>&1 | tee bp\nwait $!", bash = BIN_BASH.as_str()), - )?; + let script = format!( + r#" +set -o pipefail +set -e + +# Function to kill child processes +cleanup() {{ + echo "Terminating child processes..." + + # Ignore SIGTERM and SIGINT + trap '' SIGTERM SIGINT + + # Kill the process group of the script (negative PID value) + pkill -P $$ + exit +}} + + +# Trap SIGTERM (or other signals) and call cleanup function +trap cleanup SIGTERM SIGINT + +# Create a named pipe +mkfifo bp + +# Start background processes +cat bp | tail -1 >> ./result2.out & + +# Run main.sh in the same process group +{bash} ./main.sh "$@" 2>&1 | tee bp & + +# Wait for all background processes to finish +wait +"#, + bash = BIN_BASH.as_str(), + ); + write_file(job_dir, "wrapper.sh", &script)?; let token = client.get_token().await; let mut reserved_variables = get_reserved_variables(job, &token, db).await?; @@ -112,6 +143,7 @@ pub async fn handle_bash_job( .current_dir(job_dir) .env_clear() .envs(reserved_variables) + .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .args(cmd_args) @@ -404,6 +436,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", Command::new(NSJAIL_PATH.as_str()) .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) .envs(reserved_variables) .env("TZ", TZ_ENV.as_str()) .env("PATH", PATH_ENV.as_str()) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 0721918568..5b7c2a06c0 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -26,7 +26,7 @@ use crate::{ handle_child::handle_child, AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, - NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, TZ_ENV, + NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TZ_ENV, }; #[cfg(windows)] @@ -63,8 +63,26 @@ const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js"); const NSJAIL_CONFIG_RUN_BUN_CONTENT: &str = include_str!("../nsjail/run.bun.config.proto"); pub const BUN_LOCKB_SPLIT: &str = "\n//bun.lockb\n"; +pub const BUN_LOCKB_SPLIT_WINDOWS: &str = "\r\n//bun.lockb\r\n"; + pub const EMPTY_FILE: &str = ""; +fn split_lockfile(lockfile: &str) -> (&str, Option<&str>, bool) { + if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT) { + // Split using "\n//bun.lockb\n" + let (before, after_with_sep) = lockfile.split_at(index); + let after = &after_with_sep[BUN_LOCKB_SPLIT.len()..]; + (before, Some(after), after == EMPTY_FILE) + } else if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT_WINDOWS) { + // Split using "\r\n//bun.lockb\r\n" + let (before, after_with_sep) = lockfile.split_at(index); + let after = &after_with_sep[BUN_LOCKB_SPLIT_WINDOWS.len()..]; + (before, Some(after), after == EMPTY_FILE) + } else { + (lockfile, None, false) + } +} + pub async fn gen_bun_lockfile( mem_peak: &mut i32, canceled_by: &mut Option, @@ -180,7 +198,12 @@ pub async fn gen_bun_lockfile( file.read_to_string(&mut content).await?; } if !npm_mode { + #[cfg(any(target_os = "linux", target_os = "macos"))] content.push_str(BUN_LOCKB_SPLIT); + + #[cfg(target_os = "windows")] + content.push_str(BUN_LOCKB_SPLIT_WINDOWS); + { let file = format!("{job_dir}/bun.lockb"); if !empty_deps && tokio::fs::metadata(&file).await.is_ok() { @@ -252,6 +275,7 @@ pub async fn install_bun_lockfile( child_cmd .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) .envs(common_bun_proc_envs) .args(vec!["install"]) .stdout(Stdio::piped()) @@ -762,12 +786,15 @@ async fn compute_bundle_local_and_remote_path( } pub async fn prepare_job_dir(reqs: &str, job_dir: &str) -> Result<()> { - let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::>(); - let _ = write_file(job_dir, "package.json", &splitted[0])?; + let (pkg, lock, empty) = split_lockfile(reqs); + let _ = write_file(job_dir, "package.json", pkg)?; - if splitted[1] != EMPTY_FILE { - let _ = write_lockb(splitted[1], job_dir).await?; + if !empty { + if let Some(lock) = lock { + let _ = write_lockb(lock, job_dir).await?; + } } + Ok(()) } async fn write_lockb(splitted_lockb_2: &str, job_dir: &str) -> Result<()> { @@ -874,22 +901,23 @@ pub async fn handle_bun_job( } else if let Some(codebase) = codebase.as_ref() { pull_codebase(&job.workspace_id, codebase, job_dir).await?; } else if let Some(reqs) = requirements_o.as_ref() { - let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::>(); - if splitted.len() != 2 && !annotation.npm { + let (pkg, lock, empty) = split_lockfile(reqs); + + if lock.is_none() && !annotation.npm { return Err(error::Error::ExecutionErr( format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|") )); } - let _ = write_file(job_dir, "package.json", &splitted[0])?; - let lockb = if annotation.npm { "" } else { splitted[1] }; - if lockb != EMPTY_FILE { + let _ = write_file(job_dir, "package.json", pkg)?; + let lockb = if annotation.npm { "" } else { lock.unwrap() }; + if !empty { let mut skip_install = false; let mut create_buntar = false; let mut buntar_path = "".to_string(); if !annotation.npm { - let _ = write_lockb(&splitted[1], job_dir).await?; + let _ = write_lockb(lockb, job_dir).await?; let mut sha_path = sha2::Sha256::new(); sha_path.update(lockb.as_bytes()); @@ -1561,20 +1589,20 @@ pub async fn start_worker( if let Some(codebase) = codebase.as_ref() { pull_codebase(w_id, codebase, job_dir).await?; } else if let Some(reqs) = requirements_o { - let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::>(); - if splitted.len() != 2 { + let (pkg, lock, empty) = split_lockfile(&reqs); + if lock.is_none() { return Err(error::Error::ExecutionErr( format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|") )); } - let _ = write_file(job_dir, "package.json", &splitted[0])?; - let lockb = splitted[1]; - if lockb != EMPTY_FILE { + let _ = write_file(job_dir, "package.json", pkg)?; + let lockb = lock.unwrap(); + if !empty { let _ = write_file_binary( job_dir, "bun.lockb", &base64::engine::general_purpose::STANDARD - .decode(&splitted[1]) + .decode(lockb) .map_err(|_| { error::Error::InternalErr("Could not decode bun.lockb".to_string()) })?, diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 406f128917..e22f9be54c 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -736,8 +736,8 @@ async fn arg_value_hash_additions( .await; storage = s3_object.storage.clone(); - if let Some(s3_resource) = s3_resource_opt.ok().flatten() { - let etag = get_etag_or_empty(&s3_resource, s3_object.clone()).await; + if let Some(mut s3_resource) = s3_resource_opt.ok().flatten() { + let etag = get_etag_or_empty(&mut s3_resource, s3_object.clone()).await; tracing::warn!("Enriching s3 arg value with etag: {:?}", etag); result.insert(s3_object.s3.clone(), etag.unwrap_or_default()); // TODO: maybe inject a random value to invalidate the cache? } @@ -793,9 +793,9 @@ pub async fn get_cached_resource_value_if_valid( return None; } for (s3_file_key, s3_file_etag) in s3_etags { - if let Some(object_store_resource) = object_store_resource_opt.clone() { + if let Some(mut object_store_resource) = object_store_resource_opt.clone() { let etag = get_etag_or_empty( - &object_store_resource, + &mut object_store_resource, S3Object { s3: s3_file_key.clone(), storage: cached_resource.storage.clone(), diff --git a/backend/windmill-worker/src/dedicated_worker.rs b/backend/windmill-worker/src/dedicated_worker.rs index da3e25a699..1a332e76ef 100644 --- a/backend/windmill-worker/src/dedicated_worker.rs +++ b/backend/windmill-worker/src/dedicated_worker.rs @@ -77,7 +77,7 @@ pub async fn handle_dedicated_process( ) -> std::result::Result<(), error::Error> { //do not cache local dependencies - use crate::handle_child::process_status; + use crate::{handle_child::process_status, PROXY_ENVS}; let mut child = { let mut cmd = Command::new(command_path); @@ -85,6 +85,7 @@ pub async fn handle_dedicated_process( .env_clear() .envs(context_envs) .envs(envs) + .envs(PROXY_ENVS.clone()) .envs( reserved_variables .iter() diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index c171e52156..470e647383 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -1,3 +1,4 @@ +use crate::PROXY_ENVS; use std::{collections::HashMap, fs::DirBuilder, process::Stdio}; use itertools::Itertools; @@ -189,6 +190,7 @@ func Run(req Req) (interface{{}}, error){{ .env("BASE_INTERNAL_URL", base_internal_url) .env("GOPATH", GO_CACHE_DIR) .env("HOME", HOME_ENV.as_str()) + .envs(PROXY_ENVS.clone()) .args(vec!["build", "main.go"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 7702b73b03..55f5abea31 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -18,12 +18,8 @@ use serde_json::value::RawValue; use serde_json::Map; use serde_json::Value; use tokio::sync::Mutex; -use tokio_postgres::types::IsNull; use tokio_postgres::Client; -use tokio_postgres::{ - types::{to_sql_checked, ToSql}, - NoTls, Row, -}; +use tokio_postgres::{types::ToSql, NoTls, Row}; use tokio_postgres::{ types::{FromSql, Type}, Column, @@ -41,7 +37,7 @@ use windmill_queue::CanceledBy; use crate::common::{build_args_values, sizeof_val, OccupancyMetrics}; use crate::handle_child::run_future_with_polling_update_job_poller; use crate::{AuthedClientBackgroundTask, MAX_RESULT_SIZE}; -use bytes::{Buf, BytesMut}; +use bytes::Buf; use lazy_static::lazy_static; use urlencoding::encode; @@ -85,7 +81,7 @@ fn do_postgresql_inner<'a>( let arg_t = arg .otyp .as_ref() - .ok_or_else(|| anyhow::anyhow!("Missing otyp for pg arg"))?; + .ok_or_else(|| anyhow::anyhow!("Missing otzyp for pg arg"))?; let typ = &arg.typ; let param = convert_val(value, arg_t, typ)?; query_params.push(param); @@ -98,6 +94,11 @@ fn do_postgresql_inner<'a>( let mut res: Vec = vec![]; + let query_params = query_params + .iter() + .map(|p| &**p as &(dyn ToSql + Sync)) + .collect_vec(); + if skip_collect { client .execute_raw(&query, query_params) @@ -414,133 +415,176 @@ pub async fn do_postgresql( return Ok(raw_result); } -#[derive(Debug)] -enum PgType { - String(String), - Bool(bool), - I8(i8), - I16(i16), - I32(i32), - I64(i64), - U32(u32), - F32(f32), - F64(f64), - Uuid(Uuid), - Decimal(Decimal), - Date(chrono::NaiveDate), - Time(chrono::NaiveTime), - Timestamp(chrono::NaiveDateTime), - None(Option), - Array(Vec), - Json(serde_json::Value), - Bytea(Vec), +fn map_as_single_type( + vec: &Vec, + f: impl Fn(&Value) -> Option, +) -> anyhow::Result>> { + vec.into_iter() + .map(|v| { + // allow nulls in arrays + if matches!(v, Value::Null) { + Some(None) + } else { + f(v).map(Some) + } + }) + .collect::>>>() + .ok_or_else(|| anyhow::anyhow!("Mixed types in array")) } -impl ToSql for PgType { - fn to_sql( - &self, - ty: &Type, - out: &mut BytesMut, - ) -> Result> { - match *self { - PgType::String(ref val) => val.to_sql(ty, out), - PgType::Bool(ref val) => val.to_sql(ty, out), - PgType::I8(ref val) => val.to_sql(ty, out), - PgType::I16(ref val) => val.to_sql(ty, out), - PgType::I32(ref val) => val.to_sql(ty, out), - PgType::I64(ref val) => val.to_sql(ty, out), - PgType::U32(ref val) => val.to_sql(ty, out), - PgType::F32(ref val) => val.to_sql(ty, out), - PgType::F64(ref val) => val.to_sql(ty, out), - PgType::Uuid(ref val) => val.to_sql(ty, out), - PgType::Decimal(ref val) => val.to_sql(ty, out), - PgType::Date(ref val) => val.to_sql(ty, out), - PgType::Time(ref val) => val.to_sql(ty, out), - PgType::Timestamp(ref val) => val.to_sql(ty, out), - PgType::None(ref val) => val.to_sql(ty, out), - PgType::Array(ref val) => val.to_sql(ty, out), - PgType::Json(ref val) => val.to_sql(ty, out), - PgType::Bytea(ref val) => val.to_sql(ty, out), +fn convert_vec_val( + vec: &Vec, + arg_t: &String, +) -> windmill_common::error::Result> { + match arg_t.as_str() { + "bool" | "boolean" => Ok(Box::new(map_as_single_type(vec, |v| v.as_bool())?)), + "char" | "character" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_i64().map(|x| x as i8) + })?)), + "smallint" | "smallserial" | "int2" | "serial2" => { + Ok(Box::new(map_as_single_type(vec, |v| { + v.as_i64().map(|x| x as i16) + })?)) } + "int" | "integer" | "int4" | "serial" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_i64().map(|x| x as i32) + })?)), + "numeric" | "decimal" => Ok(Box::new(map_as_single_type(vec, |v| { + if v.is_i64() { + Decimal::from_i64(v.as_i64().unwrap()) + } else if v.is_f64() { + Decimal::from_f64(v.as_f64().unwrap()) + } else { + None + } + })?)), + "oid" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_u64().map(|x| x as u32) + })?)), + "bigint" | "bigserial" | "int8" | "serial8" => { + Ok(Box::new(map_as_single_type(vec, |v| { + v.as_u64().map(|x| x as i64) + })?)) + } + "real" | "float4" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_f64().map(|x| x as f32) + })?)), + "double" | "float8" => Ok(Box::new(map_as_single_type(vec, |v| v.as_f64())?)), + "uuid" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| Uuid::parse_str(x).ok()).flatten() + })?)), + "date" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| { + chrono::NaiveDate::parse_from_str(x, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default() + }) + })?)), + "time" | "timetz" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| { + chrono::NaiveTime::parse_from_str(x, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default() + }) + })?)), + "timestamp" | "timestamptz" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| { + chrono::NaiveDateTime::parse_from_str(x, "%Y-%m-%dT%H:%M:%S.%3fZ") + .unwrap_or_default() + }) + })?)), + "jsonb" | "json" => Ok(Box::new(vec.clone().into_iter().map(Some).collect_vec())), + "bytea" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| { + engine::general_purpose::STANDARD + .decode(x) + .unwrap_or(vec![]) + }) + })?)), + "text" | "varchar" => Ok(Box::new(map_as_single_type(vec, |v| { + v.as_str().map(|x| x.to_string()) + })?)), + _ => Err(anyhow::anyhow!("Unsupported JSON array type"))?, } - - fn accepts(_: &Type) -> bool { - true - } - - to_sql_checked!(); } -fn convert_val(value: &Value, arg_t: &String, typ: &Typ) -> windmill_common::error::Result { +fn convert_val( + value: &Value, + arg_t: &String, + typ: &Typ, +) -> windmill_common::error::Result> { match value { Value::Array(vec) if arg_t.ends_with("[]") => { let arg_t = arg_t.trim_end_matches("[]").to_string(); - let mut result = vec![]; - for val in vec { - result.push(convert_val(val, &arg_t, typ)?); - } - Ok(PgType::Array(result)) + convert_vec_val(vec, &arg_t) } - Value::Null => Ok(PgType::None(None::)), - Value::Bool(b) => Ok(PgType::Bool(b.clone())), - Value::Number(n) if matches!(typ, Typ::Str(_)) => Ok(PgType::String(n.to_string())), - Value::Number(n) if n.is_i64() && arg_t == "char" => { - Ok(PgType::I8(n.as_i64().unwrap() as i8)) - } - Value::Number(n) if n.is_i64() && (arg_t == "smallint" || arg_t == "smallserial") => { - Ok(PgType::I16(n.as_i64().unwrap() as i16)) + Value::Null => Ok(Box::new(None::)), + Value::Bool(b) => Ok(Box::new(b.clone())), + Value::Number(n) if matches!(typ, Typ::Str(_)) => Ok(Box::new(n.to_string())), + Value::Number(n) if arg_t == "char" && n.is_i64() => { + Ok(Box::new(n.as_i64().unwrap() as i8)) } Value::Number(n) - if n.is_i64() - && (arg_t == "int" - || arg_t == "integer" - || arg_t == "int4" - || arg_t == "serial") => + if (arg_t == "smallint" + || arg_t == "smallserial" + || arg_t == "int2" + || arg_t == "serial2") + && n.is_i64() => { - Ok(PgType::I32(n.as_i64().unwrap() as i32)) + Ok(Box::new(n.as_i64().unwrap() as i16)) } - Value::Number(n) if n.is_i64() && (arg_t == "numeric" || arg_t == "decimal") => Ok( - PgType::Decimal(Decimal::from_i64(n.as_i64().unwrap()).unwrap()), + Value::Number(n) + if (arg_t == "int" || arg_t == "integer" || arg_t == "int4" || arg_t == "serial") + && n.is_i64() => + { + Ok(Box::new(n.as_i64().unwrap() as i32)) + } + Value::Number(n) if (arg_t == "real" || arg_t == "float4") && n.as_f64().is_some() => { + Ok(Box::new(n.as_f64().unwrap() as f32)) + } + Value::Number(n) if (arg_t == "double" || arg_t == "float8") && n.as_f64().is_some() => { + Ok(Box::new(n.as_f64().unwrap())) + } + Value::Number(n) if (arg_t == "numeric" || arg_t == "decimal") && n.is_i64() => Ok( + Box::new(Decimal::from_i64(n.as_i64().unwrap()).unwrap_or_default()), ), - Value::Number(n) if n.is_i64() => Ok(PgType::I64(n.as_i64().unwrap())), - Value::Number(n) if n.is_u64() && arg_t == "oid" => { - Ok(PgType::U32(n.as_u64().unwrap() as u32)) - } - Value::Number(n) if n.is_u64() && (arg_t == "bigint" || arg_t == "bigserial") => { - Ok(PgType::I64(n.as_u64().unwrap() as i64)) - } - Value::Number(n) if n.is_f64() && arg_t == "real" => { - Ok(PgType::F32(n.as_f64().unwrap() as f32)) - } - Value::Number(n) if n.is_f64() && arg_t == "double" => Ok(PgType::F64(n.as_f64().unwrap())), - Value::Number(n) if n.is_f64() && (arg_t == "numeric" || arg_t == "decimal") => Ok( - PgType::Decimal(Decimal::from_f64(n.as_f64().unwrap()).unwrap()), + Value::Number(n) if (arg_t == "numeric" || arg_t == "decimal") && n.is_f64() => Ok( + Box::new(Decimal::from_f64(n.as_f64().unwrap()).unwrap_or_default()), ), - Value::Number(n) => Ok(PgType::F64(n.as_f64().unwrap())), - Value::String(s) if arg_t == "uuid" => Ok(PgType::Uuid(Uuid::parse_str(s)?)), + Value::Number(n) if arg_t == "oid" && n.is_u64() => { + Ok(Box::new(n.as_u64().unwrap() as u32)) + } + Value::Number(n) + if (arg_t == "bigint" + || arg_t == "bigserial" + || arg_t == "int8" + || arg_t == "serial8") + && n.is_u64() => + { + Ok(Box::new(n.as_u64().unwrap() as i64)) + } + Value::Number(n) if n.is_i64() => Ok(Box::new(n.as_i64().unwrap())), + Value::Number(n) => Ok(Box::new(n.as_f64().unwrap())), + Value::String(s) if arg_t == "uuid" => Ok(Box::new(Uuid::parse_str(s)?)), Value::String(s) if arg_t == "date" => { let date = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default(); - Ok(PgType::Date(date)) + Ok(Box::new(date)) } Value::String(s) if arg_t == "time" || arg_t == "timetz" => { let time = chrono::NaiveTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ").unwrap_or_default(); - Ok(PgType::Time(time)) + Ok(Box::new(time)) } Value::String(s) if arg_t == "timestamp" || arg_t == "timestamptz" => { let datetime = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S.%3fZ") .unwrap_or_default(); - Ok(PgType::Timestamp(datetime)) + Ok(Box::new(datetime)) } Value::String(s) if arg_t == "bytea" => { let bytes = engine::general_purpose::STANDARD .decode(s) .unwrap_or(vec![]); - Ok(PgType::Bytea(bytes)) + Ok(Box::new(bytes)) } - Value::Object(_) => Ok(PgType::Json(value.clone())), - Value::String(s) => Ok(PgType::String(s.clone())), + Value::Object(_) => Ok(Box::new(value.clone())), + Value::String(s) => Ok(Box::new(s.clone())), _ => Err(Error::ExecutionErr(format!( "Unsupported type in query: {:?} and signature {arg_t:?}", value @@ -624,6 +668,9 @@ pub fn pg_cell_to_json_value( Type::TS_VECTOR => get_basic(row, column, column_i, |a: StringCollector| { Ok(JSONValue::String(a.0)) })?, + Type::OID => get_basic(row, column, column_i, |a: u32| { + Ok(JSONValue::Number(serde_json::Number::from(a))) + })?, // array types Type::BOOL_ARRAY => get_array(row, column, column_i, |a: bool| Ok(JSONValue::Bool(a)))?, Type::BIT_ARRAY => get_array(row, column, column_i, |a: bit_vec::BitVec| match a.len() { @@ -655,6 +702,10 @@ pub fn pg_cell_to_json_value( Type::FLOAT8_ARRAY => { get_array(row, column, column_i, |a: f64| Ok(f64_to_json_number(a)?))? } + Type::NUMERIC_ARRAY => get_array(row, column, column_i, |a: Decimal| { + Ok(serde_json::to_value(a) + .map_err(|_| anyhow::anyhow!("Cannot convert decimal to json"))?) + })?, // these types require a custom StringCollector struct as an intermediary (see struct at bottom) Type::TS_VECTOR_ARRAY => get_array(row, column, column_i, |a: StringCollector| { Ok(JSONValue::String(a.0)) @@ -766,7 +817,7 @@ fn get_array<'a, T: FromSql<'a>>( val_to_json_val: impl Fn(T) -> Result, ) -> Result { let raw_val_array = row - .try_get::<_, Option>>(column_i) + .try_get::<_, Option>>>(column_i) .with_context(|| { format!( "conversion issue for array at column_name `{}`", @@ -777,7 +828,11 @@ fn get_array<'a, T: FromSql<'a>>( Some(val_array) => { let mut result = vec![]; for val in val_array { - result.push(val_to_json_val(val)?); + result.push( + val.map(|v| val_to_json_val(v)) + .transpose()? + .unwrap_or(Value::Null), + ); } JSONValue::Array(result) } diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index b16e25f8ae..194e621fe1 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -143,7 +143,7 @@ fn check_php_exists() -> error::Result<()> { #[cfg(feature = "enterprise")] fn check_php_exists() -> error::Result<()> { if !Path::new(PHP_PATH.as_str()).exists() { - let msg = format!("Couldn't find php at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run php jobs.", PHP_PATH.as_str()); + let msg = format!("Couldn't find php at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-ee-full` for your instance in order to run php jobs.", PHP_PATH.as_str()); return Err(error::Error::NotFound(msg)); } Ok(()) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 2c8b3f2c84..112b3c84ca 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -65,9 +65,9 @@ use crate::{ read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, - AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY, - LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, - PIP_INDEX_URL, TZ_ENV, UV_CACHE_DIR, + AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, LOCK_CACHE_DIR, + NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, TZ_ENV, + UV_CACHE_DIR, }; #[cfg(windows)] @@ -565,6 +565,7 @@ mount {{ .env_clear() // inject PYTHONPATH here - for some reason I had to do it in nsjail conf .envs(reserved_variables) + .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -1003,15 +1004,6 @@ pub async fn handle_python_reqs( if let Some(host) = PIP_TRUSTED_HOST.as_ref() { vars.push(("TRUSTED_HOST", host)); } - if let Some(http_proxy) = HTTP_PROXY.as_ref() { - vars.push(("HTTP_PROXY", http_proxy)); - } - if let Some(https_proxy) = HTTPS_PROXY.as_ref() { - vars.push(("HTTPS_PROXY", https_proxy)); - } - if let Some(no_proxy) = NO_PROXY.as_ref() { - vars.push(("NO_PROXY", no_proxy)); - } let _ = write_file( job_dir, @@ -1026,6 +1018,9 @@ pub async fn handle_python_reqs( let mut req_with_penv: Vec<(String, String)> = vec![]; for req in requirements { + if req.starts_with('#') { + continue; + } let venv_p = format!( "{PIP_CACHE_DIR}/{}", req.replace(' ', "").replace('/', "").replace(':', "") @@ -1137,6 +1132,7 @@ pub async fn handle_python_reqs( .current_dir(job_dir) .env_clear() .envs(vars) + .envs(PROXY_ENVS.clone()) .args(vec!["--config", "download.config.proto"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -1190,15 +1186,6 @@ pub async fn handle_python_reqs( } let mut envs = vec![("PATH", PATH_ENV.as_str())]; - if let Some(http_proxy) = HTTP_PROXY.as_ref() { - envs.push(("HTTP_PROXY", http_proxy)); - } - if let Some(https_proxy) = HTTPS_PROXY.as_ref() { - envs.push(("HTTPS_PROXY", https_proxy)); - } - if let Some(no_proxy) = NO_PROXY.as_ref() { - envs.push(("NO_PROXY", no_proxy)); - } envs.push(("HOME", HOME_ENV.as_str())); @@ -1209,6 +1196,7 @@ pub async fn handle_python_reqs( let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); flock_cmd .env_clear() + .envs(PROXY_ENVS.clone()) .envs(envs) .args([ "-x", @@ -1227,6 +1215,7 @@ pub async fn handle_python_reqs( pip_cmd .env_clear() .envs(envs) + .envs(PROXY_ENVS.clone()) .env("SystemRoot", SYSTEM_ROOT.as_str()) .args(&command_args[1..]) .stdout(Stdio::piped()) diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 6a0090ab55..9391fd9a69 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -20,7 +20,7 @@ use crate::{ }, handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, - RUST_CACHE_DIR, TZ_ENV, + PROXY_ENVS, RUST_CACHE_DIR, TZ_ENV, }; #[cfg(windows)] @@ -192,6 +192,7 @@ pub async fn build_rust_crate( build_rust_cmd .current_dir(job_dir) .env_clear() + .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) @@ -281,7 +282,7 @@ fn check_cargo_exists() -> Result<(), Error> { #[cfg(feature = "enterprise")] fn check_cargo_exists() -> Result<(), Error> { if !Path::new(CARGO_PATH.as_str()).exists() { - let msg = format!("Couldn't find cargo at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run rust jobs.", CARGO_PATH.as_str()); + let msg = format!("Couldn't find cargo at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-ee-full` for your instance in order to run rust jobs.", CARGO_PATH.as_str()); return Err(Error::NotFound(msg)); } Ok(()) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 83e8adff25..f4d78bf0ef 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -324,6 +324,20 @@ lazy_static::lazy_static! { pub static ref NO_PROXY: Option = std::env::var("no_proxy").ok().or(std::env::var("NO_PROXY").ok()); pub static ref HTTP_PROXY: Option = std::env::var("http_proxy").ok().or(std::env::var("HTTP_PROXY").ok()); pub static ref HTTPS_PROXY: Option = std::env::var("https_proxy").ok().or(std::env::var("HTTPS_PROXY").ok()); + + pub static ref PROXY_ENVS: Vec<(&'static str, String)> = { + let mut proxy_env = Vec::new(); + if let Some(no_proxy) = NO_PROXY.as_ref() { + proxy_env.push(("NO_PROXY", no_proxy.to_string())); + } + if let Some(http_proxy) = HTTP_PROXY.as_ref() { + proxy_env.push(("HTTP_PROXY", http_proxy.to_string())); + } + if let Some(https_proxy) = HTTPS_PROXY.as_ref() { + proxy_env.push(("HTTPS_PROXY", https_proxy.to_string())); + } + proxy_env + }; pub static ref DENO_PATH: String = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string()); pub static ref BUN_PATH: String = std::env::var("BUN_PATH").unwrap_or_else(|_| "/usr/bin/bun".to_string()); pub static ref NPM_PATH: String = std::env::var("NPM_PATH").unwrap_or_else(|_| "/usr/bin/npm".to_string()); @@ -1273,16 +1287,16 @@ pub async fn run_worker { + #[cfg(feature = "prometheus")] + if let Some(wb) = worker_busy.as_ref() { + wb.set(1); + tracing::debug!("set worker busy to 1"); + } + + occupancy_metrics.running_job_started_at = Some(Instant::now()); + last_executed_job = None; jobs_executed += 1; @@ -2067,15 +2081,10 @@ pub fn build_envs( hm }; - if let Some(ref env) = *HTTPS_PROXY { - envs.insert("HTTPS_PROXY".to_string(), env.to_string()); - } - if let Some(ref env) = *HTTP_PROXY { - envs.insert("HTTP_PROXY".to_string(), env.to_string()); - } - if let Some(ref env) = *NO_PROXY { - envs.insert("NO_PROXY".to_string(), env.to_string()); + for (k, v) in PROXY_ENVS.iter() { + envs.insert(k.to_string(), v.to_string()); } + Ok(envs) } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 3f92cfc829..a1a4fd9904 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -2403,15 +2403,15 @@ async fn push_next_flow_job } tracing::debug!(id = %flow_job.id, root_id = %job_root, "pushing job {i} of {len}"); let payload_tag = match &job_payloads { - ContinuePayload::SingleJob(payload) => payload.clone(), - ContinuePayload::BranchAllJobs(payloads) => payloads[i].clone(), + ContinuePayload::SingleJob(payload) => payload, + ContinuePayload::BranchAllJobs(payloads) => &payloads[i], ContinuePayload::ForloopJobs { flow_value, delete_after_use, .. } => { let mut fv = flow_value.clone(); if let Some(failure_module) = fv.failure_module.as_mut() { failure_module.id_append(&format!("{}-{i}", &status.step.to_string())); } - JobPayloadWithTag { + &JobPayloadWithTag { payload: JobPayload::RawFlow { value: fv, path: Some(format!("{}/forloop-{i}", flow_job.script_path())), @@ -2590,13 +2590,17 @@ async fn push_next_flow_job }; tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed perms for job {i} of {len}"); - + let tag = if flow_job.tag == "flow" || flow_job.tag == format!("flow-{}", flow_job.workspace_id) { + payload_tag.tag.clone() + } else { + Some(flow_job.tag.clone()) + }; let tx2 = PushIsolationLevel::Transaction(tx); let (uuid, mut inner_tx) = push( &db, tx2, &flow_job.workspace_id, - payload_tag.payload, + payload_tag.payload.clone(), push_args, &flow_job.created_by, &flow_job.email, @@ -2610,11 +2614,7 @@ async fn push_next_flow_job continue_on_same_worker, err, flow_job.visible_to_owner, - if flow_job.tag == "flow" || flow_job.tag == format!("flow-{}", flow_job.workspace_id) { - payload_tag.tag - } else { - Some(flow_job.tag.clone()) - }, + tag, payload_tag.timeout, Some(module.id.clone()), new_job_priority_override, @@ -3761,6 +3761,11 @@ async fn script_to_payload( module: &FlowModule, tag_override: &Option, ) -> Result { + let tag_override = if tag_override.as_ref().is_some_and(|x| x.trim().is_empty()) { + None + } else { + tag_override.clone() + }; let (payload, tag, delete_after_use, script_timeout) = if script_hash.is_none() { let (jp, tag, delete_after_use, script_timeout) = script_path_to_payload(script_path, db, &flow_job.workspace_id, Some(true)).await?; diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index c2209d9e73..f71c73f618 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -291,7 +291,7 @@ pub async fn handle_dependency_job { return await windmill.UserService.login({ diff --git a/cli/instance.ts b/cli/instance.ts index bc3f0968b7..1e86282d1a 100644 --- a/cli/instance.ts +++ b/cli/instance.ts @@ -181,7 +181,9 @@ export type InstanceSyncOptions = { instance?: string; baseUrl?: string; token?: string; + folderPerInstance?: boolean; yes?: boolean; + prefix?: string; }; export async function pickInstance( @@ -189,6 +191,21 @@ export async function pickInstance( allowNew: boolean ) { const instances = await allInstances(); + if (opts.baseUrl && opts.token && opts.instance) { + log.info("Using instance defined by --instance, --base-url and --token"); + + setClient( + opts.token, + opts.baseUrl.endsWith("/") ? opts.baseUrl.slice(0, -1) : opts.baseUrl + ); + + return { + name: opts.instance, + remote: opts.baseUrl, + token: opts.token, + prefix: opts.prefix ?? opts.instance, + }; + } if (opts.baseUrl && opts.token) { log.info("Using instance fully defined by --base-url and --token"); @@ -201,7 +218,7 @@ export async function pickInstance( name: "custom", remote: opts.baseUrl, token: opts.token, - prefix: "custom", + prefix: opts.prefix ?? "custom", }; } if (!allowNew && instances.length < 1) { @@ -297,7 +314,11 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { if (opts.includeWorkspaces) { log.info("\nPulling all workspaces"); const rootDir = Deno.cwd(); - const localWorkspaces = await getLocalWorkspaces(rootDir, instance.prefix); + const localWorkspaces = await getLocalWorkspaces( + rootDir, + instance.prefix, + opts.folderPerInstance + ); const previousActiveWorkspace = await getActiveWorkspace(undefined); const remoteWorkspaces = await wmill.listWorkspacesAsSuperAdmin({ @@ -306,7 +327,9 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) { }); for (const remoteWorkspace of remoteWorkspaces) { log.info("\nPulling workspace " + remoteWorkspace.id); - const workspaceName = instance.prefix + "_" + remoteWorkspace.id; + const workspaceName = opts?.folderPerInstance + ? instance.prefix + "/" + remoteWorkspace.id + : instance.prefix + "_" + remoteWorkspace.id; await Deno.mkdir(path.join(rootDir, workspaceName), { recursive: true, }); @@ -425,16 +448,21 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { instances = await allInstances(); const rootDir = Deno.cwd(); - const localPrefix = (await Select.prompt({ - message: "What is the prefix of the local workspaces you want to sync?", - options: [ - ...instances.map((i) => ({ - name: `${i.prefix} (${i.name} - ${i.remote})`, - value: i.prefix, - })), - ], - default: instance.prefix as unknown, - })) as unknown as string; + let localPrefix; + if (opts.prefix) { + localPrefix = opts.prefix; + } else { + localPrefix = (await Select.prompt({ + message: "What is the prefix of the local workspaces you want to sync?", + options: [ + ...instances.map((i) => ({ + name: `${i.prefix} (${i.name} - ${i.remote})`, + value: i.prefix, + })), + ], + default: instance.prefix as unknown, + })) as unknown as string; + } const remoteWorkspaces = await wmill.listWorkspacesAsSuperAdmin({ page: 1, @@ -443,7 +471,11 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { const previousActiveWorkspace = await getActiveWorkspace(undefined); - const localWorkspaces = await getLocalWorkspaces(rootDir, localPrefix); + const localWorkspaces = await getLocalWorkspaces( + rootDir, + localPrefix, + opts.folderPerInstance + ); log.info( `\nPushing all workspaces: ${localWorkspaces.map((x) => x.id).join(", ")}` @@ -521,17 +553,35 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) { } } -async function getLocalWorkspaces(rootDir: string, localPrefix: string) { +async function getLocalWorkspaces( + rootDir: string, + localPrefix: string, + folderPerInstance?: boolean +) { const localWorkspaces: { dir: string; id: string }[] = []; - for await (const dir of Deno.readDir(rootDir)) { - const dirName = dir.name; - if (dirName.startsWith(localPrefix + "_")) { + if (!(await Deno.stat(localPrefix).catch(() => null))) { + await Deno.mkdir(localPrefix); + } + if (folderPerInstance) { + for await (const dir of Deno.readDir(rootDir + "/" + localPrefix)) { + const dirName = dir.name; localWorkspaces.push({ - dir: dirName, - id: dirName.substring(localPrefix.length + 1), + dir: localPrefix + "/" + dirName, + id: dirName, }); } + log.info(localWorkspaces); + } else { + for await (const dir of Deno.readDir(rootDir)) { + const dirName = dir.name; + if (dirName.startsWith(localPrefix + "_")) { + localWorkspaces.push({ + dir: dirName, + id: dirName.substring(localPrefix.length + 1), + }); + } + } } return localWorkspaces; } @@ -650,6 +700,15 @@ const command = new Command() .option("--skip-configs", "Skip pulling configs (worker groups and SMTP)") .option("--skip-groups", "Skip pulling instance groups") .option("--include-workspaces", "Also pull workspaces") + .option("--folder-per-instance", "Create a folder per instance") + .option( + "--instance ", + "Name of the instance to pull from, override the active instance" + ) + .option( + "--prefix ", + "Prefix of the local workspaces to pull, used to create the folders when using --include-workspaces" + ) .action(instancePull as any) .command("push") @@ -662,13 +721,14 @@ const command = new Command() .option("--skip-configs", "Skip pushing configs (worker groups and SMTP)") .option("--skip-groups", "Skip pushing instance groups") .option("--include-workspaces", "Also push workspaces") + .option("--folder-per-instance", "Create a folder per instance") .option( - "--instance", + "--instance ", "Name of the instance to push to, override the active instance" ) .option( - "--base-url", - "If used with --token, will be used as the base url for the instance" + "--prefix ", + "Prefix of the local workspaces folders to push" ) .action(instancePush as any) .command("whoami") diff --git a/cli/local_encryption.ts b/cli/local_encryption.ts new file mode 100644 index 0000000000..4277093f8f --- /dev/null +++ b/cli/local_encryption.ts @@ -0,0 +1,101 @@ +import crypto from "node:crypto"; + +// Helper function to convert strings to Uint8Array (binary) +function encode(input: string): Uint8Array { + return new TextEncoder().encode(input); +} + +// Helper function to convert Uint8Array (binary) to base64 +function toBase64(arr: Uint8Array): string { + return btoa(String.fromCharCode(...arr)); +} + +// Helper function to convert base64 to Uint8Array (binary) +function fromBase64(base64: string): Uint8Array { + return new Uint8Array( + atob(base64) + .split("") + .map((char) => char.charCodeAt(0)) + ); +} + +// Function to derive a 256-bit key from any input string using SHA-256 +async function deriveKey( + keyString: string +): Promise { + const keyMaterial = encode(keyString); + const keyHash = await crypto.subtle.digest("SHA-256", keyMaterial); // Generate SHA-256 hash + // Import the hash as a CryptoKey for AES-GCM + return crypto.subtle.importKey("raw", keyHash, { name: "AES-GCM" }, false, [ + "encrypt", + "decrypt", + ]); +} + +// Encrypt function +export async function encrypt( + plaintext: string, + keyString: string +): Promise { + const key = await deriveKey(keyString); // Derive a 256-bit AES key from any input string + const iv = crypto.getRandomValues(new Uint8Array(12)); // AES-GCM needs a 12-byte IV + const encrypted = await crypto.subtle.encrypt( + { + name: "AES-GCM", + iv, + tagLength: 128, + }, + key, + encode(plaintext) // convert plaintext to binary + ); + + // Concatenate IV and encrypted data + const combined = new Uint8Array(iv.length + encrypted.byteLength); + combined.set(iv, 0); // first part is the IV + combined.set(new Uint8Array(encrypted), iv.length); // second part is the ciphertext + + // Convert to base64 for storage/transmission + return toBase64(combined); +} + +// Decrypt function +export async function decrypt( + combinedCiphertext: string, + keyString: string +): Promise { + const key = await deriveKey(keyString); // Derive the same 256-bit AES key from the input string + const combined = fromBase64(combinedCiphertext); // decode base64 to binary + + // Split the IV and the ciphertext + const iv = combined.slice(0, 12); // First 12 bytes are the IV + const ciphertext = combined.slice(12); // The rest is the encrypted data + console.log(); + + // log.info({keyString, key, ciphertext}) + // Perform decryption + const decrypted = await crypto.subtle.decrypt( + { + name: "AES-GCM", + iv, + tagLength: 128, + }, + key, + ciphertext + ); + + // Convert decrypted data from binary to string + return new TextDecoder().decode(decrypted); +} + +// // Example usage: +// const key = "any-length-key-you-want"; // Now can be any length +// const message = "This is a secret message."; + +// encrypt(message, key).then((combinedCiphertext) => { +// console.log("Encrypted message:", combinedCiphertext); + +// // Now decrypt it +// decrypt(combinedCiphertext, key).then((decryptedMessage) => { +// console.log("Decrypted message:", decryptedMessage); +// }); +// }); diff --git a/cli/main.ts b/cli/main.ts index 0cf659c601..562c3fde45 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.410.3"; +export const VERSION = "1.416.2"; const command = new Command() .name("wmill") diff --git a/cli/settings.ts b/cli/settings.ts index 328a6d0e25..f9dbac1fa6 100644 --- a/cli/settings.ts +++ b/cli/settings.ts @@ -9,6 +9,7 @@ import { deepEqual } from "./utils.ts"; import * as wmill from "./gen/services.gen.ts"; import { Config, GlobalSetting } from "./gen/types.gen.ts"; import { removeWorkerPrefix } from "./worker_groups.ts"; +import process from "node:process"; export interface SimplifiedSettings { // slack_team_id?: string; @@ -112,10 +113,10 @@ export async function pushWorkspaceSettings( workspace, requestBody: localSettings.auto_invite_enabled ? { - operator: localSettings.auto_invite_as === "operator", - invite_all: true, - auto_add: localSettings.auto_invite_mode === "add", - } + operator: localSettings.auto_invite_as === "operator", + invite_all: true, + auto_add: localSettings.auto_invite_mode === "add", + } : {}, }); } catch (_) { @@ -127,10 +128,10 @@ export async function pushWorkspaceSettings( workspace, requestBody: localSettings.auto_invite_enabled ? { - operator: localSettings.auto_invite_as === "operator", - invite_all: false, - auto_add: localSettings.auto_invite_mode === "add", - } + operator: localSettings.auto_invite_as === "operator", + invite_all: false, + auto_add: localSettings.auto_invite_mode === "add", + } : {}, }); } @@ -155,7 +156,7 @@ export async function pushWorkspaceSettings( settings.error_handler_extra_args ) || localSettings.error_handler_muted_on_cancel !== - settings.error_handler_muted_on_cancel + settings.error_handler_muted_on_cancel ) { log.debug(`Updating error handler...`); await wmill.editErrorHandler({ @@ -259,27 +260,69 @@ export async function pushWorkspaceKey( } } +const INSTANCE_SETTINGS_PATH = "instance_settings.yaml"; + export async function readInstanceSettings() { let localSettings: GlobalSetting[] = []; try { - localSettings = (await yamlParseFile( - "instance_settings.yaml" - )) as GlobalSetting[]; + localSettings = (await yamlParseFile(INSTANCE_SETTINGS_PATH)) as GlobalSetting[]; } catch { - log.warn("No instance_settings.yaml found"); + log.warn(`No ${INSTANCE_SETTINGS_PATH} found`); } return localSettings; } + +import { decrypt, encrypt } from "./local_encryption.ts"; + +const SENSITIVE_FIELD: string[] = ["license_key", "jwt_secret"] + +async function processInstanceSettings(settings: GlobalSetting[], mode: "encode" | "decode"): Promise { + const encKey = process.env.WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY; + if (encKey) { + const res: GlobalSetting[] = [] + + for (const s of settings) { + if (SENSITIVE_FIELD.includes(s.name) && typeof s.value === "string") { + res.push(await processField(s, "value", encKey, mode) as GlobalSetting); + } else if (s.name == "oauths") { + if (typeof s.value === "object") { + const oauths = s.value as { [key: string]: any }; + for (const [k, v] of Object.entries(oauths)) { + oauths[k] = await processField(v, "secret", encKey, mode); + } + res.push(s); + } else { + log.warn(`Unexpected oauths value type: ${typeof s.value}`); + res.push(s); + } + } else { + res.push(s); + } + } + return res; + } else { + log.warn("No encryption key found, skipping encryption. Recommend setting WMILL_INSTANCE_LOCAL_ENCRYPTION_KEY"); + } + return settings; +} + +async function processField(obj: { [key: string]: any }, field: string, encKey: string, mode: "encode" | "decode"): Promise<{ [key: string]: any }> { + return { + ...obj, + [field]: mode === "encode" ? await encrypt(obj[field], encKey) : await decrypt(obj[field], encKey) as any, + } +} + export async function pullInstanceSettings(preview = false) { const remoteSettings = await wmill.listGlobalSettings(); if (preview) { const localSettings: GlobalSetting[] = await readInstanceSettings(); - + const processedSettings = await processInstanceSettings(remoteSettings, "encode"); return compareInstanceObjects( - remoteSettings, + processedSettings, localSettings, "name", "setting" @@ -287,12 +330,13 @@ export async function pullInstanceSettings(preview = false) { } else { log.info("Pulling settings from instance"); + const processedSettings = await processInstanceSettings(remoteSettings, "encode"); await Deno.writeTextFile( - "instance_settings.yaml", - yamlStringify(remoteSettings as any) + INSTANCE_SETTINGS_PATH, + yamlStringify(processedSettings) ); - log.info(colors.green("Settings written to instance_settings.yaml")); + log.info(colors.green(`Settings written to ${INSTANCE_SETTINGS_PATH}`)); } } @@ -302,6 +346,7 @@ export async function pushInstanceSettings( ) { const remoteSettings = await wmill.listGlobalSettings(); let localSettings: GlobalSetting[] = await readInstanceSettings(); + localSettings = await processInstanceSettings(localSettings, "decode"); if (baseUrl) { localSettings = localSettings.filter((s) => s.name !== "base_url"); diff --git a/cli/sync.ts b/cli/sync.ts index ce539b7a1f..342dda3162 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -618,8 +618,8 @@ export async function elementsToMap( for await (const entry of readDirRecursiveWithIgnore(ignore, els)) { if (entry.isDirectory || entry.ignored) continue; const path = entry.path; - if (json && path.endsWith(".yaml")) continue; - if (!json && path.endsWith(".json")) continue; + if (json && path.endsWith(".yaml") && !isFileResource(path)) continue; + if (!json && path.endsWith(".json") && !isFileResource(path)) continue; const ext = json ? ".json" : ".yaml"; if (!skips.includeSchedules && path.endsWith(".schedule" + ext)) continue; if (!skips.includeUsers && path.endsWith(".user" + ext)) continue; diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index cb0284ff1d..16c2361a0d 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -11,7 +11,7 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - -RUN apt-get -y update && apt-get install -y curl nodejs awscli +RUN apt-get -y update && apt-get install -y curl procps nodejs awscli ENV TZ=Etc/UTC @@ -20,7 +20,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.1.27 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.32 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 54a7b15764..65c6b55a19 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -11,7 +11,7 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - -RUN apt-get -y update && apt-get install -y curl nodejs awscli +RUN apt-get -y update && apt-get install -y curl procps nodejs awscli ENV TZ=Etc/UTC @@ -19,7 +19,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.1.27 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.32 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/frontend/package-lock.json b/frontend/package-lock.json index a4f5ccbd4a..028ac3e356 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.410.3", + "version": "1.416.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.410.3", + "version": "1.416.2", "license": "AGPL-3.0", "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", diff --git a/frontend/package.json b/frontend/package.json index d64d037923..9c4f3a2a7d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.410.3", + "version": "1.416.2", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/frontend/src/lib/components/AppConnectDrawer.svelte b/frontend/src/lib/components/AppConnectDrawer.svelte index 28cb801616..4f30737736 100644 --- a/frontend/src/lib/components/AppConnectDrawer.svelte +++ b/frontend/src/lib/components/AppConnectDrawer.svelte @@ -6,7 +6,7 @@ import AppConnectInner from './AppConnectInner.svelte' import DarkModeObserver from './DarkModeObserver.svelte' - let expressOAuthSetup = false + export let expressOAuthSetup = false let drawer: Drawer let resourceType = '' @@ -18,8 +18,7 @@ let appConnectInner: AppConnectInner | undefined = undefined let rtToLoad: string | undefined = '' - export async function open(rt?: string, express?: boolean) { - expressOAuthSetup = express ?? false + export async function open(rt?: string) { rtToLoad = rt drawer.openDrawer?.() } @@ -27,7 +26,7 @@ $: appConnectInner && onRtToLoadChange(rtToLoad) function onRtToLoadChange(rtToLoad: string | undefined) { - appConnectInner?.open(rtToLoad, expressOAuthSetup) + appConnectInner?.open(rtToLoad) } const dispatch = createEventDispatcher() @@ -60,6 +59,7 @@ bind:manual on:close={drawer?.closeDrawer} on:refresh + express={expressOAuthSetup} />
{#if step > 1} diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index dbe162fd64..e568589b49 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -31,6 +31,7 @@ export let isGoogleSignin = false export let disabled = false export let manual = true + export let express = false let isValid = true @@ -81,10 +82,7 @@ let pathError = '' - let expressOAuthSetup = false - - export async function open(rt?: string, express?: boolean) { - expressOAuthSetup = express ?? false + export async function open(rt?: string) { if (!rt) { loadResourceTypes() } @@ -95,12 +93,12 @@ valueToken = undefined await loadConnects() manual = !connects?.includes(resourceType) - if (manual && expressOAuthSetup) { + if (manual && express) { dispatch('error', 'Express OAuth setup is not available for non OAuth resource types') return } if (rt) { - if (!manual && expressOAuthSetup) { + if (!manual && express) { await getScopesAndParams() step = 2 } @@ -171,7 +169,6 @@ } function popupListener(event) { - console.log('popupListener', event.data, event.origin, window.location.origin) let data = event.data if (event.origin == null || event.origin !== window.location.origin) { return @@ -187,7 +184,7 @@ value = data.res.access_token! valueToken = data.res step = 4 - if (expressOAuthSetup) { + if (express) { path = `u/${$userStore?.username}/${resourceType}_${new Date().getTime()}` next() } @@ -329,252 +326,254 @@ let editScopes = false - a.localeCompare(b)) - .map((key) => ({ - key - })) - : undefined} - bind:filteredItems={filteredConnects} - f={(x) => x.key} -/> - a[0].localeCompare(b[0]))} - bind:filteredItems={filteredConnectsManual} - f={(x) => x[0]} -/> -{#if step == 1} -
- -
- -

OAuth APIs

-
- {#if filteredConnects} - {#each filteredConnects as { key }} - - {/each} - {:else} - {#each new Array(3) as _} - - {/each} - {/if} -
- {#if connects && connects.length == 0} -
No OAuth APIs has been setup on the instance. To add oauth APIs, first sync the resource - types with the hub, then add oauth configuration. See documentation -
- {/if} - -

Others

- - {#if connectsManual && connectsManual?.length < 10} -
- Resource Types have not been synced with the hub. Go to the admins workspace to sync them (and - add a schedule to do daily): -

1. Go to the "admins" workspaces: - sync resource types -

-

- 2: Run the synchronization script: - sync resource types -

-
- {/if} - -
- {#if filteredConnectsManual} - {#each filteredConnectsManual as [key, _]} - {#if nativeLanguagesCategory.includes(key)} - - {/if} - {/each} - {/if} -
- -
-
- {#if filteredConnectsManual} - {#each filteredConnectsManual as [key, _]} - {#if !nativeLanguagesCategory.includes(key)} - - - {/if} - {/each} - {:else} - {#each new Array(9) as _} - - {/each} - {/if} -
-{:else if step == 2 && manual} - a.localeCompare(b)) + .map((key) => ({ + key + })) + : undefined} + bind:filteredItems={filteredConnects} + f={(x) => x.key} /> - - {#if apiTokenApps[resourceType]} -

Instructions

-
-
    - {#each apiTokenApps[resourceType].instructions as step} -
  1. - {@html step} -
  2. - {/each} -
-
- {#if apiTokenApps[resourceType].img} -
- connect -
- {/if} - {:else if !emptyString(resourceTypeInfo?.description)} -

{resourceTypeInfo?.name} description

-
- -
- {/if} - {#if resourceType == 'postgresql' || resourceType == 'mysql' || resourceType == 'mongodb'} - - {/if} - -

Resource description -
- - -
-

- {#if renderDescription} -
-
GH Markdown
-