chore(main): release 1.371.4 (#4158)

* chore(main): release 1.371.4

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.github/DockerfileBackendTests

@@ -1,5 +1,5 @@
 ARG DEBIAN_IMAGE=debian:bookworm-slim
-ARG RUST_IMAGE=rust:1.78-slim-bookworm
+ARG RUST_IMAGE=rust:1.79-slim-bookworm
 ARG PYTHON_IMAGE=python:3.11.4-slim-bookworm
 
 FROM ${DEBIAN_IMAGE} as downloader

.github/workflows/backend-test.yml

@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          toolchain: 1.78.0
+          toolchain: 1.79.0
       # - uses: Swatinem/rust-cache@v2
       #   with:
       #     workspaces: |

.github/workflows/build-staging-image.yml

@@ -0,0 +1,70 @@
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+name: Build windmill-staging
+on:
+  workflow_dispatch:
+
+permissions: write-all
+
+jobs:
+  build_ee:
+    runs-on: ubicloud
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Read EE repo commit hash
+        run: |
+          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"
+
+      - uses: actions/checkout@v4
+        with:
+          repository: windmill-labs/windmill-ee-private
+          path: ./windmill-ee-private
+          ref: ${{ env.ee_repo_ref }}
+          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
+          fetch-depth: 0
+
+      # - name: Set up Docker Buildx
+      #   uses: docker/setup-buildx-action@v2
+      - uses: depot/setup-action@v1
+
+      - name: Docker meta
+        id: meta-ee-public
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-staging-ee
+          flavor: |
+            latest=false
+          tags: |
+            type=sha
+            type=ref,event=branch
+
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Substitute EE code
+        run: |
+          ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private
+
+      - name: Build and push publicly ee
+        uses: depot/build-push-action@v1
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          build-args: |
+            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc
+          tags: |
+            ${{ steps.meta-ee-public.outputs.tags }}
+          labels: |
+            ${{ steps.meta-ee-public.outputs.labels }}
+            org.opencontainers.image.licenses=Windmill-Enterprise-License

.github/workflows/docker-image.yml

@@ -135,7 +135,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           build-args: |
-            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc
+            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:dev
             ${{ steps.meta-ee-public.outputs.tags }}
@@ -198,7 +198,7 @@ jobs:
           platforms: linux/amd64
           push: true
           build-args: |
-            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc
+            features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy
             PYTHON_IMAGE=python:3.12.2-slim-bookworm
           tags: |
             ${{ steps.meta-ee-public-py312.outputs.tags }}

CHANGELOG.md

@@ -1,5 +1,360 @@
 # Changelog
 
+## [1.371.4](https://github.com/windmill-labs/windmill/compare/v1.371.3...v1.371.4) (2024-07-31)
+
+
+### Bug Fixes
+
+* **frontend:** fix recompute all ([#4161](https://github.com/windmill-labs/windmill/issues/4161)) ([10c6997](https://github.com/windmill-labs/windmill/commit/10c699759e1d06d009783b9f5cba2cbf3ba6a7da))
+* **frontend:** If multiple recompute all present, interval is now in sync ([#4162](https://github.com/windmill-labs/windmill/issues/4162)) ([efeb65b](https://github.com/windmill-labs/windmill/commit/efeb65be4c91491bb4fbedfbc160da7517ebe1d6))
+* improve index usage and runs page performance ([d69aa8d](https://github.com/windmill-labs/windmill/commit/d69aa8d484de016ae7e7bbd190ce6c254818c861))
+
+## [1.371.3](https://github.com/windmill-labs/windmill/compare/v1.371.2...v1.371.3) (2024-07-30)
+
+
+### Bug Fixes
+
+* for codebase bundle, use cjs exports instead ([be8cedf](https://github.com/windmill-labs/windmill/commit/be8cedfe7fe42105a375c92536ec7f5e5a1343bf))
+
+## [1.371.2](https://github.com/windmill-labs/windmill/compare/v1.371.1...v1.371.2) (2024-07-30)
+
+
+### Bug Fixes
+
+* improve codebase handling by ignoring creating lock and bundle ([716bb71](https://github.com/windmill-labs/windmill/commit/716bb7118b0d5b8dd86799d5514f37b068613256))
+
+## [1.371.1](https://github.com/windmill-labs/windmill/compare/v1.371.0...v1.371.1) (2024-07-30)
+
+
+### Bug Fixes
+
+* database connections now scale linearly with number of subworkers ([f59046a](https://github.com/windmill-labs/windmill/commit/f59046a9249962ba7e1739de579cd8d885ec63a8))
+
+## [1.371.0](https://github.com/windmill-labs/windmill/compare/v1.370.0...v1.371.0) (2024-07-30)
+
+
+### Features
+
+* **frontend:** support array of objects in schema ([#4106](https://github.com/windmill-labs/windmill/issues/4106)) ([5992b82](https://github.com/windmill-labs/windmill/commit/5992b821803d9cdc45fa0dee100f159dd3ed557f))
+
+
+### Bug Fixes
+
+* always consider electron as external for bundles ([06433a6](https://github.com/windmill-labs/windmill/commit/06433a6ec65534e1a11085cefd6a67699e4eb0a0))
+* fix bunnative lock creation ([6a33624](https://github.com/windmill-labs/windmill/commit/6a33624416cc48e2a1ab21afbd2ac4ab48be2559))
+* submit and form persistence on app on render change ([28277da](https://github.com/windmill-labs/windmill/commit/28277da35bae1c18f16e85d093989e0d419b3601))
+* support bunnative in the CLI ([e02e644](https://github.com/windmill-labs/windmill/commit/e02e644d4ca16773bae3cc2a6b52ac08417d3905))
+
+## [1.370.0](https://github.com/windmill-labs/windmill/compare/v1.369.1...v1.370.0) (2024-07-29)
+
+
+### Features
+
+* get completed flow node result by api/download + eval list result json path optim ([#4108](https://github.com/windmill-labs/windmill/issues/4108)) ([5031a8c](https://github.com/windmill-labs/windmill/commit/5031a8cb015e3ba8d668f5de769b8d67da82c116))
+
+
+### Bug Fixes
+
+* improve native runtime with axios support ([6e91005](https://github.com/windmill-labs/windmill/commit/6e91005daa4bbe2f0bd1f8b1c2cce1f68485f9ef))
+
+## [1.369.1](https://github.com/windmill-labs/windmill/compare/v1.369.0...v1.369.1) (2024-07-29)
+
+
+### Bug Fixes
+
+* fix lang picker artefact for script editor ([bc94acb](https://github.com/windmill-labs/windmill/commit/bc94acb378b1310271dd3dda80614e0e38b5285f))
+
+## [1.369.0](https://github.com/windmill-labs/windmill/compare/v1.368.3...v1.369.0) (2024-07-29)
+
+
+### Features
+
+* add support for text/plain webhook ([#4146](https://github.com/windmill-labs/windmill/issues/4146)) ([eb6557a](https://github.com/windmill-labs/windmill/commit/eb6557a6beb5d3b2324b30f2abc117b1a122c3f7))
+* **frontend:** App bar as components ([#4103](https://github.com/windmill-labs/windmill/issues/4103)) ([fb89eed](https://github.com/windmill-labs/windmill/commit/fb89eed8fa43568ffe3a22c5a6dd0b9f38890a37))
+* remove nativets in favor of bun with native pragma ([18f22be](https://github.com/windmill-labs/windmill/commit/18f22be2bae3136f1d9ff06c74f3751504bef60b))
+* remove nativets in favor of bun with native pragma ([b02baa2](https://github.com/windmill-labs/windmill/commit/b02baa2cfb242f7f6ed64e9861a8562272fb2974))
+
+## [1.368.3](https://github.com/windmill-labs/windmill/compare/v1.368.2...v1.368.3) (2024-07-28)
+
+
+### Bug Fixes
+
+* update bun to 1.1.21 ([e45a8a4](https://github.com/windmill-labs/windmill/commit/e45a8a4591649de240a4bff0a25301a2586618bf))
+
+## [1.368.2](https://github.com/windmill-labs/windmill/compare/v1.368.1...v1.368.2) (2024-07-28)
+
+
+### Bug Fixes
+
+* add the nobundling option for bun ([c3848e2](https://github.com/windmill-labs/windmill/commit/c3848e2e3011c7e74d5ae6eb85605476ef2998f4))
+* disable prebundling for nodejs mode scripts ([1e2e907](https://github.com/windmill-labs/windmill/commit/1e2e907982eab2ea7e713d9cfbec046527616fa2))
+
+## [1.368.1](https://github.com/windmill-labs/windmill/compare/v1.368.0...v1.368.1) (2024-07-27)
+
+
+### Bug Fixes
+
+* improve runs page performance through pg indices ([99623f3](https://github.com/windmill-labs/windmill/commit/99623f31af109463b065688058d0cab8ea9910de))
+
+## [1.368.0](https://github.com/windmill-labs/windmill/compare/v1.367.2...v1.368.0) (2024-07-26)
+
+
+### Features
+
+* add FORCE_<env> to allow to override db settings ([f1d5be8](https://github.com/windmill-labs/windmill/commit/f1d5be8a2a76d5c0d37c55ba762f2cd7a6f6982f))
+* job view audit logging based on env variable ([#4131](https://github.com/windmill-labs/windmill/issues/4131)) ([12f9e56](https://github.com/windmill-labs/windmill/commit/12f9e56154b32d839f49dd967ada4952ca4c395d))
+* multi sql statement with pg fix ([#4134](https://github.com/windmill-labs/windmill/issues/4134)) ([6df9eca](https://github.com/windmill-labs/windmill/commit/6df9eca2cffef754ee27b3219f18d0ad69615c83))
+* multi statement sql ([#4104](https://github.com/windmill-labs/windmill/issues/4104)) ([23108b4](https://github.com/windmill-labs/windmill/commit/23108b4688ec9c9683371ee70a50658f6baae42f))
+* pre-bundle bun scripts ([#4132](https://github.com/windmill-labs/windmill/issues/4132)) ([78fd99c](https://github.com/windmill-labs/windmill/commit/78fd99cb6843c7f8045acd6a6a8df851ec6b35d1))
+
+
+### Bug Fixes
+
+* allow colors for bun and log error directly ([545a57f](https://github.com/windmill-labs/windmill/commit/545a57fd2c425f02d867f4183aac37131e17210f))
+* search modal improvements ([#4128](https://github.com/windmill-labs/windmill/issues/4128)) ([c270ab1](https://github.com/windmill-labs/windmill/commit/c270ab11718f8d41fe764d5d2116bb3b9aa20cb2))
+* support npmjs mode for raw deps of package.json from CLI ([8e615c9](https://github.com/windmill-labs/windmill/commit/8e615c900de46632c52d78a1ec4b6ccc2cec79ff))
+
+## [1.367.2](https://github.com/windmill-labs/windmill/compare/v1.367.1...v1.367.2) (2024-07-24)
+
+
+### Bug Fixes
+
+* move bun cache to non mounted volume to benefit from cache optimization ([92dac02](https://github.com/windmill-labs/windmill/commit/92dac027f26dfa880e22b1896b48605ae2ef2794))
+* use symlink and straight copy as fallback methods for buntar ([80d4fb0](https://github.com/windmill-labs/windmill/commit/80d4fb0352606948b01f4918e11f80a0bef544f4))
+
+## [1.367.1](https://github.com/windmill-labs/windmill/compare/v1.367.0...v1.367.1) (2024-07-24)
+
+
+### Bug Fixes
+
+* delete buntar if any issue while creating it ([204e2fa](https://github.com/windmill-labs/windmill/commit/204e2fafda681347813ab91f96cee4194db2e156))
+* generate lockfile with npm when npm mode is used ([7fbd002](https://github.com/windmill-labs/windmill/commit/7fbd0028c1f2dfbd6a5cde12e6a0e314ed31552a))
+
+## [1.367.0](https://github.com/windmill-labs/windmill/compare/v1.366.6...v1.367.0) (2024-07-24)
+
+
+### Features
+
+* apply workspace specific tags only to some workspaces ([#4107](https://github.com/windmill-labs/windmill/issues/4107)) ([14a4f12](https://github.com/windmill-labs/windmill/commit/14a4f1282671d1c6a2481123cd5a4297fbd89607))
+* job search index backed up and loaded from s3 ([#4100](https://github.com/windmill-labs/windmill/issues/4100)) ([cac39a1](https://github.com/windmill-labs/windmill/commit/cac39a105187bd005dee442c4968ebbd9ba163d7))
+* use hardlinks instead of tar to improve bun cache performances ([48e9f08](https://github.com/windmill-labs/windmill/commit/48e9f089beff10828eef7a35fb341894eb50d25d))
+* use jwks for external jwt auth ([#4089](https://github.com/windmill-labs/windmill/issues/4089)) ([d096704](https://github.com/windmill-labs/windmill/commit/d096704b27f169672d8949f8570b2b5cba25e78c))
+
+
+### Bug Fixes
+
+* cgroupv1 mem limit + granular memory reporting ([#4119](https://github.com/windmill-labs/windmill/issues/4119)) ([13e9e8d](https://github.com/windmill-labs/windmill/commit/13e9e8de9e49f9513ad0cd8ac5a3b8b356de02a9))
+* filter audit logs end user by username + complete resource filter ([#4105](https://github.com/windmill-labs/windmill/issues/4105)) ([73decb2](https://github.com/windmill-labs/windmill/commit/73decb24b23e1a998957455cf4f4148be5990cce))
+* fix webhooks urls after BASE_URL change ([d49b2d0](https://github.com/windmill-labs/windmill/commit/d49b2d0f06842e70180f931ac7b4ac683c1c65b6))
+* make result_json path stable for python executors ([97f6b2e](https://github.com/windmill-labs/windmill/commit/97f6b2e7ad9962c99e566238bde40cdba0ffc3a8))
+* preserve force json across code preview ([9ab5b2e](https://github.com/windmill-labs/windmill/commit/9ab5b2e32d78056752f654769154b90872a9ab85))
+* respect sorting when downloading csv f rom auto table ([20390c5](https://github.com/windmill-labs/windmill/commit/20390c53e82686ac6a8b7a9bc1bf9fb8e014e3c5))
+* sqlx build ([#4120](https://github.com/windmill-labs/windmill/issues/4120)) ([c6dc06b](https://github.com/windmill-labs/windmill/commit/c6dc06bf702b501225a51813a2489ce9ea3c4636))
+
+## [1.366.6](https://github.com/windmill-labs/windmill/compare/v1.366.5...v1.366.6) (2024-07-23)
+
+
+### Bug Fixes
+
+* fix copilot completion after base_url change ([ae4cbb0](https://github.com/windmill-labs/windmill/commit/ae4cbb0401b01e1e378b2a982d7a610648c96239))
+
+## [1.366.5](https://github.com/windmill-labs/windmill/compare/v1.366.4...v1.366.5) (2024-07-22)
+
+
+### Bug Fixes
+
+* fix BASE_URL build conf ([bb861cf](https://github.com/windmill-labs/windmill/commit/bb861cfedef1e5531f31b62dd73907b89ddd473a))
+
+## [1.366.4](https://github.com/windmill-labs/windmill/compare/v1.366.3...v1.366.4) (2024-07-22)
+
+
+### Bug Fixes
+
+* fix BASE_URL build conf ([6e33e4e](https://github.com/windmill-labs/windmill/commit/6e33e4e0b9fce86407cc7cfc9a4a57e03ec5199b))
+
+## [1.366.3](https://github.com/windmill-labs/windmill/compare/v1.366.2...v1.366.3) (2024-07-22)
+
+
+### Bug Fixes
+
+* fix BASE_URL build conf ([f6a948b](https://github.com/windmill-labs/windmill/commit/f6a948ba7705b5b59af87bff7d17252ec1c8e739))
+
+## [1.366.2](https://github.com/windmill-labs/windmill/compare/v1.366.1...v1.366.2) (2024-07-22)
+
+
+### Bug Fixes
+
+* fix BASE_URL build conf ([c26457c](https://github.com/windmill-labs/windmill/commit/c26457c967c07f05110ca31c987247869e267c51))
+
+## [1.366.1](https://github.com/windmill-labs/windmill/compare/v1.366.0...v1.366.1) (2024-07-22)
+
+
+### Bug Fixes
+
+* fix BASE_URL build conf ([9cb4586](https://github.com/windmill-labs/windmill/commit/9cb4586211af8159037e402dc03b85f37db8aa5f))
+
+## [1.366.0](https://github.com/windmill-labs/windmill/compare/v1.365.0...v1.366.0) (2024-07-22)
+
+
+### Features
+
+* dynamic select ([#4110](https://github.com/windmill-labs/windmill/issues/4110)) ([19d523d](https://github.com/windmill-labs/windmill/commit/19d523dd64327cc8c590737f6a14a5d10fe19aa6))
+* make components resizable directly on side ([26c30c4](https://github.com/windmill-labs/windmill/commit/26c30c4f8df6eac7c1d99770c99b8159bd3afe49))
+
+
+### Bug Fixes
+
+* add WM_SCHEDULED_FOR to contextual variables and early stop of flows ([e91a06f](https://github.com/windmill-labs/windmill/commit/e91a06fa41a1bdcc781126b936e20a83e540c9f9))
+* Allow deploying frontend in a sub directory ([#3867](https://github.com/windmill-labs/windmill/issues/3867)) ([dd75dd4](https://github.com/windmill-labs/windmill/commit/dd75dd446b5d155ac0c9a007e694732a81dc6b35))
+* improve cgroup readings ([#4030](https://github.com/windmill-labs/windmill/issues/4030)) ([ea53a12](https://github.com/windmill-labs/windmill/commit/ea53a129afc520ee804f473f63847df57e7287a2))
+* update parsers for CLI ([f65ccc0](https://github.com/windmill-labs/windmill/commit/f65ccc07de8e1367ce5acc45d4f3674593351b75))
+
+## [1.365.0](https://github.com/windmill-labs/windmill/compare/v1.364.3...v1.365.0) (2024-07-17)
+
+
+### Features
+
+* **frontend:** array of resources ([#4095](https://github.com/windmill-labs/windmill/issues/4095)) ([5542b7d](https://github.com/windmill-labs/windmill/commit/5542b7d04d0aabff29636b6e0e2e7ff6b5098206))
+
+
+### Bug Fixes
+
+* **frontend:** close the content search modal when clicking on an item ([#4098](https://github.com/windmill-labs/windmill/issues/4098)) ([c8c5c27](https://github.com/windmill-labs/windmill/commit/c8c5c2785b2c88d35f5bfe8ed65935dd658afd98))
+* **frontend:** Display 'parallel'and 'skip failure' even when a summary is set ([#4099](https://github.com/windmill-labs/windmill/issues/4099)) ([319454e](https://github.com/windmill-labs/windmill/commit/319454ecf2143703dd7aca9147a9c7d95f83c6a6))
+* **frontend:** fix flow graph when anode has multiple steps to the flow inputs ([#4097](https://github.com/windmill-labs/windmill/issues/4097)) ([fc749c6](https://github.com/windmill-labs/windmill/commit/fc749c687c3919ad57f8bdd7e0b2313c57f0d737))
+* **frontend:** fix use inputs flow preview ([#4094](https://github.com/windmill-labs/windmill/issues/4094)) ([7aa2189](https://github.com/windmill-labs/windmill/commit/7aa2189feca1b502674c5c1957f207b3dd1e7264))
+* **frontend:** improve search modal ([#4088](https://github.com/windmill-labs/windmill/issues/4088)) ([cdc7190](https://github.com/windmill-labs/windmill/commit/cdc7190d8962ca98e78d04e3a7f4da7aa3391ec3))
+
+## [1.364.3](https://github.com/windmill-labs/windmill/compare/v1.364.2...v1.364.3) (2024-07-16)
+
+
+### Bug Fixes
+
+* fix erronous not connected error message ([9a5dc97](https://github.com/windmill-labs/windmill/commit/9a5dc97b184a6b4a027baaa244c117885fb86fbb))
+* fix missing workspaceId on display result of apps ([bfbbeab](https://github.com/windmill-labs/windmill/commit/bfbbeabe4f57ed65a0190715028ed6df9a201e06))
+
+## [1.364.2](https://github.com/windmill-labs/windmill/compare/v1.364.1...v1.364.2) (2024-07-16)
+
+
+### Bug Fixes
+
+* **frontend:** Handle three significant digits for jobs that ran in less than 1 min ([#4084](https://github.com/windmill-labs/windmill/issues/4084)) ([2859d78](https://github.com/windmill-labs/windmill/commit/2859d78f3fe8c261ce4ca1af6a4a5ee53b6aeb29))
+
+## [1.364.1](https://github.com/windmill-labs/windmill/compare/v1.364.0...v1.364.1) (2024-07-15)
+
+
+### Bug Fixes
+
+* fix cli build ([1deccc4](https://github.com/windmill-labs/windmill/commit/1deccc476d4db07e1530ac5b642e619b9d7b618f))
+
+## [1.364.0](https://github.com/windmill-labs/windmill/compare/v1.363.0...v1.364.0) (2024-07-15)
+
+
+### Features
+
+* rehydrate instance settings/configs/users from CLI ([#4035](https://github.com/windmill-labs/windmill/issues/4035)) ([0f7f7c3](https://github.com/windmill-labs/windmill/commit/0f7f7c37a97efda3fa41f1856aec658425349abd))
+
+
+### Bug Fixes
+
+* improve generate flow locks ([304b90f](https://github.com/windmill-labs/windmill/commit/304b90fa2751a024f294efe8bbe587d9524f3418))
+
+## [1.363.0](https://github.com/windmill-labs/windmill/compare/v1.362.0...v1.363.0) (2024-07-15)
+
+
+### Features
+
+* **frontend:** Rich result by id component ([#4069](https://github.com/windmill-labs/windmill/issues/4069)) ([546c343](https://github.com/windmill-labs/windmill/commit/546c343811a6eb66a49670c915e984cb68f5aab1))
+
+
+### Bug Fixes
+
+* **frontend:** flow editor improvements ([#4008](https://github.com/windmill-labs/windmill/issues/4008)) ([e6dfa39](https://github.com/windmill-labs/windmill/commit/e6dfa390bbf27ed8d4d13128c33b7fd5b2ef69ab))
+* improve schema editor ([2ec8ce5](https://github.com/windmill-labs/windmill/commit/2ec8ce5b7da76d7b5429120b749c66ffb833ff68))
+* make indexer opt-in when in standalone mode ([#4076](https://github.com/windmill-labs/windmill/issues/4076)) ([3e31977](https://github.com/windmill-labs/windmill/commit/3e31977ba1de984d256dbd888b769ddd9f566b04))
+
+## [1.362.0](https://github.com/windmill-labs/windmill/compare/v1.361.1...v1.362.0) (2024-07-14)
+
+
+### Features
+
+* **frontend:** date select component ([#4064](https://github.com/windmill-labs/windmill/issues/4064)) ([3b4a376](https://github.com/windmill-labs/windmill/commit/3b4a3762087cc3f75f7ac297b236bfd98f858076))
+* Full-text search on runs using tantivy and command palette for quick actions ([#4046](https://github.com/windmill-labs/windmill/issues/4046)) ([7ea554a](https://github.com/windmill-labs/windmill/commit/7ea554a7fd315dd6f91b9cbdc12c36fdf50ede9e))
+* improve cancel all for non started jobs ([#4065](https://github.com/windmill-labs/windmill/issues/4065)) ([2747e1b](https://github.com/windmill-labs/windmill/commit/2747e1b0879a734077df363594363ea552f96f9c))
+* improve flow status viewer (show branch chosen + all iterations in for loop) ([#4074](https://github.com/windmill-labs/windmill/issues/4074)) ([b498664](https://github.com/windmill-labs/windmill/commit/b498664c76246931d77cdf6740b15160bd8379a5))
+
+
+### Bug Fixes
+
+* **frontend:** app editor improvements ([#4052](https://github.com/windmill-labs/windmill/issues/4052)) ([fd4fe15](https://github.com/windmill-labs/windmill/commit/fd4fe15f49be5cd68c4af8fdba8dbaf61cbc68cc))
+
+## [1.361.1](https://github.com/windmill-labs/windmill/compare/v1.361.0...v1.361.1) (2024-07-11)
+
+
+### Bug Fixes
+
+* improve filter jobs query ([583190f](https://github.com/windmill-labs/windmill/commit/583190f883e2a82fb6d0ab0d882283d5da5436c5))
+
+## [1.361.0](https://github.com/windmill-labs/windmill/compare/v1.360.1...v1.361.0) (2024-07-11)
+
+
+### Features
+
+* **frontend:** add support for GFM and fix max-width issue ([#4057](https://github.com/windmill-labs/windmill/issues/4057)) ([133a278](https://github.com/windmill-labs/windmill/commit/133a2789120c50f2733af4bce5290bd7365b566d))
+* **frontend:** improve table action UX ([#4056](https://github.com/windmill-labs/windmill/issues/4056)) ([a6eeb68](https://github.com/windmill-labs/windmill/commit/a6eeb68b77c49e2fa41e2bb62e72ce4446841fbc))
+
+
+### Bug Fixes
+
+* fix migration for instances starting from scratch ([b4f0b32](https://github.com/windmill-labs/windmill/commit/b4f0b32c40bc63b4063816f51c5a30679edf03f4))
+* **frontend:** fix run page ms readability ([#4059](https://github.com/windmill-labs/windmill/issues/4059)) ([0b932ca](https://github.com/windmill-labs/windmill/commit/0b932cac6f8cd959a10d057e53985d876fe158f2))
+* **frontend:** fix style panel title ([#4058](https://github.com/windmill-labs/windmill/issues/4058)) ([c853db4](https://github.com/windmill-labs/windmill/commit/c853db4e0570346b35dcb2249723e42b1af86547))
+* improve completed_job index ([e450300](https://github.com/windmill-labs/windmill/commit/e450300c225a965cde3472013b5ae7fb83200a31))
+* improve performance of cancel_all ([1d1dde0](https://github.com/windmill-labs/windmill/commit/1d1dde0d91644c8b973a4104a93d4c7cd123ac56))
+
+## [1.360.1](https://github.com/windmill-labs/windmill/compare/v1.360.0...v1.360.1) (2024-07-09)
+
+
+### Bug Fixes
+
+* fix previous_result not always working for failure steps ([23a00c5](https://github.com/windmill-labs/windmill/commit/23a00c55e95054ecebbcbb829499f5a1d622efd5))
+* **frontend:** Reload the flowStateStore when a node is restored (undo) ([#4049](https://github.com/windmill-labs/windmill/issues/4049)) ([6e52632](https://github.com/windmill-labs/windmill/commit/6e526327dd0259eb096fdaf387dfc9206936c19b))
+* improve flow version fix migration ([#4050](https://github.com/windmill-labs/windmill/issues/4050)) ([eda6629](https://github.com/windmill-labs/windmill/commit/eda6629bb381469ba27e5a479bb39f7428920aec))
+* make dedicated workers work with dates in bun ([7f49798](https://github.com/windmill-labs/windmill/commit/7f49798e3fb7a2aa528388a1718fd459de19deb8))
+* update flow path ([#4053](https://github.com/windmill-labs/windmill/issues/4053)) ([82c88ad](https://github.com/windmill-labs/windmill/commit/82c88ad3b893e3222391ade65a10dfb2ef1c0fc7))
+
+## [1.360.0](https://github.com/windmill-labs/windmill/compare/v1.359.0...v1.360.0) (2024-07-08)
+
+
+### Features
+
+* **frontend:** add custom actions header ([#4044](https://github.com/windmill-labs/windmill/issues/4044)) ([9e5a3f9](https://github.com/windmill-labs/windmill/commit/9e5a3f98bdc3b66736b099f5828a4c7060b653cd))
+
+
+### Bug Fixes
+
+* always run flow versioning edge case fix migration ([#4047](https://github.com/windmill-labs/windmill/issues/4047)) ([9b2919b](https://github.com/windmill-labs/windmill/commit/9b2919be329f687740e1169aae61073437d328a5))
+* **frontend:** fix min rows for LightWeightArgInput ([#4045](https://github.com/windmill-labs/windmill/issues/4045)) ([0d35741](https://github.com/windmill-labs/windmill/commit/0d3574158dbe963c940e06b98896f5d4ab3c3edb))
+* improve failure id assignment for parallel forloop ([5657ccd](https://github.com/windmill-labs/windmill/commit/5657ccdbe4b19756f6365579d8ef4760450d08d7))
+* persist enums change for string type ([a264a38](https://github.com/windmill-labs/windmill/commit/a264a383de5e57c25b4d3b9250f260f3ed64fc2d))
+
+## [1.359.0](https://github.com/windmill-labs/windmill/compare/v1.358.1...v1.359.0) (2024-07-08)
+
+
+### Features
+
+* **frontend:** improve table actions ([#4040](https://github.com/windmill-labs/windmill/issues/4040)) ([0ffac69](https://github.com/windmill-labs/windmill/commit/0ffac69406e1cd068a172b528037bb396fe9e092))
+
+
+### Bug Fixes
+
+* **backend:** make value of flow_version not null ([#4039](https://github.com/windmill-labs/windmill/issues/4039)) ([11b05b3](https://github.com/windmill-labs/windmill/commit/11b05b3d17897d45cda032353eecbd24ad7f2ec5))
+* flow versioning edge case ([#4037](https://github.com/windmill-labs/windmill/issues/4037)) ([729f911](https://github.com/windmill-labs/windmill/commit/729f911b436613991c351c6c48db9429d6a8e8cb))
+* **frontend:** fix navbar item overflow ([#4041](https://github.com/windmill-labs/windmill/issues/4041)) ([95c2e1b](https://github.com/windmill-labs/windmill/commit/95c2e1b21c1e0ca169513bf1fa773fe9e8dd8e81))
+* improve handling of schedules with retries and concurrency limits ([e100622](https://github.com/windmill-labs/windmill/commit/e10062234efa2b87364468a089b6a888123f73b6))
+* improve input history ([db19c86](https://github.com/windmill-labs/windmill/commit/db19c86a2d5da734fb3485056447b5bf879bd68e))
+* improve performance of content search for large repos ([299e7cf](https://github.com/windmill-labs/windmill/commit/299e7cf5cf87aedf53b97d002545031de49fc9b9))
+
 ## [1.358.1](https://github.com/windmill-labs/windmill/compare/v1.358.0...v1.358.1) (2024-07-05)
 
 

Caddyfile

@@ -2,6 +2,7 @@
         bind {$ADDRESS}
         reverse_proxy /ws/* http://lsp:3001
         # reverse_proxy /ws_mp/* http://multiplayer:3002
+        # reverse_proxy /api/srch/* http://windmill_indexer:8001
         reverse_proxy /* http://windmill_server:8000
         # tls /certs/cert.pem /certs/key.pem
 }

Dockerfile

@@ -1,5 +1,5 @@
 ARG DEBIAN_IMAGE=debian:bookworm-slim
-ARG RUST_IMAGE=rust:1.78-slim-bookworm
+ARG RUST_IMAGE=rust:1.79-slim-bookworm
 ARG PYTHON_IMAGE=python:3.11.8-slim-bookworm
 
 FROM ${RUST_IMAGE} AS rust_base
@@ -41,6 +41,7 @@ COPY /typescript-client/docs/ /frontend/static/tsdocs/
 
 RUN npm run generate-backend-client
 ENV NODE_OPTIONS "--max-old-space-size=8192"
+ARG VITE_BASE_URL ""
 RUN npm run build
 
 
@@ -154,9 +155,9 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
 COPY --from=builder /frontend/build /static_frontend
 COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
 
-COPY --from=denoland/deno:1.44.4 --chmod=755 /usr/bin/deno /usr/bin/deno
+COPY --from=denoland/deno:1.45.4 --chmod=755 /usr/bin/deno /usr/bin/deno
 
-COPY --from=oven/bun:1.1.18 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:1.1.21 /usr/local/bin/bun /usr/bin/bun
 
 COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
 COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer
@@ -172,4 +173,4 @@ RUN windmill cache
 
 EXPOSE 8000
 
-CMD ["windmill"]
+CMD ["windmill"]

README.md

@@ -3,7 +3,7 @@
 </p>
 
 <p align=center>
-Open-source developer infrastructure for internal tools (APIs, background jobs, workflows and UIs). Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs and custom UIs to trigger workflows and scripts as internal apps.
+Open-source developer infrastructure for internal tools (APIs, background jobs, workflows and UIs). Self-hostable alternative to Retool, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs and custom UIs to trigger workflows and scripts as internal apps.
 
 <p align=center>
 Scripts are turned into sharable UIs automatically, and can be composed together into flows or used into richer apps built with low-code. Supported script languages supported are: Python, TypeScript, Go, Bash, SQL, and GraphQL.
@@ -110,8 +110,8 @@ You can build your entire infra on top of Windmill!
 
 ```typescript
 //import any dependency  from npm
-import * as wmill from "https://deno.land/x/windmill@v1.136.0/mod.ts";
-import cowsay from "npm:cowsay@1.5.0";
+import * as wmill from "windmill-client"
+import * as cowsay from 'cowsay@1.5.0';
 
 // fill the type, or use the +Resource type to get a type-safe reference to a resource
 type Postgresql = {
@@ -131,7 +131,7 @@ export async function main(
   e = { nested: "object" }
   //f: wmill.Base64
 ) {
-  const email = Deno.env.get("WM_EMAIL");
+  const email = process.env["WM_EMAIL"];
   // variables are permissioned and by path
   let variable = await wmill.getVariable("f/company-folder/my_secret");
   const lastTimeRun = await wmill.getState();
@@ -146,10 +146,8 @@ export async function main(
 
 ## CLI
 
-We have a powerful CLI to interact with the windmill platform and sync your
-scripts from local files, github repos and to run scripts and flows on the
-instance from local commands. See
-[more details](https://github.com/windmill-labs/windmill/tree/main/cli).
+We have a powerful CLI to interact with the windmill platform and sync your scripts from local files, GitHub repos and to run scripts and flows on the instance from local commands. See
+[more details](https://www.windmill.dev/docs/advanced/cli).
 
 ![CLI Screencast](./cli/vhs/output/setup.gif)
 
@@ -165,22 +163,21 @@ Code extension: <https://www.windmill.dev/docs/cli_local_dev/vscode-extension>.
 
 ## Stack
 
-- Postgres as the database
-- backend in Rust with the following highly-available and horizontally scalable
-  architecture:
-  - stateless API backend
-  - workers that pull jobs from a queue in Postgres (and later, Kafka or Redis.
-    Upvote [#173](#https://github.com/windmill-labs/windmill/issues/173) if
-    interested )
-- frontend in Svelte
-- scripts executions are sandboxed using google's
-  [nsjail](https://github.com/google/nsjail)
-- javascript runtime is the
+- Postgres as the database.
+- Backend in Rust with the following highly-available and horizontally scalable.
+  Architecture:
+  - Stateless API backend.
+  - Workers that pull jobs from a queue in Postgres (and later, Kafka or Redis.
+    Upvote [#173](#https://github.com/windmill-labs/windmill/issues/173) if interested).
+- Frontend in Svelte.
+- Scripts executions are sandboxed using Google's
+  [nsjail](https://github.com/google/nsjail).
+- Javascript runtime is the
   [deno_core rust library](https://denolib.gitbook.io/guide/) (which itself uses
-  the [rusty_v8](https://github.com/denoland/rusty_v8) and hence V8 underneath)
-- typescript runtime is deno
-- python runtime is python3
-- golang runtime is 1.19.1
+  the [rusty_v8](https://github.com/denoland/rusty_v8) and hence V8 underneath).
+- TypeScript runtime is Bun and deno.
+- Python runtime is python3.
+- Golang runtime is 1.19.1.
 
 ## Fastest Self-Hostable Workflow Engine
 
@@ -399,4 +396,4 @@ running options.
 
 ## Copyright
 
-Windmill Labs, Inc 2023
+Windmill Labs, Inc 2023

backend/.gitignore

@@ -4,4 +4,5 @@ oauth.json
 oauth2.json
 windmill-api/openapi-deref.yaml
 tracing.folded
-heaptrack*
+heaptrack*
+index/

backend/.sqlx/query-061ff848f258dc880bec81d923370c905e689f37c6d931ee4559c3cfd394e168.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT flow_status FROM completed_job WHERE id = $1 AND workspace_id = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "flow_status",
-        "type_info": "Jsonb"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Uuid",
-        "Text"
-      ]
-    },
-    "nullable": [
-      true
-    ]
-  },
-  "hash": "061ff848f258dc880bec81d923370c905e689f37c6d931ee4559c3cfd394e168"
-}

backend/.sqlx/query-07834003a631acac03a52a128d3dfce162abdef0d94e5020b84fed424dfd1ae9.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO email_to_igroup (email, igroup) VALUES ($1, $2)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "07834003a631acac03a52a128d3dfce162abdef0d94e5020b84fed424dfd1ae9"
+}

backend/.sqlx/query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json

@@ -55,7 +55,8 @@
                 "graphql",
                 "powershell",
                 "mssql",
-                "php"
+                "php",
+                "bunnative"
               ]
             }
           }

backend/.sqlx/query-15105be6247457fc01b7d65767ccdde047d0f0c172c7a01eeabe3bd8206a3069.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO windmill_migrations (name) VALUES ($1) ON CONFLICT DO NOTHING",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "15105be6247457fc01b7d65767ccdde047d0f0c172c7a01eeabe3bd8206a3069"
+}

backend/.sqlx/query-16c93e721e30b72197e6143e0ebe46931ce25efc5ea0c53e160fcd98a96f4306.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM password",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "16c93e721e30b72197e6143e0ebe46931ce25efc5ea0c53e160fcd98a96f4306"
+}

backend/.sqlx/query-170f620fbd99269d194d14d56f6a3863d9db5fe736a0a34325b824d9cec9b1a0.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO completed_job AS cj\n                   ( workspace_id\n                   , id\n                   , parent_job\n                   , created_by\n                   , created_at\n                   , started_at\n                   , duration_ms\n                   , success\n                   , script_hash\n                   , script_path\n                   , args\n                   , result\n                   , raw_code\n                   , raw_lock\n                   , canceled\n                   , canceled_by\n                   , canceled_reason\n                   , job_kind\n                   , schedule_path\n                   , permissioned_as\n                   , flow_status\n                   , raw_flow\n                   , is_flow_step\n                   , is_skipped\n                   , language\n                   , email\n                   , visible_to_owner\n                   , mem_peak\n                   , tag\n                   , priority\n                )\n                SELECT  workspace_id\n                   , id\n                   , parent_job\n                   , created_by\n                   , created_at\n                   , now()\n                   , 0\n                   , false\n                   , script_hash\n                   , script_path\n                   , args\n                   , $4\n                   , raw_code\n                   , raw_lock\n                   , true\n                   , $1\n                   , canceled_reason\n                   , job_kind\n                   , schedule_path\n                   , permissioned_as\n                   , flow_status\n                   , raw_flow\n                   , is_flow_step\n                   , false\n                   , language\n                   , email\n                   , visible_to_owner\n                   , mem_peak\n                   , tag\n                   , priority FROM queue \n        WHERE id = any($2) AND running = false AND parent_job IS NULL AND workspace_id = $3 AND schedule_path IS NULL FOR UPDATE SKIP LOCKED\n        ON CONFLICT (id) DO NOTHING RETURNING id",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "UuidArray",
+        "Text",
+        "Jsonb"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "170f620fbd99269d194d14d56f6a3863d9db5fe736a0a34325b824d9cec9b1a0"
+}

backend/.sqlx/query-188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT\n                -- slack_team_id, \n                -- slack_name, \n                -- slack_command_script, \n                -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n                auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n                CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n                CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n                webhook, \n                deploy_to, \n                error_handler, \n                openai_resource_path, \n                code_completion_enabled, \n                error_handler_extra_args, \n                error_handler_muted_on_cancel, \n                large_file_storage, \n                git_sync, \n                default_app,\n                default_scripts \n            FROM workspace_settings\n            WHERE workspace_id = $1",
+  "query": "SELECT\n                -- slack_team_id, \n                -- slack_name, \n                -- slack_command_script, \n                -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n                auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n                CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n                CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n                webhook, \n                deploy_to, \n                error_handler, \n                openai_resource_path, \n                code_completion_enabled, \n                error_handler_extra_args, \n                error_handler_muted_on_cancel, \n                large_file_storage, \n                git_sync,\n                default_app,\n                default_scripts,\n                workspace.name\n            FROM workspace_settings\n            LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id\n            WHERE workspace_id = $1",
   "describe": {
     "columns": [
       {
@@ -72,6 +72,11 @@
         "ordinal": 13,
         "name": "default_scripts",
         "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 14,
+        "name": "name",
+        "type_info": "Varchar"
       }
     ],
     "parameters": {
@@ -93,8 +98,9 @@
       true,
       true,
       true,
-      true
+      true,
+      false
     ]
   },
-  "hash": "8626f698fd20f2da77edcd9912a6f840f49002353d5c900a25a3024d7634a89c"
+  "hash": "188534f4b29f6461b1a6214d060f183c830b19a403ebb7b8be55a691675010c3"
 }

backend/.sqlx/query-1a612eb0b64eddd2c5657ef73598c47886545796424f8612135b711e2b9ddb6c.json

@@ -49,7 +49,8 @@
                 "graphql",
                 "powershell",
                 "mssql",
-                "php"
+                "php",
+                "bunnative"
               ]
             }
           }

backend/.sqlx/query-234a278f20cb73f8ce10d2bfb67af58e5dd888581467c976e76f140b2c00f6d7.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO instance_group (name, summary, id, scim_display_name, external_id) VALUES ($1, $2, $3, $4, $5)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "234a278f20cb73f8ce10d2bfb67af58e5dd888581467c976e76f140b2c00f6d7"
+}

backend/.sqlx/query-2bfb918104568288bb57e64d1cf914c14cd493f96005017c299a805c48aef092.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "args",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text",
+        "Bool"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "2bfb918104568288bb57e64d1cf914c14cd493f96005017c299a805c48aef092"
+}

backend/.sqlx/query-39154d865946cb99219ae0139dcc758523610d1260ff59aace18abaf4869edd3.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM email_to_igroup",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "39154d865946cb99219ae0139dcc758523610d1260ff59aace18abaf4869edd3"
+}

backend/.sqlx/query-3b4b62161a5197f37850c8c4197ea026d8e94a3cb9cdcfa5fde19343acf81ecc.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4) WHERE id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Uuid",
+        "Text",
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "3b4b62161a5197f37850c8c4197ea026d8e94a3cb9cdcfa5fde19343acf81ecc"
+}

backend/.sqlx/query-43cc4e7d9846dbce81b61a25e8ceca636a08451db3288969ed9105fa0cb4e5b8.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT worker, worker_instance, vcpus, ping_at, started_at FROM worker_ping WHERE ping_at > now() - interval '30 days'",
+  "query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at FROM worker_ping WHERE ping_at > now() - interval '30 days'",
   "describe": {
     "columns": [
       {
@@ -20,11 +20,16 @@
       },
       {
         "ordinal": 3,
+        "name": "memory",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 4,
         "name": "ping_at",
         "type_info": "Timestamptz"
       },
       {
-        "ordinal": 4,
+        "ordinal": 5,
         "name": "started_at",
         "type_info": "Timestamptz"
       }
@@ -36,9 +41,10 @@
       false,
       false,
       true,
+      true,
       false,
       false
     ]
   },
-  "hash": "8480792eefdd0f31a49afec0f91262a6973c068da5413b154eff9b9da90db95f"
+  "hash": "43cc4e7d9846dbce81b61a25e8ceca636a08451db3288969ed9105fa0cb4e5b8"
 }

backend/.sqlx/query-4a764cdcb847b71183425a7a0e863708ef7fe2c88b28d0667988a47b9e995c0e.json

@@ -16,7 +16,7 @@
       ]
     },
     "nullable": [
-      true
+      false
     ]
   },
   "hash": "4a764cdcb847b71183425a7a0e863708ef7fe2c88b28d0667988a47b9e995c0e"

backend/.sqlx/query-51a7e36ba1cc1616a19485eb5930cf90df758e85a8cac6037d044d5bd7440681.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "51a7e36ba1cc1616a19485eb5930cf90df758e85a8cac6037d044d5bd7440681"
+}

backend/.sqlx/query-598d4be56bb8c5174b279c183fffa96f763e48316fb7420f2aad733562cab582.json

@@ -0,0 +1,68 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT email, password_hash, login_type, super_admin, verified, name, company, first_time_user, username FROM password",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "email",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "password_hash",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "login_type",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "super_admin",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 4,
+        "name": "verified",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 5,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 6,
+        "name": "company",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 7,
+        "name": "first_time_user",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 8,
+        "name": "username",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      true,
+      false,
+      false,
+      false,
+      true,
+      true,
+      false,
+      true
+    ]
+  },
+  "hash": "598d4be56bb8c5174b279c183fffa96f763e48316fb7420f2aad733562cab582"
+}

backend/.sqlx/query-5dfa6932d7c6d5006fe352da3041680b2ee1ebe9258355a0db24ce4fd26f23de.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 AND workspace_id = $4 AND canceled = false RETURNING id",
+  "query": "UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 AND workspace_id = $4 AND (canceled = false OR canceled_reason != $2) RETURNING id",
   "describe": {
     "columns": [
       {
@@ -21,5 +21,5 @@
       false
     ]
   },
-  "hash": "aa923ddc251f5daf4bf075cb251dfddf87877f12fbbc8fbf3cfd08f43bbad8a0"
+  "hash": "5dfa6932d7c6d5006fe352da3041680b2ee1ebe9258355a0db24ce4fd26f23de"
 }

backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json

@@ -69,7 +69,8 @@
                 "graphql",
                 "powershell",
                 "mssql",
-                "php"
+                "php",
+                "bunnative"
               ]
             }
           }

backend/.sqlx/query-6fa6fa8eb511119c6adb18bd4f9f174bbb48eef8952f91911b8d9b45357372d9.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM completed_job WHERE id = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "args",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text",
+        "Bool"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "6fa6fa8eb511119c6adb18bd4f9f174bbb48eef8952f91911b8d9b45357372d9"
+}

backend/.sqlx/query-8455e77a6a87bd0b8dbe55c854abfb3938cb9a440e9d53323a93ec9473417890.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id FROM queue WHERE created_at <= $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Timestamptz"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "8455e77a6a87bd0b8dbe55c854abfb3938cb9a440e9d53323a93ec9473417890"
+}

backend/.sqlx/query-8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c.json

@@ -54,7 +54,8 @@
                 "graphql",
                 "powershell",
                 "mssql",
-                "php"
+                "php",
+                "bunnative"
               ]
             }
           }

backend/.sqlx/query-8d655c34a00510699d2ad7044f7e526ba5082e5d1945c76a98404fe5d92e32ee.json

@@ -0,0 +1,15 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM queue WHERE id = any($1) AND workspace_id = $2",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "UuidArray",
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "8d655c34a00510699d2ad7044f7e526ba5082e5d1945c76a98404fe5d92e32ee"
+}

backend/.sqlx/query-8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb.json

@@ -35,7 +35,8 @@
                 "graphql",
                 "powershell",
                 "mssql",
-                "php"
+                "php",
+                "bunnative"
               ]
             }
           }

backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT created_at FROM script WHERE workspace_id = $1 AND hash = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "created_at",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b"
+}

backend/.sqlx/query-919c9432f36a6212ba79b50fbf887589bf7c37d3557cd831787b9b16b3665e47.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT MAX(created_at) FROM completed_job",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "max",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "919c9432f36a6212ba79b50fbf887589bf7c37d3557cd831787b9b16b3665e47"
+}

backend/.sqlx/query-9cd6e964ba933de247ba7ddce0acef0b70b784c2410f3a5f51288aaea6904b8f.json

@@ -0,0 +1,50 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT name, summary, array_remove(array_agg(email_to_igroup.email), null) as emails, id, scim_display_name, external_id FROM email_to_igroup RIGHT JOIN instance_group ON instance_group.name = email_to_igroup.igroup GROUP BY name",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "summary",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "emails",
+        "type_info": "VarcharArray"
+      },
+      {
+        "ordinal": 3,
+        "name": "id",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "scim_display_name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 5,
+        "name": "external_id",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      true,
+      null,
+      true,
+      true,
+      true
+    ]
+  },
+  "hash": "9cd6e964ba933de247ba7ddce0acef0b70b784c2410f3a5f51288aaea6904b8f"
+}

backend/.sqlx/query-9d25fbd21a63e6e9ccfdbf0460c8b3ca6d7fa05600067f379659b59b6f2bf418.json

@@ -54,7 +54,8 @@
                 "graphql",
                 "powershell",
                 "mssql",
-                "php"
+                "php",
+                "bunnative"
               ]
             }
           }

backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json

@@ -49,7 +49,8 @@
                 "graphql",
                 "powershell",
                 "mssql",
-                "php"
+                "php",
+                "bunnative"
               ]
             }
           }

backend/.sqlx/query-9dd0cf1627f9e767af8758f26d8bf9e96a48e45daf4a3c54f7b81a7b92d39431.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT pg_try_advisory_lock(4242)",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "pg_try_advisory_lock",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "9dd0cf1627f9e767af8758f26d8bf9e96a48e45daf4a3c54f7b81a7b92d39431"
+}

backend/.sqlx/query-9f0783aae9cf5a6de2ca783aeba7a8c2f0eb5cc11cb2f49222f5eef4a21d8c41.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO password(email, password_hash, login_type, super_admin, verified, name, company, first_time_user, username)\n            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Varchar",
+        "Varchar",
+        "Varchar",
+        "Bool",
+        "Bool",
+        "Varchar",
+        "Varchar",
+        "Bool",
+        "Varchar"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "9f0783aae9cf5a6de2ca783aeba7a8c2f0eb5cc11cb2f49222f5eef4a21d8c41"
+}

backend/.sqlx/query-a22146c0a924e5a1a99bc72775399a8c3db31b57ae9ded5bff55f44321c3f3c2.json

@@ -0,0 +1,24 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "args",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text",
+        "Bool"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a22146c0a924e5a1a99bc72775399a8c3db31b57ae9ded5bff55f44321c3f3c2"
+}

backend/.sqlx/query-a6af0760c20eb9a7db648ea5d643b4c5703b776dc6cb1c5315e26569a2daa240.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'fix_flow_versioning_2')",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "a6af0760c20eb9a7db648ea5d643b4c5703b776dc6cb1c5315e26569a2daa240"
+}

backend/.sqlx/query-b1f2ab29cb09a213a38a711d5c228f4f735a3343fbae15d2bc3510c3308d16ba.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT name, config FROM config",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "config",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      true
+    ]
+  },
+  "hash": "b1f2ab29cb09a213a38a711d5c228f4f735a3343fbae15d2bc3510c3308d16ba"
+}

backend/.sqlx/query-b30a29fa6edbeacaddad2a8b79a9aa73fcb0e8cb1f5e863f31faf98e4137b5e6.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT name, value FROM global_settings",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "value",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "b30a29fa6edbeacaddad2a8b79a9aa73fcb0e8cb1f5e863f31faf98e4137b5e6"
+}

backend/.sqlx/query-bbd5f968d7b62a55a7ebf7b98cfd411678ce544fdb9472d8d223235fb2818aaf.json

@@ -1,23 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT args FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT args FROM input WHERE id = $1 AND workspace_id = $2",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "args",
-        "type_info": "Jsonb"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Uuid",
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "bbd5f968d7b62a55a7ebf7b98cfd411678ce544fdb9472d8d223235fb2818aaf"
-}

backend/.sqlx/query-c4d032639dbfb47c84a2fbcb4cdca75af3007499a9f6c8c492a32fac5274aeb2.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO windmill_migrations (name) VALUES ('fix_flow_versioning_2')",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "c4d032639dbfb47c84a2fbcb4cdca75af3007499a9f6c8c492a32fac5274aeb2"
+}

backend/.sqlx/query-c7dbcb52f6b44c8fc0d6240b7d73717ee7f24abd54fab5b4a687490b7c0b3672.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM instance_group",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": []
+  },
+  "hash": "c7dbcb52f6b44c8fc0d6240b7d73717ee7f24abd54fab5b4a687490b7c0b3672"
+}

backend/.sqlx/query-c8ac658002423906f2a6e43e423a9e24561dfaa6b07ce89063d0910d2342a83b.json

@@ -1,30 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT leaf_jobs->$1::text as leaf_jobs, parent_job FROM queue WHERE COALESCE((SELECT root_job FROM queue WHERE id = $2), $2) = id AND workspace_id = $3",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "leaf_jobs",
-        "type_info": "Jsonb"
-      },
-      {
-        "ordinal": 1,
-        "name": "parent_job",
-        "type_info": "Uuid"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Uuid",
-        "Text"
-      ]
-    },
-    "nullable": [
-      null,
-      true
-    ]
-  },
-  "hash": "c8ac658002423906f2a6e43e423a9e24561dfaa6b07ce89063d0910d2342a83b"
-}

backend/.sqlx/query-d5a8614286c170e0d175903cd1b53ff66b37ed8110a0b67aedb9f25e6a7383e1.json

@@ -75,7 +75,8 @@
                 "graphql",
                 "powershell",
                 "mssql",
-                "php"
+                "php",
+                "bunnative"
               ]
             }
           }

backend/.sqlx/query-e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99.json

@@ -54,7 +54,8 @@
                 "graphql",
                 "powershell",
                 "mssql",
-                "php"
+                "php",
+                "bunnative"
               ]
             }
           }

backend/.sqlx/query-ec47955683d811b12e82ff3e6aeafa0df46a320c436bad5eb1acc127df138a61.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3",
+  "query": "UPDATE script SET lock = $1, created_at = now() WHERE hash = $2 AND workspace_id = $3",
   "describe": {
     "columns": [],
     "parameters": {
@@ -12,5 +12,5 @@
     },
     "nullable": []
   },
-  "hash": "d697b7311430e7bd5375ec5494179f4071e3bfe123d9600249cfa80c1103edd8"
+  "hash": "ec47955683d811b12e82ff3e6aeafa0df46a320c436bad5eb1acc127df138a61"
 }

backend/.sqlx/query-f04fa0262091c5a4abf7dddafeec1b34ed806dda70592b0030067aed46a104d9.json

@@ -0,0 +1,41 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT created_by, CONCAT(coalesce(completed_job.logs, ''), coalesce(job_logs.logs, '')) as logs, job_logs.log_offset, job_logs.log_file_index\n        FROM completed_job\n        LEFT JOIN job_logs ON job_logs.job_id = completed_job.id\n        WHERE completed_job.id = $1 AND completed_job.workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "created_by",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "logs",
+        "type_info": "Text"
+      },
+      {
+        "ordinal": 2,
+        "name": "log_offset",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 3,
+        "name": "log_file_index",
+        "type_info": "TextArray"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      null,
+      false,
+      true
+    ]
+  },
+  "hash": "f04fa0262091c5a4abf7dddafeec1b34ed806dda70592b0030067aed46a104d9"
+}

backend/.sqlx/query-f3d20b0fa17836538ec93b84b08ffc6b555371d6eeeadfc54f5bf0bcbe93d8b4.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT COALESCE((SELECT MIN(started_at) as min_started_at\n            FROM queue\n            WHERE script_path = $1 AND job_kind != 'dependencies'  AND running = true AND workspace_id = $2 AND canceled = false\n            GROUP BY script_path), $3) as min_started_at, now() AS now",
+  "query": "SELECT COALESCE((SELECT MIN(started_at) as min_started_at\n            FROM queue\n            WHERE script_path = $1 AND job_kind != 'dependencies'  AND running = true AND workspace_id = $2 AND canceled = false AND concurrent_limit > 0), $3) as min_started_at, now() AS now",
   "describe": {
     "columns": [
       {
@@ -26,5 +26,5 @@
       null
     ]
   },
-  "hash": "3901cce744c9b246b661c817e068bdb3b1ab504ff8070fcccf6c909ad75f1f6d"
+  "hash": "f3d20b0fa17836538ec93b84b08ffc6b555371d6eeeadfc54f5bf0bcbe93d8b4"
 }

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.358.1"
+version = "1.371.4"
 authors.workspace = true
 edition.workspace = true
 
@@ -13,6 +13,7 @@ members = [
     "./windmill-common",
     "./windmill-audit",
     "./windmill-git-sync",
+    "./windmill-indexer",
     "./parsers/windmill-parser",
     "./parsers/windmill-parser-ts",
     "./parsers/windmill-parser-wasm",
@@ -24,7 +25,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.358.1"
+version = "1.371.4"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -38,7 +39,7 @@ incremental = true
 
 [features]
 default = []
-enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise"]
+enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"]
 enterprise_saml = ["windmill-api/enterprise_saml"]
 stripe = ["windmill-api/stripe"]
 benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark"]
@@ -46,12 +47,13 @@ flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
 loki = ["windmill-common/loki"]
 pg_embed = ["dep:pg-embed"]
 embedding = ["windmill-api/embedding"]
-parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet"]
+parquet = ["windmill-api/parquet", "windmill-common/parquet", "windmill-worker/parquet", "windmill-indexer/parquet"]
 prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill-worker/prometheus", "windmill-queue/prometheus"]
 flow_testing = ["windmill-worker/flow_testing"]
 openidconnect = ["windmill-api/openidconnect"]
 cloud = ["windmill-queue/cloud", "windmill-worker/cloud"]
 jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"]
+tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"]
 
 [dependencies]
 anyhow.workspace = true
@@ -62,6 +64,7 @@ windmill-common = { workspace = true, default-features = false }
 windmill-git-sync.workspace = true
 windmill-api = { workspace = true, default-features = false }
 windmill-worker.workspace = true
+windmill-indexer = { workspace = true, optional = true }
 futures.workspace = true
 tracing.workspace = true
 sqlx.workspace = true
@@ -80,7 +83,8 @@ gethostname.workspace = true
 serde_json.workspace = true
 serde.workspace = true
 deno_core.workspace = true
-pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false, features = ['rt_tokio']} 
+pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false, features = ['rt_tokio']}
+
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 tikv-jemallocator = { optional = true, workspace = true }
@@ -103,6 +107,7 @@ windmill-worker = { path = "./windmill-worker" }
 windmill-common = { path = "./windmill-common", default-features = false }
 windmill-audit = { path = "./windmill-audit" }
 windmill-git-sync = { path = "./windmill-git-sync" }
+windmill-indexer = {path = "./windmill-indexer"}
 windmill-parser = { path = "./parsers/windmill-parser" }
 windmill-parser-ts = { path = "./parsers/windmill-parser-ts" }
 windmill-parser-py = { path = "./parsers/windmill-parser-py" }
@@ -159,15 +164,15 @@ tokio-util = { version = "^0", features = ["io"] }
 json-pointer = "^0"
 itertools = "^0"
 regex = "^1"
-deno_fetch = "0.182.0"
-deno_tls = "0.145.0"
-deno_console = "0.158.0"
-deno_url = "0.158.0"
-deno_webidl = "0.158.0"
-deno_web = "0.189.0"
-deno_net = "0.150.0"
-deno_core = "0.290.0"
-deno_ast = { version = "=0.39.2", features = ["transpiling"] }
+deno_fetch = "0.187.0"
+deno_tls = "0.150.0"
+deno_console = "0.163.0"
+deno_url = "0.163.0"
+deno_webidl = "0.163.0"
+deno_web = "0.194.0"
+deno_net = "0.155.0"
+deno_core = "0.299.0"
+deno_ast = { version = "=0.40.0", features = ["transpiling"] }
 async-recursion = "^1"
 swc_common = "=0.33.26"
 swc_ecma_parser = "=0.144.3"
@@ -176,7 +181,7 @@ swc_ecma_visit = "=0.99.1"
 base64 = "0.21.0"
 hmac = "0.12.1"
 sha2 = "0.10.6"
-sqlx = { version = "0.7.3", features = [
+sqlx = { version = "0.8.0", features = [
     "macros",
     "migrate",
     "uuid",
@@ -219,7 +224,7 @@ mysql_async = { version = "*", default-features = false, features = ["minimal",
 postgres-native-tls = "^0"
 native-tls = "^0"
 # samael will break compilation on MacOS. Use this fork instead to make it work
-# samael = { git="https://github.com/njaremko/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = ["xmlsec"] }
+# samael = { git="https://github.com/njaremko/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = ["xmlsec"] }
 samael = { version="0.0.14", features = ["xmlsec"] }
 gcp_auth = "0.9.0"
 rust_decimal = { version = "^1", features = ["db-postgres"]}
@@ -254,3 +259,5 @@ tikv-jemalloc-ctl = { version = "^0.5" }
 
 # 0.1.12 broken (nested dependency of swc_common) 
 triomphe = "<0.1.12"
+
+tantivy = "0.22.0"

backend/custom_migrations/fix_flow_versioning_2.sql

@@ -0,0 +1,20 @@
+INSERT INTO flow_version (workspace_id, path, value, schema, created_by, created_at) 
+SELECT workspace_id, path, value, schema, edited_by, edited_at 
+	FROM flow 
+	WHERE NOT EXISTS (SELECT 1 FROM flow_version WHERE flow_version.workspace_id = flow.workspace_id AND flow_version.path = flow.path);
+UPDATE flow
+SET versions = subquery.versions
+FROM (
+	SELECT
+		path,
+		workspace_id,
+		array_agg(id ORDER BY created_at ASC) AS versions
+	FROM
+		flow_version
+	GROUP BY
+		path,
+		workspace_id
+) subquery
+WHERE
+	flow.path = subquery.path
+	AND flow.workspace_id = subquery.workspace_id;

backend/ee-repo-ref.txt

@@ -1 +1 @@
-d414242f92cfaf9f364e9aeb321a511c3ad46fa7
+a64d2990e733aba3f8cbc6691a6090bcd481c891

backend/migrations/20240708075901_flow_versioning_nit.down.sql

@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE flow_version ALTER COLUMN value DROP NOT NULL;

backend/migrations/20240708075901_flow_versioning_nit.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+UPDATE flow_version SET value = '{"modules":[]}'::jsonb WHERE value IS NULL;
+ALTER TABLE flow_version ALTER COLUMN value SET NOT NULL;

backend/migrations/20240711065915_remove_env_id_default.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20240711065915_remove_env_id_default.up.sql

@@ -0,0 +1,5 @@
+-- Add up migration script here
+ALTER TABLE queue ALTER COLUMN env_id DROP NOT NULL;
+ALTER TABLE queue ALTER COLUMN env_id DROP DEFAULT;
+ALTER TABLE completed_job ALTER COLUMN env_id DROP NOT NULL;
+ALTER TABLE completed_job ALTER COLUMN env_id DROP DEFAULT;

backend/migrations/20240727103951_min_started_at_index.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20240727103951_min_started_at_index.up.sql

@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS queue_concurrency_started_at ON queue (workspace_id, script_path, started_at) WHERE concurrent_limit > 0 AND running = true AND job_kind != 'dependencies' AND canceled = false;-- Add up migration script here

backend/migrations/20240729064122_addbunnative.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20240729064122_addbunnative.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TYPE SCRIPT_LANG ADD VALUE IF NOT EXISTS 'bunnative';

backend/migrations/20240731093818_remove_foreign_key.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20240731093818_remove_foreign_key.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+alter table queue drop constraint if exists queue_workspace_id_fkey;
+alter table completed_job drop constraint if exists completed_job_workspace_id_fkey;

backend/parsers/windmill-parser-bash/src/lib.rs

@@ -58,6 +58,7 @@ fn parse_bash_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
                 default: default.clone().map(|x| json!(x)),
                 otyp: None,
                 has_default: default.is_some(),
+                oidx: None,
             });
         } else {
             break;
@@ -94,6 +95,7 @@ fn parse_powershell_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
                 default: default.clone(),
                 otyp: None,
                 has_default: default.is_some(),
+                oidx: None,
             });
         }
     }
@@ -130,35 +132,40 @@ non_required="${5:-}"
                         name: "token".to_string(),
                         typ: Typ::Str(None),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "image".to_string(),
                         typ: Typ::Str(None),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "digest".to_string(),
                         typ: Typ::Str(None),
                         default: Some(json!("latest with spaces")),
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "text".to_string(),
                         typ: Typ::Str(None),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "non_required".to_string(),
                         typ: Typ::Str(None),
                         default: Some(json!("")),
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     }
                 ],
                 no_main_func: None

backend/parsers/windmill-parser-go/src/lib.rs

@@ -27,7 +27,14 @@ pub fn parse_go_sig(code: &str) -> anyhow::Result<MainArgSignature> {
             .iter()
             .map(|param| {
                 let (otyp, typ) = parse_go_typ(&param.typ);
-                Arg { name: get_name(param), otyp, typ, default: None, has_default: false }
+                Arg {
+                    name: get_name(param),
+                    otyp,
+                    typ,
+                    default: None,
+                    has_default: false,
+                    oidx: None,
+                }
             })
             .collect_vec();
         Ok(MainArgSignature {
@@ -179,28 +186,32 @@ func main(x int, y string, z bool, l []string, o struct { Name string `json:"nam
                         name: "x".to_string(),
                         typ: Typ::Int,
                         has_default: false,
-                        default: None
+                        default: None,
+                        oidx: None
                     },
                     Arg {
                         otyp: Some("string".to_string()),
                         name: "y".to_string(),
                         typ: Typ::Str(None),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                     Arg {
                         otyp: Some("bool".to_string()),
                         name: "z".to_string(),
                         typ: Typ::Bool,
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                     Arg {
                         otyp: Some("[]string".to_string()),
                         name: "l".to_string(),
                         typ: Typ::List(Box::new(Typ::Str(None))),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                     Arg {
                         otyp: Some("struct { Name string `json:\"name\"` }".to_string()),
@@ -210,21 +221,24 @@ func main(x int, y string, z bool, l []string, o struct { Name string `json:"nam
                             typ: Box::new(Typ::Str(None))
                         },]),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                     Arg {
                         otyp: Some("interface{}".to_string()),
                         name: "n".to_string(),
                         typ: Typ::Object(vec![]),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                     Arg {
                         otyp: Some("map[string]interface{}".to_string()),
                         name: "m".to_string(),
                         typ: Typ::Object(vec![]),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                 ],
                 no_main_func: Some(false)

backend/parsers/windmill-parser-graphql/src/lib.rs

@@ -51,6 +51,7 @@ fn parse_graphql_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
             default: parsed_default,
             otyp: Some(typ.unwrap()),
             has_default,
+            oidx: None,
         });
     }
 
@@ -93,21 +94,24 @@ query($i: Int, $arr: [String]!, $wahoo: String = "wahoo") {
                         name: "i".to_string(),
                         typ: Typ::Int,
                         default: None,
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     },
                     Arg {
                         otyp: Some("[String]".to_string()),
                         name: "arr".to_string(),
                         typ: Typ::List(Box::new(Typ::Str(None))),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                     Arg {
                         otyp: Some("String".to_string()),
                         name: "wahoo".to_string(),
                         typ: Typ::Str(None),
                         default: Some(json!("wahoo")),
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     }
                 ],
                 no_main_func: None

backend/parsers/windmill-parser-php/src/lib.rs

@@ -85,6 +85,7 @@ pub fn parse_php_signature(
                     typ,
                     has_default: default.is_some(),
                     default,
+                    oidx: None,
                 }
             })
             .collect();
@@ -135,35 +136,40 @@ function main(string $input1 = \"hey\", bool $input2 = false, int $input3 = 3, f
                         name: "input1".to_string(),
                         typ: Typ::Str(None),
                         has_default: true,
-                        default: Some(Value::String("hey".to_string()))
+                        default: Some(Value::String("hey".to_string())),
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "input2".to_string(),
                         typ: Typ::Bool,
                         has_default: true,
-                        default: Some(Value::Bool(false))
+                        default: Some(Value::Bool(false)),
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "input3".to_string(),
                         typ: Typ::Int,
                         has_default: true,
-                        default: Some(Value::Number(Number::from(3)))
+                        default: Some(Value::Number(Number::from(3))),
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "input4".to_string(),
                         typ: Typ::Float,
                         has_default: true,
-                        default: Some(Value::Number(Number::from_f64(f64::from(4.5)).unwrap()))
+                        default: Some(Value::Number(Number::from_f64(f64::from(4.5)).unwrap())),
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "resource".to_string(),
                         typ: Typ::Resource("stripe".to_string()),
                         has_default: false,
-                        default: None
+                        default: None,
+                        oidx: None
                     }
                 ],
                 no_main_func: Some(false)

backend/parsers/windmill-parser-py/src/lib.rs

@@ -114,6 +114,7 @@ pub fn parse_python_signature(
                         typ,
                         has_default: default.is_some(),
                         default,
+                        oidx: None,
                     }
                 })
                 .collect(),
@@ -187,6 +188,9 @@ fn parse_typ(id: &str) -> Typ {
         "datetime" => Typ::Datetime,
         "datetime.datetime" => Typ::Datetime,
         "Sql" | "sql" => Typ::Sql,
+        x @ _ if x.starts_with("DynSelect_") => {
+            Typ::DynSelect(x.strip_prefix("DynSelect_").unwrap().to_string())
+        }
         _ => Typ::Resource(id.to_string()),
     }
 }
@@ -268,49 +272,56 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt
                         name: "test1".to_string(),
                         typ: Typ::Str(None),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "name".to_string(),
                         typ: Typ::Unknown,
                         default: Some(json!("<function call>")),
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "byte".to_string(),
                         typ: Typ::Bytes,
                         default: Some(json!("<function call>")),
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "f".to_string(),
                         typ: Typ::Str(None),
                         default: Some(json!("wewe")),
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "g".to_string(),
                         typ: Typ::Int,
                         default: Some(json!(21)),
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "h".to_string(),
                         typ: Typ::List(Box::new(Typ::Int)),
                         default: Some(json!([1, 2])),
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "i".to_string(),
                         typ: Typ::Bool,
                         default: Some(json!(true)),
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     },
                 ],
                 no_main_func: Some(false),
@@ -349,28 +360,32 @@ def main(test1: str,
                         name: "test1".to_string(),
                         typ: Typ::Str(None),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "name".to_string(),
                         typ: Typ::Unknown,
                         default: Some(json!("<function call>")),
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "byte".to_string(),
                         typ: Typ::Bytes,
                         default: Some(json!("<function call>")),
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "resource".to_string(),
                         typ: Typ::Resource("postgresql".to_string()),
                         default: Some(json!("$res:g/all/resource")),
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     }
                 ],
                 no_main_func: Some(false),
@@ -404,28 +419,32 @@ def main(test1: str,
                         name: "test1".to_string(),
                         typ: Typ::Str(None),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "s3o".to_string(),
                         typ: Typ::Resource("S3Object".to_string()),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "name".to_string(),
                         typ: Typ::Str(None),
                         default: Some(json!("test")),
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
                         name: "byte".to_string(),
                         typ: Typ::Bytes,
                         default: Some(json!("<function call>")),
-                        has_default: true
+                        has_default: true,
+                        oidx: None
                     }
                 ],
                 no_main_func: Some(false),
@@ -456,7 +475,8 @@ def main(test1: Literal["foo", "bar"], test2: List[Literal["foo", "bar"]]): retu
                         name: "test1".to_string(),
                         typ: Typ::Str(Some(vec!["foo".to_string(), "bar".to_string()])),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     },
                     Arg {
                         otyp: None,
@@ -466,7 +486,8 @@ def main(test1: Literal["foo", "bar"], test2: List[Literal["foo", "bar"]]): retu
                             "bar".to_string()
                         ])))),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None
                     }
                 ],
                 no_main_func: Some(false),
@@ -475,4 +496,34 @@ def main(test1: Literal["foo", "bar"], test2: List[Literal["foo", "bar"]]): retu
 
         Ok(())
     }
+
+    #[test]
+    fn test_parse_python_sig_5() -> anyhow::Result<()> {
+        let code = r#"
+
+import os
+
+def main(test1: DynSelect_foo): return
+
+"#;
+        //println!("{}", serde_json::to_string()?);
+        assert_eq!(
+            parse_python_signature(code, None)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![Arg {
+                    otyp: None,
+                    name: "test1".to_string(),
+                    typ: Typ::DynSelect("foo".to_string()),
+                    default: None,
+                    has_default: false,
+                    oidx: None
+                }],
+                no_main_func: Some(false),
+            }
+        );
+
+        Ok(())
+    }
 }

backend/parsers/windmill-parser-sql/src/lib.rs

@@ -4,7 +4,11 @@ use anyhow::anyhow;
 use regex::Regex;
 use serde_json::json;
 
-use std::collections::HashMap;
+use std::{
+    collections::{HashMap, HashSet},
+    iter::Peekable,
+    str::CharIndices,
+};
 pub use windmill_parser::{Arg, MainArgSignature, Typ};
 
 pub fn parse_mysql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
@@ -62,9 +66,32 @@ pub fn parse_db_resource(code: &str) -> Option<String> {
     cap.map(|x| x.get(1).map(|x| x.as_str().to_string()).unwrap())
 }
 
+pub fn parse_sql_blocks(code: &str) -> Vec<&str> {
+    let mut blocks = vec![];
+    let mut last_idx = 0;
+
+    run_on_sql_statement_matches(
+        code,
+        |char, _| char == ';',
+        |idx, _| {
+            blocks.push(&code[last_idx..=idx]);
+            last_idx = idx + 1;
+        },
+    );
+    if last_idx < code.len() {
+        let last_block = &code[last_idx..];
+        if RE_NONEMPTY_SQL_BLOCK.is_match(last_block) {
+            blocks.push(last_block);
+        }
+    }
+    blocks
+}
+
 lazy_static::lazy_static! {
     static ref RE_CODE_PGSQL: Regex = Regex::new(r#"(?m)\$(\d+)(?:::(\w+(?:\[\])?))?"#).unwrap();
 
+    static ref RE_NONEMPTY_SQL_BLOCK: Regex = Regex::new(r#"(?m)^\s*[^\s](?:[^-]|$)"#).unwrap();
+
     static ref RE_DB: Regex = Regex::new(r#"(?m)^-- database (\S+) *(?:\r|\n|$)"#).unwrap();
 
     // -- $1 name (type) = default
@@ -117,6 +144,7 @@ fn parse_mysql_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
             default: parsed_default,
             otyp: Some(typ),
             has_default,
+            oidx: None,
         });
     }
 
@@ -140,6 +168,7 @@ fn parse_mysql_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
                 default: parsed_default,
                 otyp: Some(typ),
                 has_default,
+                oidx: None,
             });
         }
     }
@@ -147,7 +176,105 @@ fn parse_mysql_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
     Ok(Some(args))
 }
 
+enum ParserState {
+    Normal,
+    InSingleQuote,
+    InDoubleQuote,
+    InSingleLineComment,
+    InMultiLineComment,
+}
+
+fn run_on_sql_statement_matches<
+    F1: FnMut(char, &mut Peekable<CharIndices>) -> bool,
+    F2: FnMut(usize, &mut Peekable<CharIndices>) -> (),
+>(
+    code: &str,
+    mut cond: F1,
+    mut case: F2,
+) {
+    let mut chars = code.char_indices().peekable();
+    let mut state = ParserState::Normal;
+    while let Some((idx, char)) = chars.next() {
+        match (&state, char) {
+            (ParserState::Normal, '\'') => {
+                state = ParserState::InSingleQuote;
+            }
+            (ParserState::Normal, '"') => {
+                state = ParserState::InDoubleQuote;
+            }
+            (ParserState::Normal, '-')
+                if chars.peek().is_some_and(|&(_, next_char)| next_char == '-') =>
+            {
+                state = ParserState::InSingleLineComment;
+            }
+            (ParserState::Normal, '/')
+                if chars.peek().is_some_and(|&(_, next_char)| next_char == '*') =>
+            {
+                state = ParserState::InMultiLineComment;
+            }
+            (ParserState::Normal, _) if cond(char, &mut chars) => {
+                case(idx, &mut chars);
+            }
+            (ParserState::InSingleQuote, '\'') => {
+                if chars
+                    .peek()
+                    .is_some_and(|&(_, next_char)| next_char == '\'')
+                {
+                    chars.next(); // skip the escaped single quote
+                } else {
+                    state = ParserState::Normal;
+                }
+            }
+            (ParserState::InDoubleQuote, '"') => {
+                if chars.peek().is_some_and(|&(_, next_char)| next_char == '"') {
+                    chars.next(); // skip the escaped single quote
+                } else {
+                    state = ParserState::Normal;
+                }
+            }
+            (ParserState::InSingleLineComment, '\n') => {
+                state = ParserState::Normal;
+            }
+            (ParserState::InMultiLineComment, '*')
+                if chars.peek().is_some_and(|&(_, next_char)| next_char == '/') =>
+            {
+                state = ParserState::Normal;
+            }
+            _ => {}
+        }
+    }
+}
+
+pub fn parse_pg_statement_arg_indices(code: &str) -> HashSet<i32> {
+    let mut arg_indices = HashSet::new();
+    run_on_sql_statement_matches(
+        code,
+        |char, chars| {
+            char == '$'
+                && chars
+                    .peek()
+                    .is_some_and(|&(_, next_char)| next_char.is_ascii_digit())
+        },
+        |_, chars| {
+            let mut arg_idx = String::new();
+            while let Some(&(_, char)) = chars.peek() {
+                if char.is_ascii_digit() {
+                    arg_idx.push(char);
+                    chars.next();
+                } else {
+                    break;
+                }
+            }
+            if let Ok(arg_idx) = arg_idx.parse::<i32>() {
+                arg_indices.insert(arg_idx);
+            }
+        },
+    );
+    arg_indices
+}
+
 fn parse_pg_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
+    let mut args = vec![];
     let mut hm: HashMap<i32, String> = HashMap::new();
     for cap in RE_CODE_PGSQL.captures_iter(code) {
         hm.insert(
@@ -159,39 +286,74 @@ fn parse_pg_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
                 .unwrap_or_else(|| "text".to_string()),
         );
     }
-    let mut args = vec![];
-    for i in 1..50 {
-        if hm.contains_key(&i) {
-            let typ = hm.get(&i).unwrap().to_lowercase();
-            args.push(Arg {
-                name: format!("${}", i),
-                typ: parse_pg_typ(typ.as_str()),
-                default: None,
-                otyp: Some(typ),
-                has_default: false,
-            });
-        } else {
-            break;
-        }
+    for (i, v) in hm.iter() {
+        let typ = v.to_lowercase();
+        args.push(Arg {
+            name: format!("${}", i),
+            typ: parse_pg_typ(typ.as_str()),
+            default: None,
+            otyp: Some(typ),
+            has_default: false,
+            oidx: Some(*i),
+        });
     }
+    args.sort_by(|a, b| a.oidx.unwrap().cmp(&b.oidx.unwrap()));
     for cap in RE_ARG_PGSQL.captures_iter(code) {
-        let i = cap.get(1).and_then(|x| x.as_str().parse::<i32>().ok());
-        if i.is_none() || i.unwrap() as usize > args.len() {
-            continue;
-        }
-        let name = cap.get(2).map(|x| x.as_str().to_string()).unwrap();
-        let default = cap.get(3).map(|x| x.as_str().to_string());
-        let has_default = default.is_some();
-        let oarg = args[(i.unwrap() - 1) as usize].clone();
-        let parsed_default = default.and_then(|x| parsed_default(&oarg.typ, x));
+        let i = cap
+            .get(1)
+            .and_then(|x| x.as_str().parse::<i32>().ok())
+            .map(|x| x);
 
-        args[(i.unwrap() - 1) as usize] =
-            Arg { name, typ: oarg.typ, default: parsed_default, otyp: oarg.otyp, has_default };
+        if let Some(arg_pos) = args
+            .iter()
+            .position(|x| i.is_some_and(|i| x.oidx.unwrap() == i))
+        {
+            let name = cap.get(2).map(|x| x.as_str().to_string()).unwrap();
+            let default = cap.get(3).map(|x| x.as_str().to_string());
+            let has_default = default.is_some();
+            let oarg = args[arg_pos].clone();
+            let parsed_default = default.and_then(|x| parsed_default(&oarg.typ, x));
+
+            args[arg_pos] = Arg {
+                name,
+                typ: oarg.typ,
+                default: parsed_default,
+                otyp: oarg.otyp,
+                has_default,
+                oidx: oarg.oidx,
+            };
+        }
     }
 
     Ok(Some(args))
 }
 
+pub fn parse_sql_statement_named_params(code: &str, prefix: char) -> HashSet<String> {
+    let mut arg_names = HashSet::new();
+    run_on_sql_statement_matches(
+        code,
+        |char, chars| {
+            char == prefix
+                && chars
+                    .peek()
+                    .is_some_and(|&(_, next_char)| next_char.is_alphanumeric())
+        },
+        |_, chars| {
+            let mut arg_name = String::new();
+            while let Some(&(_, char)) = chars.peek() {
+                if char.is_alphanumeric() {
+                    arg_name.push(char);
+                    chars.next();
+                } else {
+                    break;
+                }
+            }
+            arg_names.insert(arg_name);
+        },
+    );
+    arg_names
+}
+
 fn parse_bigquery_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
     let mut args: Vec<Arg> = vec![];
 
@@ -213,6 +375,7 @@ fn parse_bigquery_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
             default: parsed_default,
             otyp: Some(typ),
             has_default,
+            oidx: None,
         });
     }
 
@@ -240,6 +403,7 @@ fn parse_snowflake_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
             default: parsed_default,
             otyp: Some(typ),
             has_default,
+            oidx: None,
         });
     }
 
@@ -267,6 +431,7 @@ fn parse_mssql_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
             default: parsed_default,
             otyp: Some(typ),
             has_default,
+            oidx: None,
         });
     }
 
@@ -372,7 +537,7 @@ mod tests {
     use super::*;
 
     #[test]
-    fn test_parse_sql_sig() -> anyhow::Result<()> {
+    fn test_parse_pgsql_sig() -> anyhow::Result<()> {
         let code = r#"
 SELECT * FROM table WHERE token=$1::TEXT AND image=$2::BIGINT
 "#;
@@ -388,14 +553,237 @@ SELECT * FROM table WHERE token=$1::TEXT AND image=$2::BIGINT
                         name: "$1".to_string(),
                         typ: Typ::Str(None),
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: Some(1),
                     },
                     Arg {
                         otyp: Some("bigint".to_string()),
                         name: "$2".to_string(),
                         typ: Typ::Int,
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: Some(2),
+                    },
+                ],
+                no_main_func: None
+            }
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_pgsql_mutli_sig() -> anyhow::Result<()> {
+        let code = r#"
+-- $1 param1
+-- $2 param2
+-- $3 param3
+SELECT $3::TEXT, $1::BIGINT;
+SELECT $2::TEXT;
+"#;
+        //println!("{}", serde_json::to_string()?);
+        assert_eq!(
+            parse_pgsql_sig(code)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![
+                    Arg {
+                        otyp: Some("bigint".to_string()),
+                        name: "param1".to_string(),
+                        typ: Typ::Int,
+                        default: None,
+                        has_default: false,
+                        oidx: Some(1),
+                    },
+                    Arg {
+                        otyp: Some("text".to_string()),
+                        name: "param2".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: false,
+                        oidx: Some(2),
+                    },
+                    Arg {
+                        otyp: Some("text".to_string()),
+                        name: "param3".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: false,
+                        oidx: Some(3),
+                    },
+                ],
+                no_main_func: None
+            }
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_sql_blocks_multi_2semi() -> anyhow::Result<()> {
+        let code = r#"
+-- $1 param1
+-- $2 param2
+-- $3 param3
+SELECT '--', ';' $3::TEXT, $1::BIGINT;
+-- ;
+SELECT $2::TEXT;
+"#;
+        assert_eq!(parse_sql_blocks(code).len(), 2);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_sql_blocks_multi_1semi() -> anyhow::Result<()> {
+        let code = r#"
+-- $1 param1
+-- $2 param2
+-- $3 param3
+SELECT '--', ';' $3::TEXT, $1::BIGINT;
+-- ;
+SELECT $2::TEXT
+"#;
+        assert_eq!(
+            parse_sql_blocks(code),
+            vec![
+                r#"
+-- $1 param1
+-- $2 param2
+-- $3 param3
+SELECT '--', ';' $3::TEXT, $1::BIGINT;"#,
+                r#"
+-- ;
+SELECT $2::TEXT
+"#
+            ]
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_sql_blocks_single_1semi() -> anyhow::Result<()> {
+        let code = r#"
+-- $1 param1
+-- $2 param2
+-- $3 param3
+SELECT '--', ';' $3::TEXT, $1::BIGINT;
+-- hey
+"#;
+        assert_eq!(
+            parse_sql_blocks(code),
+            vec![
+                r#"
+-- $1 param1
+-- $2 param2
+-- $3 param3
+SELECT '--', ';' $3::TEXT, $1::BIGINT;"#,
+            ]
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_sql_blocks_single_nosemi() -> anyhow::Result<()> {
+        let code = r#"
+-- $1 param1
+-- $2 param2
+-- $3 param3
+SELECT '--', ';' $3::TEXT, $1::BIGINT
+"#;
+        assert_eq!(
+            parse_sql_blocks(code),
+            vec![
+                r#"
+-- $1 param1
+-- $2 param2
+-- $3 param3
+SELECT '--', ';' $3::TEXT, $1::BIGINT
+"#
+            ]
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_mysql_positional_sig() -> anyhow::Result<()> {
+        let code = r#"
+-- ? param1 (int) = 3
+-- ? param2 (text)
+SELECT ?, ?;
+"#;
+        assert_eq!(
+            parse_mysql_sig(code)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![
+                    Arg {
+                        otyp: Some("int".to_string()),
+                        name: "param1".to_string(),
+                        typ: Typ::Int,
+                        default: Some(json!(3)),
+                        has_default: true,
+                        oidx: None,
+                    },
+                    Arg {
+                        otyp: Some("text".to_string()),
+                        name: "param2".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: false,
+                        oidx: None,
+                    },
+                ],
+                no_main_func: None
+            }
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_mysql_sig() -> anyhow::Result<()> {
+        let code = r#"
+-- :param1 (int) = 3
+-- :param2 (text)
+-- :param3 (text)
+SELECT :param3, :param1;
+SELECT :param2;
+"#;
+        assert_eq!(
+            parse_mysql_sig(code)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![
+                    Arg {
+                        otyp: Some("int".to_string()),
+                        name: "param1".to_string(),
+                        typ: Typ::Int,
+                        default: Some(json!(3)),
+                        has_default: true,
+                        oidx: None,
+                    },
+                    Arg {
+                        otyp: Some("text".to_string()),
+                        name: "param2".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: false,
+                        oidx: None,
+                    },
+                    Arg {
+                        otyp: Some("text".to_string()),
+                        name: "param3".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: false,
+                        oidx: None,
                     },
                 ],
                 no_main_func: None
@@ -408,9 +796,10 @@ SELECT * FROM table WHERE token=$1::TEXT AND image=$2::BIGINT
     #[test]
     fn test_parse_bigquery_sig() -> anyhow::Result<()> {
         let code = r#"
--- @token (string)
+-- @token (string) = abc
 -- @image (int64)
-SELECT * FROM table WHERE token=@token AND image=@image
+SELECT * FROM table WHERE token=@token AND image=@image;
+SELECT @token;
 "#;
         //println!("{}", serde_json::to_string()?);
         assert_eq!(
@@ -423,15 +812,111 @@ SELECT * FROM table WHERE token=@token AND image=@image
                         otyp: Some("string".to_string()),
                         name: "token".to_string(),
                         typ: Typ::Str(None),
-                        default: None,
-                        has_default: false
+                        default: Some(json!("abc")),
+                        has_default: true,
+                        oidx: None,
                     },
                     Arg {
                         otyp: Some("int64".to_string()),
                         name: "image".to_string(),
                         typ: Typ::Int,
                         default: None,
-                        has_default: false
+                        has_default: false,
+                        oidx: None,
+                    },
+                ],
+                no_main_func: None
+            }
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_snowflake_sig() -> anyhow::Result<()> {
+        let code = r#"
+-- ? param1 (int) = 3
+-- ? param2 (varchar)
+SELECT ?, ?;
+-- ? param3 (varchar)
+SELECT ?;
+"#;
+        assert_eq!(
+            parse_snowflake_sig(code)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![
+                    Arg {
+                        otyp: Some("int".to_string()),
+                        name: "param1".to_string(),
+                        typ: Typ::Int,
+                        default: Some(json!(3)),
+                        has_default: true,
+                        oidx: None,
+                    },
+                    Arg {
+                        otyp: Some("varchar".to_string()),
+                        name: "param2".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: false,
+                        oidx: None,
+                    },
+                    Arg {
+                        otyp: Some("varchar".to_string()),
+                        name: "param3".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: false,
+                        oidx: None,
+                    }
+                ],
+                no_main_func: None
+            }
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_mssql_sig() -> anyhow::Result<()> {
+        let code = r#"
+-- @p1 param1 (int) = 3
+-- @p2 param2 (varchar)
+-- @p3 param3 (varchar)
+SELECT @p3, @p1;
+SELECT @p2;
+"#;
+        assert_eq!(
+            parse_mssql_sig(code)?,
+            MainArgSignature {
+                star_args: false,
+                star_kwargs: false,
+                args: vec![
+                    Arg {
+                        otyp: Some("int".to_string()),
+                        name: "param1".to_string(),
+                        typ: Typ::Int,
+                        default: Some(json!(3)),
+                        has_default: true,
+                        oidx: None,
+                    },
+                    Arg {
+                        otyp: Some("varchar".to_string()),
+                        name: "param2".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: false,
+                        oidx: None,
+                    },
+                    Arg {
+                        otyp: Some("varchar".to_string()),
+                        name: "param3".to_string(),
+                        typ: Typ::Str(None),
+                        default: None,
+                        has_default: false,
+                        oidx: None,
                     },
                 ],
                 no_main_func: None

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -204,6 +204,7 @@ fn parse_param(
                 typ,
                 default: None,
                 has_default: ident.id.optional || nullable,
+                oidx: None,
             })
         }
         // Pat::Object(ObjectPat { ... }) = todo!()
@@ -250,13 +251,13 @@ fn parse_param(
             if typ == Typ::Unknown && dflt.is_some() {
                 typ = json_to_typ(dflt.as_ref().unwrap());
             }
-            Ok(Arg { otyp: None, name, typ, default: dflt, has_default: true })
+            Ok(Arg { otyp: None, name, typ, default: dflt, has_default: true, oidx: None })
         }
         Pat::Object(ObjectPat { type_ann, .. }) => {
             let (typ, nullable) = eval_type_ann(&type_ann);
             *counter += 1;
             let name = format!("anon{}", counter);
-            Ok(Arg { otyp: None, name, typ, default: None, has_default: nullable })
+            Ok(Arg { otyp: None, name, typ, default: None, has_default: nullable, oidx: None })
         }
         _ => Err(anyhow::anyhow!(
             "parameter syntax unsupported: `{}`: {:#?}",
@@ -402,72 +403,8 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
                 (tstype_to_typ(&types[other_p]).0, true)
             } else {
                 if types.len() > 1 {
-                    let one_of_values: Vec<OneOfVariant> = types
-                        .into_iter()
-                        .map_while(|x| match &**x {
-                            TsType::TsTypeLit(TsTypeLit { members, .. }) => {
-                                let label = members
-                                    .iter()
-                                    .find_map(|y| match y {
-                                        TsTypeElement::TsPropertySignature(
-                                            TsPropertySignature { key, type_ann, .. },
-                                        ) => match (&**key, type_ann) {
-                                            (Expr::Ident(Ident { sym, .. }), type_ann) => {
-                                                if sym.to_string() == "label" {
-                                                    type_ann.as_ref().map(|z| match &*z.type_ann {
-                                                        TsType::TsLitType(TsLitType {
-                                                            lit: TsLit::Str(Str { value, .. }),
-                                                            ..
-                                                        }) => Some(value.to_string()),
-                                                        _ => None,
-                                                    })
-                                                } else {
-                                                    None
-                                                }
-                                            }
-                                            _ => None,
-                                        },
-                                        _ => None,
-                                    })
-                                    .flatten();
-
-                                match label {
-                                    Some(label) => {
-                                        let properties: Vec<ObjectProperty> = members
-                                            .into_iter()
-                                            .filter_map(|x| match x {
-                                                TsTypeElement::TsPropertySignature(
-                                                    TsPropertySignature { key, type_ann, .. },
-                                                ) => match (*key.to_owned(), type_ann) {
-                                                    (Expr::Ident(Ident { sym, .. }), type_ann) => {
-                                                        Some(ObjectProperty {
-                                                            key: sym.to_string(),
-                                                            typ: type_ann
-                                                                .as_ref()
-                                                                .map(|typ| {
-                                                                    Box::new(
-                                                                        tstype_to_typ(
-                                                                            &*typ.type_ann,
-                                                                        )
-                                                                        .0,
-                                                                    )
-                                                                })
-                                                                .unwrap_or(Box::new(Typ::Unknown)),
-                                                        })
-                                                    }
-                                                    _ => None,
-                                                },
-                                                _ => None,
-                                            })
-                                            .collect();
-                                        Some(OneOfVariant { label, properties })
-                                    }
-                                    _ => None,
-                                }
-                            }
-                            _ => None,
-                        })
-                        .collect();
+                    let one_of_values: Vec<OneOfVariant> =
+                        types.into_iter().map_while(parse_one_of_type).collect();
 
                     if one_of_values.len() == types.len() {
                         return (Typ::OneOf(one_of_values), false);
@@ -526,6 +463,10 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
                 "Base64" => (Typ::Bytes, false),
                 "Email" => (Typ::Email, false),
                 "Sql" => (Typ::Sql, false),
+                x @ _ if x.starts_with("DynSelect_") => (
+                    Typ::DynSelect(x.strip_prefix("DynSelect_").unwrap().to_string()),
+                    false,
+                ),
                 x @ _ => (Typ::Resource(to_snake_case(x)), false),
             }
         }
@@ -533,6 +474,66 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
     }
 }
 
+fn parse_one_of_type(x: &Box<TsType>) -> Option<OneOfVariant> {
+    match &**x {
+        TsType::TsTypeLit(TsTypeLit { members, .. }) => {
+            let label = one_of_label(members)?;
+            let properties = one_of_properties(members);
+            Some(OneOfVariant { label, properties })
+        }
+        _ => None,
+    }
+}
+
+fn one_of_label(members: &Vec<TsTypeElement>) -> Option<String> {
+    members.iter().find_map(|y| {
+        let TsTypeElement::TsPropertySignature(TsPropertySignature { key, type_ann, .. }) = y
+        else {
+            return None;
+        };
+
+        let Expr::Ident(Ident { sym, .. }) = &**key else {
+            return None;
+        };
+        if sym != "label" {
+            return None;
+        }
+
+        let Some(type_ann) = type_ann.as_ref() else {
+            return None;
+        };
+        let TsType::TsLitType(TsLitType { lit: TsLit::Str(Str { value, .. }), .. }) =
+            &*type_ann.type_ann
+        else {
+            return None;
+        };
+
+        Some(value.to_string())
+    })
+}
+
+fn one_of_properties(members: &Vec<TsTypeElement>) -> Vec<ObjectProperty> {
+    members
+        .iter()
+        .filter_map(|x| {
+            let TsTypeElement::TsPropertySignature(TsPropertySignature { key, type_ann, .. }) = x
+            else {
+                return None;
+            };
+
+            let Expr::Ident(Ident { sym, .. }) = *key.to_owned() else {
+                return None;
+            };
+            let typ = type_ann
+                .as_ref()
+                .map(|typ| Box::new(tstype_to_typ(&*typ.type_ann).0))
+                .unwrap_or(Box::new(Typ::Unknown));
+
+            Some(ObjectProperty { key: sym.to_string(), typ })
+        })
+        .collect()
+}
+
 fn find_undefined(types: &Vec<Box<TsType>>) -> Option<usize> {
     types.into_iter().position(|x| match **x {
         TsType::TsKeywordType(TsKeywordType { kind, .. }) => {

backend/parsers/windmill-parser-wasm/build.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 set -eou pipefail
 
-deno task wasmbuild --out ../../../cli/wasm/
+deno task wasmbuild --out ../../../cli/wasm/  -p windmill-parser-wasm

backend/parsers/windmill-parser-wasm/deno.json

@@ -1,5 +1,5 @@
 {
-    "tasks": {
-      "wasmbuild": "deno run -A https://deno.land/x/wasmbuild@0.15.4/main.ts"
-    }
-  }
+  "tasks": {
+    "wasmbuild": "deno run -A jsr:@deno/wasmbuild@0.17.2"
+  }
+}

backend/parsers/windmill-parser-wasm/pkg/package.json

@@ -3,7 +3,7 @@
   "collaborators": [
     "Ruben Fiszel <ruben@windmill.dev>"
   ],
-  "version": "1.355.4",
+  "version": "1.367.2",
   "files": [
     "windmill_parser_wasm_bg.wasm",
     "windmill_parser_wasm.js",

backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.js

@@ -6,6 +6,20 @@ heap.push(undefined, null, true, false);
 
 function getObject(idx) { return heap[idx]; }
 
+let heap_next = heap.length;
+
+function dropObject(idx) {
+    if (idx < 132) return;
+    heap[idx] = heap_next;
+    heap_next = idx;
+}
+
+function takeObject(idx) {
+    const ret = getObject(idx);
+    dropObject(idx);
+    return ret;
+}
+
 let WASM_VECTOR_LEN = 0;
 
 let cachedUint8Memory0 = null;
@@ -84,27 +98,13 @@ function getInt32Memory0() {
     return cachedInt32Memory0;
 }
 
-let heap_next = heap.length;
+let cachedFloat64Memory0 = null;
 
-function dropObject(idx) {
-    if (idx < 132) return;
-    heap[idx] = heap_next;
-    heap_next = idx;
-}
-
-function takeObject(idx) {
-    const ret = getObject(idx);
-    dropObject(idx);
-    return ret;
-}
-
-const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
-
-if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
-
-function getStringFromWasm0(ptr, len) {
-    ptr = ptr >>> 0;
-    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
+function getFloat64Memory0() {
+    if (cachedFloat64Memory0 === null || cachedFloat64Memory0.byteLength === 0) {
+        cachedFloat64Memory0 = new Float64Array(wasm.memory.buffer);
+    }
+    return cachedFloat64Memory0;
 }
 
 function addHeapObject(obj) {
@@ -116,13 +116,13 @@ function addHeapObject(obj) {
     return idx;
 }
 
-let cachedFloat64Memory0 = null;
+const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
 
-function getFloat64Memory0() {
-    if (cachedFloat64Memory0 === null || cachedFloat64Memory0.byteLength === 0) {
-        cachedFloat64Memory0 = new Float64Array(wasm.memory.buffer);
-    }
-    return cachedFloat64Memory0;
+if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
+
+function getStringFromWasm0(ptr, len) {
+    ptr = ptr >>> 0;
+    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
 }
 
 let cachedBigInt64Memory0 = null;
@@ -585,6 +585,13 @@ async function __wbg_load(module, imports) {
 function __wbg_get_imports() {
     const imports = {};
     imports.wbg = {};
+    imports.wbg.__wbg_eval_aa725d466edcea2c = function(arg0, arg1) {
+        const ret = eval(getStringFromWasm0(arg0, arg1));
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
+        takeObject(arg0);
+    };
     imports.wbg.__wbindgen_string_get = function(arg0, arg1) {
         const obj = getObject(arg1);
         const ret = typeof(obj) === 'string' ? obj : undefined;
@@ -593,13 +600,6 @@ function __wbg_get_imports() {
         getInt32Memory0()[arg0 / 4 + 1] = len1;
         getInt32Memory0()[arg0 / 4 + 0] = ptr1;
     };
-    imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
-        takeObject(arg0);
-    };
-    imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
-        const ret = new Error(getStringFromWasm0(arg0, arg1));
-        return addHeapObject(ret);
-    };
     imports.wbg.__wbindgen_boolean_get = function(arg0) {
         const v = getObject(arg0);
         const ret = typeof(v) === 'boolean' ? (v ? 1 : 0) : 2;
@@ -636,8 +636,8 @@ function __wbg_get_imports() {
         const ret = BigInt.asUintN(64, arg0);
         return addHeapObject(ret);
     };
-    imports.wbg.__wbg_eval_64ce1594989e82f8 = function(arg0, arg1) {
-        const ret = eval(getStringFromWasm0(arg0, arg1));
+    imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
+        const ret = new Error(getStringFromWasm0(arg0, arg1));
         return addHeapObject(ret);
     };
     imports.wbg.__wbindgen_jsval_loose_eq = function(arg0, arg1) {

backend/parsers/windmill-parser-wasm/tests/wasm.rs

@@ -12,7 +12,8 @@ export function main(test1?: string, test2: string = \"burkina\",
     email: Email, literal: \"test\", literal_union: \"test\" | \"test2\",
     opt_type?: string | null, opt_type_union: string | null, opt_type_union_union2: string | undefined,
     min_object: {a: string, b: number},
-    literals_with_undefined: \"foo\" | \"bar\" | undefined) {
+    literals_with_undefined: \"foo\" | \"bar\" | undefined,
+    dyn_select: DynSelect_foo) {
     console.log(42)
 }
 ";
@@ -27,77 +28,88 @@ export function main(test1?: string, test2: string = \"burkina\",
                     name: "test1".to_string(),
                     typ: Typ::Str(None),
                     default: None,
-                    has_default: true
+                    has_default: true,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "test2".to_string(),
                     typ: Typ::Str(None),
                     default: Some(json!("burkina")),
-                    has_default: true
+                    has_default: true,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "test3".to_string(),
                     typ: Typ::Resource("postgres".to_string()),
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "b64".to_string(),
                     typ: Typ::Bytes,
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "ls".to_string(),
                     typ: Typ::List(Box::new(Typ::Bytes)),
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "email".to_string(),
                     typ: Typ::Email,
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "literal".to_string(),
                     typ: Typ::Str(Some(vec!["test".to_string()])),
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "literal_union".to_string(),
                     typ: Typ::Str(Some(vec!["test".to_string(), "test2".to_string()])),
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "opt_type".to_string(),
                     typ: Typ::Str(None),
                     default: None,
-                    has_default: true
+                    has_default: true,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "opt_type_union".to_string(),
                     typ: Typ::Str(None),
                     default: None,
-                    has_default: true
+                    has_default: true,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "opt_type_union_union2".to_string(),
                     typ: Typ::Str(None),
                     default: None,
-                    has_default: true
+                    has_default: true,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
@@ -107,14 +119,24 @@ export function main(test1?: string, test2: string = \"burkina\",
                         ObjectProperty { key: "b".to_string(), typ: Box::new(Typ::Float) }
                     ]),
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "literals_with_undefined".to_string(),
                     typ: Typ::Str(Some(vec!["foo".to_string(), "bar".to_string()])),
                     default: None,
-                    has_default: true
+                    has_default: true,
+                    oidx: None
+                },
+                Arg {
+                    otyp: None,
+                    name: "dyn_select".to_string(),
+                    typ: Typ::DynSelect("foo".to_string()),
+                    default: None,
+                    has_default: false,
+                    oidx: None
                 }
             ],
             no_main_func: Some(false)
@@ -146,35 +168,40 @@ export function main(test2 = \"burkina\",
                     name: "test2".to_string(),
                     typ: Typ::Str(None),
                     default: Some(json!("burkina")),
-                    has_default: true
+                    has_default: true,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "bool".to_string(),
                     typ: Typ::Bool,
                     default: Some(json!(true)),
-                    has_default: true
+                    has_default: true,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "float".to_string(),
                     typ: Typ::Float,
                     default: Some(json!(4.2)),
-                    has_default: true
+                    has_default: true,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "int".to_string(),
                     typ: Typ::Int,
                     default: Some(json!(42)),
-                    has_default: true
+                    has_default: true,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "ls".to_string(),
                     typ: Typ::List(Box::new(Typ::Str(None))),
                     default: Some(json!(["test"])),
-                    has_default: true
+                    has_default: true,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
@@ -184,7 +211,8 @@ export function main(test2 = \"burkina\",
                         ObjectProperty { key: "b".to_string(), typ: Box::new(Typ::Int) }
                     ]),
                     default: Some(json!({"a": "test", "b": 42})),
-                    has_default: true
+                    has_default: true,
+                    oidx: None
                 }
             ],
             no_main_func: Some(false)
@@ -216,21 +244,24 @@ export function main(foo: FooBar, {a, b}: FooBar, {c, d}: FooBar = {a: \"foo\",
                     otyp: None,
                     typ: Typ::Resource("foo_bar".to_string()),
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     name: "anon1".to_string(),
                     otyp: None,
                     typ: Typ::Resource("foo_bar".to_string()),
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     name: "anon2".to_string(),
                     otyp: None,
                     typ: Typ::Resource("foo_bar".to_string()),
                     default: Some(json!({"a": "foo", "b": 42})),
-                    has_default: true
+                    has_default: true,
+                    oidx: None
                 }
             ],
             no_main_func: Some(false)
@@ -260,7 +291,8 @@ export function main(foo: (\"foo\" | \"bar\")[]) {
                     "bar".to_string()
                 ])))),
                 default: None,
-                has_default: false
+                has_default: false,
+                oidx: None
             }],
             no_main_func: Some(false)
         }
@@ -341,56 +373,64 @@ Write-Output 'Testing...'
                     name: "test_none".to_string(),
                     typ: Typ::Str(None),
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "test_string".to_string(),
                     typ: Typ::Str(None),
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "test_int".to_string(),
                     typ: Typ::Int,
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "test_decimal".to_string(),
                     typ: Typ::Float,
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "test_double".to_string(),
                     typ: Typ::Float,
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "test_single".to_string(),
                     typ: Typ::Float,
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "test_datetime_lower".to_string(),
                     typ: Typ::Datetime,
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 },
                 Arg {
                     otyp: None,
                     name: "test_datetime_upper".to_string(),
                     typ: Typ::Datetime,
                     default: None,
-                    has_default: false
+                    has_default: false,
+                    oidx: None
                 }
             ],
             no_main_func: None,

backend/parsers/windmill-parser/Cargo.toml

@@ -9,5 +9,5 @@ name = "windmill_parser"
 path = "./src/lib.rs"
 
 [dependencies]
-serde.workspace = true
+serde = { version = "1.0", features = ["derive"] }
 serde_json.workspace = true

backend/parsers/windmill-parser/src/lib.rs

@@ -44,6 +44,7 @@ pub enum Typ {
     Resource(String),
     Email,
     Sql,
+    DynSelect(String),
     Object(Vec<ObjectProperty>),
     OneOf(Vec<OneOfVariant>),
     Unknown,
@@ -56,6 +57,7 @@ pub struct Arg {
     pub typ: Typ,
     pub default: Option<serde_json::Value>,
     pub has_default: bool,
+    pub oidx: Option<i32>,
 }
 
 pub fn json_to_typ(js: &Value) -> Typ {

backend/parsers/windmill-sql-datatype-parser-wasm/Cargo.toml

@@ -16,5 +16,4 @@ wasm-bindgen-test.workspace = true
 [dependencies]
 windmill-parser.workspace = true
 windmill-parser-sql.workspace = true
-wasm-bindgen.workspace = true
-serde = { version = "1.0", features = ["derive"] }
+wasm-bindgen.workspace = true

backend/src/main.rs

@@ -23,11 +23,11 @@ use windmill_common::ee::schedule_key_renewal;
 use windmill_common::{
     global_settings::{
         BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ERROR_CHANNELS_SETTING,
-        CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, ENV_SETTINGS,
-        EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
-        HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING,
-        KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
-        PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
+        ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
+        EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
+        JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING,
+        OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
     },
@@ -54,18 +54,18 @@ use windmill_common::METRICS_ADDR;
 use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
 
 use windmill_worker::{
-    BUN_CACHE_DIR, BUN_TAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
-    GO_BIN_CACHE_DIR, GO_CACHE_DIR, HUB_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR,
-    POWERSHELL_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR,
+    BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR,
+    DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, HUB_CACHE_DIR,
+    LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR,
 };
 
 use crate::monitor::{
     initial_load, load_keep_job_dir, load_metrics_debug_enabled, load_require_preexisting_user,
-    load_tag_per_workspace_enabled, monitor_db, monitor_pool, reload_base_url_setting,
-    reload_bunfig_install_scopes_setting, reload_critical_error_channels_setting,
-    reload_extra_pip_index_url_setting, reload_hub_base_url_setting,
-    reload_job_default_timeout_setting, reload_jwt_secret_setting, reload_license_key,
-    reload_npm_config_registry_setting, reload_pip_index_url_setting,
+    load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, monitor_db, monitor_pool,
+    reload_base_url_setting, reload_bunfig_install_scopes_setting,
+    reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
+    reload_hub_base_url_setting, reload_job_default_timeout_setting, reload_jwt_secret_setting,
+    reload_license_key, reload_npm_config_registry_setting, reload_pip_index_url_setting,
     reload_retention_period_setting, reload_scim_token_setting, reload_server_config,
     reload_worker_config,
 };
@@ -150,6 +150,8 @@ async fn windmill_main() -> anyhow::Result<()> {
         _ => {}
     }
 
+    let mut enable_standalone_indexer: bool = false;
+
     let mode = std::env::var("MODE")
         .map(|x| x.to_lowercase())
         .map(|x| {
@@ -174,7 +176,20 @@ async fn windmill_main() -> anyhow::Result<()> {
                 }
                 #[cfg(feature = "enterprise")]
                 Mode::Agent
-            } else {
+            } else if &x == "indexer" {
+                tracing::info!("Binary is in 'indexer' mode");
+                #[cfg(not(feature = "tantivy"))]
+                {
+                    panic!("Indexer mode requires the tantivy feature flag");
+                }
+                #[cfg(feature = "tantivy")]
+                Mode::Indexer
+            } else if &x == "standalone+search"{
+                    enable_standalone_indexer = true;
+                    tracing::info!("Binary is in 'standalone' mode with search enabled");
+                    Mode::Standalone
+            }
+            else {
                 if &x != "standalone" {
                     tracing::error!("mode not recognized, defaulting to standalone: {x}");
                 } else {
@@ -188,7 +203,7 @@ async fn windmill_main() -> anyhow::Result<()> {
             Mode::Standalone
         });
 
-    let num_workers = if mode == Mode::Server {
+    let num_workers = if mode == Mode::Server || mode == Mode::Indexer {
         0
     } else {
         std::env::var("NUM_WORKERS")
@@ -209,7 +224,9 @@ async fn windmill_main() -> anyhow::Result<()> {
         .unwrap_or(false)
         && (mode == Mode::Server || mode == Mode::Standalone);
 
-    let server_bind_address: IpAddr = if server_mode {
+    let indexer_mode = mode == Mode::Indexer;
+
+    let server_bind_address: IpAddr = if server_mode || indexer_mode {
         std::env::var("SERVER_BIND_ADDR")
             .ok()
             .and_then(|x| x.parse().ok())
@@ -248,7 +265,7 @@ async fn windmill_main() -> anyhow::Result<()> {
     };
 
     tracing::info!("Connecting to database...");
-    let db = windmill_common::connect_db(server_mode).await?;
+    let db = windmill_common::connect_db(server_mode, indexer_mode).await?;
     tracing::info!("Database connected");
 
     let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await;
@@ -302,10 +319,10 @@ Windmill Community Edition {GIT_VERSION}
 
     let worker_mode = num_workers > 0;
 
-    if server_mode || worker_mode {
+    if server_mode || worker_mode || indexer_mode {
         let port_var = std::env::var("PORT").ok().and_then(|x| x.parse().ok());
 
-        let port = if server_mode {
+        let port = if server_mode || indexer_mode {
             port_var.unwrap_or(DEFAULT_PORT as u16)
         } else {
             port_var.unwrap_or(0)
@@ -346,11 +363,44 @@ Windmill Community Edition {GIT_VERSION}
             .await
             .expect("could not create initial server dir");
 
+        #[cfg(feature = "tantivy")]
+        let should_index_jobs =
+            mode == Mode::Indexer || (enable_standalone_indexer && mode == Mode::Standalone);
+
+        #[cfg(feature = "tantivy")]
+        let (index_reader, index_writer) = if should_index_jobs {
+            let (r, w) = windmill_indexer::indexer_ee::init_index().await?;
+            (Some(r), Some(w))
+        } else {
+            (None, None)
+        };
+
+        #[cfg(feature = "tantivy")]
+        let indexer_f = {
+            let indexer_rx = killpill_rx.resubscribe();
+            let index_writer2 = index_writer.clone();
+            async {
+                if let Some(index_writer) = index_writer2 {
+                    windmill_indexer::indexer_ee::run_indexer(db.clone(), index_writer, indexer_rx)
+                        .await;
+                }
+                Ok(())
+            }
+        };
+
+        #[cfg(not(feature = "tantivy"))]
+        let (index_reader, index_writer) = (None, None);
+
+        #[cfg(not(feature = "tantivy"))]
+        let indexer_f = async { Ok(()) as anyhow::Result<()> };
+
         let server_f = async {
             if !is_agent {
                 windmill_api::run_server(
                     db.clone(),
                     rsmq2,
+                    index_reader,
+                    index_writer,
                     addr,
                     server_killpill_rx,
                     base_internal_tx,
@@ -461,9 +511,14 @@ Windmill Community Edition {GIT_VERSION}
                                                 },
                                                 DEFAULT_TAGS_PER_WORKSPACE_SETTING => {
                                                     if let Err(e) = load_tag_per_workspace_enabled(&db).await {
-                                                        tracing::error!("Error loading default tag per workpsace: {e:#}");
+                                                        tracing::error!("Error loading default tag per workspace: {e:#}");
                                                     }
                                                 },
+                                                DEFAULT_TAGS_WORKSPACES_SETTING => {
+                                                    if let Err(e) = load_tag_per_workspace_workspaces(&db).await {
+                                                        tracing::error!("Error loading default tag per workspace workspaces: {e:#}");
+                                                    }
+                                                }
                                                 RETENTION_PERIOD_SECS_SETTING => {
                                                     reload_retention_period_setting(&db).await
                                                 },
@@ -496,14 +551,12 @@ Windmill Community Edition {GIT_VERSION}
                                                     load_require_preexisting_user(&db).await;
                                                 },
                                                 EXPOSE_METRICS_SETTING  => {
-                                                    if worker_mode {
-                                                        tracing::info!("Metrics setting changed, restarting");
-                                                        // we wait a bit randomly to avoid having all serverss and workers shutdown at same time
-                                                        let rd_delay = rand::thread_rng().gen_range(0..4);
-                                                        tokio::time::sleep(Duration::from_secs(rd_delay)).await;
-                                                        if let Err(e) = tx.send(()) {
-                                                            tracing::error!(error = %e, "Could not send killpill to server");
-                                                        }
+                                                    tracing::info!("Metrics setting changed, restarting");
+                                                    // we wait a bit randomly to avoid having all servers and workers shutdown at same time
+                                                    let rd_delay = rand::thread_rng().gen_range(0..40);
+                                                    tokio::time::sleep(Duration::from_secs(rd_delay)).await;
+                                                    if let Err(e) = tx.send(()) {
+                                                        tracing::error!(error = %e, "Could not send killpill to server");
                                                     }
                                                 },
                                                 EXPOSE_DEBUG_METRICS_SETTING => {
@@ -597,7 +650,14 @@ Windmill Community Edition {GIT_VERSION}
             schedule_key_renewal(&HTTP_CLIENT, &db).await;
         }
 
-        futures::try_join!(shutdown_signal, workers_f, monitor_f, server_f, metrics_f)?;
+        futures::try_join!(
+            shutdown_signal,
+            workers_f,
+            monitor_f,
+            server_f,
+            metrics_f,
+            indexer_f
+        )?;
     } else {
         tracing::info!("Nothing to do, exiting.");
     }
@@ -716,7 +776,8 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
         DENO_CACHE_DIR_DEPS,
         DENO_CACHE_DIR_NPM,
         BUN_CACHE_DIR,
-        BUN_TAR_CACHE_DIR,
+        BUN_DEPSTAR_CACHE_DIR,
+        BUN_BUNDLE_CACHE_DIR,
         GO_CACHE_DIR,
         GO_BIN_CACHE_DIR,
         HUB_CACHE_DIR,
@@ -729,6 +790,10 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
             .expect("could not create initial worker dir");
     }
 
+    tracing::info!(
+        "Starting {num_workers} workers and SLEEP_QUEUE={}ms",
+        *windmill_worker::SLEEP_QUEUE
+    );
     for i in 1..(num_workers + 1) {
         let db1 = db.clone();
         let instance_name = instance_name.clone();
@@ -741,7 +806,9 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
         let hostname = hostname.clone();
 
         handles.push(tokio::spawn(async move {
-            tracing::info!(worker = %worker_name, "starting worker");
+            if num_workers > 1 {
+                tracing::info!(worker = %worker_name, "starting worker {i}");
+            }
 
             let f = windmill_worker::run_worker(
                 &db1,

backend/src/monitor.rs

@@ -28,10 +28,11 @@ use windmill_common::{
     flow_status::FlowStatusModule,
     global_settings::{
         BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ERROR_CHANNELS_SETTING,
-        DEFAULT_TAGS_PER_WORKSPACE_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
-        EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
-        JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING,
-        OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
+        EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
+        HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING,
+        KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
+        PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
     },
@@ -41,10 +42,10 @@ use windmill_common::{
     users::truncate_token,
     utils::{now_from_db, rd_string},
     worker::{
-        load_worker_config, reload_custom_tags_setting, DEFAULT_TAGS_PER_WORKSPACE, SERVER_CONFIG,
-        WORKER_CONFIG,
+        load_worker_config, reload_custom_tags_setting, DEFAULT_TAGS_PER_WORKSPACE,
+        DEFAULT_TAGS_WORKSPACES, SERVER_CONFIG, WORKER_CONFIG,
     },
-    BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL,
+    BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS,
     METRICS_DEBUG_ENABLED, METRICS_ENABLED,
 };
 use windmill_queue::cancel_job;
@@ -104,8 +105,6 @@ lazy_static::lazy_static! {
         "Number of jobs in the queue",
         &["tag"]
     ).unwrap();
-
-    static ref JOB_RETENTION_SECS: Arc<RwLock<i64>> = Arc::new(RwLock::new(0));
 }
 
 pub async fn initial_load(
@@ -127,6 +126,10 @@ pub async fn initial_load(
         tracing::error!("Error loading default tag per workpsace: {e:#}");
     }
 
+    if let Err(e) = load_tag_per_workspace_workspaces(db).await {
+        tracing::error!("Error loading default tag per workpsace workspaces: {e:#}");
+    }
+
     if server_mode {
         load_require_preexisting_user(db).await;
     }
@@ -204,6 +207,28 @@ pub async fn load_tag_per_workspace_enabled(db: &DB) -> error::Result<()> {
     Ok(())
 }
 
+pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> {
+    let workspaces = load_value_from_global_settings(db, DEFAULT_TAGS_WORKSPACES_SETTING).await;
+
+    match workspaces {
+        Ok(Some(serde_json::Value::Array(t))) => {
+            let workspaces = t
+                .iter()
+                .filter_map(|x| x.as_str())
+                .map(|x| x.to_string())
+                .collect::<Vec<String>>();
+            let mut w = DEFAULT_TAGS_WORKSPACES.write().await;
+            *w = Some(workspaces);
+        }
+        Ok(None) => {
+            let mut w = DEFAULT_TAGS_WORKSPACES.write().await;
+            *w = None;
+        }
+        _ => (),
+    };
+    Ok(())
+}
+
 pub async fn load_metrics_debug_enabled(db: &DB) -> error::Result<()> {
     let metrics_enabled = load_value_from_global_settings(db, EXPOSE_DEBUG_METRICS_SETTING).await;
     match metrics_enabled {
@@ -650,6 +675,16 @@ pub async fn reload_option_setting<T: FromStr + DeserializeOwned>(
     std_env_var: &str,
     lock: Arc<RwLock<Option<T>>>,
 ) -> error::Result<()> {
+    let force_value = std::env::var(format!("FORCE_{}", std_env_var))
+        .ok()
+        .and_then(|x| x.parse::<T>().ok());
+
+    if let Some(force_value) = force_value {
+        let mut l = lock.write().await;
+        *l = Some(force_value);
+        return Ok(());
+    }
+
     let q = load_value_from_global_settings(db, setting_name).await?;
 
     let mut value = std::env::var(std_env_var)

backend/substitute_ee_code.sh

@@ -71,7 +71,7 @@ if [ "$REVERT" == "YES" ]; then
     ce_file="${ee_file/${EE_CODE_DIR}/.}"
     ce_file="${root_dirpath}/backend/${ce_file}"
     if [ "$REVERT_PREVIOUS" == "YES" ]; then
-      git checkout HEAD@{5} ${ce_file} || true
+      git checkout HEAD@{3} ${ce_file} || true
     else
       git restore --staged ${ce_file} || true
       git restore ${ce_file} || true

backend/tests/worker.rs

@@ -122,6 +122,8 @@ impl ApiServer {
         let task = tokio::task::spawn(windmill_api::run_server(
             db.clone(),
             None,
+            None,
+            None,
             addr,
             rx,
             port_tx,

backend/windmill-api/Cargo.toml

@@ -18,6 +18,7 @@ embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core"
 parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet"]
 prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus"]
 openidconnect = ["dep:openidconnect"]
+tantivy = ["dep:windmill-indexer"]
 
 [dependencies]
 windmill-queue.workspace = true
@@ -27,6 +28,7 @@ windmill-parser.workspace = true
 windmill-parser-py-imports.workspace = true
 windmill-parser-ts.workspace = true
 windmill-git-sync.workspace = true
+windmill-indexer = { workspace = true, optional = true }
 tokio.workspace = true
 anyhow.workspace = true
 argon2.workspace = true
@@ -92,4 +94,4 @@ pin-project.workspace = true
 crc.workspace = true
 http.workspace = true
 async-stream.workspace = true
-ulid.workspace = true
+ulid.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.358.1
+  version: 1.371.4
   title: Windmill API
 
   contact:
@@ -439,6 +439,44 @@ paths:
             text/plain:
               schema:
                 type: string
+  /users/overwrite:
+    post:
+      summary: global overwrite users (require super admin and EE)
+      operationId: globalUsersOverwrite
+      tags:
+        - user
+      requestBody:
+        description: List of users
+        required: true
+        content:
+          application/json:
+            schema:
+              type: array
+              items:
+                $ref: "#/components/schemas/ExportedUser"
+      responses:
+        "200":
+          description: Success message
+          content:
+            text/plain:
+              schema:
+                type: string
+  
+  /users/export:
+    get:
+      summary: global export users (require super admin and EE)
+      operationId: globalUsersExport
+      tags:
+        - user
+      responses:
+        "200":
+          description: exported users
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/ExportedUser"
 
   /w/{workspace}/users/delete/{username}:
     delete:
@@ -850,6 +888,23 @@ paths:
               schema:
                 type: string
 
+  
+  /settings/list_global:
+    get:
+      summary: list global settings
+      operationId: listGlobalSettings
+      tags:
+        - setting
+      responses:
+        "200":
+          description: list of settings
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/GlobalSetting"
+
   /users/email:
     get:
       summary: get current user email (if logged in)
@@ -1971,6 +2026,8 @@ paths:
               properties:
                 new_key:
                   type: string
+                skip_reencrypt:
+                  type: boolean
               required:
                 - new_key
 
@@ -2387,6 +2444,8 @@ paths:
           in: query
           schema:
             type: string
+        - $ref: "#/components/parameters/Page"
+        - $ref: "#/components/parameters/PerPage"
       responses:
         "200":
           description: variable list
@@ -6827,6 +6886,44 @@ paths:
             text/plain:
               schema:
                 type: string
+  /groups/export:
+    get:
+      summary: export instance groups
+      operationId: exportInstanceGroups
+      tags:
+        - group
+      responses:
+        "200":
+          description: exported instance groups
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/ExportedInstanceGroup"
+
+  /groups/overwrite:
+    post:
+      summary: overwrite instance groups
+      operationId: overwriteInstanceGroups
+      tags:
+        - group
+      requestBody:
+        description: overwrite instance groups
+        required: true
+        content:
+          application/json:
+            schema:
+              type: array
+              items:
+                $ref: "#/components/schemas/ExportedInstanceGroup"
+      responses:
+        "200":
+          description: success message
+          content:
+            text/plain:
+              schema:
+                type: string
 
   /w/{workspace}/groups/list:
     get:
@@ -7415,6 +7512,22 @@ paths:
               schema:
                 type: string
 
+  /configs/list:
+    get:
+      summary: list configs
+      operationId: listConfigs
+      tags:
+        - config
+      responses:
+        "200":
+          description: list of configs
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/Config"
+
   /w/{workspace}/acls/get/{kind}/{path}:
     get:
       summary: get granular acls
@@ -7666,6 +7779,14 @@ paths:
           required: true
           schema:
             type: string
+        - name: input
+          in: query
+          schema:
+            type: boolean
+        - name: allow_large
+          in: query
+          schema:
+            type: boolean
       responses:
         "200":
           description: args
@@ -8544,6 +8665,42 @@ paths:
               schema:
                 $ref: "#/components/schemas/ExtendedJobs"
 
+  /srch/w/{workspace}/index/search/job:
+    get:
+      summary: Search through jobs with a string query
+      operationId: searchJobsIndex
+      tags:
+        - indexSearch
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - name: search_query
+          in: query
+          required: true
+          schema:
+            type: string
+      responses:
+        "200":
+          description: search results
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  query_parse_errors:
+                    description: a list of the terms that couldn't be parsed (and thus ignored)
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        dancer:
+                          type: string
+                  hits:
+                    description: the jobs that matched the query
+                    type: array
+                    items:
+                      $ref: "#/components/schemas/JobSearchHit"
+
+
 components:
   securitySchemes:
     bearerAuth:
@@ -9147,8 +9304,6 @@ components:
           type: string
         name:
           type: string
-        args:
-          type: object
         created_by:
           type: string
         created_at:
@@ -11079,3 +11234,75 @@ components:
       required:
         - jobs
         - obscured_jobs
+    
+    ExportedUser:
+      type: object
+      properties:
+        email:
+          type: string
+        password_hash:
+          type: string
+        super_admin:
+          type: boolean
+        verified:
+          type: boolean
+        name:
+          type: string
+        company:
+          type: string
+        first_time_user:
+          type: boolean
+        username:
+          type: string
+      required:
+        - email
+        - super_admin
+        - verified
+        - first_time_user
+
+    GlobalSetting:
+      type: object
+      properties:
+        name:
+          type: string
+        value:
+          type: object
+      required:
+        - name
+        - value
+
+    Config:
+      type: object
+      properties:
+        name:
+          type: string
+        config:
+          type: object
+      required:
+        - name
+    
+    ExportedInstanceGroup:
+      type: object
+      properties:
+        name:
+          type: string
+        summary:
+          type: string
+        emails:
+          type: array
+          items:
+            type: string
+        id:
+          type: string
+        scim_display_name:
+          type: string
+        external_id:
+          type: string
+      required:
+        - name
+
+    JobSearchHit:
+      type: object
+      properties:
+        dancer:
+          type: string

backend/windmill-api/src/configs.rs

@@ -28,6 +28,7 @@ pub fn global_service() -> Router {
         .route("/list_worker_groups", get(list_worker_groups))
         .route("/update/:name", post(update_config).delete(delete_config))
         .route("/get/:name", get(get_config))
+        .route("/list", get(list_configs))
 }
 
 #[derive(Serialize, Deserialize, FromRow)]
@@ -166,3 +167,22 @@ async fn delete_config(
     }
     Ok(format!("Deleted config {name}"))
 }
+
+#[cfg(feature = "enterprise")]
+async fn list_configs(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+) -> error::JsonResult<Vec<Config>> {
+    require_super_admin(&db, &authed.email).await?;
+    let configs = sqlx::query_as!(Config, "SELECT name, config FROM config")
+        .fetch_all(&db)
+        .await?;
+    Ok(Json(configs))
+}
+
+#[cfg(not(feature = "enterprise"))]
+async fn list_configs() -> error::JsonResult<String> {
+    Err(error::Error::BadRequest(
+        "Config listing available only in the enterprise version".to_string(),
+    ))
+}

backend/windmill-api/src/db.rs

@@ -7,7 +7,6 @@
  */
 
 use futures::FutureExt;
-#[cfg(feature = "enterprise")]
 use sqlx::Executor;
 
 use sqlx::{
@@ -167,6 +166,18 @@ impl Migrate for CustomMigrator {
 pub async fn migrate(db: &DB) -> Result<(), Error> {
     let migrator = db.acquire().await?;
     let mut custom_migrator = CustomMigrator { inner: migrator };
+
+    if let Err(err) = fix_flow_versioning_migration(&mut custom_migrator, db).await {
+        tracing::error!("Could not apply flow versioning fix migration: {err:#}");
+    }
+
+    let db2 = db.clone();
+    let _ = tokio::task::spawn(async move {
+        if let Err(err) = fix_job_completed_index(&db2).await {
+            tracing::error!("Could not apply job completed index fix migration: {err:#}");
+        }
+    });
+
     match sqlx::migrate!("../migrations")
         .run_direct(&mut custom_migrator)
         .await
@@ -190,32 +201,234 @@ pub async fn migrate(db: &DB) -> Result<(), Error> {
     Ok(())
 }
 
-#[cfg(feature = "enterprise")]
-async fn windmill_migrations(migrator: &mut CustomMigrator, db: &DB) -> Result<(), Error> {
-    if std::env::var("MIGRATION_NO_BYPASSRLS").is_ok() {
-        #[cfg(feature = "enterprise")]
+async fn fix_flow_versioning_migration(
+    migrator: &mut CustomMigrator,
+    db: &DB,
+) -> Result<(), Error> {
+    let has_done_migration = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'fix_flow_versioning_2')",
+    )
+    .fetch_one(db)
+    .await?
+    .unwrap_or(false);
+
+    if !has_done_migration {
+        migrator.lock().await?;
+
+        if migrator
+            .list_applied_migrations()
+            .await?
+            .iter()
+            .any(|x| x.version == 20240630102146)
         {
-            migrator.lock().await?;
             let has_done_migration = sqlx::query_scalar!(
-                "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'bypassrls_1-2')",
+                "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'fix_flow_versioning_2')",
             )
             .fetch_one(db)
             .await?
             .unwrap_or(false);
 
             if !has_done_migration {
-                let query = include_str!("../../custom_migrations/bypassrls_1.sql");
-                tracing::info!("Applying bypassrls_1.sql");
+                let query = include_str!("../../custom_migrations/fix_flow_versioning_2.sql");
+                tracing::info!("Applying fix_flow_versioning_2.sql");
                 let mut tx: sqlx::Transaction<'_, Postgres> = db.begin().await?;
                 tx.execute(query).await?;
-                tracing::info!("Applied bypassrls_1.sql");
-                sqlx::query!("INSERT INTO windmill_migrations (name) VALUES ('bypassrls_1-2')")
-                    .execute(&mut *tx)
-                    .await?;
+                tracing::info!("Applied fix_flow_versioning_2.sql");
+                sqlx::query!(
+                    "INSERT INTO windmill_migrations (name) VALUES ('fix_flow_versioning_2')"
+                )
+                .execute(&mut *tx)
+                .await?;
                 tx.commit().await?;
             }
-            migrator.unlock().await?;
         }
+
+        migrator.unlock().await?;
+    }
+    Ok(())
+}
+
+async fn fix_job_completed_index(db: &DB) -> Result<(), Error> {
+    // let has_done_migration = sqlx::query_scalar!(
+    //     "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'fix_job_completed_index')"
+    // )
+    // .fetch_one(db)
+    // .await?
+    // .unwrap_or(false);
+    // if !has_done_migration {
+    //     tracing::info!("Applying fix_job_completed_index migration");
+    //     let mut tx = db.begin().await?;
+    //     let mut r = false;
+    //     while !r {
+    //         r = sqlx::query_scalar!("SELECT pg_try_advisory_lock(4242)")
+    //             .fetch_one(&mut *tx)
+    //             .await
+    //             .map_err(|e| {
+    //                 tracing::error!("Error acquiring fix_job_completed_index lock: {e:#}");
+    //                 sqlx::migrate::MigrateError::Execute(e)
+    //             })?
+    //             .unwrap_or(false);
+    //         if !r {
+    //             tracing::info!("PG fix_job_completed_index_migration lock already acquired by another server or worker, retrying in 5s. (look for the advisory lock in pg_lock with granted = true)");
+    //             tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+    //         }
+    //     }
+    //     // sqlx::query(
+    //     //     "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_completed_job_workspace_id_created_at_new ON completed_job (workspace_id, job_kind, is_skipped, is_flow_step, created_at DESC, started_at DESC)"
+    //     // ).execute(db).await?;
+
+    //     sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_created_at")
+    //         .execute(db)
+    //         .await?;
+
+    //     sqlx::query!("INSERT INTO windmill_migrations (name) VALUES ('fix_job_completed_index') ON CONFLICT DO NOTHING")
+    //         .execute(&mut *tx)
+    //         .await?;
+    //     let _ = sqlx::query("SELECT pg_advisory_unlock(4242)")
+    //         .execute(&mut *tx)
+    //         .await?;
+    //     tx.commit().await?;
+    // }
+
+    let migration_job_name = "fix_job_completed_index_2";
+    let has_done_migration = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)",
+        migration_job_name
+    )
+    .fetch_one(db)
+    .await?
+    .unwrap_or(false);
+    if !has_done_migration {
+        tracing::info!("Applying {migration_job_name} migration");
+        let mut tx = db.begin().await?;
+        let mut r = false;
+        while !r {
+            r = sqlx::query_scalar!("SELECT pg_try_advisory_lock(4242)")
+                .fetch_one(&mut *tx)
+                .await
+                .map_err(|e| {
+                    tracing::error!("Error acquiring {migration_job_name} lock: {e:#}");
+                    sqlx::migrate::MigrateError::Execute(e)
+                })?
+                .unwrap_or(false);
+            if !r {
+                tracing::info!("PG {migration_job_name} lock already acquired by another server or worker, retrying in 5s. (look for the advisory lock in pg_lock with granted = true)");
+                tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+            }
+        }
+        sqlx::query(
+            "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_completed_job_workspace_id_created_at_new_2 ON completed_job (workspace_id, job_kind, success, is_skipped, is_flow_step, created_at DESC)"
+        ).execute(db).await?;
+
+        sqlx::query(
+            "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_completed_job_workspace_id_started_at_new ON completed_job (workspace_id, job_kind, success, is_skipped, is_flow_step, started_at DESC)"
+        ).execute(db).await?;
+
+        sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_created_at")
+            .execute(db)
+            .await?;
+
+        sqlx::query(
+            "DROP INDEX CONCURRENTLY IF EXISTS ix_completed_job_workspace_id_created_at_new",
+        )
+        .execute(db)
+        .await?;
+
+        sqlx::query!(
+            "INSERT INTO windmill_migrations (name) VALUES ($1) ON CONFLICT DO NOTHING",
+            migration_job_name
+        )
+        .execute(&mut *tx)
+        .await?;
+        let _ = sqlx::query("SELECT pg_advisory_unlock(4242)")
+            .execute(&mut *tx)
+            .await?;
+        tx.commit().await?;
+        tracing::info!("Finished applying {migration_job_name} migration");
+    }
+
+    let migration_job_name = "fix_job_completed_index_3";
+    let has_done_migration = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)",
+        migration_job_name
+    )
+    .fetch_one(db)
+    .await?
+    .unwrap_or(false);
+    if !has_done_migration {
+        tracing::info!("Applying {migration_job_name} migration");
+        let mut tx = db.begin().await?;
+        let mut r = false;
+        while !r {
+            r = sqlx::query_scalar!("SELECT pg_try_advisory_lock(4242)")
+                .fetch_one(&mut *tx)
+                .await
+                .map_err(|e| {
+                    tracing::error!("Error acquiring {migration_job_name} lock: {e:#}");
+                    sqlx::migrate::MigrateError::Execute(e)
+                })?
+                .unwrap_or(false);
+            if !r {
+                tracing::info!("PG {migration_job_name} lock already acquired by another server or worker, retrying in 5s. (look for the advisory lock in pg_lock with granted = true)");
+                tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+            }
+        }
+
+        sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS index_completed_job_on_schedule_path")
+            .execute(db)
+            .await?;
+
+        sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS concurrency_limit_stats_queue")
+            .execute(db)
+            .await?;
+
+        sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS root_job_index")
+            .execute(db)
+            .await?;
+
+        sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS index_completed_on_created")
+            .execute(db)
+            .await?;
+
+        sqlx::query!(
+            "INSERT INTO windmill_migrations (name) VALUES ($1) ON CONFLICT DO NOTHING",
+            migration_job_name
+        )
+        .execute(&mut *tx)
+        .await?;
+        let _ = sqlx::query("SELECT pg_advisory_unlock(4242)")
+            .execute(&mut *tx)
+            .await?;
+        tx.commit().await?;
+        tracing::info!("Finished applying {migration_job_name} migration");
+    }
+
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+async fn windmill_migrations(migrator: &mut CustomMigrator, db: &DB) -> Result<(), Error> {
+    if std::env::var("MIGRATION_NO_BYPASSRLS").is_ok() {
+        migrator.lock().await?;
+        let has_done_migration = sqlx::query_scalar!(
+            "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'bypassrls_1-2')",
+        )
+        .fetch_one(db)
+        .await?
+        .unwrap_or(false);
+
+        if !has_done_migration {
+            let query = include_str!("../../custom_migrations/bypassrls_1.sql");
+            tracing::info!("Applying bypassrls_1.sql");
+            let mut tx: sqlx::Transaction<'_, Postgres> = db.begin().await?;
+            tx.execute(query).await?;
+            tracing::info!("Applied bypassrls_1.sql");
+            sqlx::query!("INSERT INTO windmill_migrations (name) VALUES ('bypassrls_1-2')")
+                .execute(&mut *tx)
+                .await?;
+            tx.commit().await?;
+        }
+        migrator.unlock().await?;
     }
     Ok(())
 }

backend/windmill-api/src/ee.rs

@@ -1,4 +1,8 @@
 use anyhow::anyhow;
+#[cfg(feature = "enterprise")]
+use std::sync::Arc;
+#[cfg(feature = "enterprise")]
+use tokio::sync::RwLock;
 
 pub async fn validate_license_key(_license_key: String) -> anyhow::Result<String> {
     // Implementation is not open source
@@ -9,8 +13,20 @@ pub async fn validate_license_key(_license_key: String) -> anyhow::Result<String
 pub async fn jwt_ext_auth(
     _w_id: Option<&String>,
     _token: &str,
-) -> Option<(crate::db::ApiAuthed, usize)> {
+    _external_jwks: Option<Arc<RwLock<ExternalJwks>>>,
+) -> anyhow::Result<(crate::db::ApiAuthed, usize)> {
     // Implementation is not open source
 
-    None
+    Err(anyhow!("External JWT auth is not open source"))
+}
+
+#[cfg(feature = "enterprise")]
+pub struct ExternalJwks;
+
+#[cfg(feature = "enterprise")]
+impl ExternalJwks {
+    pub async fn load() -> Option<Arc<RwLock<Self>>> {
+        // Implementation is not open source
+        None
+    }
 }

backend/windmill-api/src/flows.rs

@@ -730,7 +730,7 @@ async fn update_flow(
 
     sqlx::query!(
         "UPDATE flow SET versions = array_append(versions, $1) WHERE path = $2 AND workspace_id = $3",
-        version, flow_path, w_id
+        version, nf.path, w_id
     ).execute(&mut tx).await?;
 
     if is_new_path {

backend/windmill-api/src/groups.rs

@@ -50,6 +50,8 @@ pub fn global_service() -> Router {
         .route("/delete/:name", delete(delete_igroup))
         .route("/adduser/:name", post(add_user_igroup))
         .route("/removeuser/:name", post(remove_user_igroup))
+        .route("/export", get(export_igroups))
+        .route("/overwrite", post(overwrite_igroups))
 }
 
 #[derive(FromRow, Serialize, Deserialize)]
@@ -759,3 +761,118 @@ async fn remove_user(
 
     Ok(format!("Removed {} to group {}", user_username, name))
 }
+
+#[cfg(feature = "enterprise")]
+#[derive(Serialize, Deserialize)]
+struct ExportedIGroup {
+    name: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    summary: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    id: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    scim_display_name: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    external_id: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    emails: Option<Vec<String>>,
+}
+
+#[cfg(feature = "enterprise")]
+async fn export_igroups(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+) -> JsonResult<Vec<ExportedIGroup>> {
+    require_super_admin(&db, &authed.email).await?;
+    let mut tx = db.begin().await?;
+    let igroups = sqlx::query_as!(
+        ExportedIGroup,
+        "SELECT name, summary, array_remove(array_agg(email_to_igroup.email), null) as emails, id, scim_display_name, external_id FROM email_to_igroup RIGHT JOIN instance_group ON instance_group.name = email_to_igroup.igroup GROUP BY name",
+    ).fetch_all(&mut *tx).await?;
+
+    audit_log(
+        &mut *tx,
+        &authed,
+        "igroups.export",
+        ActionKind::Execute,
+        "global",
+        None,
+        None,
+    )
+    .await?;
+
+    tx.commit().await?;
+
+    Ok(Json(igroups))
+}
+
+#[cfg(not(feature = "enterprise"))]
+async fn export_igroups() -> JsonResult<String> {
+    Err(Error::BadRequest(
+        "This feature is only available in the enterprise version".to_string(),
+    ))
+}
+
+#[cfg(feature = "enterprise")]
+async fn overwrite_igroups(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Json(igroups): Json<Vec<ExportedIGroup>>,
+) -> Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+    let mut tx = db.begin().await?;
+
+    sqlx::query!("DELETE FROM email_to_igroup")
+        .execute(&mut *tx)
+        .await?;
+
+    sqlx::query!("DELETE FROM instance_group")
+        .execute(&mut *tx)
+        .await?;
+
+    for igroup in igroups.iter() {
+        sqlx::query!(
+            "INSERT INTO instance_group (name, summary, id, scim_display_name, external_id) VALUES ($1, $2, $3, $4, $5)",
+            igroup.name,
+            igroup.summary,
+            igroup.id,
+            igroup.scim_display_name,
+            igroup.external_id,
+        )
+        .execute(&mut *tx)
+        .await?;
+
+        if let Some(emails) = &igroup.emails {
+            for email in emails.iter() {
+                sqlx::query!(
+                    "INSERT INTO email_to_igroup (email, igroup) VALUES ($1, $2)",
+                    email,
+                    igroup.name,
+                )
+                .execute(&mut *tx)
+                .await?;
+            }
+        }
+    }
+
+    audit_log(
+        &mut *tx,
+        &authed,
+        "igroups.import",
+        ActionKind::Create,
+        "global",
+        None,
+        None,
+    )
+    .await?;
+
+    tx.commit().await?;
+    Ok("Imported igroups".to_string())
+}
+
+#[cfg(not(feature = "enterprise"))]
+async fn overwrite_igroups() -> JsonResult<String> {
+    Err(Error::BadRequest(
+        "This feature is only available in the enterprise version".to_string(),
+    ))
+}

backend/windmill-api/src/indexer_ee.rs

@@ -0,0 +1,5 @@
+use axum::Router;
+
+pub fn workspaced_service() -> Router {
+    Router::new()
+}

backend/windmill-api/src/inputs.rs

@@ -127,7 +127,7 @@ async fn get_input_history(
     let mut tx = user_db.begin(&authed).await?;
 
     let sql = &format!(
-        "select id, created_at, created_by, CASE WHEN args is null or pg_column_size(args) < 40000 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args, success from completed_job \
+        "select id, created_at, created_by, 'null'::jsonb as args, success from completed_job \
         where {} = $1 and job_kind = $2 and workspace_id = $3 \
         order by created_at desc limit $4 offset $5",
         r.runnable_type.column_name()
@@ -173,19 +173,48 @@ async fn get_input_history(
     Ok(Json(inputs))
 }
 
+#[derive(Deserialize)]
+struct GetArgs {
+    input: Option<bool>,
+    allow_large: Option<bool>,
+}
 async fn get_args_from_history_or_saved_input(
     authed: ApiAuthed,
     Extension(user_db): Extension<UserDB>,
+    Query(g): Query<GetArgs>,
     Path((w_id, job_or_input_id)): Path<(String, Uuid)>,
 ) -> JsonResult<Option<Value>> {
     let mut tx = user_db.begin(&authed).await?;
-    let result_o = sqlx::query_scalar!(
-        "SELECT args FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT args FROM input WHERE id = $1 AND workspace_id = $2",
+    let result_o = if let Some(input) = g.input {
+        if input {
+            sqlx::query_scalar!(
+                    "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2",
+                    job_or_input_id,
+                    w_id,
+                    g.allow_large.unwrap_or(true)
+                )
+                .fetch_optional(&mut *tx)
+                .await?
+        } else {
+            sqlx::query_scalar!(
+                "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM completed_job WHERE id = $1 AND workspace_id = $2",
+                job_or_input_id,
+                w_id,
+                g.allow_large.unwrap_or(true)
+            )
+            .fetch_optional(&mut *tx)
+            .await?
+        }
+    } else {
+        sqlx::query_scalar!(
+        "SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM completed_job WHERE id = $1 AND workspace_id = $2 UNION ALL SELECT CASE WHEN pg_column_size(args) < 40000 OR $3 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args FROM input WHERE id = $1 AND workspace_id = $2",
         job_or_input_id,
-        w_id
+        w_id,
+        g.allow_large.unwrap_or(true)
     )
     .fetch_optional(&mut *tx)
-    .await?;
+    .await?
+    };
 
     tx.commit().await?;
 
@@ -206,7 +235,7 @@ async fn list_saved_inputs(
     let mut tx = user_db.begin(&authed).await?;
 
     let rows = sqlx::query_as::<_, InputRow>(
-        "select id, workspace_id, runnable_id, runnable_type, name, CASE WHEN pg_column_size(args) < 40000 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args, created_at, created_by, is_public from input \
+        "select id, workspace_id, runnable_id, runnable_type, name, 'null'::jsonb as args, created_at, created_by, is_public from input \
          where runnable_id = $1 and runnable_type = $2 and workspace_id = $3 \
          and (is_public IS true OR created_by = $4) \
          order by created_at desc limit $5 offset $6",

backend/windmill-api/src/jobs.rs

@@ -8,6 +8,7 @@
 
 use axum::body::Body;
 use axum::http::HeaderValue;
+use quick_cache::sync::Cache;
 use serde_json::value::RawValue;
 use sqlx::Pool;
 use std::collections::HashMap;
@@ -297,11 +298,17 @@ struct JsonPath {
     pub approver: Option<String>,
 }
 async fn get_result_by_id(
+    authed: ApiAuthed,
     Extension(db): Extension<DB>,
     Path((w_id, flow_id, node_id)): Path<(String, Uuid, String)>,
     Query(JsonPath { json_path, .. }): Query<JsonPath>,
 ) -> windmill_common::error::JsonResult<Box<JsonRawValue>> {
-    let res = windmill_queue::get_result_by_id(db, w_id, flow_id, node_id, json_path).await?;
+    let res =
+        windmill_queue::get_result_by_id(db.clone(), w_id.clone(), flow_id, node_id, json_path)
+            .await?;
+
+    log_job_view(&db, Some(&authed), &w_id, &flow_id).await?;
+
     Ok(Json(res))
 }
 
@@ -622,6 +629,9 @@ async fn get_flow_job_debug_info(
                 jobs.insert(job.id().to_string(), job);
             }
         }
+
+        log_job_view(&db, opt_authed.as_ref(), &w_id, &id).await?;
+
         Ok(Json(jobs).into_response())
     } else {
         Err(error::Error::NotFound(format!(
@@ -651,6 +661,9 @@ async fn get_job(
     )
     .await?;
     job.fetch_outstanding_wait_time(&db).await?;
+
+    log_job_view(&db, opt_authed.as_ref(), &w_id, &id).await?;
+
     Ok(Json(job).into_response())
 }
 
@@ -701,7 +714,7 @@ fn generate_get_job_query(no_logs: bool, table: &str) -> String {
     return format!("SELECT 
     id, {table}.workspace_id, parent_job, created_by, {table}.created_at, started_at, script_hash, script_path, 
     CASE WHEN args is null or pg_column_size(args) < 90000 THEN args ELSE '{{\"reason\": \"WINDMILL_TOO_BIG\"}}'::jsonb END as args, 
-    {log_expr} as logs, raw_code, canceled, canceled_by, canceled_reason, job_kind, env_id,
+    {log_expr} as logs, raw_code, canceled, canceled_by, canceled_reason, job_kind,
     schedule_path, permissioned_as, flow_status, raw_flow, is_flow_step, language,
     raw_lock, email, visible_to_owner, mem_peak, tag, priority, {additional_fields}
     FROM {table}
@@ -871,6 +884,9 @@ async fn get_job_logs(
             ));
         }
         let logs = record.logs.unwrap_or_default();
+
+        log_job_view(&db, opt_authed.as_ref(), &w_id, &id).await?;
+
         #[cfg(all(feature = "enterprise", feature = "parquet"))]
         if let Some(r) = get_logs_from_store(record.log_offset, &logs, &record.log_file_index).await
         {
@@ -900,6 +916,9 @@ async fn get_job_logs(
             ));
         }
         let logs = text.logs.unwrap_or_default();
+
+        log_job_view(&db, opt_authed.as_ref(), &w_id, &id).await?;
+
         #[cfg(all(feature = "enterprise", feature = "parquet"))]
         if let Some(r) = get_logs_from_store(text.log_offset, &logs, &text.log_file_index).await {
             return r.map(content_plain);
@@ -938,6 +957,9 @@ async fn get_args(
                 "As a non logged in user, you can only see jobs ran by anonymous users".to_string(),
             ));
         }
+
+        log_job_view(&db, opt_authed.as_ref(), &w_id, &id).await?;
+
         Ok(Json(record.args.map(|x| x.0).unwrap_or_default()))
     } else {
         let record = sqlx::query_as::<_, RawArgs>(
@@ -955,6 +977,9 @@ async fn get_args(
                 "As a non logged in user, you can only see jobs ran by anonymous users".to_string(),
             ));
         }
+
+        log_job_view(&db, opt_authed.as_ref(), &w_id, &id).await?;
+
         Ok(Json(record.args.map(|x| x.0).unwrap_or_default()))
     }
 }
@@ -1298,7 +1323,93 @@ async fn cancel_jobs(
     rsmq: Option<rsmq_async::MultiplexedRsmq>,
 ) -> error::JsonResult<Vec<Uuid>> {
     let mut uuids = vec![];
+    let mut tx = db.begin().await?;
+    let trivial_jobs =  sqlx::query!("INSERT INTO completed_job AS cj
+                   ( workspace_id
+                   , id
+                   , parent_job
+                   , created_by
+                   , created_at
+                   , started_at
+                   , duration_ms
+                   , success
+                   , script_hash
+                   , script_path
+                   , args
+                   , result
+                   , raw_code
+                   , raw_lock
+                   , canceled
+                   , canceled_by
+                   , canceled_reason
+                   , job_kind
+                   , schedule_path
+                   , permissioned_as
+                   , flow_status
+                   , raw_flow
+                   , is_flow_step
+                   , is_skipped
+                   , language
+                   , email
+                   , visible_to_owner
+                   , mem_peak
+                   , tag
+                   , priority
+                )
+                SELECT  workspace_id
+                   , id
+                   , parent_job
+                   , created_by
+                   , created_at
+                   , now()
+                   , 0
+                   , false
+                   , script_hash
+                   , script_path
+                   , args
+                   , $4
+                   , raw_code
+                   , raw_lock
+                   , true
+                   , $1
+                   , canceled_reason
+                   , job_kind
+                   , schedule_path
+                   , permissioned_as
+                   , flow_status
+                   , raw_flow
+                   , is_flow_step
+                   , false
+                   , language
+                   , email
+                   , visible_to_owner
+                   , mem_peak
+                   , tag
+                   , priority FROM queue 
+        WHERE id = any($2) AND running = false AND parent_job IS NULL AND workspace_id = $3 AND schedule_path IS NULL FOR UPDATE SKIP LOCKED
+        ON CONFLICT (id) DO NOTHING RETURNING id", username, &jobs, w_id, serde_json::json!({"error": { "message": format!("Job canceled: cancel all by {username}"), "name": "Canceled", "reason": "cancel all", "canceler": username}}))
+        .fetch_all(&mut *tx)
+        .await?.into_iter().map(|x| x.id).collect::<Vec<Uuid>>();
+
+    sqlx::query!(
+        "DELETE FROM queue WHERE id = any($1) AND workspace_id = $2",
+        &jobs,
+        w_id
+    )
+    .execute(&mut *tx)
+    .await?;
+    tx.commit().await?;
+
+    // sqlx::query!(
+    //     "UPDATE queue SET canceled = true, canceled_by = $1, canceled_reason = 'cancelled all by user' WHERE id IN (SELECT id FROM queue where id = any($2) AND workspace_id = $3 AND schedule_path IS NULL FOR UPDATE SKIP LOCKED) RETURNING id",
+    //     username,
+    //     &jobs,
+    //     w_id
+    // ).execute(db).await?;
     for job_id in jobs.into_iter() {
+        if trivial_jobs.contains(&job_id) {
+            continue;
+        }
         let rsmq = rsmq.clone();
         match tokio::time::timeout(tokio::time::Duration::from_secs(5), async move {
             let tx = db.begin().await?;
@@ -1466,7 +1577,7 @@ async fn list_jobs(
     };
 
     let sql = if lq.success.is_none() && lq.label.is_none() {
-        let sqlq = list_queue_jobs_query(
+        let mut sqlq = list_queue_jobs_query(
             &w_id,
             &ListQueueQuery { order_desc: Some(true), ..lq.into() },
             UnifiedJob::queued_job_fields(),
@@ -1482,10 +1593,10 @@ async fn list_jobs(
                 offset
             )
         } else {
-            sqlq.query()?
+            sqlq.limit(per_page).offset(offset).query()?
         }
     } else {
-        sqlc.unwrap().query()?
+        sqlc.unwrap().limit(per_page).offset(offset).query()?
     };
     let mut tx = user_db.begin(&authed).await?;
 
@@ -1858,7 +1969,7 @@ pub async fn get_suspended_job_flow(
         Job::QueuedJob(job) => &job.email,
         Job::CompletedJobWithFormattedResult(job) => &job.cj.email,
     };
-    conditionally_require_authed_user(authed, flow_status.clone(), trigger_email)?;
+    conditionally_require_authed_user(authed.clone(), flow_status.clone(), trigger_email)?;
 
     let approvers_from_status = match flow_module_status {
         FlowStatusModule::Success { approvers, .. } => approvers.to_owned(),
@@ -1885,6 +1996,8 @@ pub async fn get_suspended_job_flow(
         approvers_from_status
     };
 
+    log_job_view(&db, authed.as_ref(), &w_id, &job).await?;
+
     Ok(Json(SuspendedJobFlow { job: flow, approvers }).into_response())
 }
 
@@ -3170,6 +3283,76 @@ lazy_static::lazy_static! {
         .ok()
         .and_then(|x| x.parse().ok())
         .unwrap_or(200);
+
+    static ref JOB_VIEW_AUDIT_LOGS: bool = std::env::var("JOB_VIEW_AUDIT_LOGS")
+        .ok()
+        .and_then(|x| x.parse().ok())
+        .unwrap_or(false);
+
+    static ref JOB_VIEW_CACHE: JobViewCache = JobViewCache::new(50000);
+}
+
+struct JobViewCache {
+    cache: Cache<String, std::time::Instant>,
+}
+
+impl JobViewCache {
+    fn new(items_capacity: usize) -> Self {
+        Self { cache: Cache::new(items_capacity) }
+    }
+    fn get_or_insert(&self, key: &str) -> Option<std::time::Instant> {
+        match self.cache.get(key) {
+            Some(t) if t < std::time::Instant::now() => {
+                self.cache.insert(
+                    key.to_string(),
+                    std::time::Instant::now() + std::time::Duration::from_secs(60),
+                );
+                None
+            }
+            v => {
+                self.cache.insert(
+                    key.to_string(),
+                    std::time::Instant::now() + std::time::Duration::from_secs(60),
+                );
+                v
+            }
+        }
+    }
+}
+
+async fn log_job_view(
+    db: &DB,
+    opt_authed: Option<&ApiAuthed>,
+    w_id: &str,
+    job_id: &Uuid,
+) -> error::Result<()> {
+    if *JOB_VIEW_AUDIT_LOGS {
+        let audit_author = match opt_authed {
+            Some(authed) => AuditAuthor::from(authed),
+            None => AuditAuthor {
+                username: "anonymous".to_string(),
+                username_override: None,
+                email: "anonymous".to_string(),
+            },
+        };
+        if JOB_VIEW_CACHE
+            .get_or_insert(&format!("{}_{}", job_id, audit_author.email))
+            .is_none()
+        {
+            audit_log(
+                db,
+                &audit_author,
+                "jobs.view",
+                ActionKind::Execute,
+                w_id,
+                Some(&job_id.to_string()),
+                None,
+            )
+            .await?;
+        };
+    }
+
+    Ok(())
 }
 
 pub async fn run_wait_result_job_by_path_get(
@@ -3789,22 +3972,27 @@ async fn run_dependencies_job(
     let raw_script = req.raw_scripts[0].clone();
     let script_path = raw_script.script_path;
     let ehm = HashMap::new();
+    let raw_code = raw_script.raw_code.unwrap_or_else(|| "".to_string());
+    let language = raw_script.language;
+
     let (args, raw_code) = if let Some(deps) = req.raw_deps {
         let mut hm = HashMap::new();
         hm.insert(
             "raw_deps".to_string(),
             JsonRawValue::from_string("true".to_string()).unwrap(),
         );
+        if language == ScriptLang::Bun {
+            let annotation = windmill_common::worker::get_annotation(&raw_code);
+            hm.insert(
+                "npm_mode".to_string(),
+                JsonRawValue::from_string(annotation.npm_mode.to_string()).unwrap(),
+            );
+        }
         (PushArgs { extra: Some(hm), args: &ehm }, deps)
     } else {
-        (
-            PushArgs::from(&ehm),
-            raw_script.raw_code.unwrap_or_else(|| "".to_string()),
-        )
+        (PushArgs::from(&ehm), raw_code)
     };
 
-    let language = raw_script.language;
-
     let (uuid, tx) = push(
         &db,
         PushIsolationLevel::IsolatedRoot(db.clone(), rsmq),
@@ -4316,6 +4504,7 @@ async fn get_job_update(
                 "As a non logged in user, you can only see jobs ran by anonymous users".to_string(),
             ));
         }
+        log_job_view(&db, opt_authed.as_ref(), &w_id, &job_id).await?;
         Ok(Json(JobUpdate {
             running: if !running && record.running {
                 Some(true)
@@ -4351,6 +4540,7 @@ async fn get_job_update(
                         .to_string(),
                 ));
             }
+            log_job_view(&db, opt_authed.as_ref(), &w_id, &job_id).await?;
             Ok(Json(JobUpdate {
                 running: Some(false),
                 completed: Some(true),
@@ -4598,11 +4788,11 @@ async fn get_completed_job<'a>(
     Path((w_id, id)): Path<(String, Uuid)>,
 ) -> error::Result<Response> {
     let job_o = sqlx::query_as::<_, CompletedJob>("SELECT id, workspace_id, parent_job, created_by, created_at, duration_ms, success, script_hash, script_path, 
-    CASE WHEN args is null or pg_column_size(args) < 90000 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args, CASE WHEN result is null or pg_column_size(result) < 90000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result, logs, deleted, raw_code, canceled, canceled_by, canceled_reason, job_kind, env_id,
+    CASE WHEN args is null or pg_column_size(args) < 90000 THEN args ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as args, CASE WHEN result is null or pg_column_size(result) < 90000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result, logs, deleted, raw_code, canceled, canceled_by, canceled_reason, job_kind,
     schedule_path, permissioned_as, flow_status, raw_flow, is_flow_step, language, started_at, is_skipped,
     raw_lock, email, visible_to_owner, mem_peak, tag, priority, result->'wm_labels' as labels FROM completed_job WHERE id = $1 AND workspace_id = $2")
         .bind(id)
-        .bind(w_id)
+        .bind(&w_id)
         .fetch_optional(&db)
         .await?;
 
@@ -4624,6 +4814,9 @@ async fn get_completed_job<'a>(
     // )
     // .fetch_optional(db)
     // .await.ok().flatten().flatten();
+
+    log_job_view(&db, opt_authed.as_ref(), &w_id, &id).await?;
+
     Ok(response)
 }
 
@@ -4718,6 +4911,8 @@ async fn get_completed_job_result(
         raw_result.result.map(|x| x.0),
     );
 
+    log_job_view(&db, opt_authed.as_ref(), &w_id, &id).await?;
+
     Ok(Json(result).into_response())
 }
 
@@ -4759,6 +4954,9 @@ async fn get_completed_job_result_maybe(
                 "As a non logged in user, you can only see jobs ran by anonymous users".to_string(),
             ));
         }
+
+        log_job_view(&db, opt_authed.as_ref(), &w_id, &id).await?;
+
         Ok(Json(CompletedJobResult {
             started: Some(true),
             success: Some(res.success),

backend/windmill-api/src/lib.rs

@@ -7,6 +7,8 @@
  */
 
 use crate::db::ApiAuthed;
+#[cfg(feature = "enterprise")]
+use crate::ee::ExternalJwks;
 #[cfg(feature = "embedding")]
 use crate::embeddings::load_embeddings_db;
 use crate::oauth2_ee::AllClients;
@@ -57,6 +59,7 @@ mod flows;
 mod folders;
 mod granular_acls;
 mod groups;
+mod indexer_ee;
 mod inputs;
 mod integration;
 #[cfg(feature = "parquet")]
@@ -140,9 +143,22 @@ pub async fn add_webhook_allowed_origin(
     next.run(req).await
 }
 
+#[cfg(not(feature = "tantivy"))]
+type IndexReader = ();
+
+#[cfg(not(feature = "tantivy"))]
+type IndexWriter = ();
+
+#[cfg(feature = "tantivy")]
+type IndexReader = windmill_indexer::indexer_ee::IndexReader;
+#[cfg(feature = "tantivy")]
+type IndexWriter = windmill_indexer::indexer_ee::IndexWriter;
+
 pub async fn run_server(
     db: DB,
     rsmq: Option<rsmq_async::MultiplexedRsmq>,
+    index_reader: Option<IndexReader>,
+    index_writer: Option<IndexWriter>,
     addr: SocketAddr,
     mut rx: tokio::sync::broadcast::Receiver<()>,
     port_tx: tokio::sync::oneshot::Sender<String>,
@@ -161,9 +177,13 @@ pub async fn run_server(
     }
     let user_db = UserDB::new(db.clone());
 
+    #[cfg(feature = "enterprise")]
+    let ext_jwks = ExternalJwks::load().await;
     let auth_cache = Arc::new(users::AuthCache::new(
         db.clone(),
         std::env::var("SUPERADMIN_SECRET").ok(),
+        #[cfg(feature = "enterprise")]
+        ext_jwks,
     ));
     let argon2 = Arc::new(Argon2::default());
 
@@ -177,6 +197,8 @@ pub async fn run_server(
         .layer(Extension(rsmq))
         .layer(Extension(user_db))
         .layer(Extension(auth_cache.clone()))
+        .layer(Extension(index_reader))
+        .layer(Extension(index_writer))
         .layer(CookieManagerLayer::new())
         .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())))
         .layer(DefaultBodyLimit::max(
@@ -266,6 +288,10 @@ pub async fn run_server(
                 .route_layer(from_extractor::<ApiAuthed>())
                 .route_layer(from_extractor::<users::Tokened>())
                 .nest("/jobs", jobs::global_root_service())
+                .nest(
+                    "/srch/w/:workspace_id/index",
+                    indexer_ee::workspaced_service(),
+                )
                 .nest("/oidc", oidc_ee::global_service())
                 .nest(
                     "/saml",

backend/windmill-api/src/resources.rs

@@ -567,6 +567,7 @@ pub async fn transform_json_value<'c>(
                 job.flow_step_id.clone(),
                 job.root_job.map(|x| x.to_string()),
                 None,
+                Some(job.scheduled_for.clone()),
             )
             .await;
 

backend/windmill-api/src/scripts.rs

@@ -52,7 +52,7 @@ use windmill_common::{
     utils::{
         not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
     },
-    worker::to_raw_value,
+    worker::{get_annotation, to_raw_value},
     HUB_BASE_URL,
 };
 use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
@@ -570,21 +570,34 @@ async fn create_script_internal<'c>(
     let lock = if !(ns.language == ScriptLang::Python3
         || ns.language == ScriptLang::Go
         || ns.language == ScriptLang::Bun
+        || ns.language == ScriptLang::Bunnative
         || ns.language == ScriptLang::Deno
         || ns.language == ScriptLang::Php)
+        || ns.codebase.is_some()
     {
         Some(String::new())
     } else {
         ns.lock
             .and_then(|e| if e.is_empty() { None } else { Some(e) })
     };
-    let needs_lock_gen = lock.is_none();
+    let needs_lock_gen = lock.is_none() && codebase.is_none();
     let envs = ns.envs.as_ref().map(|x| x.as_slice());
     let envs = if ns.envs.is_none() || ns.envs.as_ref().unwrap().is_empty() {
         None
     } else {
         envs
     };
+
+    let lang = if &ns.language == &ScriptLang::Bun || &ns.language == &ScriptLang::Bunnative {
+        let anns = get_annotation(&ns.content);
+        if anns.native_mode {
+            ScriptLang::Bunnative
+        } else {
+            ScriptLang::Bun
+        }
+    } else {
+        ns.language.clone()
+    };
     sqlx::query!(
         "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \
          content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \
@@ -604,7 +617,7 @@ async fn create_script_internal<'c>(
         ns.is_template.unwrap_or(false),
         extra_perms,
         lock,
-        ns.language.clone() as ScriptLang,
+        lang as ScriptLang,
         ns.kind.unwrap_or(ScriptKind::Script) as ScriptKind,
         ns.tag,
         ns.draft_only,

backend/windmill-api/src/settings.rs

@@ -40,6 +40,7 @@ pub fn global_service() -> Router {
             "/global/:key",
             post(set_global_setting).get(get_global_setting),
         )
+        .route("/list_global", get(list_global_settings))
         .route("/test_smtp", post(test_email))
         .route("/test_license_key", post(test_license_key))
         .route("/send_stats", post(send_stats))
@@ -115,14 +116,17 @@ pub async fn test_s3_bucket(
     let client = build_object_store_from_settings(test_s3_bucket).await?;
 
     let mut list = client.list(Some(&object_store::path::Path::from("".to_string())));
-    let first_file = list
-        .next()
-        .await
-        .ok_or_else(|| {
-            error::Error::InternalErr("Failed to list files in blob storage".to_string())
-        })?
-        .map_err(|e| anyhow::anyhow!("error listing bucket: {e:#}"))?;
-    tracing::info!("Listed files: {:?}", first_file);
+    let first_file = list.next().await;
+    if first_file.is_some() {
+        if let Err(e) = first_file.as_ref().unwrap() {
+            tracing::error!("error listing bucket: {e:#}");
+            error::Error::InternalErr(format!("Failed to list files in blob storage: {e:#}"));
+        }
+        tracing::info!("Listed files: {:?}", first_file.unwrap());
+    } else {
+        tracing::info!("No files in blob storage");
+    }
+
     let path = object_store::path::Path::from(format!(
         "/test-s3-bucket-{uuid}",
         uuid = uuid::Uuid::new_v4()
@@ -263,6 +267,33 @@ pub async fn get_global_setting(
     Ok(Json(value.unwrap_or_else(|| serde_json::Value::Null)))
 }
 
+#[cfg(feature = "enterprise")]
+#[derive(Deserialize, serde::Serialize)]
+struct GlobalSetting {
+    name: String,
+    value: serde_json::Value,
+}
+
+#[cfg(feature = "enterprise")]
+async fn list_global_settings(
+    Extension(db): Extension<DB>,
+    authed: ApiAuthed,
+) -> JsonResult<Vec<GlobalSetting>> {
+    require_super_admin(&db, &authed.email).await?;
+    let settings = sqlx::query_as!(GlobalSetting, "SELECT name, value FROM global_settings")
+        .fetch_all(&db)
+        .await?;
+
+    Ok(Json(settings))
+}
+
+#[cfg(not(feature = "enterprise"))]
+async fn list_global_settings() -> JsonResult<String> {
+    return Err(error::Error::BadRequest(
+        "Listing global settings not available on community edition".to_string(),
+    ));
+}
+
 pub async fn send_stats(Extension(db): Extension<DB>, authed: ApiAuthed) -> Result<String> {
     require_super_admin(&db, &authed.email).await?;
     windmill_common::stats_ee::send_stats(&"manual".to_string(), &HTTP_CLIENT, &db).await?;