chore(main): release 1.156.1 (#2170 )

* chore(main): release 1.156.1 * Apply automatic changes --------- Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
fix: fix python client
2026-09-20 16:02:28 +00:00 · 2023-08-25 13:38:58 +02:00 · 2023-08-25 13:37:10 +02:00 · 2023-08-25 10:52:02 +02:00 · 2023-08-24 19:55:52 +02:00 · 2023-08-24 19:49:02 +02:00
404 changed files with 93536 additions and 90986 deletions
@@ -33,7 +33,7 @@ ENV SQLX_OFFLINE=true

 RUN mkdir -p /frontend/build
 RUN apt-get update \
-    && apt-get install -y ca-certificates tzdata libpq5 \
+    && apt-get install -y ca-certificates tzdata libpq5 cmake\
    make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
    libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
    libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
@@ -18,7 +18,6 @@ jobs:
    runs-on: [self-hosted, new]
    container:
      image: ghcr.io/windmill-labs/backend-tests
-      options: --privileged
    services:
      postgres:
        image: postgres
@@ -27,9 +26,7 @@ jobs:
          POSTGRES_PASSWORD: changeme

        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
+          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v3
@@ -41,4 +38,8 @@ jobs:
            backend -> target
      - name: cargo test
        timeout-minutes: 10
-        run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture
+        run:
+          mkdir frontend/build && cd backend && touch
+          windmill-api/openapi-deref.yaml &&
+          DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo
+          test --all -- --nocapture
@@ -0,0 +1,84 @@
+env:
+  REGISTRY: ghcr.io
+  ECR_REGISTRY: 976079455550.dkr.ecr.us-east-1.amazonaws.com
+  IMAGE_NAME: ${{ github.repository }}-lsp
+
+name: Publish lsp
+on:
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+  packages: write
+
+jobs:
+  publish_lsp:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Sleep for 900 seconds waiting for pypi to update index
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: sleep 900
+        shell: bash
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: depot/setup-action@v1
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push publicly
+        uses: depot/build-push-action@v1
+        with:
+          context: "{{defaultContext}}:lsp"
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ steps.meta.outputs.tags }}
+          labels: |
+            ${{ steps.meta.outputs.labels }}
+            org.opencontainers.image.licenses=AGPLv3
+
+  publish_lsp_private:
+    needs: [publish_lsp]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to ECR
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.ECR_REGISTRY }}
+          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Push image to ECR
+        run: |
+          docker buildx imagetools create \
+            --tag ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest \
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
@@ -1,8 +1,3 @@
-env:
-  REGISTRY: ghcr.io
-  ECR_REGISTRY: 976079455550.dkr.ecr.us-east-1.amazonaws.com
-  IMAGE_NAME: ${{ github.repository }}-lsp
-
 name: Publish python-client
 on:
  push:
@@ -10,11 +5,6 @@ on:
      - "v*"
  workflow_dispatch:

-permissions:
-  contents: read
-  id-token: write
-  packages: write
-
 jobs:
  publish_pypi:
    runs-on: ubuntu-latest
@@ -29,70 +19,3 @@ jobs:
        run: |
          cd python-client
          ./publish.sh
-
-  publish_lsp:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Sleep for 300 seconds waiting for pypi to update index
-        if: startsWith(github.ref, 'refs/tags/v')
-        run: sleep 300
-        shell: bash
-
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - uses: depot/setup-action@v1
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          tags: |
-            type=ref,event=branch
-            type=ref,event=pr
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-
-      - name: Login to registry
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push publicly
-        uses: depot/build-push-action@v1
-        with:
-          context: "{{defaultContext}}:lsp"
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-            ${{ steps.meta.outputs.tags }}
-          labels: |
-            ${{ steps.meta.outputs.labels }}
-            org.opencontainers.image.licenses=AGPLv3
-
-  publish_lsp_private:
-    needs: [publish_lsp]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to ECR
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.ECR_REGISTRY }}
-          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
-      - name: Push image to ECR
-        run: |
-          docker buildx imagetools create \
-            --tag ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest \
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
@@ -1,6 +1,209 @@
 # Changelog


+## [1.156.1](https://github.com/windmill-labs/windmill/compare/v1.156.0...v1.156.1) (2023-08-25)
+
+
+### Bug Fixes
+
+* fix python client ([7649a53](https://github.com/windmill-labs/windmill/commit/7649a53f3c792ceba8f2a0fc8535c512b25bf969))
+
+## [1.156.0](https://github.com/windmill-labs/windmill/compare/v1.155.0...v1.156.0) (2023-08-24)
+
+
+### Features
+
+* schedule recovery handler ([#2126](https://github.com/windmill-labs/windmill/issues/2126)) ([0dcb425](https://github.com/windmill-labs/windmill/commit/0dcb425e4a9cf241ed301f794680b36a7f17cc34))
+
+## [1.155.0](https://github.com/windmill-labs/windmill/compare/v1.154.2...v1.155.0) (2023-08-24)
+
+
+### Features
+
+* add templatev2 using new eval ([13d870f](https://github.com/windmill-labs/windmill/commit/13d870f16370a74fe481a1701eda27109a776c75))
+* eval v2, blazing fast eval triggered only upon the right changes ([#2164](https://github.com/windmill-labs/windmill/issues/2164)) ([5207a7a](https://github.com/windmill-labs/windmill/commit/5207a7a6aa1520c987d26d5c1f99f653c1c81cf6))
+* remove connect in favor of eval ([e7aaa17](https://github.com/windmill-labs/windmill/commit/e7aaa177b72749ca9d0d78c452ec8e47d6514186))
+
+
+### Bug Fixes
+
+* bump bun to 0.8.0 ([4825519](https://github.com/windmill-labs/windmill/commit/4825519ac94a4992cf21fbf4a21fbea8038058d9))
+* fix tables not updating inputs on creation ([a419bc4](https://github.com/windmill-labs/windmill/commit/a419bc41bfadce1ac75383d1824ff9fef3404aad))
+* **frontend:** Fix code display + use async/await in fetch examples ([#2150](https://github.com/windmill-labs/windmill/issues/2150)) ([2f9177f](https://github.com/windmill-labs/windmill/commit/2f9177f6cec0a676c774ee426482f55227e6e388))
+* **frontend:** fix copyToClipboard on non-HTTPS site ([#2046](https://github.com/windmill-labs/windmill/issues/2046)) ([95ea0e8](https://github.com/windmill-labs/windmill/commit/95ea0e8f87195816dde3f9554b3cb92791b63a37))
+* update go to 1.12.0 and deno to 1.36.2 ([4317573](https://github.com/windmill-labs/windmill/commit/431757339bbfff6d67f484439d87255acc5c62ff))
+* update python client with by_path methods ([8a25a86](https://github.com/windmill-labs/windmill/commit/8a25a86e586485e7949bb208fa94db906e983b6c))
+
+## [1.154.2](https://github.com/windmill-labs/windmill/compare/v1.154.1...v1.154.2) (2023-08-22)
+
+
+### Bug Fixes
+
+* fix cancel job for flows in some edge cases ([58bb19a](https://github.com/windmill-labs/windmill/commit/58bb19a4471ce8cfced4b144fca40069b5ce0820))
+
+## [1.154.1](https://github.com/windmill-labs/windmill/compare/v1.154.0...v1.154.1) (2023-08-22)
+
+
+### Bug Fixes
+
+* **frontend:** Fix hub navigation ([#2151](https://github.com/windmill-labs/windmill/issues/2151)) ([d0ed8f0](https://github.com/windmill-labs/windmill/commit/d0ed8f0fefe3176b9bab621a6b3e9231254504e2))
+* show for-loop settings ([ab8a27f](https://github.com/windmill-labs/windmill/commit/ab8a27f123fbca187eee3b372d512797f8a03916))
+
+## [1.154.0](https://github.com/windmill-labs/windmill/compare/v1.153.0...v1.154.0) (2023-08-21)
+
+
+### Features
+
+* deploy folders as well in the UI deployer ([bcf5d4e](https://github.com/windmill-labs/windmill/commit/bcf5d4e5d42a7d17e2d1932b030cca101d9de9b4))
+
+
+### Bug Fixes
+
+* avoid stack-overflow on jsruntime for recursive objects ([127eea3](https://github.com/windmill-labs/windmill/commit/127eea3c8144b14b8f78a196f5c2cd245d2caad9))
+* do not require auth for OPTIONS requests ([bdd59c9](https://github.com/windmill-labs/windmill/commit/bdd59c94a9bde10e808427ef529d1b6ab6e78a45))
+
+## [1.153.0](https://github.com/windmill-labs/windmill/compare/v1.152.0...v1.153.0) (2023-08-20)
+
+
+### Features
+
+* multiline support in bash ([e1469cc](https://github.com/windmill-labs/windmill/commit/e1469cc64d672b5fc42edac313bc11a017812511))
+
+
+### Bug Fixes
+
+* update deno-client to use new Resource and Variable endpoints ([c13428a](https://github.com/windmill-labs/windmill/commit/c13428ad089999e38768b86bfd251d747759dc69))
+
+## [1.152.0](https://github.com/windmill-labs/windmill/compare/v1.151.2...v1.152.0) (2023-08-20)
+
+
+### Features
+
+* handle drift in every time referencing db times ([b9fb206](https://github.com/windmill-labs/windmill/commit/b9fb206c112798f3776ba0e6da70e86e7c769a1f))
+* prometheus metrics are now ee only ([2afea50](https://github.com/windmill-labs/windmill/commit/2afea504977f9cd08d62c5f85be1fd2cefe8a691))
+
+
+### Bug Fixes
+
+* improve progress bar UX ([85d2d47](https://github.com/windmill-labs/windmill/commit/85d2d4782779d981a131f48db6e1058fe79daeef))
+* reinit retry to undefined in flow steps ([75f4723](https://github.com/windmill-labs/windmill/commit/75f472381cfa73d77295b29a202efbd58c79918d))
+
+## [1.151.2](https://github.com/windmill-labs/windmill/compare/v1.151.1...v1.151.2) (2023-08-18)
+
+
+### Bug Fixes
+
+* **frontend:** Fix app multiselect dark mode ([#2121](https://github.com/windmill-labs/windmill/issues/2121)) ([be577e5](https://github.com/windmill-labs/windmill/commit/be577e561dff33a404bb6f29f178b01f20aa0121))
+* **frontend:** Fix JSON pane scroll issues ([#2123](https://github.com/windmill-labs/windmill/issues/2123)) ([d367716](https://github.com/windmill-labs/windmill/commit/d367716b0a8198573b26a3c82ac7e4fd9cefe753))
+
+## [1.151.1](https://github.com/windmill-labs/windmill/compare/v1.151.0...v1.151.1) (2023-08-18)
+
+
+### Bug Fixes
+
+* at UTC Time ([0193fcc](https://github.com/windmill-labs/windmill/commit/0193fcc1d7c24147e553a0e3f9f0ab8d6f5d5996))
+* improve flow progress bar ([67cb451](https://github.com/windmill-labs/windmill/commit/67cb4516c913926c1755e46bc7acf46340fdb692))
+* show help on empty cli args ([237460b](https://github.com/windmill-labs/windmill/commit/237460b121846d160a40e849bf85fabbb7c14fdc))
+
+## [1.151.0](https://github.com/windmill-labs/windmill/compare/v1.150.0...v1.151.0) (2023-08-17)
+
+
+### Features
+
+* **frontend:** Fix workspace switch + always displays confirmation modal on top of splitpanel separator ([#2115](https://github.com/windmill-labs/windmill/issues/2115)) ([eea9ce9](https://github.com/windmill-labs/windmill/commit/eea9ce93b918115e9ed6b951d000049ca66bd5fd))
+
+
+### Bug Fixes
+
+* fix python get_resource ([cb00a13](https://github.com/windmill-labs/windmill/commit/cb00a1358d0e47575d8315e70695a9693190f211))
+
+## [1.150.0](https://github.com/windmill-labs/windmill/compare/v1.149.0...v1.150.0) (2023-08-17)
+
+
+### Features
+
+* copilot tokens streaming + cancel ([#2107](https://github.com/windmill-labs/windmill/issues/2107)) ([82612c3](https://github.com/windmill-labs/windmill/commit/82612c35bd4cd15af21582f9650b615d3e12c06c))
+* graphql custom headers ([#2111](https://github.com/windmill-labs/windmill/issues/2111)) ([6733b85](https://github.com/windmill-labs/windmill/commit/6733b8552b1128663c8fb8086c85ad0406d9b999))
+
+
+### Bug Fixes
+
+* powershell icon ([#2109](https://github.com/windmill-labs/windmill/issues/2109)) ([c817af7](https://github.com/windmill-labs/windmill/commit/c817af769457a069617fafb2d3fcf38a85212690))
+* set NETRC at init and not for every job ([359845f](https://github.com/windmill-labs/windmill/commit/359845fa9dd14e8445cc95e73cc646dce1f45ddb))
+* unify clients to use server-side interpolation to retrieve full resources ([067908c](https://github.com/windmill-labs/windmill/commit/067908c0b59f1e73222cad0e5f214f3605006ef3))
+* unify clients to use server-side interpolation to retrieve full resources ([930839a](https://github.com/windmill-labs/windmill/commit/930839aad22eaeee0737f1d057b8cfb538d26d3f))
+* unify clients to use server-side interpolation to retrieve full resources ([e9c19b5](https://github.com/windmill-labs/windmill/commit/e9c19b5b985c0e03524b2d12b1f26a0e6fdc6e0b))
+
+## [1.149.0](https://github.com/windmill-labs/windmill/compare/v1.148.0...v1.149.0) (2023-08-17)
+
+
+### Features
+
+* **frontend:** Add List pagination + add loading state in tables ([#2096](https://github.com/windmill-labs/windmill/issues/2096)) ([9b15e40](https://github.com/windmill-labs/windmill/commit/9b15e409a5b902874d0cf1566b57db6fc23a87ec))
+
+
+### Bug Fixes
+
+* appgrid refresh selected on row on result changes ([0af264f](https://github.com/windmill-labs/windmill/commit/0af264f6f8d0ff018094b97a2af9fe6f02e6ccfe))
+* fix folder creation if job folder already exist ([c320ea8](https://github.com/windmill-labs/windmill/commit/c320ea865f1632e517d4c597491517da89ff77e7))
+* fix go envs passing ([ed6494f](https://github.com/windmill-labs/windmill/commit/ed6494ff7a1f6102eaad8c0052c1ac3f82d4cadf))
+* **frontend:** Fix toast when adding a user + set default vscoode the… ([#2080](https://github.com/windmill-labs/windmill/issues/2080)) ([801f2a8](https://github.com/windmill-labs/windmill/commit/801f2a8299956f0debe95bb13faef798a0ea0b08))
+
+## [1.148.0](https://github.com/windmill-labs/windmill/compare/v1.147.3...v1.148.0) (2023-08-14)
+
+
+### Features
+
+* add s3 snippets ([#2052](https://github.com/windmill-labs/windmill/issues/2052)) ([beb4a00](https://github.com/windmill-labs/windmill/commit/beb4a000e3631a1b0a27a68923361652317aec63))
+
+
+### Bug Fixes
+
+* allow multiple db schema explorers ([#2054](https://github.com/windmill-labs/windmill/issues/2054)) ([e1b4f0a](https://github.com/windmill-labs/windmill/commit/e1b4f0a8328bc62a19e693bac99589711d08d566))
+* **frontend:** Fix Dark mode in the sleep helpbox ([#2072](https://github.com/windmill-labs/windmill/issues/2072)) ([c6ef1a6](https://github.com/windmill-labs/windmill/commit/c6ef1a6d4fbe5661f6b9018121e21061952908d0))
+* handle object pat in sig of typescript ([1d8213a](https://github.com/windmill-labs/windmill/commit/1d8213a25ba90f3d4af952e03c74196f8ce908ab))
+* remove ansi codes from result ([#2069](https://github.com/windmill-labs/windmill/issues/2069)) ([a3fa174](https://github.com/windmill-labs/windmill/commit/a3fa174cd46ce1bd67a69f7781dbdfa0719d3d06))
+* script fix no resource + error handling ([#2053](https://github.com/windmill-labs/windmill/issues/2053)) ([00b1afb](https://github.com/windmill-labs/windmill/commit/00b1afb1c90773408d1dc3233a25fa93e24d4da0))
+
+## [1.147.3](https://github.com/windmill-labs/windmill/compare/v1.147.2...v1.147.3) (2023-08-13)
+
+
+### Bug Fixes
+
+* **bun:** correctly handle empty deps script bun to deploy ([46b25f9](https://github.com/windmill-labs/windmill/commit/46b25f9b550f5f8e804cabeeeb575daea46cba31))
+
+## [1.147.2](https://github.com/windmill-labs/windmill/compare/v1.147.1...v1.147.2) (2023-08-13)
+
+
+### Bug Fixes
+
+* **bun:** add npm type acquisition ([3284245](https://github.com/windmill-labs/windmill/commit/32842457fef73f654ca89c3a232265927cf40961))
+
+## [1.147.1](https://github.com/windmill-labs/windmill/compare/v1.147.0...v1.147.1) (2023-08-13)
+
+
+### Bug Fixes
+
+* **bun:** only install -p dependencies ([23164c8](https://github.com/windmill-labs/windmill/commit/23164c83494ee6f42e77b181de0df26b4fba22dc))
+* **bun:** only install when requirements are missing if using nsjail ([3bc1050](https://github.com/windmill-labs/windmill/commit/3bc1050258bd7a9ba2be739144260037d2274b87))
+
+## [1.147.0](https://github.com/windmill-labs/windmill/compare/v1.146.1...v1.147.0) (2023-08-13)
+
+
+### Features
+
+* add lsp to bun and remove experimental status ([891c9dc](https://github.com/windmill-labs/windmill/commit/891c9dc266edea4f5239f1a82c884437b7df89e4))
+
+## [1.146.1](https://github.com/windmill-labs/windmill/compare/v1.146.0...v1.146.1) (2023-08-13)
+
+
+### Bug Fixes
+
+* **bun:** windmill-client does not require set to be initalized ([993a145](https://github.com/windmill-labs/windmill/commit/993a14502fb16387b174d1af19c87d3ae65c317c))
+* enable bun to do resolution as fallback to allow specifier ([9c97828](https://github.com/windmill-labs/windmill/commit/9c978281cdbfefa7d11213a181ffcbfdfac8115e))
+* powershell escape backticks ([#2044](https://github.com/windmill-labs/windmill/issues/2044)) ([cddef1a](https://github.com/windmill-labs/windmill/commit/cddef1a50a48e7cb60a69762a579b95e0018aa17))
+* really use bun in flow builder ([#2045](https://github.com/windmill-labs/windmill/issues/2045)) ([c2281ef](https://github.com/windmill-labs/windmill/commit/c2281ef5da7aa0222e70c5f6ca91d066d79d3862))
+
 ## [1.146.0](https://github.com/windmill-labs/windmill/compare/v1.145.3...v1.146.0) (2023-08-12)


@@ -73,7 +73,7 @@ ARG features=""

 COPY --from=planner /windmill/recipe.json recipe.json

-RUN apt-get update && apt-get install -y libxml2-dev libxmlsec1-dev clang libclang-dev
+RUN apt-get update && apt-get install -y libxml2-dev libxmlsec1-dev clang libclang-dev cmake

 RUN CARGO_NET_GIT_FETCH_WITH_CLI=true RUST_BACKTRACE=1 cargo chef cook --release --features "$features" --recipe-path recipe.json

@@ -96,8 +96,8 @@ SHELL ["/bin/bash", "-c"]
 RUN apt update -y
 RUN apt install -y unzip curl

-RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/LukeChannings/deno-arm64/releases/download/v1.35.0/deno-linux-arm64.zip -o deno.zip || true
-RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.35.0/deno-x86_64-unknown-linux-gnu.zip -o deno.zip || true
+RUN [ "$TARGETPLATFORM" == "linux/arm64" ] && curl -Lsf https://github.com/LukeChannings/deno-arm64/releases/download/v1.36.2/deno-linux-arm64.zip -o deno.zip || true
+RUN [ "$TARGETPLATFORM" == "linux/amd64" ] && curl -Lsf https://github.com/denoland/deno/releases/download/v1.36.2/deno-x86_64-unknown-linux-gnu.zip -o deno.zip || true

 RUN unzip deno.zip && rm deno.zip

@@ -149,13 +149,13 @@ RUN set -eux; \
    arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \
    case "$arch" in \
    'amd64') \
-    targz='go1.19.3.linux-amd64.tar.gz'; \
+    targz='go1.21.0.linux-amd64.tar.gz'; \
    ;; \
    'arm64') \
-    targz='go1.19.3.linux-arm64.tar.gz'; \
+    targz='go1.21.0.linux-arm64.tar.gz'; \
    ;; \
    'armhf') \
-    targz='go1.19.3.linux-armv6l.tar.gz'; \
+    targz='go1.21.0.linux-armv6l.tar.gz'; \
    ;; \
    *) echo >&2 "error: unsupported architecture '$arch' (likely packaging update needed)"; exit 1 ;; \
    esac; \
@@ -177,7 +177,7 @@ RUN chmod 755 /usr/bin/deno

 COPY --from=nsjail /nsjail/nsjail /bin/nsjail

-COPY --from=oven/bun:0.7.3 /usr/local/bin/bun /usr/bin/bun
+COPY --from=oven/bun:0.8.0 /usr/local/bin/bun /usr/bin/bun

 # add the docker client to call docker from a worker if enabled
 COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
@@ -170,9 +170,10 @@ variables from your instance if necessary. See more:

 ### Sandboxing

-Windmill uses [nsjail](https://github.com/google/nsjail) on top of the deno
-sandboxing. It is production multi-tenant grade secure. Do not take our word for
-it, take [fly.io's one](https://fly.io/blog/sandboxing-and-workload-isolation/).
+Windmill uses [nsjail](https://github.com/google/nsjail) and deno's sandboxing
+capabilities. It is production multi-tenant grade secure. Do not take our word
+for it, take
+[fly.io's one](https://fly.io/blog/sandboxing-and-workload-isolation/).

 ### Secrets, credentials and sensitive values

@@ -335,10 +336,11 @@ it being synced automatically everyday.
 | PORT                                          | 8000                                       | Exposed port                                                                                                                                                                                       | Server                |
 | NUM_WORKERS                                   | 1                                          | The number of worker per Worker instance (Set to 0 for API/Server instances, Set to 1 for normal workers, and > 1 for workers dedicated to native jobs)                                            | Worker                |
 | DISABLE_SERVER                                | false                                      | Disable the external API, operate as a worker only instance                                                                                                                                        | Worker                |
-| METRICS_ADDR                                  | None                                       | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
+| METRICS_ADDR                                  | None                                       | (ee only) The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                      | All                   |
 | JSON_FMT                                      | false                                      | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
 | BASE_URL                                      | http://localhost:8000                      | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
-| TIMEOUT                                       | 300                                        | The maximum time of execution of a script. When reached, the job is failed as having timedout.                                                                                                     | Worker                |
+| TIMEOUT                                       | 60 _ 60 _ 24 \* 7 (1 week)                 | The maximum time of execution of a script. When reached, the job is failed as having timedout.                                                                                                     |
+| SCRIPT_TOKEN_EXPIRY                           | 900                                        | The default duration period of the ephemeral-token generated at the beginning of a script                                                                                                          | Worker                |
 | ZOMBIE_JOB_TIMEOUT                            | 30                                         | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s)                                  | Server                |
 | RESTART_ZOMBIE_JOBS                           | true                                       | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed                                                                              | Server                |
 | SLEEP_QUEUE                                   | 50                                         | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker                |
@@ -59,7 +59,6 @@
                "bash",
                "postgresql",
                "nativets",
-                "Nativets",
                "bun",
                "mysql",
                "bigquery",
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "SELECT COALESCE(j.min_started_at, q.min_started_at) AS min_started_at, COALESCE(completed_count, 0) + COALESCE(running_count, 0) AS total_count\n            FROM\n                (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as completed_count\n                FROM completed_job\n                WHERE script_path = $1 AND started_at + INTERVAL '1 MILLISECOND' * duration_ms > (now() - INTERVAL '1 second' * $2)\n                GROUP BY script_path) as j\n            FULL OUTER JOIN\n                (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as running_count\n                FROM queue\n                WHERE script_path = $1 AND running = true\n                GROUP BY script_path) as q\n            ON q.script_path = j.script_path",
+  "query": "SELECT COALESCE(j.min_started_at, q.min_started_at) AS min_started_at, COALESCE(completed_count, 0) + COALESCE(running_count, 0) AS total_count\n            FROM\n                (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as completed_count\n                FROM completed_job\n                WHERE script_path = $1 AND started_at + INTERVAL '1 MILLISECOND' * duration_ms > (now() - INTERVAL '1 second' * $2) AND workspace_id = $3\n                GROUP BY script_path) as j\n            FULL OUTER JOIN\n                (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as running_count\n                FROM queue\n                WHERE script_path = $1 AND running = true AND workspace_id = $3\n                GROUP BY script_path) as q\n            ON q.script_path = j.script_path",
  "describe": {
    "columns": [
      {
@@ -17,7 +17,8 @@
    "parameters": {
      "Left": [
        "Text",
-        "Float8"
+        "Float8",
+        "Text"
      ]
    },
    "nullable": [
@@ -25,5 +26,5 @@
      true
    ]
  },
-  "hash": "077b925c0a9f19a3859ca7eb0bc9049fc7c303daf469c049cf03570d22310d74"
+  "hash": "09eb148c69fec39d01bcc96a0976ea0ead76049350250142d086015228d36338"
 }
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4 WHERE path = $5 AND workspace_id = $6 RETURNING *",
+  "query": "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4, on_failure_times = $5, on_failure_exact = $6, on_failure_extra_args = $7, on_recovery = $8, on_recovery_times = $9, on_recovery_extra_args = $10 WHERE path = $11 AND workspace_id = $12 RETURNING *",
  "describe": {
    "columns": [
      {
@@ -72,6 +72,36 @@
        "ordinal": 13,
        "name": "on_failure",
        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
      }
    ],
    "parameters": {
@@ -80,6 +110,12 @@
        "Varchar",
        "Jsonb",
        "Varchar",
+        "Int4",
+        "Bool",
+        "Json",
+        "Varchar",
+        "Int4",
+        "Json",
        "Text",
        "Text"
      ]
@@ -98,8 +134,14 @@
      false,
      true,
      false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
      true
    ]
  },
-  "hash": "1e0769c59ada78848e034250c11be0ca2e8c49b96a0066b9ab67f5a205ec456d"
+  "hash": "0be97088e8ca1e8be9bf4ea45f830cceedd598a3221c522a1f9026d2d48a0ffb"
 }
@@ -66,7 +66,6 @@
                "bash",
                "postgresql",
                "nativets",
-                "Nativets",
                "bun",
                "mysql",
                "bigquery",
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "SELECT  usr.username  \n            FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username \n            WHERE group_ = $1 AND usr.workspace_id = $2 AND usr_to_group.workspace_id = $2",
+  "query": "SELECT  usr.username  \n            FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username AND usr_to_group.workspace_id = $2\n            WHERE group_ = $1 AND usr.workspace_id = $2 AND usr_to_group.workspace_id = $2",
  "describe": {
    "columns": [
      {
@@ -19,5 +19,5 @@
      false
    ]
  },
-  "hash": "c2d6cb56c1dea4498e2aab9ea9301dbbaa127602a38f57f5add4108fdc209b1a"
+  "hash": "3fdfcab1a54c166b1d8d43215d61268a251160db4630f0342522091668f36af0"
 }
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "SELECT resource.*, (now() > account.expires_at) as is_expired, account.refresh_token != '' as is_refreshed,\n        account.refresh_error,\n        variable.path IS NOT NULL as is_linked,\n        variable.is_oauth as \"is_oauth?\",\n        variable.account\n        FROM resource\n        LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = resource.workspace_id\n        LEFT JOIN account ON variable.account = account.id AND account.workspace_id = resource.workspace_id\n        WHERE resource.path = $1 AND resource.workspace_id = $2",
+  "query": "SELECT resource.*, (now() > account.expires_at) as is_expired, account.refresh_token != '' as is_refreshed,\n        account.refresh_error,\n        variable.path IS NOT NULL as is_linked,\n        variable.is_oauth as \"is_oauth?\",\n        variable.account\n        FROM resource\n        LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = $2\n        LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $2\n        WHERE resource.path = $1 AND resource.workspace_id = $2",
  "describe": {
    "columns": [
      {
@@ -85,5 +85,5 @@
      true
    ]
  },
-  "hash": "7f443f5be90622eed3e878ada0ae8ca71902adb88861bc8a39713fc8942e652c"
+  "hash": "41f2c271514ee254739c3a097871526adcecdd8729f28c15e0db8cd28eaa8cf0"
 }
@@ -0,0 +1,38 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT success, result, started_at FROM completed_job WHERE workspace_id = $1 AND schedule_path = $2 AND script_path = $3 AND id != $4 ORDER BY created_at DESC LIMIT $5",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "success",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 1,
+        "name": "result",
+        "type_info": "Jsonb"
+      },
+      {
+        "ordinal": 2,
+        "name": "started_at",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Text",
+        "Uuid",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false,
+      true,
+      false
+    ]
+  },
+  "hash": "4a3917318d967d96c4b6e70786a034302f04da9d1e9eec59b1eb6052c2d947be"
+}
@@ -17,7 +17,6 @@
                "bash",
                "postgresql",
                "nativets",
-                "Nativets",
                "bun",
                "mysql",
                "bigquery",
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE\n        completed_job.schedule_path = schedule.path AND schedule.workspace_id = completed_job.workspace_id AND parent_job IS NULL ORDER BY created_at DESC LIMIT 20) AS jobs ) t\n        WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3",
+  "query": "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE\n        completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY started_at DESC LIMIT 20) AS jobs ) t\n        WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3",
  "describe": {
    "columns": [
      {
@@ -75,6 +75,36 @@
      },
      {
        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 20,
        "name": "jobs",
        "type_info": "JsonArray"
      }
@@ -101,8 +131,14 @@
      true,
      false,
      true,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
      null
    ]
  },
-  "hash": "dc4316a8646dc642ed12c12d8673aa1ed44c551b3174c49655defacdf49880c7"
+  "hash": "7a00843921db4ac1eb20a1045e83363587c1a2cb62d2df25d5564ae78a03b097"
 }
@@ -0,0 +1,47 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value, account, (now() > account.expires_at) as is_expired, is_secret, path from variable\n        LEFT JOIN account ON variable.account = account.id WHERE variable.path = $1 AND variable.workspace_id = $2",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "account",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 2,
+        "name": "is_expired",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 3,
+        "name": "is_secret",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 4,
+        "name": "path",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text"
+      ]
+    },
+    "nullable": [
+      false,
+      true,
+      null,
+      false,
+      false
+    ]
+  },
+  "hash": "7a92837ad2a1181580c1e0d059fd6bc0ece84d3c315635de52bd2b364c238a71"
+}
@@ -72,6 +72,36 @@
        "ordinal": 13,
        "name": "on_failure",
        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
      }
    ],
    "parameters": {
@@ -95,6 +125,12 @@
      false,
      true,
      false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
      true
    ]
  },
@@ -72,6 +72,36 @@
        "ordinal": 13,
        "name": "on_failure",
        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
      }
    ],
    "parameters": {
@@ -96,6 +126,12 @@
      false,
      true,
      false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
      true
    ]
  },
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, is_flow, args, enabled, email, on_failure) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING *",
+  "query": "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, is_flow, args, enabled, email, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) RETURNING *",
  "describe": {
    "columns": [
      {
@@ -72,6 +72,36 @@
        "ordinal": 13,
        "name": "on_failure",
        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
      }
    ],
    "parameters": {
@@ -86,7 +116,13 @@
        "Jsonb",
        "Bool",
        "Varchar",
-        "Varchar"
+        "Varchar",
+        "Int4",
+        "Bool",
+        "Json",
+        "Varchar",
+        "Int4",
+        "Json"
      ]
    },
    "nullable": [
@@ -103,8 +139,14 @@
      false,
      true,
      false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
      true
    ]
  },
-  "hash": "80e632cb6618cde2bac2c12d0afc796a4a1768f902d0540af33fe6f442cf583a"
+  "hash": "9b9afb9d5ef57b70de1207c4546b5547e579d978d899325d821a5aee1df1a9fc"
 }
@@ -28,7 +28,6 @@
                "bash",
                "postgresql",
                "nativets",
-                "Nativets",
                "bun",
                "mysql",
                "bigquery",
@@ -72,6 +72,36 @@
        "ordinal": 13,
        "name": "on_failure",
        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
      }
    ],
    "parameters": {
@@ -94,6 +124,12 @@
      false,
      true,
      false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
      true
    ]
  },
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n        app.extra_perms, app_version.value, \n        app_version.created_at, app_version.created_by,\n        app.draft_only, draft.value as \"draft?\"\n        from app\n        INNER JOIN app_version ON\n        app_version.id = app.versions[array_upper(app.versions, 1)]\n        LEFT JOIN draft ON \n        app.path = draft.path AND app.workspace_id = draft.workspace_id AND draft.typ = 'app' \n        WHERE app.path = $1 AND app.workspace_id = $2",
+  "query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n        app.extra_perms, app_version.value, \n        app_version.created_at, app_version.created_by,\n        app.draft_only, draft.value as \"draft?\"\n        from app\n        INNER JOIN app_version ON\n        app_version.id = app.versions[array_upper(app.versions, 1)]\n        LEFT JOIN draft ON \n        app.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'app' \n        WHERE app.path = $1 AND app.workspace_id = $2",
  "describe": {
    "columns": [
      {
@@ -79,5 +79,5 @@
      false
    ]
  },
-  "hash": "f07e4fb9d3b346c6c0a93133d428691a2e17a0c7eef5c593732ad7297d366ad4"
+  "hash": "aedba9cff117a7069dfd56602110b7f66903d6afb4f87fbffe0175982ea8741f"
 }
@@ -72,6 +72,36 @@
        "ordinal": 13,
        "name": "on_failure",
        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
      }
    ],
    "parameters": {
@@ -95,6 +125,12 @@
      false,
      true,
      false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
      true
    ]
  },
@@ -72,6 +72,36 @@
        "ordinal": 13,
        "name": "on_failure",
        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
      }
    ],
    "parameters": {
@@ -95,6 +125,12 @@
      false,
      true,
      false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
      true
    ]
  },
@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE queue SET canceled = true,  canceled_by = $2, scheduled_for = now(), suspend = 0 WHERE workspace_id = $1 AND schedule_path IS NULL RETURNING id",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Uuid"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Varchar"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "e25829ed40f35a0f59a9838d030a7f888ca88441e069e73a52d75450aecae70e"
+}
@@ -72,6 +72,36 @@
        "ordinal": 13,
        "name": "on_failure",
        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
      }
    ],
    "parameters": {
@@ -95,6 +125,12 @@
      false,
      true,
      false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
      true
    ]
  },
@@ -72,6 +72,36 @@
        "ordinal": 13,
        "name": "on_failure",
        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 14,
+        "name": "on_recovery",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 15,
+        "name": "on_failure_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 16,
+        "name": "on_failure_exact",
+        "type_info": "Bool"
+      },
+      {
+        "ordinal": 17,
+        "name": "on_failure_extra_args",
+        "type_info": "Json"
+      },
+      {
+        "ordinal": 18,
+        "name": "on_recovery_times",
+        "type_info": "Int4"
+      },
+      {
+        "ordinal": 19,
+        "name": "on_recovery_extra_args",
+        "type_info": "Json"
      }
    ],
    "parameters": {
@@ -93,6 +123,12 @@
      false,
      true,
      false,
+      true,
+      true,
+      true,
+      true,
+      true,
+      true,
      true
    ]
  },
@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.146.0"
+version = "1.156.1"
 authors.workspace = true
 edition.workspace = true

@@ -22,7 +22,7 @@ members = [
 ]

 [workspace.package]
-version = "1.146.0"
+version = "1.156.1"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"

@@ -131,17 +131,18 @@ tokio-util = { version = "^0", features = ["io"] }
 json-pointer = "^0"
 itertools = "^0"
 regex = "^1"
-deno_fetch = "0.133.0"
-deno_console = "0.109.0"
-deno_url = "0.109.0"
-deno_webidl = "0.109.0"
-deno_web = "0.140.0"
-deno_core = "0.191.0"
-deno_ast = { version = "0.27.2", features = ["transpiling"] }
+deno_fetch = "0.139.0"
+deno_console = "0.115.0"
+deno_url = "0.115.0"
+deno_webidl = "0.115.0"
+deno_web = "0.146.0"
+deno_core = "0.200.0"
+deno_ast = { version = "0.28.0", features = ["transpiling"] }
 async-recursion = "^1"
-swc_common = "0.31.12"
-swc_ecma_parser = "0.136.2"
-swc_ecma_ast = "0.106.2"
+swc_common = "0.31.21"
+swc_ecma_parser = "0.137.15"
+swc_ecma_ast = "0.107.7"
+swc_ecma_visit = "0.93.7"
 base64 = "0.21.0"
 hmac = "0.12.1"
 sha2 = "0.10.6"
@@ -0,0 +1 @@
+-- Add down migration script here
@@ -0,0 +1,15 @@
+-- Add up migration script here
+UPDATE script SET content = 'import wmill from "https://deno.land/x/wmill@v1.150.0/main.ts";
+export async function main() {
+  await run(
+    "workspace", "add", "__automation", "admins", Deno.env.get("BASE_INTERNAL_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
+
+  await run("hub", "pull");
+}
+
+async function run(...cmd: string[]) {
+  console.log("Running \"" + cmd.join('' '') + "\"");
+  await wmill.parse(cmd);
+}', summary = 'Synchronize Hub Resource types with instance',
+description = 'Basic administrative script to sync latest resource types from hub to share to every workspace. Recommended to run at least once. On a schedule by default.'
+WHERE hash = -28028598712388162 AND workspace_id = 'admins';
@@ -0,0 +1 @@
+-- Add down migration script here
@@ -0,0 +1,15 @@
+-- Add up migration script here
+UPDATE script SET content = 'import wmill from "https://deno.land/x/wmill@v1.151.0/main.ts";
+export async function main() {
+  await run(
+    "workspace", "add", "__automation", "admins", Deno.env.get("BASE_INTERNAL_URL") + "/", "--token", Deno.env.get("WM_TOKEN"));
+
+  await run("hub", "pull");
+}
+
+async function run(...cmd: string[]) {
+  console.log("Running \"" + cmd.join('' '') + "\"");
+  await wmill.parse(cmd);
+}', summary = 'Synchronize Hub Resource types with instance',
+description = 'Basic administrative script to sync latest resource types from hub to share to every workspace. Recommended to run at least once. On a schedule by default.'
+WHERE hash = -28028598712388162 AND workspace_id = 'admins';
@@ -0,0 +1,2 @@
+-- Add down migration script here
+ALTER TABLE schedule DROP COLUMN on_recovery;
@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE schedule ADD COLUMN on_recovery VARCHAR(1000);
@@ -0,0 +1 @@
+-- Add down migration script here
@@ -0,0 +1,25 @@
+-- Add up migration script here
+CREATE INDEX IF NOT EXISTS scheduled_root_job ON completed_job (workspace_id, schedule_path, started_at) WHERE parent_job is NULL;
+CREATE INDEX IF NOT EXISTS root_job_index_by_path ON completed_job (workspace_id, script_path, job_kind, created_at) WHERE parent_job is NULL;
+CREATE INDEX IF NOT EXISTS root_job_index ON completed_job (workspace_id, job_kind, created_at) WHERE parent_job is NULL;
+DROP INDEX IF EXISTS index_completed_on_script_hash;
+DROP INDEX IF EXISTS index_completed_on_script_path;
+DROP INDEX IF EXISTS index_completed_on_schedule_path;
+DROP INDEX IF EXISTS index_completed_on_workspace_id;
+DROP INDEX IF EXISTS index_completed_on_created_at;
+
+DROP INDEX IF EXISTS index_queue_on_script_path;
+DROP INDEX IF EXISTS index_queue_on_script_hash;
+DROP INDEX IF EXISTS index_queue_on_workspace_id;
+DROP INDEX IF EXISTS index_queue_on_scheduled_for;
+DROP INDEX IF EXISTS index_queue_on_tag;
+DROP INDEX IF EXISTS index_queue_on_running;
+DROP INDEX IF EXISTS index_queue_on_created;
+
+
+CREATE INDEX IF NOT EXISTS root_queue_index_by_path ON queue (workspace_id, created_at);
+CREATE INDEX IF NOT EXISTS root_queue_index ON queue (job_kind, tag, scheduled_for, created_at) WHERE running is false;
+CREATE INDEX IF NOT EXISTS root_queue_index_suspended ON queue (job_kind, tag, suspend_until, suspend, scheduled_for, created_at) WHERE suspend_until is not null;
+
+CREATE INDEX IF NOT EXISTS concurrency_limit_stats_queue ON queue (workspace_id, script_path, started_at) WHERE concurrent_limit is not null;
+CREATE INDEX IF NOT EXISTS concurrency_limit_stats_completed_job ON completed_job (workspace_id, script_path, started_at);
@@ -0,0 +1,7 @@
+-- Add down migration script here
+ALTER TABLE schedule 
+DROP COLUMN on_failure_times,
+DROP COLUMN on_failure_exact,
+DROP COLUMN on_failure_extra_args,
+DROP COLUMN on_recovery_times,
+DROP COLUMN on_recovery_extra_args;
@@ -0,0 +1,7 @@
+-- Add up migration script here
+ALTER TABLE schedule
+ADD COLUMN on_failure_times INTEGER,
+ADD COLUMN on_failure_exact BOOLEAN,
+ADD COLUMN on_failure_extra_args json,
+ADD COLUMN on_recovery_times INTEGER,
+ADD COLUMN on_recovery_extra_args json;
@@ -32,6 +32,9 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
    "smb" => "pysmb",
    "PIL" => "Pillow",
    "googleapiclient" => "google-api-python-client",
+    "dateutil" => "python-dateutil",
+    "mailparser" => "mail-parser",
+    "mailparser-reply" => "mail-parser-reply",
 };

 fn replace_import(x: String) -> String {
@@ -145,19 +148,20 @@ pub async fn parse_python_imports(
            .collect();
        for n in nimports.iter() {
            let nested = if n.starts_with("relative:") {
+                let rpath = n.replace("relative:", "");
                let code = sqlx::query_scalar!(
                    r#"
                    SELECT content FROM script WHERE path = $1 AND workspace_id = $2
                    AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND 
                    workspace_id = $2)
                    "#,
-                    n.replace("relative:", ""),
+                    &rpath,
                    w_id
                )
                .fetch_optional(db)
                .await?
                .unwrap_or_else(|| "".to_string());
-                parse_python_imports(&code, w_id, path, db).await?
+                parse_python_imports(&code, w_id, &rpath, db).await?
            } else {
                vec![replace_import(n.to_string())]
            };
@@ -2556,4 +2556,27 @@ import numpy as np
 '{}',
 '',
 '',
-'f/foo/baz', -28028598712388161, 'python3', '');
+'f/foo/baz', -28028598712388161, 'python3', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+import difffolder
+from .bar import main
+',
+'{}',
+'',
+'',
+'f/foobar/baz', -28028598712388160, 'python3', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+import innerdifffolder
+',
+'{}',
+'',
+'',
+'f/foobar/bar', -28028598712388159, 'python3', '');
@@ -57,7 +57,7 @@ from f.foo.bar import main1
 from .baz import main2
 from ..foo.bar import main3
 import f.foo.bar as bar
-
+from ..foobar.baz import main4

 def main():
    pass
@@ -65,7 +65,17 @@ def main():
 ";
        let r = parse_python_imports(code, "test-workspace", "f/foo/bar", &db).await?;
        println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, ["numpy", "pandas", "pandas2", "requests"]);
+        assert_eq!(
+            r,
+            [
+                "difffolder",
+                "innerdifffolder",
+                "numpy",
+                "pandas",
+                "pandas2",
+                "requests"
+            ]
+        );

        Ok(())
    }
@@ -18,6 +18,7 @@ windmill-parser.workspace = true
 swc_common.workspace = true
 swc_ecma_parser.workspace = true
 swc_ecma_ast.workspace = true
+swc_ecma_visit.workspace = true
 serde_json.workspace = true
 anyhow.workspace = true
 convert_case.workspace = true
@@ -9,21 +9,78 @@ use regex::Regex;
 */
 // use deno_core::{serde_v8, v8, JsRuntime, RuntimeOptions};
 use serde_json::Value;
+use std::collections::HashSet;
+use swc_ecma_visit::{noop_visit_type, Visit, VisitWith};
 use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectProperty, Typ};

 use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Span, Spanned};
 use swc_ecma_ast::{
    ArrayLit, AssignPat, BigInt, BindingIdent, Bool, Decl, ExportDecl, Expr, FnDecl, Ident, Lit,
-    ModuleDecl, ModuleItem, Number, ObjectLit, Param, Pat, Str, TsArrayType, TsEntityName,
-    TsKeywordType, TsKeywordTypeKind, TsLit, TsLitType, TsOptionalType, TsParenthesizedType,
-    TsPropertySignature, TsType, TsTypeElement, TsTypeLit, TsTypeRef, TsUnionOrIntersectionType,
-    TsUnionType,
+    MemberExpr, MemberProp, ModuleDecl, ModuleItem, Number, ObjectLit, ObjectPat, Param, Pat, Str,
+    TsArrayType, TsEntityName, TsKeywordType, TsKeywordTypeKind, TsLit, TsLitType, TsOptionalType,
+    TsParenthesizedType, TsPropertySignature, TsType, TsTypeAnn, TsTypeElement, TsTypeLit,
+    TsTypeRef, TsUnionOrIntersectionType, TsUnionType,
 };
-use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsConfig};
+use swc_ecma_parser::{lexer::Lexer, EsConfig, Parser, StringInput, Syntax, TsConfig};

 #[cfg(target_arch = "wasm32")]
 use wasm_bindgen::prelude::*;

+struct OutputFinder {
+    idents: HashSet<(String, String)>,
+}
+
+impl Visit for OutputFinder {
+    noop_visit_type!();
+
+    fn visit_member_expr(&mut self, m: &MemberExpr) {
+        m.obj.visit_with(self);
+        if let MemberProp::Computed(c) = &m.prop {
+            c.visit_with(self);
+        }
+        match m {
+            MemberExpr { obj, prop: MemberProp::Ident(Ident { sym, .. }), .. } => {
+                match *obj.to_owned() {
+                    Expr::Ident(Ident { sym: sym_i, .. }) => {
+                        self.idents.insert((sym_i.to_string(), sym.to_string()));
+                    }
+                    _ => (),
+                }
+            }
+            _ => (),
+        }
+    }
+}
+
+pub fn parse_expr_for_ids(code: &str) -> anyhow::Result<Vec<(String, String)>> {
+    let cm: Lrc<SourceMap> = Default::default();
+    let fm = cm.new_source_file(FileName::Custom("main.ts".into()), code.into());
+    let lexer = Lexer::new(
+        // We want to parse ecmascript
+        Syntax::Es(EsConfig { jsx: false, ..Default::default() }),
+        // EsVersion defaults to es5
+        Default::default(),
+        StringInput::from(&*fm),
+        None,
+    );
+
+    let mut parser = Parser::new_from(lexer);
+
+    let mut err_s = "".to_string();
+    for e in parser.take_errors() {
+        err_s += &e.into_kind().msg().to_string();
+    }
+
+    let expr = parser
+        .parse_module()
+        .map_err(|_| anyhow::anyhow!("Error while parsing code, it is invalid TypeScript"))?;
+
+    let mut visitor = OutputFinder { idents: HashSet::new() };
+    swc_ecma_visit::visit_module(&mut visitor, &expr);
+
+    Ok(visitor.idents.into_iter().collect())
+}
+
 pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> anyhow::Result<MainArgSignature> {
    let cm: Lrc<SourceMap> = Default::default();
    let fm = cm.new_source_file(FileName::Custom("main.ts".into()), code.into());
@@ -56,13 +113,14 @@ pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> anyhow::Result<MainA
        _ => None,
    });

+    let mut c: u16 = 0;
    if let Some(params) = params {
        let r = MainArgSignature {
            star_args: false,
            star_kwargs: false,
            args: params
                .into_iter()
-                .map(|x| parse_param(x, &cm, skip_dflt))
+                .map(|x| parse_param(x, &cm, skip_dflt, &mut c))
                .collect::<anyhow::Result<Vec<Arg>>>()?,
        };
        Ok(r)
@@ -74,7 +132,12 @@ pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> anyhow::Result<MainA
    }
 }

-fn parse_param(x: Param, cm: &Lrc<SourceMap>, skip_dflt: bool) -> anyhow::Result<Arg> {
+fn parse_param(
+    x: Param,
+    cm: &Lrc<SourceMap>,
+    skip_dflt: bool,
+    counter: &mut u16,
+) -> anyhow::Result<Arg> {
    let r = match x.pat {
        Pat::Ident(ident) => {
            let (name, typ, nullable) = binding_ident_to_arg(&ident);
@@ -86,15 +149,25 @@ fn parse_param(x: Param, cm: &Lrc<SourceMap>, skip_dflt: bool) -> anyhow::Result
                has_default: ident.id.optional || nullable,
            })
        }
+        // Pat::Object(ObjectPat { ... }) = todo!()
        Pat::Assign(AssignPat { left, right, .. }) => {
-            let (name, mut typ, _nullable) =
-                left.as_ident().map(binding_ident_to_arg).ok_or_else(|| {
-                    anyhow::anyhow!(
-                        "parameter syntax unsupported: `{}`",
+            let (name, mut typ, _nullable) = match *left {
+                Pat::Ident(ident) => binding_ident_to_arg(&ident),
+                Pat::Object(ObjectPat { type_ann, .. }) => {
+                    let (typ, nullable) = eval_type_ann(&type_ann);
+                    *counter += 1;
+                    let name = format!("anon{}", counter);
+                    (name, typ, nullable)
+                }
+                _ => {
+                    return Err(anyhow::anyhow!(
+                        "parameter syntax unsupported: `{}`: {:#?}",
                        cm.span_to_snippet(left.span())
-                            .unwrap_or_else(|_| cm.span_to_string(left.span()))
-                    )
-                })?;
+                            .unwrap_or_else(|_| cm.span_to_string(left.span())),
+                        *left
+                    ))
+                }
+            };

            let dflt = if skip_dflt {
                None
@@ -122,10 +195,17 @@ fn parse_param(x: Param, cm: &Lrc<SourceMap>, skip_dflt: bool) -> anyhow::Result
            }
            Ok(Arg { otyp: None, name, typ, default: dflt, has_default: true })
        }
+        Pat::Object(ObjectPat { type_ann, .. }) => {
+            let (typ, nullable) = eval_type_ann(&type_ann);
+            *counter += 1;
+            let name = format!("anon{}", counter);
+            Ok(Arg { otyp: None, name, typ, default: None, has_default: nullable })
+        }
        _ => Err(anyhow::anyhow!(
-            "parameter syntax unsupported: `{}`",
+            "parameter syntax unsupported: `{}`: {:#?}",
            cm.span_to_snippet(x.span())
-                .unwrap_or_else(|_| cm.span_to_string(x.span()))
+                .unwrap_or_else(|_| cm.span_to_string(x.span())),
+            x.pat
        )),
    };
    r
@@ -144,11 +224,14 @@ fn eval_span(span: Span, cm: &Lrc<SourceMap>) -> Option<Value> {
    }
 }

-fn binding_ident_to_arg(BindingIdent { id, type_ann }: &BindingIdent) -> (String, Typ, bool) {
-    let (typ, nullable) = type_ann
+fn eval_type_ann(type_ann: &Option<Box<TsTypeAnn>>) -> (Typ, bool) {
+    return type_ann
        .as_ref()
        .map(|x| tstype_to_typ(&*x.type_ann))
        .unwrap_or((Typ::Unknown, false));
+}
+fn binding_ident_to_arg(BindingIdent { id, type_ann }: &BindingIdent) -> (String, Typ, bool) {
+    let (typ, nullable) = eval_type_ann(type_ann);
    (id.sym.to_string(), typ, nullable)
 }

@@ -161,7 +244,6 @@ fn to_snake_case(s: &str) -> String {
 }

 fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
-    // log(&format!("{:?}", ts_type));
    match ts_type {
        TsType::TsKeywordType(t) => (
            match t.kind {
@@ -283,8 +365,8 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
 extern "C" {
    pub fn eval(s: &str) -> JsValue;
    pub fn alert(s: &str);
-    // #[wasm_bindgen(js_namespace = console)]
-    // fn log(s: &str);
+    #[wasm_bindgen(js_namespace = console)]
+    fn log(s: &str);

 }

@@ -3,7 +3,7 @@
  "collaborators": [
    "Ruben Fiszel <ruben@windmill.dev>"
  ],
-  "version": "1.144.4",
+  "version": "1.154.3",
  "files": [
    "windmill_parser_wasm_bg.wasm",
    "windmill_parser_wasm.js",
@@ -14,4 +14,4 @@
  "sideEffects": [
    "./snippets/*"
  ]
-}
+}
@@ -9,6 +9,11 @@ export function parse_deno(code: string): string;
 * @param {string} code
 * @returns {string}
 */
+export function parse_outputs(code: string): string;
+/**
+* @param {string} code
+* @returns {string}
+*/
 export function parse_bash(code: string): string;
 /**
 * @param {string} code
@@ -56,6 +61,7 @@ export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembl
 export interface InitOutput {
  readonly memory: WebAssembly.Memory;
  readonly parse_deno: (a: number, b: number, c: number) => void;
+  readonly parse_outputs: (a: number, b: number, c: number) => void;
  readonly parse_bash: (a: number, b: number, c: number) => void;
  readonly parse_powershell: (a: number, b: number, c: number) => void;
  readonly parse_go: (a: number, b: number, c: number) => void;
@@ -6,6 +6,20 @@ heap.push(undefined, null, true, false);

 function getObject(idx) { return heap[idx]; }

+let heap_next = heap.length;
+
+function dropObject(idx) {
+    if (idx < 132) return;
+    heap[idx] = heap_next;
+    heap_next = idx;
+}
+
+function takeObject(idx) {
+    const ret = getObject(idx);
+    dropObject(idx);
+    return ret;
+}
+
 let WASM_VECTOR_LEN = 0;

 let cachedUint8Memory0 = null;
@@ -83,20 +97,6 @@ function getInt32Memory0() {
    return cachedInt32Memory0;
 }

-let heap_next = heap.length;
-
-function dropObject(idx) {
-    if (idx < 132) return;
-    heap[idx] = heap_next;
-    heap_next = idx;
-}
-
-function takeObject(idx) {
-    const ret = getObject(idx);
-    dropObject(idx);
-    return ret;
-}
-
 function addHeapObject(obj) {
    if (heap_next === heap.length) heap.push(heap.length + 1);
    const idx = heap_next;
@@ -220,6 +220,29 @@ export function parse_deno(code) {
    }
 }

+/**
+* @param {string} code
+* @returns {string}
+*/
+export function parse_outputs(code) {
+    let deferred2_0;
+    let deferred2_1;
+    try {
+        const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
+        const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        const len0 = WASM_VECTOR_LEN;
+        wasm.parse_outputs(retptr, ptr0, len0);
+        var r0 = getInt32Memory0()[retptr / 4 + 0];
+        var r1 = getInt32Memory0()[retptr / 4 + 1];
+        deferred2_0 = r0;
+        deferred2_1 = r1;
+        return getStringFromWasm0(r0, r1);
+    } finally {
+        wasm.__wbindgen_add_to_stack_pointer(16);
+        wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
+    }
+}
+
 /**
 * @param {string} code
 * @returns {string}
@@ -469,6 +492,13 @@ async function __wbg_load(module, imports) {
 function __wbg_get_imports() {
    const imports = {};
    imports.wbg = {};
+    imports.wbg.__wbg_eval_bfffb337c5ad9d0f = function(arg0, arg1) {
+        const ret = eval(getStringFromWasm0(arg0, arg1));
+        return addHeapObject(ret);
+    };
+    imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
+        takeObject(arg0);
+    };
    imports.wbg.__wbindgen_string_get = function(arg0, arg1) {
        const obj = getObject(arg1);
        const ret = typeof(obj) === 'string' ? obj : undefined;
@@ -477,13 +507,6 @@ function __wbg_get_imports() {
        getInt32Memory0()[arg0 / 4 + 1] = len1;
        getInt32Memory0()[arg0 / 4 + 0] = ptr1;
    };
-    imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
-        takeObject(arg0);
-    };
-    imports.wbg.__wbg_eval_fc6855b8339aec25 = function(arg0, arg1) {
-        const ret = eval(getStringFromWasm0(arg0, arg1));
-        return addHeapObject(ret);
-    };
    imports.wbg.__wbindgen_boolean_get = function(arg0) {
        const v = getObject(arg0);
        const ret = typeof(v) === 'boolean' ? (v ? 1 : 0) : 2;
@@ -2,6 +2,7 @@
 /* eslint-disable */
 export const memory: WebAssembly.Memory;
 export function parse_deno(a: number, b: number, c: number): void;
+export function parse_outputs(a: number, b: number, c: number): void;
 export function parse_bash(a: number, b: number, c: number): void;
 export function parse_powershell(a: number, b: number, c: number): void;
 export function parse_go(a: number, b: number, c: number): void;
@@ -1,6 +1,7 @@
-use serde_json;
+use serde_json::{self, json};
 use wasm_bindgen::prelude::*;
 use windmill_parser::MainArgSignature;
+use windmill_parser_ts::parse_expr_for_ids;

 fn wrap_sig(r: anyhow::Result<MainArgSignature>) -> String {
    if let Ok(r) = r {
@@ -15,6 +16,17 @@ pub fn parse_deno(code: &str) -> String {
    wrap_sig(windmill_parser_ts::parse_deno_signature(code, false))
 }

+#[wasm_bindgen]
+pub fn parse_outputs(code: &str) -> String {
+    let parsed = parse_expr_for_ids(code);
+    let r = if let Ok(parsed) = parsed {
+        json!({ "outputs": parsed })
+    } else {
+        json!({"error": parsed.err().unwrap().to_string()})
+    };
+    return serde_json::to_string(&r).unwrap();
+}
+
 #[wasm_bindgen]
 pub fn parse_bash(code: &str) -> String {
    wrap_sig(windmill_parser_bash::parse_bash_sig(code))
@@ -1,7 +1,7 @@
 use serde_json::json;
 use wasm_bindgen_test::wasm_bindgen_test;
 use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ};
-use windmill_parser_ts::parse_deno_signature;
+use windmill_parser_ts::{parse_deno_signature, parse_expr_for_ids};

 #[wasm_bindgen_test]
 fn test_parse_deno_sig() -> anyhow::Result<()> {
@@ -191,7 +191,8 @@ type FooBar = {
    a: string,
    b: number,
 }
-export function main(foo: FooBar) {
+export function main(foo: FooBar, {a, b}: FooBar, {c, d}: FooBar = {a: \"foo\", b: 42}) {
+
 }
 ";
    assert_eq!(
@@ -199,13 +200,29 @@ export function main(foo: FooBar) {
        MainArgSignature {
            star_args: false,
            star_kwargs: false,
-            args: vec![Arg {
-                name: "foo".to_string(),
-                otyp: None,
-                typ: Typ::Resource("foo_bar".to_string()),
-                default: None,
-                has_default: false
-            }]
+            args: vec![
+                Arg {
+                    name: "foo".to_string(),
+                    otyp: None,
+                    typ: Typ::Resource("foo_bar".to_string()),
+                    default: None,
+                    has_default: false
+                },
+                Arg {
+                    name: "anon1".to_string(),
+                    otyp: None,
+                    typ: Typ::Resource("foo_bar".to_string()),
+                    default: None,
+                    has_default: false
+                },
+                Arg {
+                    name: "anon2".to_string(),
+                    otyp: None,
+                    typ: Typ::Resource("foo_bar".to_string()),
+                    default: Some(json!({"a": "foo", "b": 42})),
+                    has_default: true
+                }
+            ]
        }
    );

@@ -239,3 +256,22 @@ export function main(foo: (\"foo\" | \"bar\")[]) {

    Ok(())
 }
+
+#[wasm_bindgen_test]
+fn test_parse_extract_ident() -> anyhow::Result<()> {
+    let code = "
+    let foo = 3;
+    bar
+    baroof.foob.ar
+    foobar[barfoo.x]
+";
+    assert_eq!(
+        parse_expr_for_ids(code)?,
+        vec![
+            ("baroof".to_string(), "foob".to_string()),
+            ("barfoo".to_string(), "x".to_string())
+        ]
+    );
+
+    Ok(())
+}
@@ -22,7 +22,8 @@ use tokio::{
 use windmill_api::{LICENSE_KEY, OAUTH_CLIENTS, SMTP_CLIENT};
 use windmill_common::{utils::rd_string, METRICS_ADDR};
 use windmill_worker::{
-    BUN_CACHE_DIR, BUN_TMP_CACHE_DIR, DENO_CACHE_DIR, DENO_TMP_CACHE_DIR, GO_BIN_CACHE_DIR,
+    BUN_CACHE_DIR, BUN_TMP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
+    DENO_TMP_CACHE_DIR, DENO_TMP_CACHE_DIR_DEPS, DENO_TMP_CACHE_DIR_NPM, GO_BIN_CACHE_DIR,
    GO_CACHE_DIR, GO_TMP_CACHE_DIR, HUB_CACHE_DIR, HUB_TMP_CACHE_DIR, LOCK_CACHE_DIR,
    PIP_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR,
 };
@@ -243,8 +244,12 @@ Windmill Community Edition {GIT_VERSION}

        let metrics_f = async {
            match metrics_addr {
-                Some(addr) => {
-                    windmill_common::serve_metrics(addr, rx.resubscribe(), num_workers > 0)
+                Some(_addr) => {
+                    #[cfg(not(feature = "enterprise"))]
+                    panic!("Metrics are only available in the Enterprise Edition");
+
+                    #[cfg(feature = "enterprise")]
+                    windmill_common::serve_metrics(_addr, rx.resubscribe(), num_workers > 0)
                        .await
                        .map_err(anyhow::Error::from)
                }
@@ -343,12 +348,16 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
        LOCK_CACHE_DIR,
        PIP_CACHE_DIR,
        DENO_CACHE_DIR,
+        DENO_CACHE_DIR_DEPS,
+        DENO_CACHE_DIR_NPM,
        BUN_CACHE_DIR,
        GO_CACHE_DIR,
        GO_BIN_CACHE_DIR,
        HUB_CACHE_DIR,
        TAR_PIP_TMP_CACHE_DIR,
        DENO_TMP_CACHE_DIR,
+        DENO_TMP_CACHE_DIR_DEPS,
+        DENO_TMP_CACHE_DIR_NPM,
        BUN_TMP_CACHE_DIR,
        GO_TMP_CACHE_DIR,
        HUB_TMP_CACHE_DIR,
@@ -10,7 +10,7 @@ use windmill_common::{
    METRICS_ENABLED,
 };
 use windmill_worker::{
-    create_token_for_owner, handle_job_error, AuthedClient, SESSION_TOKEN_EXPIRY,
+    create_token_for_owner, handle_job_error, AuthedClient, SCRIPT_TOKEN_EXPIRY,
 };

 lazy_static::lazy_static! {
@@ -116,7 +116,7 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
            &job.workspace_id,
            &job.permissioned_as,
            "ephemeral-zombie-jobs",
-            *SESSION_TOKEN_EXPIRY,
+            *SCRIPT_TOKEN_EXPIRY,
            &job.email,
        )
        .await
@@ -13,6 +13,59 @@ INSERT INTO workspace_key(workspace_id, kind, key) VALUES

 insert INTO token(token, email, label, super_admin) VALUES ('SECRET_TOKEN', 'test@windmill.dev', 'test token', true);

+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main(fail: boolean = true) {
+  if (fail) {
+    throw new Error("Failed")
+  }
+
+  return "OK"
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
+'',
+'',
+'f/system/failing_script', -28028598712388162, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main() {
+  return "Error handler";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
+'',
+'',
+'f/system/schedule_error_handler', -28028598712388161, 'deno', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+export async function main() {
+  return "Recovery handler";
+}
+',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
+'',
+'',
+'f/system/schedule_recovery_handler', -28028598712388160, 'deno', '');
+
+INSERT INTO public.flow(workspace_id, edited_by, value, schema, summary, description, path) VALUES (
+'test-workspace',
+'system',
+'{"modules": [{"id": "a", "value": {"path": "f/system/failing_script", "type": "script", "input_transforms": {"fail": {"expr": "flow_input.fail", "type": "javascript"}}}}]}',
+'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean","format":""}},"required":[],"type":"object"}',
+'',
+'',
+'f/system/failing_flow'
+);
+
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_admin;
 GRANT ALL PRIVILEGES ON TABLE workspace_key TO windmill_user;

@@ -1,19 +1,26 @@
 use std::sync::Arc;

+use chrono::Timelike;
+use futures::StreamExt;
 use futures::{stream, Stream};
 use serde::Deserialize;
 use serde_json::json;
 use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, Transaction};
-use tokio::sync::RwLock;
+use tokio::{
+    sync::RwLock,
+    time::{timeout, Duration},
+};
 use windmill_api::jobs::{CompletedJob, Job};
-use windmill_api_client::types::{CreateFlowBody, RawScript};
+use windmill_api_client::types::{
+    CreateFlowBody, EditSchedule, NewSchedule, RawScript, ScriptArgs,
+};
 use windmill_common::{
    flow_status::{FlowStatus, FlowStatusModule},
    flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
    jobs::{JobPayload, RawCode},
    scripts::ScriptLang,
 };
-use windmill_queue::get_queued_job;
+use windmill_queue::{get_queued_job, PushIsolationLevel};

 async fn initialize_tracing() {
    use std::sync::Once;
@@ -839,8 +846,10 @@ impl RunJob {

    async fn push(self, db: &Pool<Postgres>) -> Uuid {
        let RunJob { payload, args } = self;
+        let tx = PushIsolationLevel::IsolatedRoot(db.clone(), None);
        let (uuid, tx) = windmill_queue::push::<rsmq_async::MultiplexedRsmq>(
-            (None, db.begin().await.unwrap()).into(),
+            &db,
+            tx,
            "test-workspace",
            payload,
            args,
@@ -924,11 +933,19 @@ fn spawn_test_worker(
    tokio::sync::broadcast::Sender<()>,
    tokio::task::JoinHandle<()>,
 ) {
+    for x in [windmill_worker::LOCK_CACHE_DIR] {
+        std::fs::DirBuilder::new()
+            .recursive(true)
+            .create(x)
+            .expect("could not create initial worker dir");
+    }
+
    let (tx, rx) = tokio::sync::broadcast::channel(1);
    let db = db.to_owned();
    let worker_instance: &str = "test worker instance";
    let worker_name: String = next_worker_name();
    let ip: &str = Default::default();
+
    let future = async move {
        let base_internal_url = format!("http://localhost:{}", port);
        windmill_worker::run_worker::<rsmq_async::MultiplexedRsmq>(
@@ -2519,15 +2536,15 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
    let listen_first_job = str.next();
    in_test_worker(&db, listen_first_job, port).await;

-    client
+    let modules = client
        .get_flow_by_path("test-workspace", "g/all/flow_lock_all")
        .await
        .unwrap()
        .into_inner()
        .subtype_0
        .value
-        .modules
-        .into_iter()
+        .modules;
+    modules.into_iter()
        .for_each(|m| {
            assert!(matches!(
                m.value,
@@ -2561,3 +2578,274 @@ async fn test_rust_client(db: Pool<Postgres>) {
    .await
    .unwrap();
 }
+
+#[sqlx::test(fixtures("base"))]
+async fn test_script_schedule_handlers(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    let mut args = std::collections::HashMap::new();
+    args.insert("fail".to_string(), json!(true));
+
+    let now = chrono::Utc::now();
+    // add 5 seconds to now
+    let then = now
+        .checked_add_signed(chrono::Duration::seconds(5))
+        .unwrap();
+
+    let schedule = NewSchedule {
+        args: ScriptArgs::from(args),
+        enabled: Some(true),
+        is_flow: false,
+        on_failure: Some("script/f/system/schedule_error_handler".to_string()),
+        on_failure_times: None,
+        on_failure_exact: None,
+        on_failure_extra_args: None,
+        on_recovery: Some("script/f/system/schedule_recovery_handler".to_string()),
+        on_recovery_times: None,
+        on_recovery_extra_args: None,
+        path: "f/system/failing_script_schedule".to_string(),
+        script_path: "f/system/failing_script".to_string(),
+        timezone: "UTC".to_string(),
+        schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
+    };
+
+    let _ = client.create_schedule("test-workspace", &schedule).await;
+
+    let mut str = listen_for_completed_jobs(&db).await;
+
+    let db2 = db.clone();
+    in_test_worker(
+        &db,
+        async move {
+            str.next().await; // completed error job
+
+            let uuid = timeout(Duration::from_millis(5000), str.next()).await; // error handler
+
+            if uuid.is_err() {
+                panic!("schedule error handler was not run within 5 s");
+            }
+
+            let uuid = uuid.unwrap().unwrap();
+
+            let completed_job =
+                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
+                    .bind(uuid)
+                    .fetch_one(&db2)
+                    .await
+                    .unwrap();
+
+            if completed_job.script_path.is_none()
+                || completed_job.script_path != Some("f/system/schedule_error_handler".to_string())
+            {
+                panic!(
+                    "a script was run after main job execution but was not schedule error handler"
+                );
+            }
+        },
+        port,
+    )
+    .await;
+
+    let mut args = std::collections::HashMap::new();
+    args.insert("fail".to_string(), json!(false));
+    let now = chrono::Utc::now();
+    let then = now
+        .checked_add_signed(chrono::Duration::seconds(5))
+        .unwrap();
+    client
+        .update_schedule(
+            "test-workspace",
+            "f/system/failing_script_schedule",
+            &EditSchedule {
+                args: ScriptArgs::from(args),
+                on_failure: Some("script/f/system/schedule_error_handler".to_string()),
+                on_failure_times: None,
+                on_failure_exact: None,
+                on_failure_extra_args: None,
+                on_recovery: Some("script/f/system/schedule_recovery_handler".to_string()),
+                on_recovery_times: None,
+                on_recovery_extra_args: None,
+                timezone: "UTC".to_string(),
+                schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
+            },
+        )
+        .await
+        .unwrap();
+
+    let mut str = listen_for_completed_jobs(&db).await;
+
+    let db2 = db.clone();
+    in_test_worker(
+        &db,
+        async move {
+            str.next().await; // completed working job
+            let uuid = timeout(Duration::from_millis(5000), str.next()).await; // recovery handler
+
+            if uuid.is_err() {
+                panic!("schedule recovery handler was not run within 5 s");
+            }
+
+            let uuid = uuid.unwrap().unwrap();
+            
+            let completed_job =
+                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
+                    .bind(uuid)
+                    .fetch_one(&db2)
+                    .await
+                    .unwrap();
+
+            if completed_job.script_path.is_none()
+                || completed_job.script_path
+                    != Some("f/system/schedule_recovery_handler".to_string())
+            {
+                panic!("a script was run after main job execution but was not schedule recovery handler");
+            }
+        },
+        port,
+    )
+    .await;
+}
+
+
+#[sqlx::test(fixtures("base"))]
+async fn test_flow_schedule_handlers(db: Pool<Postgres>) {
+    initialize_tracing().await;
+    let server = ApiServer::start(db.clone()).await;
+    let port = server.addr.port();
+
+    let client = windmill_api_client::create_client(
+        &format!("http://localhost:{port}"),
+        "SECRET_TOKEN".to_string(),
+    );
+
+    let mut args = std::collections::HashMap::new();
+    args.insert("fail".to_string(), json!(true));
+
+    let now = chrono::Utc::now();
+    // add 5 seconds to now
+    let then = now
+        .checked_add_signed(chrono::Duration::seconds(5))
+        .unwrap();
+
+    let schedule = NewSchedule {
+        args: ScriptArgs::from(args),
+        enabled: Some(true),
+        is_flow: true,
+        on_failure: Some("script/f/system/schedule_error_handler".to_string()),
+        on_failure_times: None,
+        on_failure_exact: None,
+        on_failure_extra_args: None,
+        on_recovery: Some("script/f/system/schedule_recovery_handler".to_string()),
+        on_recovery_times: None,
+        on_recovery_extra_args: None,
+        path: "f/system/failing_flow_schedule".to_string(),
+        script_path: "f/system/failing_flow".to_string(),
+        timezone: "UTC".to_string(),
+        schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
+    };
+
+    let _ = client.create_schedule("test-workspace", &schedule).await;
+
+    let mut str = listen_for_completed_jobs(&db).await;
+
+    let db2 = db.clone();
+    in_test_worker(
+        &db,
+        async move {
+            str.next().await; // completed error step
+            str.next().await; // completed error flow
+
+            let uuid = timeout(Duration::from_millis(5000), str.next()).await; // error handler
+
+            if uuid.is_err() {
+                panic!("schedule error handler was not run within 5 s");
+            }
+
+            let uuid = uuid.unwrap().unwrap();
+
+            let completed_job =
+                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
+                    .bind(uuid)
+                    .fetch_one(&db2)
+                    .await
+                    .unwrap();
+
+            if completed_job.script_path.is_none()
+                || completed_job.script_path != Some("f/system/schedule_error_handler".to_string())
+            {
+                panic!(
+                    "a script was run after main job execution but was not schedule error handler"
+                );
+            }
+        },
+        port,
+    )
+    .await;
+
+    let mut args = std::collections::HashMap::new();
+    args.insert("fail".to_string(), json!(false));
+    let now = chrono::Utc::now();
+    let then = now
+        .checked_add_signed(chrono::Duration::seconds(5))
+        .unwrap();
+    client
+        .update_schedule(
+            "test-workspace",
+            "f/system/failing_flow_schedule",
+            &EditSchedule {
+                args: ScriptArgs::from(args),
+                on_failure: Some("script/f/system/schedule_error_handler".to_string()),
+                on_failure_times: None,
+                on_failure_exact: None,
+                on_failure_extra_args: None,
+                on_recovery: Some("script/f/system/schedule_recovery_handler".to_string()),
+                on_recovery_times: None,
+                on_recovery_extra_args: None,
+                timezone: "UTC".to_string(),
+                schedule: format!("{} {} * * * *", then.second(), then.minute()).to_string(),
+            },
+        )
+        .await
+        .unwrap();
+
+    let mut str = listen_for_completed_jobs(&db).await;
+
+    let db2 = db.clone();
+    in_test_worker(
+        &db,
+        async move {
+            str.next().await; // completed working step
+            str.next().await; // completed working flow
+            let uuid = timeout(Duration::from_millis(5000), str.next()).await; // recovery handler
+
+            if uuid.is_err() {
+                panic!("schedule recovery handler was not run within 5 s");
+            }
+
+            let uuid = uuid.unwrap().unwrap();
+            
+            let completed_job =
+                sqlx::query_as::<_, CompletedJob>("SELECT * FROM completed_job WHERE id = $1")
+                    .bind(uuid)
+                    .fetch_one(&db2)
+                    .await
+                    .unwrap();
+
+            if completed_job.script_path.is_none()
+                || completed_job.script_path
+                    != Some("f/system/schedule_recovery_handler".to_string())
+            {
+                panic!("a script was run after main job execution but was not schedule recovery handler");
+            }
+        },
+        port,
+    )
+    .await;
+}
@@ -72,4 +72,5 @@ rsmq_async.workspace = true
 regex.workspace = true
 bytes.workspace = true
 mail-send.workspace = true
-samael = { workspace = true, optional = true }
+samael = { workspace = true, optional = true }
+async-recursion.workspace = true
@@ -1,7 +1,7 @@
 openapi: "3.0.3"

 info:
-  version: 1.146.0
+  version: 1.156.1
  title: Windmill API

  contact:
@@ -1444,6 +1444,23 @@ paths:
              schema:
                $ref: "#/components/schemas/ListableVariable"

+  /w/{workspace}/variables/get_value/{path}:
+    get:
+      summary: get variable value
+      operationId: getVariableValue
+      tags:
+        - variable
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: variable
+          content:
+            application/json:
+              schema:
+                type: string
+
  /w/{workspace}/variables/exists/{path}:
    get:
      summary: does variable exists at path
@@ -1854,6 +1871,22 @@ paths:
              schema:
                $ref: "#/components/schemas/Resource"

+  /w/{workspace}/resources/get_value_interpolated/{path}:
+    get:
+      summary: get resource interpolated (variables and resources are fully unrolled)
+      operationId: getResourceValueInterpolated
+      tags:
+        - resource
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: resource value
+          content:
+            application/json:
+              schema: {}
+
  /w/{workspace}/resources/get_value/{path}:
    get:
      summary: get resource value
@@ -2295,8 +2328,7 @@ paths:
          schema:
            type: string
        - name: first_parent_hash
-          description:
-            mask to filter scripts whom first direct parent has exact hash
+          description: mask to filter scripts whom first direct parent has exact hash
          in: query
          schema:
            type: string
@@ -2464,8 +2496,7 @@ paths:

  /workers/custom_tags:
    get:
-      summary:
-        get all instance custom tags (tags are used to dispatch jobs to
+      summary: get all instance custom tags (tags are used to dispatch jobs to
        different worker groups)
      operationId: getCustomTags
      tags:
@@ -2516,8 +2547,7 @@ paths:

  /w/{workspace}/scripts/delete/h/{hash}:
    post:
-      summary:
-        delete script by hash (erase content but keep hash, require admin)
+      summary: delete script by hash (erase content but keep hash, require admin)
      operationId: deleteScriptByHash
      tags:
        - script
@@ -2709,16 +2739,14 @@ paths:
            type: string
            format: date-time
        - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
+          description: schedule the script to execute in the number of seconds starting now
          in: query
          schema:
            type: integer
        - $ref: "#/components/parameters/ParentJob"
        - $ref: "#/components/parameters/NewJobId"
        - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
          in: query
          schema:
            type: boolean
@@ -3596,8 +3624,7 @@ paths:
            type: string
            format: date-time
        - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
+          description: schedule the script to execute in the number of seconds starting now
          in: query
          schema:
            type: integer
@@ -3605,8 +3632,7 @@ paths:
        - $ref: "#/components/parameters/NewJobId"
        - $ref: "#/components/parameters/IncludeHeader"
        - name: invisible_to_owner
-          description:
-            make the run invisible to the the flow owner (default false)
+          description: make the run invisible to the the flow owner (default false)
          in: query
          schema:
            type: boolean
@@ -3644,8 +3670,7 @@ paths:
            type: string
            format: date-time
        - name: scheduled_in_secs
-          description:
-            schedule the script to execute in the number of seconds starting now
+          description: schedule the script to execute in the number of seconds starting now
          in: query
          schema:
            type: integer
@@ -3653,8 +3678,7 @@ paths:
        - $ref: "#/components/parameters/NewJobId"
        - $ref: "#/components/parameters/IncludeHeader"
        - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
          in: query
          schema:
            type: boolean
@@ -3685,8 +3709,7 @@ paths:
        - $ref: "#/components/parameters/WorkspaceId"
        - $ref: "#/components/parameters/IncludeHeader"
        - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
          in: query
          schema:
            type: boolean
@@ -3719,8 +3742,7 @@ paths:
        - $ref: "#/components/parameters/WorkspaceId"
        - $ref: "#/components/parameters/IncludeHeader"
        - name: invisible_to_owner
-          description:
-            make the run invisible to the the script owner (default false)
+          description: make the run invisible to the the script owner (default false)
          in: query
          schema:
            type: boolean
@@ -3777,6 +3799,45 @@ paths:
                items:
                  $ref: "#/components/schemas/QueuedJob"

+  /w/{workspace}/jobs/queue/count:
+    get:
+      summary: get queue count
+      operationId: getQueueCount
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: queue count
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  database_length:
+                    type: integer
+                required:
+                  - database_length
+
+  /w/{workspace}/jobs/queue/cancel_all:
+    post:
+      summary: cancel all jobs
+      operationId: cancelAll
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: uuids of canceled jobs
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+
  /w/{workspace}/jobs/completed/list:
    get:
      summary: list all available completed jobs
@@ -3835,6 +3896,8 @@ paths:
        - $ref: "#/components/parameters/ScriptExactHash"
        - $ref: "#/components/parameters/StartedBefore"
        - $ref: "#/components/parameters/StartedAfter"
+        - $ref: "#/components/parameters/CreatedOrStartedBefore"
+        - $ref: "#/components/parameters/CreatedOrStartedAfter"
        - $ref: "#/components/parameters/JobKinds"
        - $ref: "#/components/parameters/ArgsFilter"
        - $ref: "#/components/parameters/Tag"
@@ -3864,6 +3927,20 @@ paths:
                items:
                  $ref: "#/components/schemas/Job"

+  /jobs/db_clock:
+    get:
+      summary: get db clock
+      operationId: getDbClock
+      tags:
+        - job
+      responses:
+        "200":
+          description: the timestamp of the db that can be used to compute the drift
+          content:
+            application/json:
+              schema:
+                type: integer
+
  /w/{workspace}/jobs_u/get/{id}:
    get:
      summary: get job
@@ -4091,8 +4168,7 @@ paths:

  /w/{workspace}/jobs/resume_urls/{id}/{resume_id}:
    get:
-      summary:
-        get resume urls given a job_id, resume_id and a nonce to resume a flow
+      summary: get resume urls given a job_id, resume_id and a nonce to resume a flow
      operationId: getResumeUrls
      tags:
        - job
@@ -4605,8 +4681,7 @@ paths:
        - $ref: "#/components/parameters/WorkspaceId"
        - name: only_member_of
          in: query
-          description:
-            only list the groups the user is member of (default false)
+          description: only list the groups the user is member of (default false)
          schema:
            type: boolean
      responses:
@@ -4794,8 +4869,7 @@ paths:
        - $ref: "#/components/parameters/WorkspaceId"
        - name: only_member_of
          in: query
-          description:
-            only list the folders the user is member of (default false)
+          description: only list the folders the user is member of (default false)
          schema:
            type: boolean
      responses:
@@ -5433,8 +5507,7 @@ components:
        type: integer
    PerPage:
      name: per_page
-      description:
-        number of items to return for a given page (default 30, max 100)
+      description: number of items to return for a given page (default 30, max 100)
      in: query
      schema:
        type: integer
@@ -5518,14 +5591,32 @@ components:
        type: string
    StartedBefore:
      name: started_before
-      description: filter on created before (inclusive) timestamp
+      description: filter on started before (inclusive) timestamp
      in: query
      schema:
        type: string
        format: date-time
    StartedAfter:
      name: started_after
-      description: filter on created after (exclusive) timestamp
+      description: filter on started after (exclusive) timestamp
+      in: query
+      schema:
+        type: string
+        format: date-time
+    CreatedOrStartedAfter:
+      name: created_or_started_after
+      description:
+        filter on created_at for non non started job and started_at otherwise
+        after (exclusive) timestamp
+      in: query
+      schema:
+        type: string
+        format: date-time
+    CreatedOrStartedBefore:
+      name: created_or_started_before
+      description:
+        filter on created_at for non non started job and started_at otherwise
+        before (inclusive) timestamp
      in: query
      schema:
        type: string
@@ -5550,8 +5641,7 @@ components:
        type: boolean
    ArgsFilter:
      name: args
-      description:
-        filter on jobs containing those args as a json subset (@> in postgres)
+      description: filter on jobs containing those args as a json subset (@> in postgres)
      in: query
      schema:
        type: string
@@ -5563,8 +5653,7 @@ components:
        type: string
    ResultFilter:
      name: result
-      description:
-        filter on jobs containing those result as a json subset (@> in postgres)
+      description: filter on jobs containing those result as a json subset (@> in postgres)
      in: query
      schema:
        type: string
@@ -5926,6 +6015,7 @@ components:
              "script",
              "preview",
              "dependencies",
+              "flowdependencies",
              "flow",
              "flowpreview",
              "script_hub",
@@ -6030,6 +6120,7 @@ components:
              "preview",
              "dependencies",
              "flow",
+              "flowdependencies",
              "flowpreview",
              "script_hub",
              "identity",
@@ -6720,6 +6811,18 @@ components:
        on_failure:
          # a reference to a script path, flow path, or webhook (script/<path>, flow/<path>)
          type: string
+        on_failure_times:
+          type: number
+        on_failure_exact:
+          type: boolean
+        on_failure_extra_args:
+          $ref: "#/components/schemas/ScriptArgs"
+        on_recovery:
+          type: string
+        on_recovery_times:
+          type: number
+        on_recovery_extra_args:
+          $ref: "#/components/schemas/ScriptArgs"
      required:
        - path
        - edited_by
@@ -6773,6 +6876,18 @@ components:
        on_failure:
          # a reference to a script path, flow path, or webhook (script/<path>, flow/<path>)
          type: string
+        on_failure_times:
+          type: number
+        on_failure_exact:
+          type: boolean
+        on_failure_extra_args:
+          $ref: "#/components/schemas/ScriptArgs"
+        on_recovery:
+          type: string
+        on_recovery_times:
+          type: number
+        on_recovery_extra_args:
+          $ref: "#/components/schemas/ScriptArgs"
      required:
        - path
        - schedule
@@ -6793,6 +6908,18 @@ components:
        on_failure:
          # a reference to a script path, flow path, or webhook (script/<path>, flow/<path>)
          type: string
+        on_failure_times:
+          type: number
+        on_failure_exact:
+          type: boolean
+        on_failure_extra_args:
+          $ref: "#/components/schemas/ScriptArgs"
+        on_recovery:
+          type: string
+        on_recovery_times:
+          type: number
+        on_recovery_extra_args:
+          $ref: "#/components/schemas/ScriptArgs"
      required:
        - schedule
        - timezone
@@ -8,8 +8,8 @@ use std::collections::HashMap;
 * LICENSE-AGPL for a copy of the license.
 */
 use crate::{
-    db::{UserDB, DB},
-    users::{require_owner_of_path, Authed, OptAuthed},
+    db::{ApiAuthed, DB},
+    users::{require_owner_of_path, OptAuthed},
    variables::build_crypt,
    webhook_util::{WebhookMessage, WebhookShared},
    HTTP_CLIENT,
@@ -30,6 +30,7 @@ use std::str;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
    apps::ListAppQuery,
+    db::UserDB,
    error::{to_anyhow, Error, JsonResult, Result},
    jobs::{get_payload_tag_from_prefixed_path, JobPayload, RawCode},
    users::username_to_permissioned_as,
@@ -37,7 +38,7 @@ use windmill_common::{
        http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
    },
 };
-use windmill_queue::{push, QueueTransaction};
+use windmill_queue::push;

 pub fn workspaced_service() -> Router {
    Router::new()
@@ -159,7 +160,7 @@ pub struct EditApp {
 }

 async fn list_apps(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Query(pagination): Query<Pagination>,
@@ -220,7 +221,7 @@ async fn list_apps(
 }

 async fn get_app(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<AppWithLastVersion> {
@@ -245,7 +246,7 @@ async fn get_app(
 }

 async fn get_app_w_draft(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<AppWithLastVersionAndDraft> {
@@ -262,7 +263,7 @@ async fn get_app_w_draft(
        INNER JOIN app_version ON
        app_version.id = app.versions[array_upper(app.versions, 1)]
        LEFT JOIN draft ON 
-        app.path = draft.path AND app.workspace_id = draft.workspace_id AND draft.typ = 'app' 
+        app.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'app' 
        WHERE app.path = $1 AND app.workspace_id = $2"#,
        path.to_owned(),
        &w_id
@@ -276,7 +277,7 @@ async fn get_app_w_draft(
 }

 async fn get_app_by_id(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, id)): Path<(String, i64)>,
 ) -> JsonResult<AppWithLastVersion> {
@@ -341,7 +342,7 @@ async fn get_public_app_by_secret(
 }

 async fn get_secret_id(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
@@ -369,7 +370,7 @@ async fn get_secret_id(
 }

 async fn create_app(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path(w_id): Path<String>,
@@ -461,7 +462,7 @@ async fn create_app(
    Ok((StatusCode::CREATED, app.path))
 }

-async fn list_hub_apps(Authed { email, .. }: Authed) -> JsonResult<serde_json::Value> {
+async fn list_hub_apps(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serde_json::Value> {
    let flows = list_elems_from_hub(
        &HTTP_CLIENT,
        "https://hub.windmill.dev/searchUiData?approved=true",
@@ -472,7 +473,7 @@ async fn list_hub_apps(Authed { email, .. }: Authed) -> JsonResult<serde_json::V
 }

 pub async fn get_hub_app_by_id(
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
    Path(id): Path<i32>,
 ) -> JsonResult<serde_json::Value> {
    let value = http_get_from_hub(
@@ -489,7 +490,7 @@ pub async fn get_hub_app_by_id(
 }

 async fn delete_app(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -539,7 +540,7 @@ async fn delete_app(
 }

 async fn update_app(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -709,7 +710,6 @@ async fn execute_component(
    };

    let path = path.to_path();
-    let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into();

    let policy = if let Some(static_fields) = payload.clone().force_viewer_static_fields {
        let mut hm = HashMap::new();
@@ -738,7 +738,7 @@ async fn execute_component(
            path,
            &w_id
        )
-        .fetch_optional(&mut tx)
+        .fetch_optional(&db)
        .await?;

        let policy = not_found_if_none(policy_o, "App", path)?;
@@ -782,15 +782,16 @@ async fn execute_component(
            (payload, args, None)
        }
        ExecuteApp { args, component, raw_code: None, path: Some(path), .. } => {
-            let (payload, tag) =
-                get_payload_tag_from_prefixed_path(path, tx.transaction_mut(), &w_id).await?;
+            let (payload, tag) = get_payload_tag_from_prefixed_path(path, &db, &w_id).await?;
            let args = build_args(policy, component, path.to_string(), args)?;
            (payload, args, tag)
        }
        _ => unreachable!(),
    };
+    let tx = windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);

    let (uuid, tx) = push(
+        &db,
        tx,
        &w_id,
        job_payload,
@@ -841,7 +842,7 @@ fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> {
    Ok((permissioned_as, email))
 }

-pub async fn require_is_writer(authed: &Authed, path: &str, w_id: &str, db: DB) -> Result<()> {
+pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: DB) -> Result<()> {
    return crate::users::require_is_writer(
        authed,
        path,
@@ -12,9 +12,9 @@ use axum::{
    Extension, Json, Router,
 };
 use windmill_audit::{AuditLog, ListAuditLogQuery};
-use windmill_common::{error::JsonResult, utils::Pagination};
+use windmill_common::{db::UserDB, error::JsonResult, utils::Pagination};

-use crate::{db::UserDB, users::Authed};
+use crate::db::ApiAuthed;

 pub fn workspaced_service() -> Router {
    Router::new()
@@ -23,7 +23,7 @@ pub fn workspaced_service() -> Router {
 }

 async fn get_audit(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(id): Path<i32>,
 ) -> JsonResult<AuditLog> {
@@ -32,7 +32,7 @@ async fn get_audit(
    Ok(Json(audit))
 }
 async fn list_audit(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Query(pagination): Query<Pagination>,
@@ -14,14 +14,14 @@ use axum::{
 use hyper::{HeaderMap, StatusCode};
 use serde::Deserialize;
 use windmill_common::{
+    db::UserDB,
    error::{JsonResult, Result},
    utils::{not_found_if_none, StripPath},
 };

 use crate::{
-    db::{UserDB, DB},
+    db::{ApiAuthed, DB},
    jobs::add_include_headers,
-    users::Authed,
 };

 const KEEP_LAST: i64 = 8;
@@ -37,7 +37,7 @@ pub fn global_service() -> Router {
 }

 pub async fn new_payload(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<StatusCode> {
@@ -120,7 +120,7 @@ pub async fn update_payload(
 }

 pub async fn get_payload(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<serde_json::Value> {
@@ -6,10 +6,11 @@
 * LICENSE-AGPL for a copy of the license.
 */

-use sqlx::{Pool, Postgres, Transaction};
-use windmill_common::error::Error;
-
-use crate::users::Authed;
+use sqlx::{Pool, Postgres};
+use windmill_common::{
+    db::{Authable, Authed},
+    error::Error,
+};

 pub type DB = Pool<Postgres>;

@@ -28,84 +29,58 @@ pub async fn migrate(db: &DB) -> Result<(), Error> {
    Ok(())
 }

-#[derive(Clone)]
-pub struct UserDB {
-    db: DB,
+#[derive(Clone, Debug)]
+pub struct ApiAuthed {
+    pub email: String,
+    pub username: String,
+    pub is_admin: bool,
+    pub is_operator: bool,
+    pub groups: Vec<String>,
+    // (folder name, can write, is owner)
+    pub folders: Vec<(String, bool, bool)>,
+    pub scopes: Option<Vec<String>>,
 }

-impl UserDB {
-    pub fn new(db: DB) -> Self {
-        Self { db }
+impl From<ApiAuthed> for Authed {
+    fn from(value: ApiAuthed) -> Self {
+        Self {
+            email: value.email,
+            username: value.username,
+            is_admin: value.is_admin,
+            is_operator: value.is_operator,
+            groups: value.groups,
+            folders: value.folders,
+            scopes: value.scopes,
+        }
    }
+}

-    pub async fn begin(
-        self,
-        authed: &Authed,
-    ) -> Result<Transaction<'static, Postgres>, sqlx::Error> {
-        let mut tx = self.db.begin().await?;
-        let user = if authed.is_admin {
-            "windmill_admin"
-        } else {
-            "windmill_user"
-        };
+impl Authable for ApiAuthed {
+    fn is_admin(&self) -> bool {
+        self.is_admin
+    }

-        sqlx::query(&format!("SET LOCAL ROLE {}", user))
-            .execute(&mut *tx)
-            .await?;
+    fn is_operator(&self) -> bool {
+        self.is_operator
+    }

-        sqlx::query!(
-            "SELECT set_config('session.user', $1, true)",
-            authed.username
-        )
-        .fetch_optional(&mut *tx)
-        .await?;
+    fn groups(&self) -> &[String] {
+        &self.groups
+    }

-        sqlx::query!(
-            "SELECT set_config('session.groups', $1, true)",
-            &authed.groups.join(",")
-        )
-        .fetch_optional(&mut *tx)
-        .await?;
+    fn folders(&self) -> &[(String, bool, bool)] {
+        &self.folders
+    }

-        sqlx::query!(
-            "SELECT set_config('session.pgroups', $1, true)",
-            &authed
-                .groups
-                .iter()
-                .map(|x| format!("g/{}", x))
-                .collect::<Vec<_>>()
-                .join(",")
-        )
-        .fetch_optional(&mut *tx)
-        .await?;
+    fn scopes(&self) -> Option<&[std::string::String]> {
+        self.scopes.as_ref().map(|x| x.as_slice())
+    }

-        let (folders_write, folders_read): &(Vec<_>, Vec<_>) =
-            &authed.folders.clone().into_iter().partition(|x| x.1);
-
-        let mut folders_read = folders_read.clone();
-        folders_read.extend(folders_write.clone());
-        sqlx::query!(
-            "SELECT set_config('session.folders_read', $1, true)",
-            folders_read
-                .iter()
-                .map(|x| x.0.clone())
-                .collect::<Vec<_>>()
-                .join(",")
-        )
-        .fetch_optional(&mut *tx)
-        .await?;
-
-        sqlx::query!(
-            "SELECT set_config('session.folders_write', $1, true)",
-            folders_write
-                .iter()
-                .map(|x| x.0.clone())
-                .collect::<Vec<_>>()
-                .join(",")
-        )
-        .fetch_optional(&mut *tx)
-        .await?;
+    fn email(&self) -> &str {
+        &self.email
+    }

-        Ok(tx)
+    fn username(&self) -> &str {
+        &self.username
    }
 }
@@ -7,8 +7,8 @@
 */

 use crate::{
-    db::{UserDB, DB},
-    users::{maybe_refresh_folders, require_owner_of_path, Authed},
+    db::{ApiAuthed, DB},
+    users::{maybe_refresh_folders, require_owner_of_path},
 };

 use axum::{
@@ -18,7 +18,7 @@ use axum::{
 };
 use hyper::StatusCode;
 use serde::{Deserialize, Serialize};
-use windmill_common::{error::Result, utils::StripPath};
+use windmill_common::{db::UserDB, error::Result, utils::StripPath};

 pub fn workspaced_service() -> Router {
    Router::new()
@@ -43,7 +43,7 @@ pub struct Draft {
 }

 pub async fn require_writer_of_path(
-    authed: &Authed,
+    authed: &ApiAuthed,
    path: &str,
    w_id: &str,
    db: DB,
@@ -63,7 +63,7 @@ pub async fn require_writer_of_path(
 }

 async fn create_draft(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
@@ -95,7 +95,7 @@ async fn create_draft(
 }

 async fn delete_draft(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, kind, path)): Path<(String, DraftType, StripPath)>,
 ) -> Result<String> {
@@ -115,7 +115,7 @@ async fn delete_draft(
 }

 // async fn get_draft(
-//     authed: Authed,
+//     authed: ApiAuthed,
 //     Extension(user_db): Extension<UserDB>,
 //     Path((w_id, path)): Path<(String, StripPath)>,
 // ) -> JsonResult<Draft> {
@@ -6,7 +6,7 @@
 * LICENSE-AGPL for a copy of the license.
 */

-use crate::{db::DB, users::Authed};
+use crate::db::{ApiAuthed, DB};
 use axum::{
    extract::{Extension, Path},
    routing::post,
@@ -39,7 +39,7 @@ pub struct Favorite {
 }

 async fn star(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
    Json(Favorite { favorite_kind, path }): Json<Favorite>,
@@ -58,7 +58,7 @@ async fn star(
 }

 async fn unstar(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
    Json(Favorite { favorite_kind, path }): Json<Favorite>,
@@ -6,10 +6,11 @@
 * LICENSE-AGPL for a copy of the license.
 */

+use crate::db::ApiAuthed;
 use crate::{
-    db::{UserDB, DB},
+    db::DB,
    schedule::clear_schedule,
-    users::{maybe_refresh_folders, require_owner_of_path, Authed},
+    users::{maybe_refresh_folders, require_owner_of_path},
    webhook_util::{WebhookMessage, WebhookShared},
    HTTP_CLIENT,
 };
@@ -18,6 +19,7 @@ use axum::{
    routing::{delete, get, post},
    Json, Router,
 };
+
 use hyper::StatusCode;
 use serde::{Deserialize, Serialize};
 use sql_builder::prelude::*;
@@ -25,6 +27,7 @@ use sql_builder::SqlBuilder;
 use sqlx::{Postgres, Transaction};
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
+    db::UserDB,
    error::{self, to_anyhow, Error, JsonResult, Result},
    flows::{Flow, ListFlowQuery, ListableFlow, NewFlow},
    jobs::JobPayload,
@@ -34,7 +37,7 @@ use windmill_common::{
        http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
    },
 };
-use windmill_queue::{push, schedule::push_scheduled_job, QueueTransaction};
+use windmill_queue::{push, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction};

 pub fn workspaced_service() -> Router {
    Router::new()
@@ -56,7 +59,7 @@ pub fn global_service() -> Router {
 }

 async fn list_flows(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Query(pagination): Query<Pagination>,
@@ -120,7 +123,7 @@ async fn list_flows(
    Ok(Json(rows))
 }

-async fn list_hub_flows(Authed { email, .. }: Authed) -> JsonResult<serde_json::Value> {
+async fn list_hub_flows(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serde_json::Value> {
    let flows = list_elems_from_hub(
        &HTTP_CLIENT,
        "https://hub.windmill.dev/searchFlowData?approved=true",
@@ -131,7 +134,7 @@ async fn list_hub_flows(Authed { email, .. }: Authed) -> JsonResult<serde_json::
 }

 async fn list_paths(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
 ) -> JsonResult<Vec<String>> {
@@ -149,7 +152,7 @@ async fn list_paths(
 }

 pub async fn get_hub_flow_by_id(
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
    Path(id): Path<i32>,
 ) -> JsonResult<serde_json::Value> {
    let value = http_get_from_hub(
@@ -185,7 +188,7 @@ async fn check_path_conflict<'c>(
 }

 async fn create_flow(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
@@ -246,7 +249,9 @@ async fn create_flow(
        WebhookMessage::CreateFlow { workspace: w_id.clone(), path: nf.path.clone() },
    );

+    let tx = PushIsolationLevel::Transaction(tx);
    let (dependency_job_uuid, mut tx) = push(
+        &db,
        tx,
        &w_id,
        JobPayload::FlowDependencies { path: nf.path.clone() },
@@ -305,7 +310,7 @@ async fn check_schedule_conflict<'c>(
    Ok(())
 }

-pub async fn require_is_writer(authed: &Authed, path: &str, w_id: &str, db: DB) -> Result<()> {
+pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: DB) -> Result<()> {
    return crate::users::require_is_writer(
        authed,
        path,
@@ -318,7 +323,7 @@ pub async fn require_is_writer(authed: &Authed, path: &str, w_id: &str, db: DB)
 }

 async fn update_flow(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Extension(db): Extension<DB>,
@@ -393,7 +398,7 @@ async fn update_flow(
        clear_schedule(tx.transaction_mut(), &schedule.path, true, &w_id).await?;

        if schedule.enabled {
-            tx = push_scheduled_job(tx, schedule).await?;
+            tx = push_scheduled_job(&db, tx, schedule).await?;
        }
    }

@@ -430,7 +435,10 @@ async fn update_flow(
        },
    );

+    let tx = PushIsolationLevel::Transaction(tx);
+
    let (dependency_job_uuid, mut tx) = push(
+        &db,
        tx,
        &w_id,
        JobPayload::FlowDependencies { path: nf.path.clone() },
@@ -473,7 +481,7 @@ async fn update_flow(
 }

 async fn get_flow_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<Flow> {
@@ -508,7 +516,7 @@ pub struct FlowWDraft {
 }

 async fn get_flow_by_path_w_draft(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<FlowWDraft> {
@@ -518,7 +526,7 @@ async fn get_flow_by_path_w_draft(
    let flow_o = sqlx::query_as::<_, FlowWDraft>(
        "SELECT flow.path, flow.summary, flow,description, flow.schema, flow.value, flow.extra_perms, flow.draft_only, draft.value as draft, flow.tag FROM flow
        LEFT JOIN draft ON 
-        flow.path = draft.path AND flow.workspace_id = draft.workspace_id AND draft.typ = 'flow' 
+        flow.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'flow' 
        WHERE flow.path = $1 AND flow.workspace_id = $2",
    )
    .bind(path)
@@ -555,7 +563,7 @@ struct Archived {
 }

 async fn archive_flow_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -593,7 +601,7 @@ async fn archive_flow_by_path(
 }

 async fn delete_flow_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -8,9 +8,11 @@

 use std::sync::Arc;

+use crate::db::ApiAuthed;
+
 use crate::{
-    db::{UserDB, DB},
-    users::{AuthCache, Authed, Tokened},
+    db::DB,
+    users::{AuthCache, Tokened},
    webhook_util::{WebhookMessage, WebhookShared},
 };
 use axum::{
@@ -22,6 +24,7 @@ use lazy_static::lazy_static;
 use regex::Regex;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
+    db::UserDB,
    error::{self, to_anyhow, JsonResult, Result},
    users::username_to_permissioned_as,
    utils::{not_found_if_none, paginate, Pagination},
@@ -74,7 +77,7 @@ pub struct Owner {
 }

 async fn list_folders(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Query(pagination): Query<Pagination>,
@@ -96,7 +99,7 @@ async fn list_folders(
    Ok(Json(rows))
 }
 async fn list_foldernames(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Query(pagination): Query<Pagination>,
@@ -145,7 +148,7 @@ lazy_static! {
 }

 async fn create_folder(
-    authed: Authed,
+    authed: ApiAuthed,
    Tokened { token }: Tokened,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
@@ -163,7 +166,7 @@ async fn create_folder(
    check_name_conflict(&mut tx, &w_id, &ng.name).await?;
    cache.invalidate(&w_id, token).await;
    let owner = username_to_permissioned_as(&authed.username);
-    let owners = &ng.owners.unwrap_or(vec![owner.clone()]);
+    let owners = &ng.owners.unwrap_or_else(|| vec![owner.clone()]);

    if let Some(extra_perms) = ng.extra_perms.clone() {
        for o in owners {
@@ -220,13 +223,13 @@ async fn create_folder(
 }

 pub async fn is_owner_api(
-    authed: Authed,
+    authed: ApiAuthed,
    Path((_w_id, name)): Path<(String, String)>,
 ) -> JsonResult<bool> {
    Ok(Json(is_owner(&authed, &name)))
 }

-pub fn is_owner(Authed { is_admin, folders, .. }: &Authed, name: &str) -> bool {
+pub fn is_owner(ApiAuthed { is_admin, folders, .. }: &ApiAuthed, name: &str) -> bool {
    if *is_admin {
        true
    } else {
@@ -234,7 +237,7 @@ pub fn is_owner(Authed { is_admin, folders, .. }: &Authed, name: &str) -> bool {
    }
 }

-pub fn require_is_owner(authed: &Authed, name: &str) -> Result<()> {
+pub fn require_is_owner(authed: &ApiAuthed, name: &str) -> Result<()> {
    if is_owner(authed, name) {
        Ok(())
    } else {
@@ -246,7 +249,7 @@ pub fn require_is_owner(authed: &Authed, name: &str) -> Result<()> {
 }

 async fn update_folder(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, name)): Path<(String, String)>,
@@ -343,7 +346,7 @@ pub async fn get_folderopt<'c>(
 }

 async fn get_folder(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, name)): Path<(String, String)>,
 ) -> JsonResult<Folder> {
@@ -365,7 +368,7 @@ struct FolderUsage {
    pub variables: i64,
 }
 async fn get_folder_usage(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, name)): Path<(String, String)>,
 ) -> JsonResult<FolderUsage> {
@@ -446,7 +449,7 @@ async fn get_folder_usage(
 }

 async fn delete_folder(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, name)): Path<(String, String)>,
@@ -483,7 +486,7 @@ async fn delete_folder(
 }

 async fn add_owner(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, name)): Path<(String, String)>,
@@ -550,7 +553,7 @@ pub async fn get_folders_for_user(
 }

 async fn remove_owner(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, name)): Path<(String, String)>,
@@ -6,18 +6,18 @@
 * LICENSE-AGPL for a copy of the license.
 */

-use crate::{
-    db::{UserDB, DB},
-    users::{require_owner_of_path, Authed},
-};
+use crate::{db::DB, users::require_owner_of_path};
 use axum::{
    extract::{Extension, Path},
    routing::{get, post},
    Json, Router,
 };

+use crate::db::ApiAuthed;
+
 use serde::{Deserialize, Serialize};
 use windmill_common::{
+    db::UserDB,
    error::{Error, JsonResult, Result},
    utils::{not_found_if_none, StripPath},
 };
@@ -36,7 +36,7 @@ pub struct GranularAcl {
 }

 async fn add_granular_acl(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -83,7 +83,7 @@ async fn add_granular_acl(
 }

 async fn remove_granular_acl(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -135,7 +135,7 @@ async fn remove_granular_acl(
 }

 async fn get_granular_acls(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<serde_json::Value> {
@@ -6,18 +6,16 @@
 * LICENSE-AGPL for a copy of the license.
 */

-use crate::{
-    db::{UserDB, DB},
-    users::{get_groups_for_user, Authed},
-    utils::require_super_admin,
-};
+use crate::db::ApiAuthed;
+use crate::{db::DB, users::get_groups_for_user, utils::require_super_admin};
+
 use axum::{
    extract::{Extension, Path, Query},
    routing::{delete, get, post},
    Json, Router,
 };
 use windmill_audit::{audit_log, ActionKind};
-use windmill_common::users::username_to_permissioned_as;
+use windmill_common::{db::UserDB, users::username_to_permissioned_as};
 use windmill_common::{
    error::{Error, JsonResult, Result},
    utils::{not_found_if_none, paginate, Pagination},
@@ -113,7 +111,7 @@ struct QueryListGroup {
    pub only_member_of: Option<bool>,
 }
 async fn list_group_names(
-    Authed { username, email, .. }: Authed,
+    ApiAuthed { username, email, .. }: ApiAuthed,
    Extension(db): Extension<DB>,
    Query(QueryListGroup { only_member_of }): Query<QueryListGroup>,
    Path(w_id): Path<String>,
@@ -158,7 +156,7 @@ async fn check_name_conflict<'c>(
 }

 pub async fn is_owner(
-    Authed { username, is_admin, groups, .. }: Authed,
+    ApiAuthed { username, is_admin, groups, .. }: ApiAuthed,
    Extension(db): Extension<DB>,
    Path((w_id, name)): Path<(String, String)>,
 ) -> JsonResult<bool> {
@@ -203,7 +201,7 @@ pub async fn require_is_owner(
 }

 async fn create_group(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Json(ng): Json<NewGroup>,
@@ -247,12 +245,13 @@ async fn create_group(
 }

 async fn create_igroup(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Json(ng): Json<NewGroup>,
 ) -> Result<String> {
+    require_super_admin(&db, &authed.email).await?;
    let mut tx = db.begin().await?;
-    require_super_admin(&mut tx, &authed.email).await?;
+
    sqlx::query!(
        "INSERT INTO instance_group (name, summary) VALUES ($1, $2) ON CONFLICT DO NOTHING",
        ng.name,
@@ -277,12 +276,12 @@ async fn create_igroup(
 }

 async fn delete_igroup(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Json(ng): Json<NewGroup>,
 ) -> Result<String> {
-    let mut tx = db.begin().await?;
-    require_super_admin(&mut tx, &authed.email).await?;
+    require_super_admin(&db, &authed.email).await?;
+    let mut tx: Transaction<'_, Postgres> = db.begin().await?;
    sqlx::query!("DELETE FROM instance_group WHERE name = $1", ng.name,)
        .execute(&mut *tx)
        .await?;
@@ -319,7 +318,7 @@ pub async fn get_group_opt<'c>(
 }

 async fn get_group(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, name)): Path<(String, String)>,
 ) -> JsonResult<GroupInfo> {
@@ -339,7 +338,7 @@ async fn get_group(

    let members = sqlx::query_scalar!(
        "SELECT  usr.username  
-            FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username 
+            FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username AND usr_to_group.workspace_id = $2
            WHERE group_ = $1 AND usr.workspace_id = $2 AND usr_to_group.workspace_id = $2",
        name,
        w_id
@@ -358,7 +357,7 @@ async fn get_group(
 }

 async fn delete_group(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, name)): Path<(String, String)>,
@@ -406,7 +405,7 @@ async fn delete_group(
 }

 async fn update_group(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, name)): Path<(String, String)>,
@@ -442,7 +441,7 @@ async fn update_group(
 }

 async fn add_user(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, name)): Path<(String, String)>,
@@ -479,14 +478,14 @@ async fn add_user(
 }

 async fn add_user_igroup(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(name): Path<String>,
    Json(Email { email }): Json<Email>,
 ) -> Result<String> {
-    let mut tx = db.begin().await?;
+    require_super_admin(&db, &authed.email).await?;

-    require_super_admin(&mut tx, &authed.email).await?;
+    let mut tx: Transaction<'_, Postgres> = db.begin().await?;

    let group_opt = sqlx::query_scalar!("SELECT name FROM instance_group WHERE name = $1", name)
        .fetch_optional(&mut *tx)
@@ -521,10 +520,10 @@ struct IGroup {
    name: String,
    emails: Option<Vec<String>>,
 }
-async fn list_igroups(authed: Authed, Extension(db): Extension<DB>) -> JsonResult<Vec<IGroup>> {
-    let mut tx = db.begin().await?;
+async fn list_igroups(authed: ApiAuthed, Extension(db): Extension<DB>) -> JsonResult<Vec<IGroup>> {
+    require_super_admin(&db, &authed.email).await?;
+    let mut tx: Transaction<'_, Postgres> = db.begin().await?;

-    require_super_admin(&mut tx, &authed.email).await?;
    let groups = sqlx::query_as!(
        IGroup,
        "SELECT name, array_remove(array_agg(email_to_igroup.email), null) as emails FROM email_to_igroup RIGHT JOIN instance_group ON instance_group.name = email_to_igroup.igroup GROUP BY name"
@@ -549,15 +548,14 @@ async fn get_igroup(Path(name): Path<String>, Extension(db): Extension<DB>) -> J
 }

 async fn remove_user_igroup(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(name): Path<String>,
    Json(Email { email }): Json<Email>,
 ) -> Result<String> {
+    require_super_admin(&db, &authed.email).await?;
    let mut tx = db.begin().await?;

-    require_super_admin(&mut tx, &authed.email).await?;
-
    let group_opt = sqlx::query_scalar!("SELECT name FROM instance_group WHERE name = $1", name,)
        .fetch_optional(&mut *tx)
        .await?;
@@ -587,7 +585,7 @@ async fn remove_user_igroup(
 }

 async fn remove_user(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, name)): Path<(String, String)>,
@@ -6,7 +6,7 @@
 * LICENSE-AGPL for a copy of the license.
 */

-use crate::{db::UserDB, jobs::CompletedJob, users::Authed};
+use crate::{db::ApiAuthed, jobs::CompletedJob};
 use axum::{
    extract::{Path, Query},
    routing::{get, post},
@@ -21,6 +21,7 @@ use std::{
    vec,
 };
 use windmill_common::{
+    db::UserDB,
    error::JsonResult,
    jobs::JobKind,
    scripts::to_i64,
@@ -102,7 +103,7 @@ pub struct Input {
 }

 async fn get_input_history(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Query(pagination): Query<Pagination>,
@@ -115,7 +116,7 @@ async fn get_input_history(
    let sql = &format!(
        "select * from completed_job \
        where {} = $1 and job_kind = $2 and workspace_id = $3 \
-        order by args, started_at desc limit $4 offset $5",
+        order by created_at desc limit $4 offset $5",
        r.runnable_type.column_name()
    );

@@ -158,7 +159,7 @@ async fn get_input_history(
 }

 async fn list_saved_inputs(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Query(pagination): Query<Pagination>,
@@ -209,7 +210,7 @@ pub struct CreateInput {
 }

 async fn create_input(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Query(r): Query<RunnableParams>,
@@ -245,7 +246,7 @@ pub struct UpdateInput {
 }

 async fn update_input(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Json(input): Json<UpdateInput>,
@@ -266,7 +267,7 @@ async fn update_input(
 }

 async fn delete_input(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, i_id)): Path<(String, Uuid)>,
 ) -> JsonResult<String> {
@@ -6,9 +6,11 @@
 * LICENSE-AGPL for a copy of the license.
 */

+use crate::db::ApiAuthed;
+
 use crate::{
-    db::{UserDB, DB},
-    users::{check_scopes, require_owner_of_path, Authed, OptAuthed},
+    db::DB,
+    users::{check_scopes, require_owner_of_path, OptAuthed},
    utils::require_super_admin,
    variables::get_workspace_key,
    workers::{CUSTOM_TAGS, CUSTOM_TAGS_PER_WORKSPACE},
@@ -28,9 +30,11 @@ use hyper::{header::CONTENT_TYPE, http, HeaderMap, Request, StatusCode};
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use sql_builder::{prelude::*, quote, SqlBuilder};
 use sqlx::{query_scalar, types::Uuid, FromRow, Postgres, Transaction};
+use tower_http::cors::{Any, CorsLayer};
 use urlencoding::encode;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
+    db::UserDB,
    error::{self, to_anyhow, Error},
    flow_status::{Approval, FlowStatus, FlowStatusModule},
    flows::FlowValue,
@@ -40,64 +44,108 @@ use windmill_common::{
    users::username_to_permissioned_as,
    utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
 };
-use windmill_queue::{get_queued_job, push, QueueTransaction};
+use windmill_queue::{get_queued_job, push, PushIsolationLevel};

 pub fn workspaced_service() -> Router {
+    let cors = CorsLayer::new()
+        .allow_methods([http::Method::GET, http::Method::POST])
+        .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION])
+        .allow_origin(Any);
+
    Router::new()
        .route(
            "/run/f/*script_path",
-            post(run_flow_by_path).head(|| async { "" }),
+            post(run_flow_by_path)
+                .head(|| async { "" })
+                .layer(cors.clone()),
        )
        .route(
            "/run/p/*script_path",
-            post(run_job_by_path).head(|| async { "" }),
+            post(run_job_by_path)
+                .head(|| async { "" })
+                .layer(cors.clone()),
        )
        .route(
            "/run_wait_result/p/*script_path",
            post(run_wait_result_script_by_path)
                .get(run_wait_result_job_by_path_get)
-                .head(|| async { "" }),
+                .head(|| async { "" })
+                .layer(cors.clone()),
        )
        .route(
            "/run_wait_result/h/:hash",
-            post(run_wait_result_script_by_hash).head(|| async { "" }),
+            post(run_wait_result_script_by_hash)
+                .head(|| async { "" })
+                .layer(cors.clone()),
        )
        .route(
            "/run_wait_result/f/*script_path",
            post(run_wait_result_flow_by_path)
                .get(run_wait_result_flow_by_path_get)
-                .head(|| async { "" }),
+                .head(|| async { "" })
+                .layer(cors.clone()),
        )
        .route(
            "/openai_sync/p/*script_path",
-            post(openai_sync_script_by_path).head(|| async { "" }),
+            post(openai_sync_script_by_path)
+                .head(|| async { "" })
+                .layer(cors.clone()),
        )
        .route(
            "/openai_sync/f/*script_path",
-            post(openai_sync_flow_by_path).head(|| async { "" }),
+            post(openai_sync_flow_by_path)
+                .head(|| async { "" })
+                .layer(cors.clone()),
+        )
+        .route(
+            "/run/h/:hash",
+            post(run_job_by_hash)
+                .head(|| async { "" })
+                .layer(cors.clone()),
        )
-        .route("/run/h/:hash", post(run_job_by_hash).head(|| async { "" }))
        .route("/run/preview", post(run_preview_job))
        .route("/add_noop_jobs/:n", post(add_noop_jobs))
        .route("/run/preview_flow", post(run_preview_flow_job))
        .route("/list", get(list_jobs))
        .route("/queue/list", get(list_queue_jobs))
        .route("/queue/count", get(count_queue_jobs))
-        .route("/completed/list", get(list_completed_jobs))
-        .route("/completed/get/:id", get(get_completed_job))
-        .route("/completed/get_result/:id", get(get_completed_job_result))
+        .route("/queue/cancel_all", post(cancel_all))
+        .route(
+            "/completed/list",
+            get(list_completed_jobs).layer(cors.clone()),
+        )
+        .route(
+            "/completed/get/:id",
+            get(get_completed_job).layer(cors.clone()),
+        )
+        .route(
+            "/completed/get_result/:id",
+            get(get_completed_job_result).layer(cors.clone()),
+        )
        .route(
            "/completed/get_result_maybe/:id",
-            get(get_completed_job_result_maybe),
+            get(get_completed_job_result_maybe).layer(cors.clone()),
+        )
+        .route(
+            "/completed/delete/:id",
+            post(delete_completed_job).layer(cors.clone()),
+        )
+        .route(
+            "/flow/resume/:id",
+            post(resume_suspended_flow_as_owner).layer(cors.clone()),
        )
-        .route("/completed/delete/:id", post(delete_completed_job))
-        .route("/flow/resume/:id", post(resume_suspended_flow_as_owner))
        .route(
            "/job_signature/:job_id/:resume_id",
-            get(create_job_signature),
+            get(create_job_signature).layer(cors.clone()),
+        )
+        .route(
+            "/resume_urls/:job_id/:resume_id",
+            get(get_resume_urls).layer(cors.clone()),
+        )
+        .route(
+            "/result_by_id/:job_id/:node_id",
+            get(get_result_by_id).layer(cors.clone()),
        )
-        .route("/resume_urls/:job_id/:resume_id", get(get_resume_urls))
-        .route("/result_by_id/:job_id/:node_id", get(get_result_by_id))
 }

 pub fn global_service() -> Router {
@@ -135,6 +183,10 @@ pub fn global_service() -> Router {
        .route("/queue/force_cancel/:id", post(force_cancel))
 }

+pub fn global_root_service() -> Router {
+    Router::new().route("/db_clock", get(get_db_clock))
+}
+
 async fn get_result_by_id(
    Extension(db): Extension<DB>,
    Path((w_id, flow_id, node_id)): Path<(String, Uuid, String)>,
@@ -143,6 +195,10 @@ async fn get_result_by_id(
    Ok(Json(res))
 }

+async fn get_db_clock(Extension(db): Extension<DB>) -> windmill_common::error::JsonResult<i64> {
+    Ok(Json(now_from_db(&db).await?.timestamp_millis()))
+}
+
 async fn cancel_job_api(
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    OptAuthed(opt_authed): OptAuthed,
@@ -249,8 +305,8 @@ pub async fn get_path_for_hash<'c>(
    Ok(path)
 }

-pub async fn get_path_tag_and_limits_for_hash<'c>(
-    db: &mut Transaction<'c, Postgres>,
+pub async fn get_path_tag_and_limits_for_hash(
+    db: &DB,
    w_id: &str,
    hash: i64,
 ) -> error::Result<(String, Option<String>, Option<i32>, Option<i32>)> {
@@ -259,7 +315,7 @@ pub async fn get_path_tag_and_limits_for_hash<'c>(
        hash,
        w_id
    )
-    .fetch_one(&mut **db)
+    .fetch_one(db)
    .await
    .map_err(|e| {
        Error::InternalErr(format!(
@@ -402,12 +458,12 @@ lazy_static::lazy_static! {
 impl RunJobQuery {
    async fn get_scheduled_for<'c>(
        &self,
-        db: &mut Transaction<'c, Postgres>,
+        db: &DB,
    ) -> error::Result<Option<chrono::DateTime<chrono::Utc>>> {
        if let Some(scheduled_for) = self.scheduled_for {
            Ok(Some(scheduled_for))
        } else if let Some(scheduled_in_secs) = self.scheduled_in_secs {
-            let now = now_from_db(&mut **db).await?;
+            let now = now_from_db(db).await?;
            Ok(Some(now + chrono::Duration::seconds(scheduled_in_secs)))
        } else {
            Ok(None)
@@ -455,6 +511,10 @@ pub struct ListQueueQuery {
    pub created_by: Option<String>,
    pub started_before: Option<chrono::DateTime<chrono::Utc>>,
    pub started_after: Option<chrono::DateTime<chrono::Utc>>,
+    pub created_before: Option<chrono::DateTime<chrono::Utc>>,
+    pub created_after: Option<chrono::DateTime<chrono::Utc>>,
+    pub created_or_started_before: Option<chrono::DateTime<chrono::Utc>>,
+    pub created_or_started_after: Option<chrono::DateTime<chrono::Utc>>,
    pub running: Option<bool>,
    pub schedule_path: Option<String>,
    pub parent_job: Option<String>,
@@ -505,6 +565,23 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
        sqlb.and_where_ge("started_at", format!("to_timestamp({})", dt.timestamp()));
    }

+    if let Some(dt) = &lq.created_before {
+        sqlb.and_where_le("created_at", format!("to_timestamp({})", dt.timestamp()));
+    }
+    if let Some(dt) = &lq.created_after {
+        sqlb.and_where_ge("created_at", format!("to_timestamp({})", dt.timestamp()));
+    }
+
+    if let Some(dt) = &lq.created_or_started_after {
+        let ts = dt.timestamp();
+        sqlb.and_where(format!("(started_at IS NOT NULL AND started_at >= to_timestamp({})) OR (started_at IS NULL AND created_at >= to_timestamp({}))", ts, ts));
+    }
+
+    if let Some(dt) = &lq.created_or_started_before {
+        let ts = dt.timestamp();
+        sqlb.and_where(format!("(started_at IS NOT NULL AND started_at < to_timestamp({})) OR (started_at IS NULL AND created_at < to_timestamp({}))", ts, ts));
+    }
+
    if let Some(s) = &lq.suspended {
        if *s {
            sqlb.and_where_gt("suspend", 0);
@@ -579,6 +656,23 @@ async fn list_queue_jobs(
    Ok(Json(jobs))
 }

+async fn cancel_all(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+) -> error::JsonResult<Vec<Uuid>> {
+    require_admin(authed.is_admin, &authed.username)?;
+
+    let uuids = sqlx::query_scalar!(
+        "UPDATE queue SET canceled = true,  canceled_by = $2, scheduled_for = now(), suspend = 0 WHERE workspace_id = $1 AND schedule_path IS NULL RETURNING id",
+        w_id,
+        authed.username
+    )
+    .fetch_all(&db)
+    .await?;
+    Ok(Json(uuids))
+}
+
 #[derive(Serialize, Debug, FromRow)]
 struct QueueStats {
    database_length: i64,
@@ -600,7 +694,7 @@ async fn count_queue_jobs(
 }

 async fn list_jobs(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Query(pagination): Query<Pagination>,
@@ -619,6 +713,10 @@ async fn list_jobs(
            created_by: lq.created_by,
            started_before: lq.started_before,
            started_after: lq.started_after,
+            created_before: lq.created_before,
+            created_after: lq.created_after,
+            created_or_started_before: lq.created_or_started_before,
+            created_or_started_after: lq.created_or_started_after,
            running: None,
            parent_job: lq.parent_job,
            order_desc: Some(true),
@@ -713,7 +811,7 @@ async fn list_jobs(
 }

 pub async fn resume_suspended_flow_as_owner(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path((_w_id, flow_id)): Path<(String, Uuid)>,
    QueryOrBody(value): QueryOrBody<serde_json::Value>,
@@ -1036,7 +1134,7 @@ pub async fn get_suspended_job_flow(
 }

 pub async fn create_job_signature(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, job_id, resume_id)): Path<(String, Uuid, u32)>,
    Query(approver): Query<QueryApprover>,
@@ -1081,7 +1179,7 @@ fn build_resume_url(
 }

 pub async fn get_resume_urls(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, job_id, resume_id)): Path<(String, Uuid, u32)>,
    Query(approver): Query<QueryApprover>,
@@ -1434,7 +1532,8 @@ fn check_tag_available_for_workspace(w_id: &str, tag: &Option<String>) -> error:
 }

 pub async fn run_flow_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Path((w_id, flow_path)): Path<(String, StripPath)>,
@@ -1445,20 +1544,21 @@ pub async fn run_flow_by_path(
    let flow_path = flow_path.to_path();
    check_scopes(&authed, || format!("run:flow/{flow_path}"))?;

-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
    let tag = sqlx::query_scalar!(
        "SELECT tag from flow WHERE path = $1 and workspace_id = $2",
        flow_path,
        w_id
    )
-    .fetch_optional(&mut tx)
+    .fetch_optional(&db)
    .await?
    .flatten();
    check_tag_available_for_workspace(&w_id, &tag)?;
-    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
+    let scheduled_for = run_query.get_scheduled_for(&db).await?;
    let args = run_query.add_include_headers(headers, args.unwrap_or_default());
    let args = add_raw_string(raw_string, args);
+    let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);
    let (uuid, tx) = push(
+        &db,
        tx,
        &w_id,
        JobPayload::Flow(flow_path.to_string()),
@@ -1485,7 +1585,8 @@ pub async fn run_flow_by_path(
 }

 pub async fn run_job_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Path((w_id, script_path)): Path<(String, StripPath)>,
@@ -1496,15 +1597,16 @@ pub async fn run_job_by_path(
    let script_path = script_path.to_path();
    check_scopes(&authed, || format!("run:script/{script_path}"))?;

-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
-    let (job_payload, tag) =
-        script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
-    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
+    let (job_payload, tag) = script_path_to_payload(script_path, &db, &w_id).await?;
+    let scheduled_for = run_query.get_scheduled_for(&db).await?;
    let args = run_query.add_include_headers(headers, args.unwrap_or_default());
    let args = add_raw_string(raw_string, args);
+
    check_tag_available_for_workspace(&w_id, &tag)?;
+    let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);

    let (uuid, tx) = push(
+        &db,
        tx,
        &w_id,
        job_payload,
@@ -1535,7 +1637,7 @@ struct Guard {
    id: Uuid,
    w_id: String,
    db: UserDB,
-    authed: Authed,
+    authed: ApiAuthed,
 }

 impl Drop for Guard {
@@ -1567,7 +1669,7 @@ impl Drop for Guard {
 }

 async fn run_wait_result<T>(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    timeout: i32,
    uuid: Uuid,
@@ -1632,7 +1734,7 @@ async fn run_wait_result<T>(
    }
 }

-pub async fn check_queue_too_long(db: DB, queue_limit: Option<i64>) -> error::Result<()> {
+pub async fn check_queue_too_long(db: &DB, queue_limit: Option<i64>) -> error::Result<()> {
    if let Some(limit) = queue_limit {
        let count = sqlx::query_scalar!(
            "SELECT COUNT(*) FROM queue WHERE  canceled = false AND (scheduled_for <= now()
@@ -1640,7 +1742,7 @@ pub async fn check_queue_too_long(db: DB, queue_limit: Option<i64>) -> error::Re
            AND (   suspend <= 0
                 OR suspend_until <= now())))",
        )
-        .fetch_one(&db)
+        .fetch_one(db)
        .await?
        .unwrap_or(0);

@@ -1677,7 +1779,7 @@ lazy_static::lazy_static! {

 pub async fn run_wait_result_job_by_path_get(
    method: hyper::http::Method,
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Extension(user_db): Extension<UserDB>,
    Extension(db): Extension<DB>,
@@ -1698,16 +1800,16 @@ pub async fn run_wait_result_job_by_path_get(
        serde_json::Map::new()
    };

-    check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
+    check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
    let script_path = script_path.to_path();
    check_scopes(&authed, || format!("run:script/{script_path}"))?;

-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
-    let (job_payload, tag) =
-        script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
+    let (job_payload, tag) = script_path_to_payload(script_path, &db, &w_id).await?;
    check_tag_available_for_workspace(&w_id, &tag)?;
+    let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);

    let (uuid, tx) = push(
+        &db,
        tx,
        &w_id,
        job_payload,
@@ -1743,7 +1845,7 @@ pub async fn run_wait_result_job_by_path_get(

 pub async fn run_wait_result_flow_by_path_get(
    method: hyper::http::Method,
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Extension(user_db): Extension<UserDB>,
    Extension(db): Extension<DB>,
@@ -1781,7 +1883,7 @@ pub async fn run_wait_result_flow_by_path_get(
 }

 pub async fn run_wait_result_script_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Extension(db): Extension<DB>,
@@ -1830,7 +1932,7 @@ fn convert_from_openai_json(
 }

 pub async fn openai_sync_script_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Extension(db): Extension<DB>,
@@ -1858,7 +1960,7 @@ async fn run_wait_result_script_by_path_internal(
    db: sqlx::Pool<Postgres>,
    run_query: RunJobQuery,
    script_path: StripPath,
-    authed: Authed,
+    authed: ApiAuthed,
    rsmq: Option<rsmq_async::MultiplexedRsmq>,
    user_db: UserDB,
    w_id: String,
@@ -1866,19 +1968,19 @@ async fn run_wait_result_script_by_path_internal(
    args: Option<serde_json::Map<String, serde_json::Value>>,
    raw_string: Option<String>,
 ) -> Result<Json<serde_json::Value>, Error> {
-    check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
+    check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
    let script_path = script_path.to_path();
    check_scopes(&authed, || format!("run:script/{script_path}"))?;

-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
-    let (job_payload, tag) =
-        script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
+    let (job_payload, tag) = script_path_to_payload(script_path, &db, &w_id).await?;

    let args = run_query.add_include_headers(headers, args.unwrap_or_default());
    let args = add_raw_string(raw_string, args);
    check_tag_available_for_workspace(&w_id, &tag)?;
+    let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);

    let (uuid, tx) = push(
+        &db,
        tx,
        &w_id,
        job_payload,
@@ -1913,7 +2015,7 @@ async fn run_wait_result_script_by_path_internal(
 }

 pub async fn run_wait_result_script_by_hash(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Extension(db): Extension<DB>,
@@ -1922,19 +2024,20 @@ pub async fn run_wait_result_script_by_hash(
    headers: HeaderMap,
    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::JsonResult<serde_json::Value> {
-    check_queue_too_long(db, run_query.queue_limit).await?;
+    check_queue_too_long(&db, run_query.queue_limit).await?;

    let hash = script_hash.0;
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
    let (path, tag, concurrent_limit, concurrency_time_window_s) =
-        get_path_tag_and_limits_for_hash(tx.transaction_mut(), &w_id, hash).await?;
+        get_path_tag_and_limits_for_hash(&db, &w_id, hash).await?;
    check_scopes(&authed, || format!("run:script/{path}"))?;

    let args = run_query.add_include_headers(headers, args.unwrap_or_default());
    let args = add_raw_string(raw_string, args);
    check_tag_available_for_workspace(&w_id, &tag)?;
+    let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);

    let (uuid, tx) = push(
+        &db,
        tx,
        &w_id,
        JobPayload::ScriptHash {
@@ -1974,7 +2077,7 @@ pub async fn run_wait_result_script_by_hash(
 }

 pub async fn openai_sync_flow_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Extension(db): Extension<DB>,
@@ -1999,7 +2102,7 @@ pub async fn openai_sync_flow_by_path(
 }

 pub async fn run_wait_result_flow_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Extension(db): Extension<DB>,
@@ -2018,7 +2121,7 @@ async fn run_wait_result_flow_by_path_internal(
    db: sqlx::Pool<Postgres>,
    run_query: RunJobQuery,
    flow_path: StripPath,
-    authed: Authed,
+    authed: ApiAuthed,
    rsmq: Option<rsmq_async::MultiplexedRsmq>,
    user_db: UserDB,
    headers: HeaderMap,
@@ -2026,13 +2129,12 @@ async fn run_wait_result_flow_by_path_internal(
    raw_string: Option<String>,
    w_id: String,
 ) -> Result<Json<serde_json::Value>, Error> {
-    check_queue_too_long(db, run_query.queue_limit).await?;
+    check_queue_too_long(&db, run_query.queue_limit).await?;

    let flow_path = flow_path.to_path();
    check_scopes(&authed, || format!("run:flow/{flow_path}"))?;

-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
-    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
+    let scheduled_for = run_query.get_scheduled_for(&db).await?;
    let args = run_query.add_include_headers(headers, args.unwrap_or_default());
    let args = add_raw_string(raw_string, args);
    let tag = sqlx::query_scalar!(
@@ -2040,12 +2142,14 @@ async fn run_wait_result_flow_by_path_internal(
        flow_path,
        w_id
    )
-    .fetch_optional(&mut tx)
+    .fetch_optional(&db)
    .await?
    .flatten();
    check_tag_available_for_workspace(&w_id, &tag)?;
+    let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);

    let (uuid, tx) = push(
+        &db,
        tx,
        &w_id,
        JobPayload::Flow(flow_path.to_string()),
@@ -2080,7 +2184,8 @@ async fn run_wait_result_flow_by_path_internal(
 }

 async fn run_preview_job(
-    authed: Authed,
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Path(w_id): Path<String>,
@@ -2094,12 +2199,13 @@ async fn run_preview_job(
            "Operators cannot run preview jobs for security reasons".to_string(),
        ));
    }
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
-    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
+    let scheduled_for = run_query.get_scheduled_for(&db).await?;
    let args = run_query.add_include_headers(headers, preview.args.unwrap_or_default());
    check_tag_available_for_workspace(&w_id, &preview.tag)?;
+    let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);

    let (uuid, tx) = push(
+        &db,
        tx,
        &w_id,
        match preview.kind {
@@ -2138,17 +2244,18 @@ async fn run_preview_job(
 }

 async fn add_noop_jobs(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Path((w_id, n)): Path<(String, i32)>,
 ) -> error::JsonResult<Vec<String>> {
-    require_super_admin(&mut db.begin().await?, &authed.email).await?;
-    let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into();
+    require_super_admin(&db, &authed.email).await?;
+    let mut tx = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);

    let mut uuids: Vec<String> = Vec::new();
    for _ in 0..n {
        let (uuid, ntx) = push(
+            &db,
            tx,
            &w_id,
            JobPayload::Noop,
@@ -2170,15 +2277,21 @@ async fn add_noop_jobs(
            None,
        )
        .await?;
-        tx = ntx;
+        tx = PushIsolationLevel::Transaction(ntx);
        uuids.push(uuid.to_string());
    }
-    tx.commit().await?;
+    match tx {
+        PushIsolationLevel::Transaction(tx) => {
+            tx.commit().await?;
+        }
+        _ => (),
+    }

    Ok(Json(uuids))
 }
 async fn run_preview_flow_job(
-    authed: Authed,
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Path(w_id): Path<String>,
@@ -2192,12 +2305,13 @@ async fn run_preview_flow_job(
            "Operators cannot run preview jobs for security reasons".to_string(),
        ));
    }
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
-    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
+    let scheduled_for = run_query.get_scheduled_for(&db).await?;
    let args = run_query.add_include_headers(headers, raw_flow.args.unwrap_or_default());
    check_tag_available_for_workspace(&w_id, &raw_flow.tag)?;
+    let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);

    let (uuid, tx) = push(
+        &db,
        tx,
        &w_id,
        JobPayload::RawFlow { value: raw_flow.value, path: raw_flow.path },
@@ -2225,7 +2339,9 @@ async fn run_preview_flow_job(
 }

 pub async fn run_job_by_hash(
-    authed: Authed,
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Path((w_id, script_hash)): Path<(String, ScriptHash)>,
@@ -2234,17 +2350,18 @@ pub async fn run_job_by_hash(
    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::Result<(StatusCode, String)> {
    let hash = script_hash.0;
-    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
    let (path, tag, concurrent_limit, concurrency_time_window_s) =
-        get_path_tag_and_limits_for_hash(tx.transaction_mut(), &w_id, hash).await?;
+        get_path_tag_and_limits_for_hash(&db, &w_id, hash).await?;
    check_scopes(&authed, || format!("run:script/{path}"))?;

-    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
+    let scheduled_for = run_query.get_scheduled_for(&db).await?;
    let args = run_query.add_include_headers(headers, args.unwrap_or_default());
    let args = add_raw_string(raw_string, args);
    check_tag_available_for_workspace(&w_id, &tag)?;
+    let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into(), rsmq);

    let (uuid, tx) = push(
+        &db,
        tx,
        &w_id,
        JobPayload::ScriptHash {
@@ -2381,6 +2498,14 @@ fn list_completed_jobs_query(
    if let Some(dt) = &lq.started_after {
        sqlb.and_where_ge("started_at", format!("to_timestamp({})", dt.timestamp()));
    }
+
+    if let Some(dt) = &lq.created_or_started_before {
+        sqlb.and_where_le("started_at", format!("to_timestamp({})", dt.timestamp()));
+    }
+    if let Some(dt) = &lq.created_or_started_after {
+        sqlb.and_where_ge("started_at", format!("to_timestamp({})", dt.timestamp()));
+    }
+
    if let Some(sk) = &lq.is_skipped {
        sqlb.and_where_eq("is_skipped", sk);
    }
@@ -2412,6 +2537,10 @@ pub struct ListCompletedQuery {
    pub created_by: Option<String>,
    pub started_before: Option<chrono::DateTime<chrono::Utc>>,
    pub started_after: Option<chrono::DateTime<chrono::Utc>>,
+    pub created_before: Option<chrono::DateTime<chrono::Utc>>,
+    pub created_after: Option<chrono::DateTime<chrono::Utc>>,
+    pub created_or_started_before: Option<chrono::DateTime<chrono::Utc>>,
+    pub created_or_started_after: Option<chrono::DateTime<chrono::Utc>>,
    pub success: Option<bool>,
    pub parent_job: Option<String>,
    pub order_desc: Option<bool>,
@@ -2428,7 +2557,7 @@ pub struct ListCompletedQuery {
 }

 async fn list_completed_jobs(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
    Query(pagination): Query<Pagination>,
@@ -2541,7 +2670,7 @@ async fn get_completed_job_result_maybe(
 }

 async fn delete_completed_job(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, id)): Path<(String, Uuid)>,
 ) -> error::JsonResult<CompletedJob> {
@@ -6,16 +6,16 @@
 * LICENSE-AGPL for a copy of the license.
 */

+use crate::db::ApiAuthed;
 use crate::oauth2::AllClients;
 use crate::saml::{SamlSsoLogin, ServiceProviderExt};
 use crate::scim::has_scim_token;
 use crate::tracing_init::MyOnFailure;
 use crate::workers::ALL_TAGS;
 use crate::{
-    db::UserDB,
    oauth2::{build_oauth_clients, SlackVerifier},
    tracing_init::{MyMakeSpan, MyOnResponse},
-    users::{Authed, OptAuthed},
+    users::OptAuthed,
    webhook_util::WebhookShared,
 };
 use anyhow::Context;
@@ -24,7 +24,7 @@ use axum::extract::DefaultBodyLimit;
 use axum::{middleware::from_extractor, routing::get, Extension, Router};
 use db::DB;
 use git_version::git_version;
-use hyper::Method;
+use hyper::{http, Method};
 use mail_send::SmtpClientBuilder;
 use reqwest::Client;
 use std::{net::SocketAddr, sync::Arc};
@@ -34,6 +34,7 @@ use tower_http::{
    cors::{Any, CorsLayer},
    trace::TraceLayer,
 };
+use windmill_common::db::UserDB;
 use windmill_common::utils::rd_string;

 use windmill_common::error::AppError;
@@ -185,6 +186,7 @@ pub async fn run_server(

    let cors = CorsLayer::new()
        .allow_methods([Method::GET, Method::POST])
+        .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION])
        .allow_origin(Any);

    #[cfg(feature = "enterprise")]
@@ -212,7 +214,7 @@ pub async fn run_server(
                        .nest("/folders", folders::workspaced_service())
                        .nest("/groups", groups::workspaced_service())
                        .nest("/inputs", inputs::workspaced_service())
-                        .nest("/jobs", jobs::workspaced_service().layer(cors.clone()))
+                        .nest("/jobs", jobs::workspaced_service())
                        .nest("/oauth", oauth2::workspaced_service())
                        .nest("/resources", resources::workspaced_service())
                        .nest("/schedules", schedule::workspaced_service())
@@ -231,13 +233,14 @@ pub async fn run_server(
                    "/users",
                    users::global_service().layer(Extension(argon2.clone())),
                )
+                .nest("/jobs", jobs::global_root_service())
                .nest("/workers", workers::global_service())
                .nest("/scripts", scripts::global_service())
                .nest("/groups", groups::global_service())
                .nest("/flows", flows::global_service())
                .nest("/apps", apps::global_service().layer(cors.clone()))
                .nest("/schedules", schedule::global_service())
-                .route_layer(from_extractor::<Authed>())
+                .route_layer(from_extractor::<ApiAuthed>())
                .route_layer(from_extractor::<users::Tokened>())
                .nest(
                    "/saml",
@@ -33,15 +33,17 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use sqlx::{Postgres, Transaction};
 use tower_cookies::{Cookie, Cookies};
 use windmill_audit::{audit_log, ActionKind};
+use windmill_common::db::UserDB;
 use windmill_common::jobs::JobPayload;
 use windmill_common::users::username_to_permissioned_as;
 use windmill_common::utils::{not_found_if_none, now_from_db};

+use crate::db::ApiAuthed;
 use crate::saml::SamlSsoLogin;
-use crate::users::{login_externally, Authed, LoginUserInfo};
+use crate::users::{login_externally, LoginUserInfo};
 use crate::webhook_util::{InstanceEvent, WebhookShared};
 use crate::{
-    db::{UserDB, DB},
+    db::DB,
    variables::{build_crypt, encrypt},
    workspaces::WorkspaceSettings,
 };
@@ -49,7 +51,7 @@ use crate::{BASE_URL, HTTP_CLIENT, IS_SECURE, OAUTH_CLIENTS, SLACK_SIGNING_SECRE
 use windmill_common::error::{self, to_anyhow, Error};
 use windmill_common::oauth2::*;

-use windmill_queue::QueueTransaction;
+use windmill_queue::PushIsolationLevel;

 use std::{fs, str};

@@ -340,7 +342,7 @@ struct CreateAccount {
    expires_in: i64,
 }
 async fn create_account(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Json(payload): Json<CreateAccount>,
@@ -364,7 +366,7 @@ async fn create_account(
 }

 async fn delete_account(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Query((w_id, id)): Query<(String, i32)>,
 ) -> error::Result<String> {
@@ -455,7 +457,7 @@ async fn connect_slack(cookies: Cookies) -> error::Result<Redirect> {
 }

 async fn disconnect(
-    authed: Authed,
+    authed: ApiAuthed,
    Path((w_id, id)): Path<(String, i32)>,
    Extension(user_db): Extension<UserDB>,
 ) -> error::Result<String> {
@@ -474,7 +476,7 @@ async fn disconnect(
 }

 async fn disconnect_slack(
-    authed: Authed,
+    authed: ApiAuthed,
    Path(w_id): Path<String>,
    Extension(user_db): Extension<UserDB>,
 ) -> error::Result<String> {
@@ -502,14 +504,14 @@ struct VariablePath {
    path: String,
 }
 async fn refresh_token(
-    authed: Authed,
+    authed: ApiAuthed,
    Path((w_id, id)): Path<(String, i32)>,
    Extension(user_db): Extension<UserDB>,
    Json(VariablePath { path }): Json<VariablePath>,
 ) -> error::Result<String> {
    let tx = user_db.begin(&authed).await?;

-    _refresh_token(tx, &path, w_id, id).await?;
+    _refresh_token(tx, &path, &w_id, id).await?;

    Ok(format!("Token at path {path} refreshed"))
 }
@@ -517,7 +519,7 @@ async fn refresh_token(
 pub async fn _refresh_token<'c>(
    mut tx: Transaction<'c, Postgres>,
    path: &str,
-    w_id: String,
+    w_id: &str,
    id: i32,
 ) -> error::Result<String> {
    let account = sqlx::query!(
@@ -635,7 +637,7 @@ async fn connect_callback(

 async fn connect_slack_callback(
    Path(w_id): Path<String>,
-    authed: Authed,
+    authed: ApiAuthed,
    cookies: Cookies,
    Extension(user_db): Extension<UserDB>,
    Json(callback): Json<OAuthCallback>,
@@ -783,13 +785,12 @@ async fn slack_command(
        }
    }

-    let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into();
    let settings = sqlx::query_as!(
        WorkspaceSettings,
        "SELECT * FROM workspace_settings WHERE slack_team_id = $1",
        form.team_id,
    )
-    .fetch_optional(&mut tx)
+    .fetch_optional(&db)
    .await?;

    if let Some(settings) = settings {
@@ -800,7 +801,7 @@ async fn slack_command(
                let path = path.strip_prefix("script/").unwrap_or_else(|| path);
                let (script_hash, tag, concurrent_limit, concurrency_time_window_s) =
                    windmill_common::get_latest_deployed_hash_for_path(
-                        tx.transaction_mut(),
+                        &db,
                        &settings.workspace_id,
                        path,
                    )
@@ -821,8 +822,10 @@ async fn slack_command(
                "response_url".to_string(),
                serde_json::Value::String(form.response_url),
            );
+            let tx = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);

            let (uuid, tx) = windmill_queue::push(
+                &db,
                tx,
                &settings.workspace_id,
                payload,
@@ -852,7 +855,6 @@ async fn slack_command(
            ));
        }
    }
-    tx.commit().await?;

    return Ok(format!(
        "workspace not properly configured (did you set the script to trigger in the settings?)"
@@ -1,4 +1,8 @@
-use crate::{db::DB, users::Authed, variables::build_crypt, HTTP_CLIENT};
+use crate::{
+    db::{ApiAuthed, DB},
+    variables::build_crypt,
+    HTTP_CLIENT,
+};

 use axum::{
    body::{Bytes, StreamBody},
@@ -9,6 +13,7 @@ use axum::{
    Router,
 };
 use magic_crypt::MagicCryptTrait;
+use serde_json::json;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::error::{to_anyhow, Error};

@@ -26,6 +31,17 @@ struct OpenaiResource {
    organization_id: Option<String>,
 }

+fn create_openai_json_error(msg: String) -> Error {
+    Error::OpenAIError(
+        serde_json::to_string(&json!({
+            "error": {
+                "message": msg
+            }
+        }))
+        .unwrap(),
+    )
+}
+
 struct Variable {
    value: String,
    is_secret: bool,
@@ -57,7 +73,7 @@ lazy_static::lazy_static! {
 }

 async fn proxy(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path((w_id, openai_path)): Path<(String, String)>,
    body: Bytes,
@@ -72,7 +88,7 @@ async fn proxy(
    tx.commit().await?;

    if openai_resource_path.is_none() {
-        return Err(Error::InternalErr(
+        return Err(create_openai_json_error(
            "OpenAI resource not configured".to_string(),
        ));
    }
@@ -92,7 +108,7 @@ async fn proxy(
    tx.commit().await?;

    if resource.is_none() {
-        return Err(Error::InternalErr(
+        return Err(create_openai_json_error(
            "OpenAI resource missing value".to_string(),
        ));
    }
@@ -133,7 +149,7 @@ async fn proxy(
        request = request.header("OpenAI-Organization", org_id);
    }

-    let resp = request.send().await.map_err(to_anyhow)?;
+    let response = request.send().await.map_err(to_anyhow)?;

    tx = db.begin().await?;
    audit_log(
@@ -148,13 +164,19 @@ async fn proxy(
    .await?;
    tx.commit().await?;

+    if response.error_for_status_ref().is_err() {
+        return Err(Error::OpenAIError(
+            response.text().await.unwrap_or("".to_string()),
+        ));
+    }
+
    let mut headers = HeaderMap::new();
-    for (k, v) in resp.headers().iter() {
+    for (k, v) in response.headers().iter() {
        headers.insert(k, v.clone());
    }

-    let status_code = resp.status();
-    let stream = resp.bytes_stream();
+    let status_code = response.status();
+    let stream = response.bytes_stream();

    Ok((status_code, headers, StreamBody::new(stream)))
 }
@@ -6,8 +6,8 @@
 * LICENSE-AGPL for a copy of the license.
 */
 use crate::{
-    db::{UserDB, DB},
-    users::{require_owner_of_path, Authed},
+    db::{ApiAuthed, DB},
+    users::require_owner_of_path,
    webhook_util::{WebhookMessage, WebhookShared},
 };
 use axum::{
@@ -26,6 +26,7 @@ use std::str;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
    apps::ListAppQuery,
+    db::UserDB,
    error::{Error, JsonResult, Result},
    utils::{not_found_if_none, paginate, Pagination, StripPath},
 };
@@ -66,7 +67,7 @@ pub struct EditApp {
 }

 async fn list_apps(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Query(pagination): Query<Pagination>,
@@ -113,7 +114,7 @@ async fn list_apps(
 }

 async fn get_data(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, _version, path)): Path<(String, u16, StripPath)>,
 ) -> Result<Response> {
@@ -139,7 +140,7 @@ async fn get_data(
 }

 async fn create_app(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path(w_id): Path<String>,
@@ -199,7 +200,7 @@ async fn create_app(
 }

 async fn delete_app(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -234,7 +235,7 @@ async fn delete_app(
 }

 async fn update_app(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -7,8 +7,8 @@
 */

 use crate::{
-    db::{UserDB, DB},
-    users::{maybe_refresh_folders, require_owner_of_path, Authed},
+    db::{ApiAuthed, DB},
+    users::{maybe_refresh_folders, require_owner_of_path},
    webhook_util::{WebhookMessage, WebhookShared},
 };
 use axum::{
@@ -18,10 +18,12 @@ use axum::{
 };
 use hyper::StatusCode;
 use serde::{Deserialize, Serialize};
+use serde_json::Value;
 use sql_builder::{bind::Bind, SqlBuilder};
 use sqlx::{FromRow, Postgres, Transaction};
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
+    db::UserDB,
    error::{Error, JsonResult, Result},
    utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
 };
@@ -32,6 +34,10 @@ pub fn workspaced_service() -> Router {
        .route("/get/*path", get(get_resource))
        .route("/exists/*path", get(exists_resource))
        .route("/get_value/*path", get(get_resource_value))
+        .route(
+            "/get_value_interpolated/*path",
+            get(get_resource_value_interpolated),
+        )
        .route("/update/*path", post(update_resource))
        .route("/update_value/*path", post(update_resource_value))
        .route("/delete/*path", delete(delete_resource))
@@ -112,7 +118,7 @@ pub struct ListResourceQuery {
    resource_type_exclude: Option<String>,
 }
 async fn list_resources(
-    authed: Authed,
+    authed: ApiAuthed,
    Query(lq): Query<ListResourceQuery>,
    Query(pagination): Query<Pagination>,
    Extension(user_db): Extension<UserDB>,
@@ -170,7 +176,7 @@ async fn list_resources(
 }

 async fn get_resource(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<ListableResource> {
@@ -185,8 +191,8 @@ async fn get_resource(
        variable.is_oauth as \"is_oauth?\",
        variable.account
        FROM resource
-        LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = resource.workspace_id
-        LEFT JOIN account ON variable.account = account.id AND account.workspace_id = resource.workspace_id
+        LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = $2
+        LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $2
        WHERE resource.path = $1 AND resource.workspace_id = $2",
        path.to_owned(),
        &w_id
@@ -218,7 +224,7 @@ async fn exists_resource(
 }

 async fn get_resource_value(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<Option<serde_json::Value>> {
@@ -238,6 +244,84 @@ async fn get_resource_value(
    Ok(Json(value))
 }

+async fn get_resource_value_interpolated(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<Option<serde_json::Value>> {
+    let path = path.to_path();
+    let mut tx = user_db.clone().begin(&authed).await?;
+
+    let value_o = sqlx::query_scalar!(
+        "SELECT value from resource WHERE path = $1 AND workspace_id = $2",
+        path.to_owned(),
+        &w_id
+    )
+    .fetch_optional(&mut *tx)
+    .await?;
+    tx.commit().await?;
+
+    let value = not_found_if_none(value_o, "Resource", path)?;
+    if let Some(value) = value {
+        Ok(Json(Some(
+            transform_json_value(&authed, &user_db, &w_id, value).await?,
+        )))
+    } else {
+        Ok(Json(None))
+    }
+}
+
+use async_recursion::async_recursion;
+
+#[async_recursion]
+pub async fn transform_json_value<'c>(
+    authed: &ApiAuthed,
+    user_db: &UserDB,
+    workspace: &str,
+    v: Value,
+) -> Result<Value> {
+    match v {
+        Value::String(y) if y.starts_with("$var:") => {
+            let path = y.strip_prefix("$var:").unwrap();
+            let tx: Transaction<'_, Postgres> = user_db.clone().begin(authed).await?;
+            let v =
+                crate::variables::get_value_internal(tx, workspace, path, &authed.username).await?;
+            Ok(Value::String(v))
+        }
+        Value::String(y) if y.starts_with("$res:") => {
+            let path = y.strip_prefix("$res:").unwrap();
+            if path.split("/").count() < 2 {
+                return Err(Error::InternalErr(format!("Invalid resource path: {path}")));
+            }
+            let mut tx: Transaction<'_, Postgres> = user_db.clone().begin(authed).await?;
+            let v = sqlx::query_scalar!(
+                "SELECT value from resource WHERE path = $1 AND workspace_id = $2",
+                path,
+                &workspace
+            )
+            .fetch_optional(&mut *tx)
+            .await?;
+            tx.commit().await?;
+            let v = not_found_if_none(v, "Resource", path)?;
+            if let Some(v) = v {
+                transform_json_value(authed, user_db, workspace, v).await
+            } else {
+                Ok(Value::Null)
+            }
+        }
+        Value::Object(mut m) => {
+            for (a, b) in m.clone().into_iter() {
+                m.insert(
+                    a.clone(),
+                    transform_json_value(authed, user_db, workspace, b).await?,
+                );
+            }
+            Ok(Value::Object(m))
+        }
+        a @ _ => Ok(a),
+    }
+}
+
 async fn check_path_conflict<'c>(
    tx: &mut Transaction<'c, Postgres>,
    w_id: &str,
@@ -262,10 +346,10 @@ async fn check_path_conflict<'c>(

 #[derive(Deserialize)]
 struct CreateResourceQuery {
-    update_if_exists: Option<bool>
+    update_if_exists: Option<bool>,
 }
 async fn create_resource(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Extension(db): Extension<DB>,
@@ -281,7 +365,7 @@ async fn create_resource(
    if !update_if_exists {
        check_path_conflict(&mut tx, &w_id, &resource.path).await?;
    }
-    
+
    sqlx::query!(
        "INSERT INTO resource
            (workspace_id, path, value, description, resource_type)
@@ -319,7 +403,7 @@ async fn create_resource(
 }

 async fn delete_resource(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -362,7 +446,7 @@ async fn delete_resource(
 }

 async fn update_resource(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Extension(db): Extension<DB>,
@@ -444,7 +528,7 @@ struct UpdateResource {
 }

 async fn update_resource_value(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -516,7 +600,7 @@ async fn list_resource_types_names(
 }

 async fn get_resource_type(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, name)): Path<(String, String)>,
 ) -> JsonResult<ResourceType> {
@@ -553,7 +637,7 @@ async fn exists_resource_type(
 }

 async fn create_resource_type(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path(w_id): Path<String>,
@@ -620,7 +704,7 @@ async fn check_rt_path_conflict<'c>(
 }

 async fn delete_resource_type(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, name)): Path<(String, String)>,
@@ -656,7 +740,7 @@ async fn delete_resource_type(
 }

 async fn update_resource_type(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, name)): Path<(String, String)>,
@@ -7,8 +7,8 @@
 */

 use crate::{
-    db::{UserDB, DB},
-    users::{maybe_refresh_folders, Authed},
+    db::{ApiAuthed, DB},
+    users::maybe_refresh_folders,
 };
 use axum::{
    extract::{Extension, Path, Query},
@@ -22,6 +22,7 @@ use sqlx::{Postgres, Transaction};
 use std::str::FromStr;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
+    db::UserDB,
    error::{Error, JsonResult, Result},
    jobs::JobKind,
    schedule::Schedule,
@@ -55,6 +56,12 @@ pub struct NewSchedule {
    pub args: Option<serde_json::Value>,
    pub enabled: Option<bool>,
    pub on_failure: Option<String>,
+    pub on_failure_times: Option<i32>,
+    pub on_failure_exact: Option<bool>,
+    pub on_failure_extra_args: Option<serde_json::Value>,
+    pub on_recovery: Option<String>,
+    pub on_recovery_times: Option<i32>,
+    pub on_recovery_extra_args: Option<serde_json::Value>,
 }

 async fn check_path_conflict<'c>(
@@ -80,7 +87,7 @@ async fn check_path_conflict<'c>(
 }

 async fn create_schedule(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
@@ -104,7 +111,7 @@ async fn create_schedule(
    let schedule = sqlx::query_as!(
        Schedule,
        "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, \
-         is_flow, args, enabled, email, on_failure) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING *",
+         is_flow, args, enabled, email, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) RETURNING *",
        w_id,
        ns.path,
        ns.schedule,
@@ -115,7 +122,13 @@ async fn create_schedule(
        ns.args,
        ns.enabled.unwrap_or(false),
        &authed.email,
-        ns.on_failure
+        ns.on_failure,
+        ns.on_failure_times,
+        ns.on_failure_exact,
+        ns.on_failure_extra_args,
+        ns.on_recovery,
+        ns.on_recovery_times,
+        ns.on_recovery_extra_args,
    )
    .fetch_one(&mut tx)
    .await
@@ -141,7 +154,7 @@ async fn create_schedule(
    .await?;

    if ns.enabled.unwrap_or(true) {
-        tx = push_scheduled_job(tx, schedule).await?
+        tx = push_scheduled_job(&db, tx, schedule).await?
    }
    tx.commit().await?;

@@ -149,7 +162,7 @@ async fn create_schedule(
 }

 async fn edit_schedule(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
@@ -177,12 +190,18 @@ async fn edit_schedule(
    clear_schedule(tx.transaction_mut(), path, is_flow, &w_id).await?;
    let schedule = sqlx::query_as!(
        Schedule,
-        "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4 WHERE path \
-         = $5 AND workspace_id = $6 RETURNING *",
+        "UPDATE schedule SET schedule = $1, timezone = $2, args = $3, on_failure = $4, on_failure_times = $5, on_failure_exact = $6, on_failure_extra_args = $7, on_recovery = $8, on_recovery_times = $9, on_recovery_extra_args = $10 WHERE path \
+         = $11 AND workspace_id = $12 RETURNING *",
        es.schedule,
        es.timezone,
        es.args,
        es.on_failure,
+        es.on_failure_times,
+        es.on_failure_exact,
+        es.on_failure_extra_args,
+        es.on_recovery,
+        es.on_recovery_times,
+        es.on_recovery_extra_args,
        path,
        w_id,
    )
@@ -207,7 +226,7 @@ async fn edit_schedule(
    .await?;

    if schedule.enabled {
-        tx = push_scheduled_job(tx, schedule).await?;
+        tx = push_scheduled_job(&db, tx, schedule).await?;
    }
    tx.commit().await?;

@@ -223,7 +242,7 @@ pub struct ListScheduleQuery {
 }

 async fn list_schedule(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Query(lsq): Query<ListScheduleQuery>,
@@ -267,11 +286,17 @@ pub struct ScheduleWJobs {
    pub email: String,
    pub error: Option<String>,
    pub on_failure: Option<String>,
+    pub on_failure_times: Option<i32>,
+    pub on_failure_exact: Option<bool>,
+    pub on_failure_extra_args: Option<serde_json::Value>,
+    pub on_recovery: Option<String>,
+    pub on_recovery_times: Option<i32>,
+    pub on_recovery_extra_args: Option<serde_json::Value>,
    pub jobs: Option<Vec<serde_json::Value>>,
 }

 async fn list_schedule_with_jobs(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Query(pagination): Query<Pagination>,
@@ -280,7 +305,7 @@ async fn list_schedule_with_jobs(
    let (per_page, offset) = paginate(pagination);
    let rows = sqlx::query_as!(ScheduleWJobs,
        "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE
-        completed_job.schedule_path = schedule.path AND schedule.workspace_id = completed_job.workspace_id AND parent_job IS NULL ORDER BY created_at DESC LIMIT 20) AS jobs ) t
+        completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY started_at DESC LIMIT 20) AS jobs ) t
        WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3",
        w_id,
        per_page as i64,
@@ -303,7 +328,7 @@ async fn list_schedule_with_jobs(
 //    ) t;

 async fn get_schedule(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<Schedule> {
@@ -352,7 +377,8 @@ pub async fn preview_schedule(
 }

 pub async fn set_enabled(
-    authed: Authed,
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -388,7 +414,7 @@ pub async fn set_enabled(
    .await?;

    if payload.enabled {
-        tx = push_scheduled_job(tx, schedule).await?;
+        tx = push_scheduled_job(&db, tx, schedule).await?;
    }
    tx.commit().await?;

@@ -399,7 +425,7 @@ pub async fn set_enabled(
 }

 async fn delete_schedule(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
@@ -462,6 +488,12 @@ pub struct EditSchedule {
    pub timezone: String,
    pub args: Option<serde_json::Value>,
    pub on_failure: Option<String>,
+    pub on_failure_times: Option<i32>,
+    pub on_failure_exact: Option<bool>,
+    pub on_failure_extra_args: Option<serde_json::Value>,
+    pub on_recovery: Option<String>,
+    pub on_recovery_times: Option<i32>,
+    pub on_recovery_extra_args: Option<serde_json::Value>,
 }

 pub async fn clear_schedule<'c>(
@@ -7,9 +7,9 @@
 */

 use crate::{
-    db::{UserDB, DB},
+    db::{ApiAuthed, DB},
    schedule::clear_schedule,
-    users::{maybe_refresh_folders, require_owner_of_path, AuthCache, Authed},
+    users::{maybe_refresh_folders, require_owner_of_path, AuthCache},
    webhook_util::{WebhookMessage, WebhookShared},
    HTTP_CLIENT,
 };
@@ -31,6 +31,7 @@ use std::{
 };
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
+    db::UserDB,
    error::{Error, JsonResult, Result},
    jobs::JobPayload,
    schedule::Schedule,
@@ -43,7 +44,7 @@ use windmill_common::{
        list_elems_from_hub, not_found_if_none, paginate, require_admin, Pagination, StripPath,
    },
 };
-use windmill_queue::{self, schedule::push_scheduled_job, QueueTransaction};
+use windmill_queue::{self, schedule::push_scheduled_job, PushIsolationLevel, QueueTransaction};

 const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;

@@ -103,7 +104,7 @@ pub fn workspaced_service() -> Router {
 }

 async fn list_scripts(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Query(pagination): Query<Pagination>,
@@ -192,7 +193,7 @@ async fn list_scripts(
    Ok(Json(rows))
 }

-async fn list_hub_scripts(Authed { email, .. }: Authed) -> JsonResult<serde_json::Value> {
+async fn list_hub_scripts(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<serde_json::Value> {
    let asks = list_elems_from_hub(
        &HTTP_CLIENT,
        "https://hub.windmill.dev/searchData?approved=true",
@@ -209,7 +210,7 @@ fn hash_script(ns: &NewScript) -> i64 {
 }

 async fn create_script(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
    Extension(webhook): Extension<WebhookShared>,
@@ -429,7 +430,7 @@ async fn create_script(
            clear_schedule(tx.transaction_mut(), &schedule.path, false, &w_id).await?;

            if schedule.enabled {
-                tx = push_scheduled_job(tx, schedule).await?;
+                tx = push_scheduled_job(&db, tx, schedule).await?;
            }
        }
    } else {
@@ -488,6 +489,8 @@ async fn create_script(
        );
    }

+    let mut tx = PushIsolationLevel::Transaction(tx);
+
    if needs_lock_gen {
        let dependencies = match ns.language {
            ScriptLang::Python3 => {
@@ -498,6 +501,7 @@ async fn create_script(
            _ => ns.content,
        };
        let (_, new_tx) = windmill_queue::push(
+            &db,
            tx,
            &w_id,
            JobPayload::Dependencies { hash, dependencies, language: ns.language, path: ns.path },
@@ -519,20 +523,30 @@ async fn create_script(
            None,
        )
        .await?;
-        tx = new_tx;
+        tx = PushIsolationLevel::Transaction(new_tx);
    }

-    tx.commit().await?;
+    match tx {
+        PushIsolationLevel::Transaction(tx) => tx.commit().await?,
+        _ => {
+            return Err(Error::InternalErr(
+                "Expected a transaction here".to_string(),
+            ));
+        }
+    }

    Ok((StatusCode::CREATED, format!("{}", hash)))
 }

-pub async fn get_hub_script_by_path(authed: Authed, Path(path): Path<StripPath>) -> Result<String> {
+pub async fn get_hub_script_by_path(
+    authed: ApiAuthed,
+    Path(path): Path<StripPath>,
+) -> Result<String> {
    windmill_common::scripts::get_hub_script_by_path(&authed.email, path, &HTTP_CLIENT).await
 }

 pub async fn get_full_hub_script_by_path(
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
    Path(path): Path<StripPath>,
 ) -> JsonResult<HubScript> {
    Ok(Json(
@@ -541,7 +555,7 @@ pub async fn get_full_hub_script_by_path(
 }

 async fn get_script_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<Script> {
@@ -564,7 +578,7 @@ async fn get_script_by_path(
 }

 async fn get_script_by_path_w_draft(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<ScriptWDraft> {
@@ -589,7 +603,7 @@ async fn get_script_by_path_w_draft(
 }

 async fn list_paths(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
 ) -> JsonResult<Vec<String>> {
@@ -619,7 +633,7 @@ async fn get_tokened_raw_script_by_path(
 }

 async fn raw_script_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
@@ -742,7 +756,7 @@ async fn get_deployment_status(
    Ok(Json(status))
 }

-pub async fn require_is_writer(authed: &Authed, path: &str, w_id: &str, db: DB) -> Result<()> {
+pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: DB) -> Result<()> {
    return crate::users::require_is_writer(
        authed,
        path,
@@ -757,7 +771,7 @@ pub async fn require_is_writer(authed: &Authed, path: &str, w_id: &str, db: DB)
 }

 async fn archive_script_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(webhook): Extension<WebhookShared>,
    Extension(user_db): Extension<UserDB>,
    Extension(db): Extension<DB>,
@@ -796,7 +810,7 @@ async fn archive_script_by_path(
 }

 async fn archive_script_by_hash(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, hash)): Path<(String, ScriptHash)>,
@@ -832,7 +846,7 @@ async fn archive_script_by_hash(
 }

 async fn delete_script_by_hash(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Extension(db): Extension<DB>,
@@ -872,7 +886,7 @@ async fn delete_script_by_hash(
 }

 async fn delete_script_by_path(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Extension(db): Extension<DB>,
@@ -10,8 +10,10 @@

 use std::{sync::Arc, time::Duration};

+use crate::db::ApiAuthed;
+
 use crate::{
-    db::{UserDB, DB},
+    db::DB,
    folders::get_folders_for_user,
    utils::require_super_admin,
    webhook_util::{InstanceEvent, WebhookShared},
@@ -40,6 +42,7 @@ use tower_cookies::{Cookie, Cookies};
 use tracing::{Instrument, Span};
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
+    db::UserDB,
    error::{self, to_anyhow, Error, JsonResult, Result},
    users::SUPERADMIN_SECRET_EMAIL,
    utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath},
@@ -100,7 +103,7 @@ pub fn make_unauthed_service() -> Router {
 }

 pub struct AuthCache {
-    cache: Cache<(String, String), Authed>,
+    cache: Cache<(String, String), ApiAuthed>,
    db: DB,
    superadmin_secret: Option<String>,
 }
@@ -114,7 +117,7 @@ impl AuthCache {
        self.cache.remove(&(w_id.to_string(), token)).await;
    }

-    pub async fn get_authed(&self, w_id: Option<String>, token: &str) -> Option<Authed> {
+    pub async fn get_authed(&self, w_id: Option<String>, token: &str) -> Option<ApiAuthed> {
        let key = (
            w_id.as_ref().unwrap_or(&"".to_string()).to_string(),
            token.to_string(),
@@ -171,7 +174,7 @@ impl AuthCache {
                                                .ok()
                                                .unwrap_or_default();

-                                        Some(Authed {
+                                        Some(ApiAuthed {
                                            email: email,
                                            username: name.to_string(),
                                            is_admin,
@@ -191,7 +194,7 @@ impl AuthCache {
                                        .await
                                        .ok()
                                        .unwrap_or_default();
-                                        Some(Authed {
+                                        Some(ApiAuthed {
                                            email: email,
                                            username: format!("group-{name}"),
                                            is_admin: false,
@@ -204,7 +207,7 @@ impl AuthCache {
                                } else {
                                    let groups = vec![];
                                    let folders = vec![];
-                                    Some(Authed {
+                                    Some(ApiAuthed {
                                        email: email,
                                        username: owner,
                                        is_admin: super_admin,
@@ -248,7 +251,7 @@ impl AuthCache {
                                            .await
                                            .ok()
                                            .unwrap_or_default();
-                                            Some(Authed {
+                                            Some(ApiAuthed {
                                                email,
                                                username,
                                                is_admin: is_admin || super_admin,
@@ -258,7 +261,7 @@ impl AuthCache {
                                                scopes,
                                            })
                                        }
-                                        None if super_admin => Some(Authed {
+                                        None if super_admin => Some(ApiAuthed {
                                            email: email.clone(),
                                            username: email,
                                            is_admin: super_admin,
@@ -270,7 +273,7 @@ impl AuthCache {
                                        None => None,
                                    }
                                } else {
-                                    Some(Authed {
+                                    Some(ApiAuthed {
                                        email: email.to_string(),
                                        username: email,
                                        is_admin: super_admin,
@@ -296,7 +299,7 @@ impl AuthCache {
                    .map(|x| x == token)
                    .unwrap_or(false)
                {
-                    Some(Authed {
+                    Some(ApiAuthed {
                        email: SUPERADMIN_SECRET_EMAIL.to_string(),
                        username: "superadmin_secret".to_string(),
                        is_admin: true,
@@ -361,6 +364,9 @@ where
        parts: &mut Parts,
        state: &S,
    ) -> std::result::Result<Self, Self::Rejection> {
+        if parts.method == http::Method::OPTIONS {
+            return Ok(Tokened { token: "".to_string() });
+        };
        let already_tokened = parts.extensions.get::<Tokened>();
        if let Some(tokened) = already_tokened {
            Ok(tokened.clone())
@@ -377,19 +383,12 @@ where
    }
 }

-#[derive(Clone, Debug)]
-pub struct Authed {
-    pub email: String,
-    pub username: String,
-    pub is_admin: bool,
-    pub is_operator: bool,
-    pub groups: Vec<String>,
-    // (folder name, can write, is owner)
-    pub folders: Vec<(String, bool, bool)>,
-    pub scopes: Option<Vec<String>>,
-}
-
-pub async fn maybe_refresh_folders(path: &str, w_id: &str, authed: Authed, db: &DB) -> Authed {
+pub async fn maybe_refresh_folders(
+    path: &str,
+    w_id: &str,
+    authed: ApiAuthed,
+    db: &DB,
+) -> ApiAuthed {
    if authed.is_admin {
        return authed;
    }
@@ -408,14 +407,14 @@ pub async fn maybe_refresh_folders(path: &str, w_id: &str, authed: Authed, db: &
            .await
            .ok()
            .unwrap_or_default();
-        Authed { folders, ..authed }
+        ApiAuthed { folders, ..authed }
    } else {
        authed
    }
 }

 #[async_trait]
-impl<S> FromRequestParts<S> for Authed
+impl<S> FromRequestParts<S> for ApiAuthed
 where
    S: Send + Sync,
 {
@@ -425,7 +424,18 @@ where
        parts: &mut Parts,
        state: &S,
    ) -> std::result::Result<Self, Self::Rejection> {
-        let already_authed = parts.extensions.get::<Authed>();
+        if parts.method == http::Method::OPTIONS {
+            return Ok(ApiAuthed {
+                email: "".to_owned(),
+                username: "".to_owned(),
+                is_admin: false,
+                is_operator: false,
+                groups: Vec::new(),
+                folders: Vec::new(),
+                scopes: None,
+            });
+        };
+        let already_authed = parts.extensions.get::<ApiAuthed>();
        if let Some(authed) = already_authed {
            Ok(authed.clone())
        } else {
@@ -475,7 +485,7 @@ where
    }
 }

-pub fn check_scopes<F>(authed: &Authed, required: F) -> error::Result<()>
+pub fn check_scopes<F>(authed: &ApiAuthed, required: F) -> error::Result<()>
 where
    F: FnOnce() -> String,
 {
@@ -489,7 +499,7 @@ where
 }

 #[derive(Clone, Debug)]
-pub struct OptAuthed(pub Option<Authed>);
+pub struct OptAuthed(pub Option<ApiAuthed>);

 #[async_trait]
 impl<S> FromRequestParts<S> for OptAuthed
@@ -502,7 +512,7 @@ where
        parts: &mut Parts,
        state: &S,
    ) -> std::result::Result<Self, Self::Rejection> {
-        Authed::from_request_parts(parts, state)
+        ApiAuthed::from_request_parts(parts, state)
            .await
            .map(|authed| Self(Some(authed)))
            .or_else(|_| Ok(Self(None)))
@@ -673,7 +683,7 @@ struct WorkspaceUsername {
 }

 async fn exists_username(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
    Json(WorkspaceUsername { username }): Json<WorkspaceUsername>,
@@ -692,7 +702,7 @@ async fn exists_username(
 }

 async fn list_users(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
 ) -> JsonResult<Vec<UserWithUsage>> {
@@ -704,7 +714,7 @@ async fn list_users(
             , LATERAL (
                SELECT COALESCE(SUM(duration_ms + 1000)/1000 , 0) executions
                  FROM completed_job
-                 WHERE workspace_id = usr.workspace_id
+                 WHERE workspace_id = $1
                   AND job_kind NOT IN ('flow', 'flowpreview')
                   AND email = usr.email
                   AND now() - '5 week'::interval < created_at 
@@ -724,12 +734,11 @@ async fn list_users(
 }

 async fn list_users_as_super_admin(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Query(pagination): Query<Pagination>,
 ) -> JsonResult<Vec<GlobalUserInfo>> {
-    let mut tx = db.begin().await?;
-    require_super_admin(&mut tx, &authed.email).await?;
+    require_super_admin(&db, &authed.email).await?;
    let per_page = pagination.per_page.unwrap_or(10000).max(1);
    let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page;

@@ -740,14 +749,13 @@ async fn list_users_as_super_admin(
        per_page as i32,
        offset as i32
    )
-    .fetch_all(&mut *tx)
+    .fetch_all(&db)
    .await?;
-    tx.commit().await?;
    Ok(Json(rows))
 }

 // async fn list_invite_codes(
-//     authed: Authed,
+//     authed: ApiAuthed,
 //     Extension(db): Extension<DB>,
 //     Query(pagination): Query<Pagination>
 // ) -> JsonResult<Vec<InviteCode>> {
@@ -763,7 +771,7 @@ async fn list_users_as_super_admin(
 // }

 async fn list_usernames(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
 ) -> JsonResult<Vec<String>> {
@@ -782,7 +790,7 @@ async fn list_usernames(
 }

 async fn list_invites(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
 ) -> JsonResult<Vec<WorkspaceInvite>> {
    let mut tx = db.begin().await?;
@@ -840,7 +848,7 @@ async fn logout(
 async fn whoami(
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
-    Authed { username, email, is_admin, groups, folders, .. }: Authed,
+    ApiAuthed { username, email, is_admin, groups, folders, .. }: ApiAuthed,
 ) -> JsonResult<UserInfo> {
    let user = get_user(&w_id, &username, &db).await?;
    if let Some(user) = user {
@@ -872,7 +880,7 @@ async fn whoami(

 async fn global_whoami(
    Extension(db): Extension<DB>,
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
    Tokened { token }: Tokened,
 ) -> JsonResult<GlobalUserInfo> {
    let user = sqlx::query_as!(
@@ -912,11 +920,14 @@ async fn exists_email(Extension(db): Extension<DB>, Path(email): Path<String>) -
    Ok(Json(exists))
 }

-async fn get_email(Authed { email, .. }: Authed) -> Result<String> {
+async fn get_email(ApiAuthed { email, .. }: ApiAuthed) -> Result<String> {
    Ok(email)
 }

-async fn get_usage(Extension(db): Extension<DB>, Authed { email, .. }: Authed) -> Result<String> {
+async fn get_usage(
+    Extension(db): Extension<DB>,
+    ApiAuthed { email, .. }: ApiAuthed,
+) -> Result<String> {
    let usage = sqlx::query_scalar!(
        "
    SELECT usage.usage FROM usage 
@@ -1001,7 +1012,7 @@ pub async fn get_groups_for_user(
    Ok(groups)
 }
 pub async fn is_owner_of_path(
-    authed: Authed,
+    authed: ApiAuthed,
    Path((_w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<bool> {
    let path = path.to_path();
@@ -1012,7 +1023,7 @@ pub async fn is_owner_of_path(
    }
 }

-pub fn require_owner_of_path(authed: &Authed, path: &str) -> Result<()> {
+pub fn require_owner_of_path(authed: &ApiAuthed, path: &str) -> Result<()> {
    if authed.is_admin {
        return Ok(());
    }
@@ -1042,7 +1053,10 @@ pub fn require_owner_of_path(authed: &Authed, path: &str) -> Result<()> {
    }
 }

-pub fn get_perm_in_extra_perms_for_authed(v: serde_json::Value, authed: &Authed) -> Option<bool> {
+pub fn get_perm_in_extra_perms_for_authed(
+    v: serde_json::Value,
+    authed: &ApiAuthed,
+) -> Option<bool> {
    match v {
        serde_json::Value::Object(obj) => {
            let mut keys = vec![format!("u/{}", authed.username)];
@@ -1067,7 +1081,7 @@ pub fn get_perm_in_extra_perms_for_authed(v: serde_json::Value, authed: &Authed)
 }

 pub async fn require_is_writer(
-    authed: &Authed,
+    authed: &ApiAuthed,
    path: &str,
    w_id: &str,
    db: DB,
@@ -1138,7 +1152,7 @@ async fn whois(
 }

 // async fn create_invite_code(
-//     Authed { email, .. }: Authed,
+//     ApiAuthed { email, .. }: ApiAuthed,
 //     Extension(db): Extension<DB>,
 //     Json(nu): Json<NewInviteCode>,
 // ) -> Result<(StatusCode, String)> {
@@ -1165,7 +1179,7 @@ async fn whois(
 // }

 async fn decline_invite(
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
    Extension(db): Extension<DB>,
    Json(nu): Json<DeclineInvite>,
 ) -> Result<(StatusCode, String)> {
@@ -1215,7 +1229,7 @@ lazy_static! {
 }

 async fn accept_invite(
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
    Extension(webhook): Extension<WebhookShared>,
    Extension(db): Extension<DB>,
    Json(nu): Json<AcceptInvite>,
@@ -1360,7 +1374,7 @@ async fn add_user_to_workspace<'c>(
 }

 async fn update_workspace_user(
-    Authed { username, is_admin, .. }: Authed,
+    ApiAuthed { username, is_admin, .. }: ApiAuthed,
    Extension(db): Extension<DB>,
    Path((w_id, username_to_update)): Path<(String, String)>,
    Json(eu): Json<EditWorkspaceUser>,
@@ -1417,15 +1431,14 @@ async fn update_workspace_user(
 }

 async fn update_user(
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
    Path(email_to_update): Path<String>,
    Extension(db): Extension<DB>,
    Json(eu): Json<EditUser>,
 ) -> Result<String> {
+    require_super_admin(&db, &email).await?;
    let mut tx = db.begin().await?;

-    require_super_admin(&mut tx, &email).await?;
-
    if let Some(sa) = eu.is_super_admin {
        sqlx::query_scalar!(
            "UPDATE password SET super_admin = $1 WHERE email = $2",
@@ -1451,14 +1464,13 @@ async fn update_user(
 }

 async fn delete_user(
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
    Path(email_to_delete): Path<String>,
    Extension(db): Extension<DB>,
 ) -> Result<String> {
+    require_super_admin(&db, &email).await?;
    let mut tx = db.begin().await?;

-    require_super_admin(&mut tx, &email).await?;
-
    sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete)
        .execute(&mut *tx)
        .await?;
@@ -1506,15 +1518,15 @@ lazy_static::lazy_static! {
 }

 async fn create_user(
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(webhook): Extension<WebhookShared>,
    Extension(argon2): Extension<Arc<Argon2<'_>>>,
    Json(mut nu): Json<NewUser>,
 ) -> Result<(StatusCode, String)> {
+    require_super_admin(&db, &email).await?;
    let mut tx = db.begin().await?;

-    require_super_admin(&mut tx, &email).await?;
    nu.email = nu.email.to_lowercase();

    if nu.email == SUPERADMIN_SECRET_EMAIL {
@@ -1596,7 +1608,7 @@ pub async fn send_email_if_possible_intern(subject: &str, content: &str, to: &st
 }

 async fn delete_workspace_user(
-    Authed { username, is_admin, .. }: Authed,
+    ApiAuthed { username, is_admin, .. }: ApiAuthed,
    Extension(db): Extension<DB>,
    Path((w_id, username_to_delete)): Path<(String, String)>,
 ) -> Result<String> {
@@ -1647,7 +1659,7 @@ async fn delete_workspace_user(
 async fn set_password(
    Extension(db): Extension<DB>,
    Extension(argon2): Extension<Arc<Argon2<'_>>>,
-    Authed { username, email, .. }: Authed,
+    ApiAuthed { username, email, .. }: ApiAuthed,
    Json(EditPassword { password }): Json<EditPassword>,
 ) -> Result<String> {
    let mut tx = db.begin().await?;
@@ -1891,7 +1903,7 @@ async fn login(

 async fn refresh_token(
    Extension(db): Extension<DB>,
-    authed: Authed,
+    authed: ApiAuthed,
    cookies: Cookies,
 ) -> Result<String> {
    let mut tx = db.begin().await?;
@@ -1946,7 +1958,7 @@ pub async fn create_session_token<'c>(

 async fn create_token(
    Extension(db): Extension<DB>,
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
    Json(new_token): Json<NewToken>,
 ) -> Result<(StatusCode, String)> {
    let token = rd_string(30);
@@ -1988,12 +2000,11 @@ async fn create_token(

 async fn impersonate(
    Extension(db): Extension<DB>,
-    Authed { email, username, .. }: Authed,
+    ApiAuthed { email, username, .. }: ApiAuthed,
    Json(new_token): Json<NewToken>,
 ) -> Result<(StatusCode, String)> {
    let token = rd_string(30);
-    let mut tx = db.begin().await?;
-    require_super_admin(&mut tx, &email).await?;
+    require_super_admin(&db, &email).await?;

    if new_token.impersonate_email.is_none() {
        return Err(Error::BadRequest(
@@ -2007,9 +2018,11 @@ async fn impersonate(
        "SELECT super_admin FROM password WHERE email = $1",
        impersonated
    )
-    .fetch_optional(&mut *tx)
+    .fetch_optional(&db)
    .await?
    .unwrap_or(false);
+    let mut tx = db.begin().await?;
+
    sqlx::query!(
        "INSERT INTO token
            (token, email, label, expiration, super_admin)
@@ -2040,7 +2053,7 @@ async fn impersonate(

 async fn list_tokens(
    Extension(db): Extension<DB>,
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
 ) -> JsonResult<Vec<TruncatedToken>> {
    let rows = sqlx::query_as!(
        TruncatedToken,
@@ -2056,7 +2069,7 @@ async fn list_tokens(

 async fn delete_token(
    Extension(db): Extension<DB>,
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
    Path(token_prefix): Path<String>,
 ) -> Result<String> {
    let mut tx = db.begin().await?;
@@ -2095,7 +2108,7 @@ async fn delete_token(
 async fn leave_workspace(
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
-    Authed { username, .. }: Authed,
+    ApiAuthed { username, .. }: ApiAuthed,
 ) -> Result<String> {
    let mut tx = db.begin().await?;
    sqlx::query!(
@@ -2136,7 +2149,7 @@ struct Runnable {

 async fn get_all_runnables(
    Extension(db): Extension<UserDB>,
-    authed: Authed,
+    authed: ApiAuthed,
    Tokened { token }: Tokened,
    Extension(cache): Extension<Arc<AuthCache>>,
 ) -> JsonResult<Vec<Runnable>> {
@@ -6,21 +6,18 @@
 * LICENSE-AGPL for a copy of the license.
 */

-use sqlx::{Postgres, Transaction};
 use windmill_common::{
    error::{self, Error},
    users::SUPERADMIN_SECRET_EMAIL,
+    DB,
 };

-pub async fn require_super_admin<'c>(
-    db: &mut Transaction<'c, Postgres>,
-    email: &str,
-) -> error::Result<()> {
+pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> {
    if email == SUPERADMIN_SECRET_EMAIL {
        return Ok(());
    }
    let is_admin = sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
-        .fetch_optional(&mut **db)
+        .fetch_optional(db)
        .await
        .map_err(|e| Error::InternalErr(format!("fetching super admin: {e}")))?
        .unwrap_or(false);
@@ -7,9 +7,9 @@
 */

 use crate::{
-    db::{UserDB, DB},
+    db::{ApiAuthed, DB},
    oauth2::_refresh_token,
-    users::{maybe_refresh_folders, require_owner_of_path, Authed},
+    users::{maybe_refresh_folders, require_owner_of_path},
    webhook_util::{WebhookMessage, WebhookShared},
 };

@@ -22,6 +22,7 @@ use hyper::StatusCode;
 use serde_json::Value;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
+    db::UserDB,
    error::{Error, JsonResult, Result},
    utils::{not_found_if_none, StripPath},
    variables::{get_reserved_variables, ContextualVariable, CreateVariable, ListableVariable},
@@ -41,6 +42,7 @@ pub fn workspaced_service() -> Router {
        .route("/list", get(list_variables))
        .route("/list_contextual", get(list_contextual_variables))
        .route("/get/*path", get(get_variable))
+        .route("/get_value/*path", get(get_value))
        .route("/exists/*path", get(exists_variable))
        .route("/update/*path", post(update_variable))
        .route("/delete/*path", delete(delete_variable))
@@ -50,7 +52,7 @@ pub fn workspaced_service() -> Router {

 async fn list_contextual_variables(
    Path(w_id): Path<String>,
-    Authed { username, email, .. }: Authed,
+    ApiAuthed { username, email, .. }: ApiAuthed,
 ) -> JsonResult<Vec<ContextualVariable>> {
    Ok(Json(
        get_reserved_variables(
@@ -71,7 +73,7 @@ async fn list_contextual_variables(
 }

 async fn list_variables(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
 ) -> JsonResult<Vec<ListableVariable>> {
@@ -84,8 +86,8 @@ async fn list_variables(
         resource.path IS NOT NULL as is_linked,
         account.refresh_token != '' as is_refreshed
         from variable
-         LEFT JOIN account ON variable.account = account.id AND account.workspace_id = variable.workspace_id
-         LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = variable.workspace_id
+         LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $1
+         LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = $1
         WHERE variable.workspace_id = $1 ORDER BY path",
    )
    .bind(&w_id)
@@ -102,7 +104,7 @@ struct GetVariableQuery {
 }

 async fn get_variable(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Query(q): Query<GetVariableQuery>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -116,7 +118,7 @@ async fn get_variable(
        account.refresh_token != '' as is_refreshed
        from variable
        LEFT JOIN account ON variable.account = account.id
-        LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = variable.workspace_id
+        LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = $2
        WHERE variable.path = $1 AND variable.workspace_id = $2
        LIMIT 1",
    )
@@ -143,7 +145,7 @@ async fn get_variable(
        let value = variable.value.unwrap_or_else(|| "".to_string());
        ListableVariable {
            value: if variable.is_expired.unwrap_or(false) && variable.account.is_some() {
-                Some(_refresh_token(tx, &variable.path, w_id, variable.account.unwrap()).await?)
+                Some(_refresh_token(tx, &variable.path, &w_id, variable.account.unwrap()).await?)
            } else if !value.is_empty() && decrypt_secret {
                let mc = build_crypt(&mut tx, &w_id).await?;
                tx.commit().await?;
@@ -164,6 +166,63 @@ async fn get_variable(
    Ok(Json(r))
 }

+async fn get_value(
+    authed: ApiAuthed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<String> {
+    let path = path.to_path();
+    let tx = user_db.begin(&authed).await?;
+    return get_value_internal(tx, &w_id, &path, &authed.username)
+        .await
+        .map(Json);
+}
+
+pub async fn get_value_internal<'c>(
+    mut tx: Transaction<'c, Postgres>,
+    w_id: &str,
+    path: &str,
+    username: &str,
+) -> Result<String> {
+    let variable_o = sqlx::query!(
+        "SELECT value, account, (now() > account.expires_at) as is_expired, is_secret, path from variable
+        LEFT JOIN account ON variable.account = account.id WHERE variable.path = $1 AND variable.workspace_id = $2", path, w_id
+    )
+    .fetch_optional(&mut *tx)
+    .await?;
+
+    let variable = not_found_if_none(variable_o, "Variable", &path)?;
+
+    let r = if variable.is_secret {
+        audit_log(
+            &mut *tx,
+            username,
+            "variables.decrypt_secret",
+            ActionKind::Execute,
+            &w_id,
+            Some(&variable.path),
+            None,
+        )
+        .await?;
+        let value = variable.value;
+        if variable.is_expired.unwrap_or(false) && variable.account.is_some() {
+            _refresh_token(tx, &variable.path, &w_id, variable.account.unwrap()).await?
+        } else if !value.is_empty() {
+            let mc = build_crypt(&mut tx, &w_id).await?;
+            tx.commit().await?;
+
+            mc.decrypt_base64_to_string(value)
+                .map_err(|e| Error::InternalErr(e.to_string()))?
+        } else {
+            "".to_string()
+        }
+    } else {
+        variable.value
+    };
+
+    Ok(r)
+}
+
 async fn exists_variable(
    Extension(db): Extension<DB>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -205,7 +264,7 @@ async fn check_path_conflict<'c>(
 }

 async fn create_variable(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
@@ -280,7 +339,7 @@ async fn encrypt_value(
 }

 async fn delete_variable(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -337,7 +396,7 @@ struct AlreadyEncrypted {
 }

 async fn update_variable(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Extension(db): Extension<DB>,
@@ -6,7 +6,6 @@
 * LICENSE-AGPL for a copy of the license.
 */

-use crate::{db::UserDB, users::Authed};
 use axum::{
    extract::{Extension, Query},
    routing::get,
@@ -18,6 +17,7 @@ use regex::Regex;
 use serde::{Deserialize, Serialize};
 use sqlx::FromRow;
 use windmill_common::{
+    db::UserDB,
    error::JsonResult,
    utils::{paginate, Pagination},
 };
@@ -28,6 +28,8 @@ use std::sync::atomic::Ordering;
 #[cfg(feature = "benchmark")]
 use windmill_queue::IDLE_WORKERS;

+use crate::db::ApiAuthed;
+
 #[cfg(not(feature = "benchmark"))]
 pub fn global_service() -> Router {
    Router::new()
@@ -92,7 +94,7 @@ struct EnableWorkerQuery {
 }

 async fn list_worker_pings(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Query(pagination): Query<Pagination>,
 ) -> JsonResult<Vec<WorkerPing>> {
@@ -10,12 +10,13 @@
 use std::str::FromStr;

 use crate::BASE_URL;
+use crate::db::ApiAuthed;
 use crate::{
    apps::AppWithLastVersion,
-    db::{UserDB, DB},
+    db::DB,
    folders::Folder,
    resources::{Resource, ResourceType},
-    users::{Authed, WorkspaceInvite, VALID_USERNAME, send_email_if_possible},
+    users::{WorkspaceInvite, VALID_USERNAME, send_email_if_possible},
    utils::require_super_admin,
    variables::build_crypt,
    webhook_util::{InstanceEvent, WebhookShared}
@@ -34,6 +35,7 @@ use magic_crypt::MagicCryptTrait;
 #[cfg(feature = "enterprise")]
 use stripe::CustomerId;
 use windmill_audit::{audit_log, ActionKind};
+use windmill_common::db::UserDB;
 use windmill_common::schedule::Schedule;
 use windmill_common::users::username_to_permissioned_as;
 use windmill_common::{
@@ -220,7 +222,7 @@ pub struct EditErrorHandler {
 }

 async fn list_pending_invites(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Path(w_id): Path<String>,
 ) -> JsonResult<Vec<WorkspaceInvite>> {
@@ -243,14 +245,14 @@ pub struct PremiumWorkspaceInfo {
    pub usage: Option<i32>,
 }
 async fn premium_info(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
 ) -> JsonResult<PremiumWorkspaceInfo> {
    require_admin(authed.is_admin, &authed.username)?;
    let mut tx = db.begin().await?;
    let row = sqlx::query_as::<_, PremiumWorkspaceInfo>(
-        "SELECT premium, usage.usage FROM workspace LEFT JOIN usage ON workspace.id = usage.id AND usage.is_workspace IS true WHERE workspace.id = $1",
+        "SELECT premium, usage.usage FROM workspace LEFT JOIN usage ON workspace.id = $1 AND usage.is_workspace IS true WHERE workspace.id = $1",
    )
    .bind(w_id)
    .fetch_one(&mut *tx)
@@ -267,7 +269,7 @@ struct PlanQuery {

 #[cfg(feature = "enterprise")]
 async fn stripe_checkout(
-    authed: Authed,
+    authed: ApiAuthed,
    Path(w_id): Path<String>,
    Query(plan): Query<PlanQuery>,
 ) -> Result<Redirect> {
@@ -311,7 +313,7 @@ async fn stripe_checkout(

 #[cfg(feature = "enterprise")]
 async fn stripe_portal(
-    authed: Authed,
+    authed: ApiAuthed,
    Path(w_id): Path<String>,
    Extension(db): Extension<DB>,
 ) -> Result<Redirect> {
@@ -337,7 +339,7 @@ async fn stripe_portal(
 }

 // async fn stripe_usage(
-//     authed: Authed,
+//     authed: ApiAuthed,
 //     Path(w_id): Path<String>,
 //     Extension(db): Extension<DB>,
 //     Extension(base_url): Extension<Arc<BaseUrl>>,
@@ -381,7 +383,7 @@ async fn stripe_portal(
 // }

 async fn exists_workspace(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
    Json(WorkspaceId { id }): Json<WorkspaceId>,
 ) -> JsonResult<bool> {
@@ -398,7 +400,7 @@ async fn exists_workspace(
 }

 async fn list_workspaces(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(user_db): Extension<UserDB>,
 ) -> JsonResult<Vec<Workspace>> {
    let mut tx = user_db.begin(&authed).await?;
@@ -415,7 +417,7 @@ async fn list_workspaces(
 }

 async fn get_settings(
-    authed: Authed,
+    authed: ApiAuthed,
    Path(w_id): Path<String>,
    Extension(user_db): Extension<UserDB>,
 ) -> JsonResult<WorkspaceSettings> {
@@ -438,7 +440,7 @@ struct DeployTo {
    deploy_to: Option<String>,
 }
 async fn get_deploy_to(
-    authed: Authed,
+    authed: ApiAuthed,
    Path(w_id): Path<String>,
    Extension(user_db): Extension<UserDB>,
 ) -> JsonResult<DeployTo> {
@@ -457,10 +459,10 @@ async fn get_deploy_to(
 }

 async fn edit_slack_command(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
-    Authed { is_admin, username, .. }: Authed,
+    ApiAuthed { is_admin, username, .. }: ApiAuthed,
    Json(es): Json<EditCommandScript>,
 ) -> Result<String> {
    require_admin(is_admin, &username)?;
@@ -499,10 +501,10 @@ async fn edit_slack_command(

 #[cfg(feature = "enterprise")]
 async fn edit_deploy_to(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
-    Authed { is_admin, username, .. }: Authed,
+    ApiAuthed { is_admin, username, .. }: ApiAuthed,
    Json(es): Json<EditDeployTo>,
 ) -> Result<String> {
    require_admin(is_admin, &username)?;
@@ -546,16 +548,16 @@ async fn edit_deploy_to() -> Result<String> {

 const BANNED_DOMAINS: &str = include_str!("../banned_domains.txt");

-async fn is_allowed_auto_domain(Authed { email, .. }: Authed) -> JsonResult<bool> {
+async fn is_allowed_auto_domain(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult<bool> {
    let domain = email.split('@').last().unwrap();
    return Ok(Json(!BANNED_DOMAINS.contains(domain)));
 }

 async fn edit_auto_invite(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
-    Authed { is_admin, email, username, .. }: Authed,
+    ApiAuthed { is_admin, email, username, .. }: ApiAuthed,
    Json(ea): Json<EditAutoInvite>,
 ) -> Result<String> {
    require_admin(is_admin, &username)?;
@@ -620,10 +622,10 @@ async fn edit_auto_invite(
 }

 async fn edit_webhook(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
-    Authed { is_admin, username, .. }: Authed,
+    ApiAuthed { is_admin, username, .. }: ApiAuthed,
    Json(ew): Json<EditWebhook>,
 ) -> Result<String> {
    require_admin(is_admin, &username)?;
@@ -662,10 +664,10 @@ async fn edit_webhook(
 }

 async fn edit_openai_resource_path(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
-    Authed { is_admin, username, .. }: Authed,
+    ApiAuthed { is_admin, username, .. }: ApiAuthed,
    Json(eo): Json<EditOpenaiResourcePath>,
 ) -> Result<String> {
    require_admin(is_admin, &username)?;
@@ -726,10 +728,10 @@ async fn exists_openai_resource_path(


 async fn edit_error_handler(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
-    Authed { is_admin, username, .. }: Authed,
+    ApiAuthed { is_admin, username, .. }: ApiAuthed,
    Json(ee): Json<EditErrorHandler>,
 ) -> Result<String> {
    require_admin(is_admin, &username)?;
@@ -781,15 +783,16 @@ async fn edit_error_handler(
 }

 async fn list_workspaces_as_super_admin(
-    authed: Authed,
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Query(pagination): Query<Pagination>,
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
 ) -> JsonResult<Vec<Workspace>> {
-    let mut tx = user_db.begin(&authed).await?;
-    require_super_admin(&mut tx, &email).await?;
+    require_super_admin(&db, &email).await?;
    let (per_page, offset) = paginate(pagination);

+    let mut tx = user_db.begin(&authed).await?;
    let workspaces = sqlx::query_as!(
        Workspace,
        "SELECT * FROM workspace LIMIT $1 OFFSET $2",
@@ -804,7 +807,7 @@ async fn list_workspaces_as_super_admin(

 async fn user_workspaces(
    Extension(db): Extension<DB>,
-    Authed { email, .. }: Authed,
+    ApiAuthed { email, .. }: ApiAuthed,
 ) -> JsonResult<WorkspaceList> {
    let mut tx = db.begin().await?;
    let workspaces = sqlx::query_as!(
@@ -841,15 +844,16 @@ lazy_static::lazy_static! {
 }

 async fn create_workspace(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Json(nw): Json<CreateWorkspace>,
 ) -> Result<String> {

-    let mut tx: Transaction<'_, Postgres> = db.begin().await?;
    if *CREATE_WORKSPACE_REQUIRE_SUPERADMIN {
-        require_super_admin(&mut tx, &authed.email).await?;
+        require_super_admin(&db, &authed.email).await?;
    }
+    let mut tx: Transaction<'_, Postgres> = db.begin().await?;
+
    check_name_conflict(&mut tx, &nw.id).await?;
    sqlx::query!(
        "INSERT INTO workspace
@@ -937,10 +941,10 @@ async fn create_workspace(
 }

 async fn edit_workspace(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
-    Authed { is_admin, username, .. }: Authed,
+    ApiAuthed { is_admin, username, .. }: ApiAuthed,
    Json(ew): Json<EditWorkspace>,
 ) -> Result<String> {
    require_admin(is_admin, &username)?;
@@ -972,7 +976,7 @@ async fn edit_workspace(
 async fn archive_workspace(
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
-    Authed { is_admin, username, email, .. }: Authed,
+    ApiAuthed { is_admin, username, email, .. }: ApiAuthed,
 ) -> Result<String> {
    require_admin(is_admin, &username)?;
    let mut tx = db.begin().await?;
@@ -998,7 +1002,7 @@ async fn archive_workspace(
 async fn unarchive_workspace(
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
-    Authed { is_admin, username, email, .. }: Authed,
+    ApiAuthed { is_admin, username, email, .. }: ApiAuthed,
 ) -> Result<String> {
    require_admin(is_admin, &username)?;
    let mut tx = db.begin().await?;
@@ -1024,7 +1028,7 @@ async fn unarchive_workspace(
 async fn delete_workspace(
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
-    Authed { username, email, .. }: Authed,
+    ApiAuthed { username, email, .. }: ApiAuthed,
 ) -> Result<String> {
    let w_id = match w_id.as_str() {
        "starter" => Err(Error::BadRequest(
@@ -1036,7 +1040,7 @@ async fn delete_workspace(
        _ => Ok(w_id),
    }?;
    let mut tx = db.begin().await?;
-    require_super_admin(&mut tx, &email).await?;
+    require_super_admin(&db, &email).await?;

    sqlx::query!("DELETE FROM script WHERE workspace_id = $1", &w_id)
        .execute(&mut *tx)
@@ -1142,7 +1146,7 @@ pub async fn invite_user_to_all_auto_invite_worspaces(db: &DB, email: &str) -> R
 }

 async fn invite_user(
-    Authed { username, is_admin, .. }: Authed,
+    ApiAuthed { username, is_admin, .. }: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(webhook): Extension<WebhookShared>,
    Path(w_id): Path<String>,
@@ -1191,7 +1195,7 @@ If you do not have an account on {}, login with SSO or ask an admin to create an
 }

 async fn add_user(
-    Authed { username, email, is_admin, .. }: Authed,
+    ApiAuthed { username, email, is_admin, .. }: ApiAuthed,
    Extension(db): Extension<DB>,
    Extension(webhook): Extension<WebhookShared>,
    Path(w_id): Path<String>,
@@ -1255,7 +1259,7 @@ If you do not have an account on {}, login with SSO or ask an admin to create an
 }

 async fn delete_invite(
-    Authed { username, is_admin, .. }: Authed,
+    ApiAuthed { username, is_admin, .. }: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
    Json(nu): Json<NewWorkspaceInvite>,
@@ -1413,7 +1417,7 @@ where
 }

 async fn tarball_workspace(
-    authed: Authed,
+    authed: ApiAuthed,
    Extension(db): Extension<DB>,
    Path(w_id): Path<String>,
    Query(ArchiveQueryParams {
@@ -0,0 +1,137 @@
+use sqlx::{Pool, Postgres, Transaction};
+
+pub type DB = Pool<Postgres>;
+
+#[derive(Clone, Debug)]
+pub struct Authed {
+    pub email: String,
+    pub username: String,
+    pub is_admin: bool,
+    pub is_operator: bool,
+    pub groups: Vec<String>,
+    // (folder name, can write, is owner)
+    pub folders: Vec<(String, bool, bool)>,
+    pub scopes: Option<Vec<String>>,
+}
+
+#[derive(Clone)]
+pub struct UserDB {
+    db: DB,
+}
+
+pub trait Authable {
+    fn email(&self) -> &str;
+    fn username(&self) -> &str;
+    fn is_admin(&self) -> bool;
+    fn is_operator(&self) -> bool;
+    fn groups(&self) -> &[String];
+    fn folders(&self) -> &[(String, bool, bool)];
+    fn scopes(&self) -> Option<&[String]>;
+}
+
+impl Authable for Authed {
+    fn is_admin(&self) -> bool {
+        self.is_admin
+    }
+
+    fn is_operator(&self) -> bool {
+        self.is_operator
+    }
+
+    fn groups(&self) -> &[String] {
+        &self.groups
+    }
+
+    fn folders(&self) -> &[(String, bool, bool)] {
+        &self.folders
+    }
+
+    fn scopes(&self) -> Option<&[std::string::String]> {
+        self.scopes.as_ref().map(|x| x.as_slice())
+    }
+
+    fn email(&self) -> &str {
+        &self.email
+    }
+
+    fn username(&self) -> &str {
+        &self.username
+    }
+}
+
+impl UserDB {
+    pub fn new(db: DB) -> Self {
+        Self { db }
+    }
+
+    pub async fn begin<T>(self, authed: &T) -> Result<Transaction<'static, Postgres>, sqlx::Error>
+    where
+        T: Authable,
+    {
+        let mut tx = self.db.begin().await?;
+        let user = if authed.is_admin() {
+            "windmill_admin"
+        } else {
+            "windmill_user"
+        };
+
+        sqlx::query(&format!("SET LOCAL ROLE {}", user))
+            .execute(&mut *tx)
+            .await?;
+
+        sqlx::query!(
+            "SELECT set_config('session.user', $1, true)",
+            authed.username()
+        )
+        .fetch_optional(&mut *tx)
+        .await?;
+
+        sqlx::query!(
+            "SELECT set_config('session.groups', $1, true)",
+            &authed.groups().join(",")
+        )
+        .fetch_optional(&mut *tx)
+        .await?;
+
+        sqlx::query!(
+            "SELECT set_config('session.pgroups', $1, true)",
+            &authed
+                .groups()
+                .iter()
+                .map(|x| format!("g/{}", x))
+                .collect::<Vec<_>>()
+                .join(",")
+        )
+        .fetch_optional(&mut *tx)
+        .await?;
+
+        let (folders_write, folders_read): &(Vec<_>, Vec<_>) =
+            &authed.folders().clone().into_iter().partition(|x| x.1);
+
+        let mut folders_read = folders_read.clone();
+        folders_read.extend(folders_write.clone());
+        sqlx::query!(
+            "SELECT set_config('session.folders_read', $1, true)",
+            folders_read
+                .iter()
+                .map(|x| x.0.clone())
+                .collect::<Vec<_>>()
+                .join(",")
+        )
+        .fetch_optional(&mut *tx)
+        .await?;
+
+        sqlx::query!(
+            "SELECT set_config('session.folders_write', $1, true)",
+            folders_write
+                .iter()
+                .map(|x| x.0.clone())
+                .collect::<Vec<_>>()
+                .join(",")
+        )
+        .fetch_optional(&mut *tx)
+        .await?;
+
+        Ok(tx)
+    }
+}
@@ -64,6 +64,8 @@ pub enum Error {
    JsonErr(serde_json::Value),
    #[error("Custom Status Code: {0:#?}")]
    CustomStatusCode(StatusCode, serde_json::Value),
+    #[error("{0}")]
+    OpenAIError(String),
 }

 impl Error {
@@ -87,7 +89,9 @@ impl IntoResponse for Error {
            Self::NotAuthorized(_) => axum::http::StatusCode::UNAUTHORIZED,
            Self::RequireAdmin(_) => axum::http::StatusCode::FORBIDDEN,
            Self::CustomStatusCode(code, _) => code,
-            Self::SqlErr(_) | Self::BadRequest(_) => axum::http::StatusCode::BAD_REQUEST,
+            Self::SqlErr(_) | Self::BadRequest(_) | Self::OpenAIError(_) => {
+                axum::http::StatusCode::BAD_REQUEST
+            }
            _ => axum::http::StatusCode::INTERNAL_SERVER_ERROR,
        };
        tracing::error!(error = e.to_string());
@@ -1,5 +1,5 @@
 use serde::{Deserialize, Serialize};
-use sqlx::{Postgres, Transaction};
+use sqlx::{Pool, Postgres, Transaction};
 use uuid::Uuid;

 use crate::{
@@ -201,9 +201,11 @@ pub struct RawCode {

 type Tag = String;

-pub async fn script_path_to_payload<'c>(
+pub type DB = Pool<Postgres>;
+
+pub async fn script_path_to_payload(
    script_path: &str,
-    db: &mut Transaction<'c, Postgres>,
+    db: &DB,
    w_id: &str,
 ) -> error::Result<(JobPayload, Option<Tag>)> {
    let (job_payload, tag) = if script_path.starts_with("hub/") {
@@ -248,13 +250,13 @@ pub async fn script_hash_to_tag_and_limits<'c>(
    ))
 }

-pub async fn get_payload_tag_from_prefixed_path<'c>(
+pub async fn get_payload_tag_from_prefixed_path(
    path: &str,
-    db: &mut Transaction<'c, Postgres>,
+    db: &DB,
    w_id: &str,
 ) -> Result<(JobPayload, Option<String>), Error> {
    let (payload, tag) = if path.starts_with("script/") {
-        script_path_to_payload(path.strip_prefix("script/").unwrap(), db, w_id).await?
+        script_path_to_payload(path.strip_prefix("script/").unwrap(), &db, w_id).await?
    } else if path.starts_with("flow/") {
        (
            JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string()),
@@ -9,8 +9,10 @@
 use std::net::SocketAddr;

 use error::Error;
+use sqlx::{Pool, Postgres};

 pub mod apps;
+pub mod db;
 pub mod error;
 pub mod external_ip;
 pub mod flow_status;
@@ -155,8 +157,10 @@ pub async fn connect(

 type Tag = String;

-pub async fn get_latest_deployed_hash_for_path<'c>(
-    db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
+pub type DB = Pool<Postgres>;
+
+pub async fn get_latest_deployed_hash_for_path(
+    db: &DB,
    w_id: &str,
    script_path: &str,
 ) -> error::Result<(scripts::ScriptHash, Option<Tag>, Option<i32>, Option<i32>)> {
@@ -167,12 +171,17 @@ pub async fn get_latest_deployed_hash_for_path<'c>(
        script_path,
        w_id
    )
-    .fetch_optional(&mut **db)
+    .fetch_optional(db)
    .await?;

    let script = utils::not_found_if_none(r_o, "script", script_path)?;

-    Ok((scripts::ScriptHash(script.hash), script.tag, script.concurrent_limit, script.concurrency_time_window_s))
+    Ok((
+        scripts::ScriptHash(script.hash),
+        script.tag,
+        script.concurrent_limit,
+        script.concurrency_time_window_s,
+    ))
 }

 pub async fn get_latest_hash_for_path<'c>(
@@ -192,5 +201,10 @@ pub async fn get_latest_hash_for_path<'c>(

    let script = utils::not_found_if_none(r_o, "script", script_path)?;

-    Ok((scripts::ScriptHash(script.hash), script.tag, script.concurrent_limit, script.concurrency_time_window_s))
+    Ok((
+        scripts::ScriptHash(script.hash),
+        script.tag,
+        script.concurrent_limit,
+        script.concurrency_time_window_s,
+    ))
 }
@@ -27,6 +27,12 @@ pub struct Schedule {
    #[serde(skip_serializing_if = "Option::is_none")]
    pub error: Option<String>,
    pub on_failure: Option<String>,
+    pub on_failure_times: Option<i32>,
+    pub on_failure_exact: Option<bool>,
+    pub on_failure_extra_args: Option<serde_json::Value>,
+    pub on_recovery: Option<String>,
+    pub on_recovery_times: Option<i32>,
+    pub on_recovery_extra_args: Option<serde_json::Value>,
 }

 pub fn schedule_to_user(path: &str) -> String {
@@ -73,7 +73,7 @@ pub fn get_reserved_variables(
    flow_path: Option<String>,
    schedule_path: Option<String>,
    step_id: Option<String>,
-) -> [ContextualVariable; 14] {
+) -> [ContextualVariable; 15] {
    let state_path = {
        let flow_path = flow_path
            .clone()
@@ -117,6 +117,27 @@ pub fn get_reserved_variables(
        }
    };

+    let joined_schedule_path = schedule_path
+        .clone()
+        .unwrap_or("manual".to_string())
+        .split("/")
+        .collect::<Vec<&str>>()
+        .join("_");
+    let ts = chrono::Utc::now().timestamp_millis();
+    let object_path = if let Some(flow_path) = flow_path.clone() {
+        let flow_path = flow_path.split("/").collect::<Vec<&str>>().join("_");
+        let step_id = step_id.clone().unwrap_or("".to_string());
+        format!("{flow_path}/{joined_schedule_path}/{step_id}/{ts}_{job_id}")
+    } else {
+        let joined_script_path = path
+            .clone()
+            .unwrap_or("".to_string())
+            .split("/")
+            .collect::<Vec<&str>>()
+            .join("_");
+        format!("{joined_script_path}/{joined_schedule_path}/{ts}_{job_id}")
+    };
+
    [
        ContextualVariable {
            name: "WM_WORKSPACE".to_string(),
@@ -192,5 +213,10 @@ pub fn get_reserved_variables(
            value: step_id.unwrap_or_else(|| "".to_string()),
            description: "The node id in a flow (like 'a', 'b', or 'f')".to_string(),
        },
+        ContextualVariable {
+            name: "WM_OBJECT_PATH".to_string(),
+            value: object_path,
+            description: "Script or flow step execution unique path, useful for storing results in an external service".to_string(),
+        }
    ]
 }
@@ -21,6 +21,7 @@ use ulid::Ulid;
 use uuid::Uuid;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
+    db::{Authed, UserDB},
    error::{self, Error},
    flow_status::{
        FlowStatus, FlowStatusModule, JobResult, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL,
@@ -33,7 +34,7 @@ use windmill_common::{
    schedule::{schedule_to_user, Schedule},
    scripts::{ScriptHash, ScriptLang},
    users::{username_to_permissioned_as, SUPERADMIN_SECRET_EMAIL},
-    METRICS_ENABLED,
+    DB, METRICS_ENABLED,
 };

 use crate::{
@@ -99,11 +100,8 @@ lazy_static::lazy_static! {
        WHERE id = (
            SELECT id
            FROM queue
-            WHERE ((running = false
-                   AND scheduled_for <= now())
-               OR (suspend_until IS NOT NULL
-                   AND (   suspend <= 0
-                        OR suspend_until <= now()))) 
+            WHERE ((running = false AND scheduled_for <= now())
+               OR (suspend_until IS NOT NULL AND (suspend <= 0 OR suspend_until <= now()))) 
                {}
            ORDER BY scheduled_for, created_at
            FOR UPDATE SKIP LOCKED
@@ -142,7 +140,11 @@ pub async fn cancel_job<'c: 'async_recursion>(
        return Ok((tx, None));
    }
    let job_running = job_running.unwrap();
-    if job_running.running && !force_cancel {
+    if job_running.running
+        && job_running.job_kind != JobKind::Flow
+        && job_running.job_kind != JobKind::FlowPreview
+        && !force_cancel
+    {
        sqlx::query!(
        "UPDATE queue SET  canceled = true, canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 \
         AND workspace_id = $4 ",
@@ -362,19 +364,35 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(

    tx = delete_job(tx, &queued_job.workspace_id, job_id).await?;
    if !queued_job.is_flow_step
-        && queued_job.job_kind != JobKind::Flow
-        && queued_job.job_kind != JobKind::FlowPreview
        && queued_job.schedule_path.is_some()
        && queued_job.script_path.is_some()
    {
-        tx = handle_maybe_scheduled_job(
+        tx = apply_schedule_handlers(
            tx,
            db,
            queued_job.schedule_path.as_ref().unwrap(),
            queued_job.script_path.as_ref().unwrap(),
            &queued_job.workspace_id,
            success,
-            if success { None } else { Some(&result) },
+            &result,
+            job_id,
+            queued_job.started_at.unwrap_or(chrono::Utc::now()),
+        )
+        .await?;
+    }
+    if !queued_job.is_flow_step
+        && queued_job.job_kind != JobKind::Flow
+        && queued_job.job_kind != JobKind::FlowPreview
+        && queued_job.schedule_path.is_some()
+        && queued_job.script_path.is_some()
+    {
+        // script only
+        tx = handle_maybe_scheduled_job(
+            tx,
+            db,
+            queued_job.schedule_path.as_ref().unwrap(),
+            queued_job.script_path.as_ref().unwrap(),
+            &queued_job.workspace_id,
        )
        .await?;
    }
@@ -439,17 +457,17 @@ pub async fn run_error_handler<R: rsmq_async::RsmqConnection + Clone + Send>(
    let w_id = &queued_job.workspace_id;
    let script_w_id = if is_global { "admins" } else { w_id }; // script workspace id
    let job_id = queued_job.id;
-    let mut tx: QueueTransaction<'_, _> = (rsmq, db.begin().await?).into();
-    let (job_payload, tag) =
-        script_path_to_payload(&error_handler_path, tx.transaction_mut(), script_w_id).await?;
+    let (job_payload, tag) = script_path_to_payload(&error_handler_path, db, script_w_id).await?;
    let mut args = result.as_object().unwrap().clone();
    args.insert("workspace_id".to_string(), json!(w_id));
    args.insert("job_id".to_string(), json!(job_id));
    args.insert("path".to_string(), json!(queued_job.script_path));
    args.insert("is_flow".to_string(), json!(queued_job.raw_flow.is_some()));
    args.insert("email".to_string(), json!(queued_job.email));
+    let tx = PushIsolationLevel::IsolatedRoot(db.clone(), rsmq);

    let (uuid, tx) = push(
+        &db,
        tx,
        script_w_id,
        job_payload,
@@ -541,8 +559,6 @@ pub async fn handle_maybe_scheduled_job<'c, R: rsmq_async::RsmqConnection + Clon
    schedule_path: &str,
    script_path: &str,
    w_id: &str,
-    success: bool,
-    result: Option<&serde_json::Value>,
 ) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
    let schedule = get_schedule_opt(tx.transaction_mut(), w_id, schedule_path).await?;

@@ -556,46 +572,8 @@ pub async fn handle_maybe_scheduled_job<'c, R: rsmq_async::RsmqConnection + Clon
    let schedule = schedule.unwrap();

    if schedule.enabled && script_path == schedule.script_path {
-        if !success {
-            if let Some(on_failure_path) = schedule.on_failure.clone() {
-                let on_failure_result = handle_on_failure(
-                    tx,
-                    schedule_path,
-                    script_path,
-                    w_id,
-                    &on_failure_path,
-                    result,
-                    &schedule.email,
-                    &schedule_to_user(&schedule.path),
-                    username_to_permissioned_as(&schedule.edited_by),
-                )
-                .await;
-
-                match on_failure_result {
-                    Ok(ntx) => {
-                        tx = ntx;
-                    }
-                    Err(err) => {
-                        sqlx::query!(
-                        "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3",
-                        format!("Could not trigger error handler: {err}"),
-                        &schedule.workspace_id,
-                        &schedule.path
-                    )
-                    .execute(db)
-                    .await?;
-                        tracing::warn!(
-                            "Could not trigger error handler for {}: {}",
-                            schedule_path,
-                            err
-                        );
-                        return Err(err);
-                    }
-                }
-            }
-        }
-
        let res = push_scheduled_job(
+            db,
            tx,
            Schedule {
                workspace_id: w_id.to_owned(),
@@ -614,6 +592,12 @@ pub async fn handle_maybe_scheduled_job<'c, R: rsmq_async::RsmqConnection + Clon
                email: schedule.email,
                error: None,
                on_failure: schedule.on_failure,
+                on_failure_times: schedule.on_failure_times,
+                on_failure_exact: schedule.on_failure_exact,
+                on_failure_extra_args: schedule.on_failure_extra_args,
+                on_recovery: schedule.on_recovery,
+                on_recovery_times: schedule.on_recovery_times,
+                on_recovery_extra_args: schedule.on_recovery_extra_args,
            },
        )
        .await;
@@ -637,29 +621,214 @@ pub async fn handle_maybe_scheduled_job<'c, R: rsmq_async::RsmqConnection + Clon
    }
 }

-async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>(
+#[derive(Clone)]
+struct CompletedJobSubset {
+    success: bool,
+    result: Option<serde_json::Value>,
+    started_at: chrono::DateTime<chrono::Utc>,
+}
+async fn apply_schedule_handlers<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>(
    mut tx: QueueTransaction<'c, R>,
+    db: &Pool<Postgres>,
    schedule_path: &str,
    script_path: &str,
    w_id: &str,
+    success: bool,
+    result: &serde_json::Value,
+    job_id: Uuid,
+    started_at: DateTime<Utc>,
+) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
+    let schedule = get_schedule_opt(tx.transaction_mut(), w_id, schedule_path).await?;
+
+    if schedule.is_none() {
+        tracing::error!(
+            "Schedule {schedule_path} in {w_id} not found. Impossible to apply schedule handlers"
+        );
+        return Ok(tx);
+    }
+
+    let schedule = schedule.unwrap();
+
+    if !success {
+        if let Some(on_failure_path) = schedule.on_failure.clone() {
+            let times = schedule.on_failure_times.unwrap_or(1).max(1);
+            let exact = schedule.on_failure_exact.unwrap_or(false);
+            if times > 1 || exact {
+                let past_jobs = sqlx::query_as!(
+                    CompletedJobSubset,
+                    "SELECT success, result, started_at FROM completed_job WHERE workspace_id = $1 AND schedule_path = $2 AND script_path = $3 AND id != $4 ORDER BY created_at DESC LIMIT $5",
+                    &schedule.workspace_id,
+                    &schedule.path,
+                    &schedule.script_path,
+                    job_id,
+                    if exact { times } else { times - 1 } as i64,
+                ).fetch_all(&mut tx).await?;
+
+                let match_times = if exact {
+                    past_jobs.len() == times as usize
+                        && past_jobs[..(times - 1) as usize].iter().all(|j| !j.success)
+                        && past_jobs[(times - 1) as usize].success
+                } else {
+                    past_jobs.len() == ((times - 1) as usize)
+                        && past_jobs.iter().all(|j| !j.success)
+                };
+
+                if !match_times {
+                    return Ok(tx);
+                }
+            }
+
+            let on_failure_result = handle_on_failure(
+                db,
+                tx,
+                schedule_path,
+                script_path,
+                schedule.is_flow,
+                w_id,
+                &on_failure_path,
+                result,
+                times,
+                started_at,
+                schedule.on_failure_extra_args,
+                &schedule.email,
+                &schedule_to_user(&schedule.path),
+                username_to_permissioned_as(&schedule.edited_by),
+            )
+            .await;
+
+            match on_failure_result {
+                Ok(ntx) => {
+                    tx = ntx;
+                }
+                Err(err) => {
+                    sqlx::query!(
+                    "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3",
+                    format!("Could not trigger error handler: {err}"),
+                    &schedule.workspace_id,
+                    &schedule.path
+                )
+                .execute(db)
+                .await?;
+                    tracing::warn!(
+                        "Could not trigger error handler for {}: {}",
+                        schedule_path,
+                        err
+                    );
+                    return Err(err);
+                }
+            }
+        }
+    } else {
+        if let Some(on_recovery_path) = schedule.on_recovery.clone() {
+            let times = schedule.on_recovery_times.unwrap_or(1).max(1);
+            let past_jobs = sqlx::query_as!(
+                CompletedJobSubset,
+                "SELECT success, result, started_at FROM completed_job WHERE workspace_id = $1 AND schedule_path = $2 AND script_path = $3 AND id != $4 ORDER BY created_at DESC LIMIT $5",
+                &schedule.workspace_id,
+                &schedule.path,
+                &schedule.script_path,
+                job_id,
+                times as i64,
+            ).fetch_all(&mut tx).await?;
+
+            if past_jobs.len() < times as usize {
+                return Ok(tx);
+            }
+
+            let n_times_successful = past_jobs[..(times - 1) as usize].iter().all(|j| j.success);
+
+            if !n_times_successful {
+                return Ok(tx);
+            }
+
+            let failed_job = past_jobs[past_jobs.len() - 1].clone();
+
+            if !failed_job.success {
+                let on_recovery_result = handle_on_recovery(
+                    db,
+                    tx,
+                    schedule_path,
+                    script_path,
+                    schedule.is_flow,
+                    w_id,
+                    &on_recovery_path,
+                    failed_job,
+                    result,
+                    times,
+                    started_at,
+                    schedule.on_recovery_extra_args,
+                    &schedule.email,
+                    &schedule_to_user(&schedule.path),
+                    username_to_permissioned_as(&schedule.edited_by),
+                )
+                .await;
+
+                match on_recovery_result {
+                    Ok(ntx) => {
+                        tx = ntx;
+                    }
+                    Err(err) => {
+                        sqlx::query!(
+                            "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3",
+                            format!("Could not trigger recovery handler: {err}"),
+                            &schedule.workspace_id,
+                            &schedule.path
+                        )
+                        .execute(db)
+                        .await?;
+                        tracing::warn!(
+                            "Could not trigger recovery handler for {}: {}",
+                            schedule_path,
+                            err
+                        );
+                        return Err(err);
+                    }
+                }
+            }
+        }
+    }
+
+    Ok(tx)
+}
+
+async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>(
+    db: &Pool<Postgres>,
+    tx: QueueTransaction<'c, R>,
+    schedule_path: &str,
+    script_path: &str,
+    is_flow: bool,
+    w_id: &str,
    on_failure_path: &str,
-    result: Option<&serde_json::Value>,
+    result: &serde_json::Value,
+    failed_times: i32,
+    started_at: DateTime<Utc>,
+    extra_args: Option<serde_json::Value>,
    username: &str,
    email: &str,
    permissioned_as: String,
 ) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
-    let (payload, tag) =
-        get_payload_tag_from_prefixed_path(on_failure_path, tx.transaction_mut(), w_id).await?;
+    let (payload, tag) = get_payload_tag_from_prefixed_path(on_failure_path, db, w_id).await?;

-    let mut args = result
-        .map(|x| x.clone())
-        .unwrap_or_else(|| json!({}))
-        .as_object()
-        .unwrap()
-        .clone();
+    let mut args = result.clone().as_object().unwrap().clone();
    args.insert("schedule_path".to_string(), json!(schedule_path));
    args.insert("path".to_string(), json!(script_path));
+    args.insert("is_flow".to_string(), json!(is_flow));
+    args.insert("started_at".to_string(), json!(started_at));
+    args.insert("failed_times".to_string(), json!(failed_times));
+
+    if let Some(args_v) = extra_args {
+        if let serde_json::Value::Object(args_m) = args_v {
+            args.extend(args_m);
+        } else {
+            return Err(error::Error::ExecutionErr(
+                "args of scripts needs to be dict".to_string(),
+            ));
+        }
+    }
+
+    let tx = PushIsolationLevel::Transaction(tx);
    let (uuid, tx) = push(
+        &db,
        tx,
        w_id,
        payload,
@@ -689,6 +858,82 @@ async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c
    return Ok(tx);
 }

+async fn handle_on_recovery<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c>(
+    db: &Pool<Postgres>,
+    tx: QueueTransaction<'c, R>,
+    schedule_path: &str,
+    script_path: &str,
+    is_flow: bool,
+    w_id: &str,
+    on_recovery_path: &str,
+    error_job: CompletedJobSubset,
+    successful_job_result: &serde_json::Value,
+    successful_times: i32,
+    successful_job_started_at: DateTime<Utc>,
+    extra_args: Option<serde_json::Value>,
+    username: &str,
+    email: &str,
+    permissioned_as: String,
+) -> windmill_common::error::Result<QueueTransaction<'c, R>> {
+    let (payload, tag) = get_payload_tag_from_prefixed_path(on_recovery_path, db, w_id).await?;
+
+    let mut args = error_job
+        .result
+        .unwrap_or(json!({}))
+        .as_object()
+        .unwrap()
+        .clone();
+    args.insert("error_started_at".to_string(), json!(error_job.started_at));
+    args.insert("schedule_path".to_string(), json!(schedule_path));
+    args.insert("path".to_string(), json!(script_path));
+    args.insert("is_flow".to_string(), json!(is_flow));
+    args.insert("success_result".to_string(), successful_job_result.clone());
+    args.insert("success_times".to_string(), json!(successful_times));
+    args.insert(
+        "success_started_at".to_string(),
+        json!(successful_job_started_at),
+    );
+    if let Some(args_v) = extra_args {
+        if let serde_json::Value::Object(args_m) = args_v {
+            args.extend(args_m);
+        } else {
+            return Err(error::Error::ExecutionErr(
+                "args of scripts needs to be dict".to_string(),
+            ));
+        }
+    }
+    let tx = PushIsolationLevel::Transaction(tx);
+    let (uuid, tx) = push(
+        &db,
+        tx,
+        w_id,
+        payload,
+        args,
+        username,
+        email,
+        permissioned_as,
+        None,
+        None,
+        None,
+        None,
+        None,
+        false,
+        false,
+        None,
+        true,
+        tag,
+        None,
+        None,
+    )
+    .await?;
+    tracing::info!(
+        "Pushed on_recovery job {} for {} to queue",
+        uuid,
+        schedule_path
+    );
+    return Ok(tx);
+}
+
 pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
    db: &Pool<Postgres>,
    rsmq: Option<R>,
@@ -733,16 +978,17 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
            FROM
                (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as completed_count
                FROM completed_job
-                WHERE script_path = $1 AND started_at + INTERVAL '1 MILLISECOND' * duration_ms > (now() - INTERVAL '1 second' * $2)
+                WHERE script_path = $1 AND started_at + INTERVAL '1 MILLISECOND' * duration_ms > (now() - INTERVAL '1 second' * $2) AND workspace_id = $3
                GROUP BY script_path) as j
            FULL OUTER JOIN
                (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as running_count
                FROM queue
-                WHERE script_path = $1 AND running = true
+                WHERE script_path = $1 AND running = true AND workspace_id = $3
                GROUP BY script_path) as q
            ON q.script_path = j.script_path",
            job_script_path,
            f64::from(job_custom_concurrency_time_window_s),
+            &pulled_job.workspace_id
        )
        .fetch_one(&mut tx)
        .await
@@ -1004,9 +1250,40 @@ pub async fn get_queued_job<'c>(
    Ok(r)
 }

+pub enum PushIsolationLevel<'c, R: rsmq_async::RsmqConnection + Send + 'c> {
+    IsolatedRoot(DB, Option<R>),
+    Isolated(UserDB, Authed, Option<R>),
+    Transaction(QueueTransaction<'c, R>),
+}
+
+#[macro_export]
+macro_rules! fetch_scalar_isolated {
+    ( $query:expr, $tx:expr) => {
+        match $tx {
+            PushIsolationLevel::IsolatedRoot(db, rmsq) => {
+                let r = $query.fetch_optional(&db).await;
+                $tx = PushIsolationLevel::IsolatedRoot(db, rmsq);
+                r
+            }
+            PushIsolationLevel::Isolated(db, user, rsmq) => {
+                let mut ntx = db.clone().begin(&user).await?;
+                let r = $query.fetch_optional(&mut *ntx).await;
+                $tx = PushIsolationLevel::Isolated(db, user, rsmq);
+                r
+            }
+            PushIsolationLevel::Transaction(mut tx) => {
+                let r = $query.fetch_optional(&mut tx).await;
+                $tx = PushIsolationLevel::Transaction(tx);
+                r
+            }
+        }
+    };
+}
+
 // #[instrument(level = "trace", skip_all)]
 pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
-    mut tx: QueueTransaction<'c, R>,
+    _db: &Pool<Postgres>,
+    mut tx: PushIsolationLevel<'c, R>,
    workspace_id: &str,
    job_payload: JobPayload,
    args: serde_json::Map<String, serde_json::Value>,
@@ -1027,30 +1304,12 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
    flow_step_id: Option<String>,
 ) -> Result<(Uuid, QueueTransaction<'c, R>), Error> {
    let args_json = serde_json::Value::Object(args);
-    let job_id: Uuid = if let Some(job_id) = job_id {
-        let conflicting_id = sqlx::query_scalar!(
-            "SELECT 1 FROM queue WHERE id = $1 UNION ALL select 1 FROM completed_job WHERE id = $1",
-            job_id
-        )
-        .fetch_optional(&mut tx)
-        .await?;
-
-        if conflicting_id.is_some() {
-            return Err(Error::BadRequest(format!(
-                "Job with id {job_id} already exists"
-            )));
-        }
-
-        job_id
-    } else {
-        Ulid::new().into()
-    };

    #[cfg(feature = "enterprise")]
    {
        let premium_workspace = *CLOUD_HOSTED
            && sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", workspace_id)
-                .fetch_one(&mut tx)
+                .fetch_one(_db)
                .await
                .map_err(|e| {
                    Error::InternalErr(format!("fetching if {workspace_id} is premium: {e}"))
@@ -1069,7 +1328,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
                    if premium_workspace { workspace_id } else { email },
                    premium_workspace
                )
-                .fetch_one(&mut tx)
+                .fetch_one(_db)
                .await
                .map_err(|e| Error::InternalErr(format!("updating usage: {e}")))?
        } else if *CLOUD_HOSTED && !premium_workspace {
@@ -1081,7 +1340,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
     AND id = $1",
                email
            )
-            .fetch_optional(&mut tx)
+            .fetch_optional(_db)
            .await?
            .flatten()
            .unwrap_or(0)
@@ -1092,7 +1351,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
        if *CLOUD_HOSTED && !premium_workspace {
            let is_super_admin =
                sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
-                    .fetch_optional(&mut tx)
+                    .fetch_optional(_db)
                    .await?
                    .unwrap_or(false);

@@ -1107,7 +1366,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
                }
                let in_queue =
                    sqlx::query_scalar!("SELECT COUNT(id) FROM queue WHERE email = $1", email)
-                        .fetch_one(&mut tx)
+                        .fetch_one(_db)
                        .await?
                        .unwrap_or(0);

@@ -1121,7 +1380,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
                    "SELECT COUNT(id) FROM queue WHERE running = true AND email = $1",
                    email
                )
-                .fetch_one(&mut tx)
+                .fetch_one(_db)
                .await?
                .unwrap_or(0);

@@ -1145,16 +1404,15 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
        concurrency_time_window_s,
    ) = match job_payload {
        JobPayload::ScriptHash { hash, path, concurrent_limit, concurrency_time_window_s } => {
-            let language = sqlx::query_scalar!(
+            let language = fetch_scalar_isolated!(sqlx::query_scalar!(
                    "SELECT language as \"language: ScriptLang\" FROM script WHERE hash = $1 AND workspace_id = $2",
                    hash.0,
                    workspace_id
-                )
-                .fetch_one(&mut tx)
-                .await
-                .map_err(|e| {
+                ), tx)
+                .ok().flatten()
+                .ok_or_else(||{
                    Error::InternalErr(format!(
-                        "fetching language for hash {hash} in {workspace_id}: {e}"
+                        "fetching language for hash {hash} in {workspace_id}"
                    ))
                })?;
            (
@@ -1209,13 +1467,14 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
            None,
        ),
        JobPayload::FlowDependencies { path } => {
-            let value_json = sqlx::query_scalar!(
-                "SELECT value FROM flow WHERE path = $1 AND workspace_id = $2",
-                path,
-                workspace_id
-            )
-            .fetch_optional(&mut tx)
-            .await?
+            let value_json = fetch_scalar_isolated!(
+                sqlx::query_scalar!(
+                    "SELECT value FROM flow WHERE path = $1 AND workspace_id = $2",
+                    path,
+                    workspace_id
+                ),
+                tx
+            )?
            .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", path)))?;
            let value = serde_json::from_value::<FlowValue>(value_json).map_err(|err| {
                Error::InternalErr(format!(
@@ -1244,13 +1503,14 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
            None,
        ),
        JobPayload::Flow(flow) => {
-            let value_json = sqlx::query_scalar!(
-                "SELECT value FROM flow WHERE path = $1 AND workspace_id = $2",
-                flow,
-                workspace_id
-            )
-            .fetch_optional(&mut tx)
-            .await?
+            let value_json = fetch_scalar_isolated!(
+                sqlx::query_scalar!(
+                    "SELECT value FROM flow WHERE path = $1 AND workspace_id = $2",
+                    flow,
+                    workspace_id
+                ),
+                tx
+            )?
            .ok_or_else(|| Error::InternalErr(format!("not found flow at path {:?}", flow)))?;
            let value = serde_json::from_value::<FlowValue>(value_json).map_err(|err| {
                Error::InternalErr(format!(
@@ -1349,6 +1609,33 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
        })
    };

+    let mut tx = match tx {
+        PushIsolationLevel::Isolated(user_db, authed, rsmq) => {
+            (rsmq, user_db.begin(&authed).await?).into()
+        }
+        PushIsolationLevel::IsolatedRoot(db, rsmq) => (rsmq, db.begin().await?).into(),
+        PushIsolationLevel::Transaction(tx) => tx,
+    };
+
+    let job_id: Uuid = if let Some(job_id) = job_id {
+        let conflicting_id = sqlx::query_scalar!(
+            "SELECT 1 FROM queue WHERE id = $1 UNION ALL select 1 FROM completed_job WHERE id = $1",
+            job_id
+        )
+        .fetch_optional(&mut tx)
+        .await?;
+
+        if conflicting_id.is_some() {
+            return Err(Error::BadRequest(format!(
+                "Job with id {job_id} already exists"
+            )));
+        }
+
+        job_id
+    } else {
+        Ulid::new().into()
+    };
+
    let uuid = sqlx::query_scalar!(
        "INSERT INTO queue
            (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, 
@@ -7,11 +7,13 @@
 */

 use crate::push;
+use crate::PushIsolationLevel;
 use crate::QueueTransaction;
 use sqlx::{query_scalar, Postgres, Transaction};
 use std::str::FromStr;
 use windmill_common::jobs::JobPayload;
 use windmill_common::schedule::schedule_to_user;
+use windmill_common::DB;
 use windmill_common::{
    error::{self, Result},
    schedule::Schedule,
@@ -20,6 +22,7 @@ use windmill_common::{
 };

 pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
+    db: &DB,
    mut tx: QueueTransaction<'c, R>,
    schedule: Schedule,
 ) -> Result<QueueTransaction<'c, R>> {
@@ -96,8 +99,9 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
    )
    .execute(&mut tx)
    .await?;
-
+    let tx = PushIsolationLevel::Transaction(tx);
    let (_, tx) = push(
+        &db,
        tx,
        &schedule.workspace_id,
        payload,
@@ -3,6 +3,7 @@ use deno_fetch::FetchPermissions;
 use deno_web::{BlobStore, TimersPermission};
 use std::env;
 use std::path::PathBuf;
+use std::sync::Arc;

 pub struct PermissionsContainer;

@@ -48,7 +49,10 @@ fn main() {
        deno_webidl::deno_webidl::init_ops_and_esm(),
        deno_url::deno_url::init_ops_and_esm(),
        deno_console::deno_console::init_ops_and_esm(),
-        deno_web::deno_web::init_ops_and_esm::<PermissionsContainer>(BlobStore::default(), None),
+        deno_web::deno_web::init_ops_and_esm::<PermissionsContainer>(
+            Arc::new(BlobStore::default()),
+            None,
+        ),
        deno_fetch::deno_fetch::init_ops_and_esm::<PermissionsContainer>(Default::default()),
        fetch::init_ops_and_esm(),
    ];
@@ -65,7 +69,7 @@ fn main() {
            startup_snapshot: None,
            extensions: exts,
            compression_cb: None,
-            snapshot_module_load_cb: None,
+            with_runtime_cb: None,
        },
    );
 }
@@ -5,11 +5,13 @@ const bo = await Bun.build({
  external: ["*"],
 });

+const fs = require("fs/promises");
+
 if (!bo.success) {
  bo.logs.forEach((l) => console.log(l));
  process.exit(1);
 } else {
-  let content = await Bun.file("./out/main.js").text();
+  let content = await fs.readFile("./out/main.js", { encoding: "utf8" });
  const imports = new Bun.Transpiler().scanImports(content);

  const { intersect } = require("semver-intersect");
@@ -86,6 +86,20 @@ mount {
    is_bind: true
 }

+mount {
+    src: "{JOB_DIR}/result.json"
+    dst: "/tmp/result.json"
+    rw: true
+    is_bind: true
+}
+
+mount {
+    src: "{JOB_DIR}/result.out"
+    dst: "/tmp/result.out"
+    rw: true
+    is_bind: true
+}
+
 iface_no_lo: true

 {SHARED_MOUNT}
@@ -67,7 +67,7 @@ mount {
    src: "{JOB_DIR}/bun.lockb"
    dst: "/tmp/bun/bun.lockb"
    is_bind: true
-    mandatory: true
+    mandatory: false
 }


@@ -75,7 +75,7 @@ mount {
    src: "{JOB_DIR}/node_modules"
    dst: "/tmp/bun/node_modules"
    is_bind: true
-    mandatory: true
+    mandatory: false
 }


@@ -0,0 +1,269 @@
+use std::{collections::HashMap, process::Stdio};
+
+use itertools::Itertools;
+use regex::Regex;
+use serde_json::{json, Value};
+use tokio::process::Command;
+use windmill_common::{error::Error, jobs::QueuedJob};
+
+const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto");
+
+use crate::{
+    common::{
+        get_reserved_variables, handle_child, read_file, read_file_content, set_logs,
+        transform_json_value, write_file,
+    },
+    AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
+};
+
+lazy_static::lazy_static! {
+
+    pub static ref ANSI_ESCAPE_RE: Regex = Regex::new(r"\x1b\[[0-9;]*m").unwrap();
+}
+
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn handle_bash_job(
+    logs: &mut String,
+    job: &QueuedJob,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    client: &AuthedClientBackgroundTask,
+    content: &str,
+    job_dir: &str,
+    shared_mount: &str,
+    base_internal_url: &str,
+    worker_name: &str,
+    envs: HashMap<String, String>,
+) -> Result<serde_json::Value, Error> {
+    logs.push_str("\n\n--- BASH CODE EXECUTION ---\n");
+    set_logs(logs, &job.id, db).await;
+    write_file(
+        job_dir,
+        "main.sh",
+        &format!("set -e\n{content}\necho \"\"\nsleep 0.02"),
+    )
+    .await?;
+    let token = client.get_token().await;
+    let mut reserved_variables = get_reserved_variables(job, &token, db).await?;
+    reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
+
+    let client = client.get_authed().await;
+    let hm = match transform_json_value(
+        "args",
+        &client,
+        &job.workspace_id,
+        job.args.clone().unwrap_or_else(|| json!({})),
+    )
+    .await?
+    {
+        Value::Object(ref hm) => hm.clone(),
+        _ => serde_json::Map::new(),
+    };
+    let args_owned = windmill_parser_bash::parse_bash_sig(&content)?
+        .args
+        .iter()
+        .map(|arg| {
+            hm.get(&arg.name)
+                .and_then(|v| match v {
+                    Value::String(s) => Some(s.clone()),
+                    _ => serde_json::to_string(v).ok(),
+                })
+                .unwrap_or_else(String::new)
+        })
+        .collect::<Vec<String>>();
+    let args = args_owned.iter().map(|s| &s[..]).collect::<Vec<&str>>();
+    let _ = write_file(job_dir, "result.json", "").await?;
+    let _ = write_file(job_dir, "result.out", "").await?;
+
+    let child = if !*DISABLE_NSJAIL {
+        let _ = write_file(
+            job_dir,
+            "run.config.proto",
+            &NSJAIL_CONFIG_RUN_BASH_CONTENT
+                .replace("{JOB_DIR}", job_dir)
+                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
+                .replace("{SHARED_MOUNT}", shared_mount),
+        )
+        .await?;
+        let mut cmd_args = vec!["--config", "run.config.proto", "--", "/bin/bash", "main.sh"];
+        cmd_args.extend(args);
+        Command::new(NSJAIL_PATH.as_str())
+            .current_dir(job_dir)
+            .env_clear()
+            .envs(reserved_variables)
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .args(cmd_args)
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?
+    } else {
+        let mut cmd_args = vec!["main.sh"];
+        cmd_args.extend(&args);
+        Command::new("/bin/bash")
+            .current_dir(job_dir)
+            .env_clear()
+            .envs(envs)
+            .envs(reserved_variables)
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .env("HOME", HOME_ENV.as_str())
+            .args(cmd_args)
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?
+    };
+    handle_child(
+        &job.id,
+        db,
+        logs,
+        child,
+        !*DISABLE_NSJAIL,
+        worker_name,
+        &job.workspace_id,
+        "bash run",
+        job.timeout,
+    )
+    .await?;
+
+    let result_json_path = format!("{job_dir}/result.json");
+    if let Ok(metadata) = tokio::fs::metadata(&result_json_path).await {
+        if metadata.len() > 0 {
+            return Ok(read_file(&result_json_path).await?);
+        }
+    }
+
+    let result_out_path = format!("{job_dir}/result.out");
+    if let Ok(metadata) = tokio::fs::metadata(&result_out_path).await {
+        if metadata.len() > 0 {
+            let result = read_file_content(&result_out_path).await?;
+            return Ok(json!(result));
+        }
+    }
+
+    //for now bash jobs have an empty result object
+    let last_line = serde_json::json!(logs
+        .lines()
+        .last()
+        .map(|x| ANSI_ESCAPE_RE.replace_all(x, "").to_string())
+        .unwrap_or_else(String::new));
+    Ok(last_line)
+}
+
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn handle_powershell_job(
+    logs: &mut String,
+    job: &QueuedJob,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    client: &AuthedClientBackgroundTask,
+    content: &str,
+    job_dir: &str,
+    shared_mount: &str,
+    base_internal_url: &str,
+    worker_name: &str,
+    envs: HashMap<String, String>,
+) -> Result<serde_json::Value, Error> {
+    logs.push_str("\n\n--- POWERSHELL CODE EXECUTION ---\n");
+    set_logs(logs, &job.id, db).await;
+    let pwsh_args = {
+        let client = client.get_authed().await;
+        let hm = match transform_json_value(
+            "args",
+            &client,
+            &job.workspace_id,
+            job.args.clone().unwrap_or_else(|| json!({})),
+        )
+        .await?
+        {
+            Value::Object(ref hm) => hm.clone(),
+            _ => serde_json::Map::new(),
+        };
+
+        let args_owned = windmill_parser_bash::parse_powershell_sig(&content)?
+            .args
+            .iter()
+            .map(|arg| {
+                (
+                    arg.name.clone(),
+                    hm.get(&arg.name)
+                        .and_then(|v| match v {
+                            Value::String(s) => Some(s.clone()),
+                            _ => serde_json::to_string(v).ok(),
+                        })
+                        .unwrap_or_else(String::new),
+                )
+            })
+            .collect::<Vec<(String, String)>>();
+        args_owned
+            .iter()
+            .map(|(n, v)| format!("--{n} {v}"))
+            .join(" ")
+    };
+
+    let content = content
+        .replace('$', r"\$") // escape powershell variables
+        .replace("`", r"\`"); // escape powershell backticks
+
+    write_file(job_dir, "main.sh", &format!("set -e\ncat > script.ps1 << EOF\n{content}\nEOF\npwsh -File script.ps1 {pwsh_args}\necho \"\"\nsleep 0.02")).await?;
+    let token = client.get_token().await;
+    let mut reserved_variables = get_reserved_variables(job, &token, db).await?;
+    reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
+
+    let _ = write_file(job_dir, "result.json", "").await?;
+    let _ = write_file(job_dir, "result.out", "").await?;
+
+    let child = if !*DISABLE_NSJAIL {
+        let _ = write_file(
+            job_dir,
+            "run.config.proto",
+            &NSJAIL_CONFIG_RUN_BASH_CONTENT
+                .replace("{JOB_DIR}", job_dir)
+                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
+                .replace("{SHARED_MOUNT}", shared_mount),
+        )
+        .await?;
+        let cmd_args = vec!["--config", "run.config.proto", "--", "/bin/bash", "main.sh"];
+        Command::new(NSJAIL_PATH.as_str())
+            .current_dir(job_dir)
+            .env_clear()
+            .envs(reserved_variables)
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .args(cmd_args)
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?
+    } else {
+        let cmd_args = vec!["main.sh"];
+        Command::new("/bin/bash")
+            .current_dir(job_dir)
+            .env_clear()
+            .envs(envs)
+            .envs(reserved_variables)
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .env("HOME", HOME_ENV.as_str())
+            .args(cmd_args)
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?
+    };
+    handle_child(
+        &job.id,
+        db,
+        logs,
+        child,
+        !*DISABLE_NSJAIL,
+        worker_name,
+        &job.workspace_id,
+        "bash/powershell run",
+        job.timeout,
+    )
+    .await?;
+
+    let last_line = serde_json::json!(logs
+        .lines()
+        .last()
+        .map(|x| ANSI_ESCAPE_RE.replace_all(x, "").to_string())
+        .unwrap_or_else(String::new));
+    Ok(last_line)
+}
@@ -6,7 +6,7 @@ use windmill_queue::HTTP_CLIENT;

 use serde::Deserialize;

-use crate::{transform_json_value, AuthedClient, JobCompleted};
+use crate::{common::transform_json_value, AuthedClient, JobCompleted};

 use gcp_auth::{AuthenticationManager, CustomServiceAccount};

@@ -5,10 +5,12 @@ use itertools::Itertools;
 use uuid::Uuid;

 use crate::{
-    common::{read_result, set_logs},
-    create_args_and_out_file, get_reserved_variables, handle_child, write_file, write_file_binary,
+    common::{
+        create_args_and_out_file, get_reserved_variables, handle_child, read_result, set_logs,
+        write_file, write_file_binary,
+    },
    AuthedClientBackgroundTask, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER,
-    NPM_CONFIG_REGISTRY, NSJAIL_CONFIG_RUN_BUN_CONTENT, NSJAIL_PATH, PATH_ENV,
+    NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV,
 };
 use tokio::{fs::File, io::AsyncReadExt, process::Command};
 use windmill_common::error::Result;
@@ -22,7 +24,10 @@ const RELATIVE_BUN_LOADER: &str = include_str!("../loader.bun.ts");

 const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.ts");

+const NSJAIL_CONFIG_RUN_BUN_CONTENT: &str = include_str!("../nsjail/run.bun.config.proto");
+
 const BUN_LOCKB_SPLIT: &str = "\n//bun.lockb\n";
+const EMPTY_FILE: &str = "<empty>";

 pub async fn gen_lockfile(
    logs: &mut String,
@@ -100,10 +105,15 @@ pub async fn gen_lockfile(
        }
        content.push_str(BUN_LOCKB_SPLIT);
        {
-            let mut file = File::open(format!("{job_dir}/bun.lockb")).await?;
-            let mut buf = vec![];
-            file.read_to_end(&mut buf).await?;
-            content.push_str(&base64::engine::general_purpose::STANDARD.encode(&buf));
+            let file = format!("{job_dir}/bun.lockb");
+            if tokio::fs::metadata(&file).await.is_ok() {
+                let mut file = File::open(&file).await?;
+                let mut buf = vec![];
+                file.read_to_end(&mut buf).await?;
+                content.push_str(&base64::engine::general_purpose::STANDARD.encode(&buf));
+            } else {
+                content.push_str(&EMPTY_FILE);
+            }
        }
        Ok(Some(content))
    } else {
@@ -170,20 +180,23 @@ pub async fn handle_bun_job(
        let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::<Vec<&str>>();
        if splitted.len() != 2 {
            return Err(error::Error::ExecutionErr(
-                "Invalid requirements, expectd to find //bun.lockb split pattern in reqs"
-                    .to_string(),
+                format!("Invalid requirements, expectd to find //bun.lockb split pattern in reqs. Found: |{reqs}|")
            ));
        }
        let _ = write_file(job_dir, "package.json", &splitted[0]).await?;
-        let _ = write_file_binary(
-            job_dir,
-            "bun.lockb",
-            &base64::engine::general_purpose::STANDARD
-                .decode(&splitted[1])
-                .map_err(|_| error::Error::InternalErr("Could not decode bun.lockb".to_string()))?,
-        )
-        .await?;
-
+        let lockb = splitted[1];
+        if lockb != EMPTY_FILE {
+            let _ = write_file_binary(
+                job_dir,
+                "bun.lockb",
+                &base64::engine::general_purpose::STANDARD
+                    .decode(&splitted[1])
+                    .map_err(|_| {
+                        error::Error::InternalErr("Could not decode bun.lockb".to_string())
+                    })?,
+            )
+            .await?;
+        }
        install_lockfile(
            logs,
            &job.id,
@@ -194,7 +207,7 @@ pub async fn handle_bun_job(
            common_bun_proc_envs.clone(),
        )
        .await?;
-    } else {
+    } else if !*DISABLE_NSJAIL {
        logs.push_str("\n\n--- BUN INSTALL ---\n");
        let _ = gen_lockfile(
            logs,
@@ -210,6 +223,7 @@ pub async fn handle_bun_job(
        )
        .await?;
    }
+
    // let mut start = Instant::now();
    logs.push_str("\n\n--- BUN CODE EXECUTION ---\n");
    set_logs(&logs, &job.id, &db).await;
@@ -325,17 +339,25 @@ plugin(p)
                "--",
                &BUN_PATH,
                "run",
+                "-i",
+                "--prefer-offline",
                "-r",
                "/tmp/bun/loader.bun.ts",
                "/tmp/bun/wrapper.ts",
-                "--no-install",
            ])
            .stdout(Stdio::piped())
            .stderr(Stdio::piped())
            .spawn()?
    } else {
        let script_path = format!("{job_dir}/wrapper.ts");
-        let args = vec!["run", "-r", "./loader.bun.ts", &script_path, "--no-install"];
+        let args = vec![
+            "run",
+            "-i",
+            "--prefer-offline",
+            "-r",
+            "./loader.bun.ts",
+            &script_path,
+        ];
        Command::new(&*BUN_PATH)
            .current_dir(job_dir)
            .env_clear()
@@ -369,6 +391,7 @@ plugin(p)
 fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String, String> {
    let mut deno_envs: HashMap<String, String> = HashMap::from([
        (String::from("PATH"), PATH_ENV.clone()),
+        (String::from("DISABLE_COLORS"), "0".to_string()),
        (String::from("DO_NOT_TRACK"), "1".to_string()),
        (
            String::from("BASE_URL"),
@@ -1,14 +1,124 @@
+use async_recursion::async_recursion;
+use serde_json::{json, Value};
 use sqlx::{Pool, Postgres};
 use tokio::{fs::File, io::AsyncReadExt};
-use windmill_common::error::{self};
+use windmill_common::{
+    error::{self, Error},
+    jobs::QueuedJob,
+};
 use windmill_queue::CLOUD_HOSTED;

-use crate::MAX_RESULT_SIZE;
+use anyhow::Result;
+use std::{
+    borrow::Borrow, collections::HashMap, io, os::unix::process::ExitStatusExt, panic,
+    time::Duration,
+};

-pub async fn read_result(job_dir: &str) -> error::Result<serde_json::Value> {
-    let mut file = File::open(format!("{job_dir}/result.json")).await?;
+use tracing::{trace_span, Instrument};
+use uuid::Uuid;
+use windmill_common::variables;
+
+use tokio::{
+    io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
+    process::Child,
+    sync::{broadcast, watch},
+    time::{interval, sleep, Instant, MissedTickBehavior},
+};
+
+use futures::{
+    future::{self, ready, FutureExt},
+    stream, StreamExt,
+};
+
+use crate::{AuthedClient, MAX_RESULT_SIZE, TIMEOUT_DURATION, WHITELIST_ENVS};
+
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn create_args_and_out_file(
+    client: &AuthedClient,
+    job: &QueuedJob,
+    job_dir: &str,
+) -> Result<(), Error> {
+    let args = if let Some(args) = &job.args {
+        Some(transform_json_value("args", client, &job.workspace_id, args.clone()).await?)
+    } else {
+        None
+    };
+    let ser_args = serde_json::to_string(&args).map_err(|e| Error::ExecutionErr(e.to_string()))?;
+    write_file(job_dir, "args.json", &ser_args).await?;
+    write_file(job_dir, "result.json", "").await?;
+    Ok(())
+}
+
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn write_file(dir: &str, path: &str, content: &str) -> error::Result<File> {
+    let path = format!("{}/{}", dir, path);
+    let mut file = File::create(&path).await?;
+    file.write_all(content.as_bytes()).await?;
+    file.flush().await?;
+    Ok(file)
+}
+
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn write_file_binary(dir: &str, path: &str, content: &[u8]) -> error::Result<File> {
+    let path = format!("{}/{}", dir, path);
+    let mut file = File::create(&path).await?;
+    file.write_all(content).await?;
+    file.flush().await?;
+    Ok(file)
+}
+
+#[async_recursion]
+pub async fn transform_json_value(
+    name: &str,
+    client: &AuthedClient,
+    workspace: &str,
+    v: Value,
+) -> error::Result<Value> {
+    match v {
+        Value::String(y) if y.starts_with("$var:") => {
+            let path = y.strip_prefix("$var:").unwrap();
+            client
+                .get_client()
+                .get_variable_value(workspace, path)
+                .await
+                .map_err(|_| Error::NotFound(format!("Variable {path} not found for `{name}`")))
+                .map(|v| json!(v.into_inner()))
+        }
+        Value::String(y) if y.starts_with("$res:") => {
+            let path = y.strip_prefix("$res:").unwrap();
+            if path.split("/").count() < 2 {
+                return Err(Error::InternalErr(format!(
+                    "Argument `{name}` is an invalid resource path: {path}",
+                )));
+            }
+            Ok(client
+                .get_client()
+                .get_resource_value_interpolated(workspace, path)
+                .await
+                .map_err(|_| Error::NotFound(format!("Resource {path} not found for `{name}`")))?
+                .into_inner())
+        }
+        Value::Object(mut m) => {
+            for (a, b) in m.clone().into_iter() {
+                m.insert(
+                    a.clone(),
+                    transform_json_value(&a, client, workspace, b).await?,
+                );
+            }
+            Ok(Value::Object(m))
+        }
+        a @ _ => Ok(a),
+    }
+}
+
+pub async fn read_file_content(path: &str) -> error::Result<String> {
+    let mut file = File::open(path).await?;
    let mut content = "".to_string();
    file.read_to_string(&mut content).await?;
+    Ok(content)
+}
+pub async fn read_file(path: &str) -> error::Result<serde_json::Value> {
+    let content = read_file_content(path).await?;
    if *CLOUD_HOSTED && content.len() > MAX_RESULT_SIZE {
        return Err(error::Error::ExecutionErr("Result is too large for the cloud app (limit 2MB). 
        If using this script as part of the flow, use the shared folder to pass heavy data between steps.".to_owned()));
@@ -16,6 +126,9 @@ pub async fn read_result(job_dir: &str) -> error::Result<serde_json::Value> {
    serde_json::from_str(&content)
        .map_err(|e| error::Error::ExecutionErr(format!("Error parsing result: {e}")))
 }
+pub async fn read_result(job_dir: &str) -> error::Result<serde_json::Value> {
+    return read_file(&format!("{job_dir}/result.json")).await;
+}

 #[tracing::instrument(level = "trace", skip_all)]
 pub async fn set_logs(logs: &str, id: &uuid::Uuid, db: &Pool<Postgres>) {
@@ -39,3 +152,435 @@ pub fn capitalize(s: &str) -> String {
        Some(f) => f.to_uppercase().collect::<String>() + c.as_str(),
    }
 }
+
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn get_reserved_variables(
+    job: &QueuedJob,
+    token: &str,
+    db: &sqlx::Pool<sqlx::Postgres>,
+) -> Result<HashMap<String, String>, Error> {
+    let flow_path = if let Some(uuid) = job.parent_job {
+        sqlx::query_scalar!("SELECT script_path FROM queue WHERE id = $1", uuid)
+            .fetch_optional(db)
+            .await?
+            .flatten()
+    } else {
+        None
+    };
+
+    let variables = variables::get_reserved_variables(
+        &job.workspace_id,
+        token,
+        &job.email,
+        &job.created_by,
+        &job.id.to_string(),
+        &job.permissioned_as,
+        job.script_path.clone(),
+        job.parent_job.map(|x| x.to_string()),
+        flow_path,
+        job.schedule_path.clone(),
+        job.flow_step_id.clone(),
+    )
+    .to_vec();
+
+    let mut r: HashMap<String, String> = variables
+        .into_iter()
+        .map(|rv| (rv.name, rv.value))
+        .collect();
+
+    if let Some(ref envs) = *WHITELIST_ENVS {
+        for e in envs {
+            r.insert(e.0.clone(), e.1.clone());
+        }
+    }
+
+    Ok(r)
+}
+
+async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
+    if pid.is_none() {
+        return -1;
+    }
+    let pid = if nsjail {
+        // This is a bit hacky, but the process id of the nsjail process is the pid of nsjail + 1.
+        // Ideally, we would get the number from fork() itself. This works in MOST cases.
+        pid.unwrap() + 1
+    } else {
+        pid.unwrap()
+    };
+
+    if let Ok(file) = File::open(format!("/proc/{}/status", pid)).await {
+        let mut lines = BufReader::new(file).lines();
+        while let Some(line) = lines.next_line().await.unwrap_or(None) {
+            if line.starts_with("VmHWM:") {
+                return line
+                    .split_whitespace()
+                    .nth(1)
+                    .and_then(|s| s.parse::<i32>().ok())
+                    .unwrap_or(-1);
+            };
+        }
+        -2
+    } else {
+        -3
+    }
+}
+/// - wait until child exits and return with exit status
+/// - read lines from stdout and stderr and append them to the "queue"."logs"
+///   quitting early if output exceedes MAX_LOG_SIZE characters (not bytes)
+/// - update the `last_line` and `logs` strings with the program output
+/// - update "queue"."last_ping" every five seconds
+/// - kill process if we exceed timeout or "queue"."canceled" is set
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn handle_child(
+    job_id: &Uuid,
+    db: &Pool<Postgres>,
+    logs: &mut String,
+    mut child: Child,
+    nsjail: bool,
+    worker_name: &str,
+    _w_id: &str,
+    child_name: &str,
+    custom_timeout: Option<i32>,
+) -> error::Result<()> {
+    let start = Instant::now();
+    let update_job_interval = Duration::from_millis(500);
+    let write_logs_delay = Duration::from_millis(500);
+
+    let pid = child.id();
+    #[cfg(target_os = "linux")]
+    if let Some(pid) = pid {
+        //set the highest oom priority
+        let mut file = File::create(format!("/proc/{pid}/oom_score_adj")).await?;
+        let _ = file.write_all(b"1000").await;
+    } else {
+        tracing::info!("could not get child pid");
+    }
+    let (set_too_many_logs, mut too_many_logs) = watch::channel::<bool>(false);
+    let (tx, mut rx) = broadcast::channel::<()>(3);
+    let mut rx2 = tx.subscribe();
+
+    let output = child_joined_output_stream(&mut child);
+
+    let job_id = job_id.clone();
+
+    /* the cancellation future is polled on by `wait_on_child` while
+     * waiting for the child to exit normally */
+    let update_job = async {
+        let db = db.clone();
+
+        let mut interval = interval(update_job_interval);
+        interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
+
+        let mut i = 1;
+        loop {
+            tokio::select!(
+                _ = rx.recv() => break,
+                _ = interval.tick() => {
+                    // update the last_ping column every 5 seconds
+                    i+=1;
+                    if i % 10 == 0 {
+                        sqlx::query!(
+                            "UPDATE worker_ping SET ping_at = now() WHERE worker = $1",
+                            &worker_name
+                        )
+                        .execute(&db)
+                        .await
+                        .expect("update worker ping");
+                    }
+                    let mem_peak = get_mem_peak(pid, nsjail).await;
+                    tracing::info!("{job_id} still running. mem peak: {}kB", mem_peak);
+                    let mem_peak = if mem_peak > 0 { Some(mem_peak) } else { None };
+                    if sqlx::query_scalar!("UPDATE queue SET mem_peak = GREATEST($1, mem_peak), last_ping = now() WHERE id = $2 RETURNING canceled", mem_peak, job_id)
+                        .fetch_optional(&db)
+                        .await
+                        .map(|v| Some(true) == v)
+                        .unwrap_or_else(|err| {
+                            tracing::error!(%job_id, %err, "error checking cancelation for job {job_id}: {err}");
+                            false
+                        })
+                    {
+                        break;
+                    }
+                },
+            );
+        }
+    };
+
+    #[derive(PartialEq, Debug)]
+    enum KillReason {
+        TooManyLogs,
+        Timeout,
+        Cancelled,
+    }
+    /* a future that completes when the child process exits */
+    let wait_on_child = async {
+        let db = db.clone();
+
+        #[cfg(not(feature = "enterprise"))]
+        let instance_timeout_duration = *TIMEOUT_DURATION;
+
+        #[cfg(feature = "enterprise")]
+        let premium_workspace = *CLOUD_HOSTED
+            && sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", _w_id)
+                .fetch_one(&db)
+                .await
+                .map_err(|e| {
+                    tracing::error!(%e, "error getting premium workspace for job {job_id}: {e}");
+                })
+                .unwrap_or(false);
+
+        #[cfg(feature = "enterprise")]
+        let instance_timeout_duration = if premium_workspace {
+            *TIMEOUT_DURATION * 6 //30mins
+        } else {
+            *TIMEOUT_DURATION
+        };
+
+        let timeout_duration = if let Some(custom_timeout) = custom_timeout {
+            Duration::min(
+                instance_timeout_duration,
+                Duration::from_secs(custom_timeout as u64),
+            )
+        } else {
+            instance_timeout_duration
+        };
+
+        let kill_reason = tokio::select! {
+            biased;
+            result = child.wait() => return result.map(Ok),
+            Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs,
+            _ = sleep(timeout_duration) => KillReason::Timeout,
+            _ = update_job => KillReason::Cancelled,
+        };
+        tx.send(()).expect("rx should never be dropped");
+        drop(tx);
+
+        let set_reason = async {
+            if kill_reason == KillReason::Timeout {
+                if let Err(err) = sqlx::query(
+                    r#"
+                       UPDATE queue
+                          SET canceled = true
+                            , canceled_by = 'timeout'
+                            , canceled_reason = $1
+                        WHERE id = $2
+                    "#,
+                )
+                .bind(format!("duration > {}", TIMEOUT_DURATION.as_secs()))
+                .bind(job_id)
+                .execute(&db)
+                .await
+                {
+                    tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}");
+                }
+            }
+        };
+
+        /* send SIGKILL and reap child process */
+        let (_, kill) = future::join(set_reason, child.kill()).await;
+        kill.map(|()| Err(kill_reason))
+    };
+
+    /* a future that reads output from the child and appends to the database */
+    let lines = async move {
+        let max_log_size = if *CLOUD_HOSTED {
+            MAX_RESULT_SIZE
+        } else {
+            usize::MAX
+        };
+        /* log_remaining is zero when output limit was reached */
+        let mut log_remaining = max_log_size.saturating_sub(logs.chars().count());
+        let mut result = io::Result::Ok(());
+        let mut output = output.take_until(rx2.recv()).boxed();
+        /* `do_write` resolves the task, but does not contain the Result.
+         * It's useful to know if the task completed. */
+        let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle();
+
+        while let Some(line) =  output.by_ref().next().await {
+
+            let do_write_ = do_write.shared();
+
+            let mut read_lines = stream::once(async { line })
+                .chain(output.by_ref())
+                /* after receiving a line, continue until some delay has passed
+                 * _and_ the previous database write is complete */
+                .take_until(future::join(sleep(write_logs_delay), do_write_.clone()))
+                .boxed();
+
+            /* Read up until an error is encountered,
+             * handle log lines first and then the error... */
+            let mut joined = String::new();
+
+            while let Some(line) = read_lines.next().await {
+
+                match line {
+                    Ok(_) if log_remaining == 0 => (),
+                    Ok(line) => {
+                        if line.is_empty() {
+                            continue;
+                        }
+                        append_with_limit(&mut joined, &line, &mut log_remaining);
+                        if log_remaining == 0 {
+                            tracing::info!(%job_id, "Too many logs lines for job {job_id}");
+                            let _ = set_too_many_logs.send(true);
+                            joined.push_str(&format!(
+                                "Job logs or result reached character limit of {MAX_RESULT_SIZE}; killing job."
+                            ));
+                            /* stop reading and drop our streams fairly quickly */
+                            break;
+                        }
+                    }
+                    Err(err) => {
+                        result = Err(err);
+                        break;
+                    }
+                }
+            }
+
+            logs.push_str(&joined);
+
+
+            /* Ensure the last flush completed before starting a new one.
+             *
+             * This shouldn't pause since `take_until()` reads lines until `do_write`
+             * resolves. We only stop reading lines before `take_until()` resolves if we reach
+             * EOF or a read error.  In those cases, waiting on a database query to complete is
+             * fine because we're done. */
+
+            if let Some(Ok(p)) = do_write_
+                .then(|()| write_result)
+                .await
+                .err()
+                .map(|err| err.try_into_panic())
+            {
+                panic::resume_unwind(p);
+            }
+
+            (do_write, write_result) = tokio::spawn(append_logs(job_id, joined, db.clone())).remote_handle();
+
+            if let Err(err) = result {
+                tracing::error!(%job_id, %err, "error reading output for job {job_id}: {err}");
+                break;
+            }
+
+            if *set_too_many_logs.borrow() {
+                break;
+            }
+
+        }
+
+        /* drop our end of the pipe */
+        drop(output);
+
+        if let Some(Ok(p)) = do_write
+            .then(|()| write_result)
+            .await
+            .err()
+            .map(|err| err.try_into_panic())
+        {
+            panic::resume_unwind(p);
+        }
+    }.instrument(trace_span!("child_lines"));
+
+    let (wait_result, _) = tokio::join!(wait_on_child, lines);
+
+    tracing::info!(%job_id, "child process '{child_name}' for {job_id} took {}ms", start.elapsed().as_millis());
+    match wait_result {
+        _ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!(
+            "logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)"
+        ))),
+        Ok(Ok(status)) => {
+            if status.success() {
+                Ok(())
+            } else if let Some(code) = status.code() {
+                Err(error::Error::ExitStatus(code))
+            } else {
+                Err(error::Error::ExecutionErr(format!(
+                    "process terminated by signal: {:#?}, stopped_signal: {:#?}, core_dumped: {}",
+                    status.signal(),
+                    status.stopped_signal(),
+                    status.core_dumped()
+                )))
+            }
+        }
+        Ok(Err(kill_reason)) => Err(Error::ExecutionErr(format!(
+            "job process killed because {kill_reason:#?}"
+        ))),
+        Err(err) => Err(Error::ExecutionErr(format!("job process io error: {err}"))),
+    }
+}
+
+/// takes stdout and stderr from Child, panics if either are not present
+///
+/// builds a stream joining both stdout and stderr each read line by line
+fn child_joined_output_stream(
+    child: &mut Child,
+) -> impl stream::FusedStream<Item = io::Result<String>> {
+    let stderr = child
+        .stderr
+        .take()
+        .expect("child did not have a handle to stdout");
+
+    let stdout = child
+        .stdout
+        .take()
+        .expect("child did not have a handle to stdout");
+
+    let stdout = BufReader::new(stdout).lines();
+    let stderr = BufReader::new(stderr).lines();
+    stream::select(lines_to_stream(stderr), lines_to_stream(stdout))
+}
+
+fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
+    mut lines: tokio::io::Lines<R>,
+) -> impl futures::Stream<Item = io::Result<String>> {
+    stream::poll_fn(move |cx| {
+        std::pin::Pin::new(&mut lines)
+            .poll_next_line(cx)
+            .map(|result| result.transpose())
+    })
+}
+
+// as a detail, `BufReader::lines()` removes \n and \r\n from the strings it yields,
+// so this pushes \n to thd destination string in each call
+fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) {
+    if *limit > 0 {
+        dst.push('\n');
+    }
+    *limit -= 1;
+
+    let src_len = src.chars().count();
+    if src_len <= *limit {
+        dst.push_str(&src);
+        *limit -= src_len;
+    } else {
+        let byte_pos = src
+            .char_indices()
+            .skip(*limit)
+            .next()
+            .map(|(byte_pos, _)| byte_pos)
+            .unwrap_or(0);
+        dst.push_str(&src[0..byte_pos]);
+        *limit = 0;
+    }
+}
+
+/* TODO retry this? */
+#[tracing::instrument(level = "trace", skip_all)]
+async fn append_logs(job_id: uuid::Uuid, logs: impl AsRef<str>, db: impl Borrow<Pool<Postgres>>) {
+    if logs.as_ref().is_empty() {
+        return;
+    }
+
+    if let Err(err) = sqlx::query!(
+        "UPDATE queue SET logs = concat(logs, $1::text) WHERE id = $2",
+        logs.as_ref(),
+        job_id,
+    )
+    .execute(db.borrow())
+    .await
+    {
+        tracing::error!(%job_id, %err, "error updating logs for job {job_id}: {err}");
+    }
+}
@@ -0,0 +1,258 @@
+use std::{collections::HashMap, process::Stdio};
+
+use itertools::Itertools;
+
+use crate::{
+    common::{
+        create_args_and_out_file, get_reserved_variables, handle_child, read_result, set_logs,
+        write_file,
+    },
+    AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, NPM_CONFIG_REGISTRY,
+    PATH_ENV,
+};
+use tokio::process::Command;
+use windmill_common::{error::Result, BASE_URL};
+use windmill_common::{
+    error::{self},
+    jobs::QueuedJob,
+};
+use windmill_parser::Typ;
+
+lazy_static::lazy_static! {
+
+    static ref DENO_FLAGS: Option<Vec<String>> = std::env::var("DENO_FLAGS")
+        .ok()
+        .map(|x| x.split(' ').map(|x| x.to_string()).collect());
+
+    static ref DENO_EXTRA_IMPORT_MAP: String = std::env::var("DENO_EXTRA_IMPORT_MAP")
+        .ok()
+        .map(|x| x.split(',').map(|x| {
+            let mut splitted = x.split("=");
+            let key = splitted.next().unwrap();
+            let value = splitted.next().unwrap();
+            format!(",\n \"{key}\": \"{value}\"")
+        }).join("\n")).unwrap_or_else(|| String::new());
+
+    static ref DENO_AUTH_TOKENS: String = std::env::var("DENO_AUTH_TOKENS")
+        .ok()
+        .map(|x| format!(";{x}"))
+        .unwrap_or_else(|| String::new());
+
+
+}
+fn get_common_deno_proc_envs(token: &str, base_internal_url: &str) -> HashMap<String, String> {
+    let hostname_base = BASE_URL.split("://").last().unwrap_or("localhost");
+    let hostname_internal = base_internal_url.split("://").last().unwrap_or("localhost");
+    let deno_auth_tokens_base = DENO_AUTH_TOKENS.as_str();
+    let deno_auth_tokens =
+        format!("{token}@{hostname_base};{token}@{hostname_internal}{deno_auth_tokens_base}",);
+
+    let mut deno_envs: HashMap<String, String> = HashMap::from([
+        (String::from("PATH"), PATH_ENV.clone()),
+        (String::from("DENO_AUTH_TOKENS"), deno_auth_tokens),
+        (
+            String::from("BASE_INTERNAL_URL"),
+            base_internal_url.to_string(),
+        ),
+    ]);
+
+    if let Some(ref s) = *NPM_CONFIG_REGISTRY {
+        deno_envs.insert(String::from("NPM_CONFIG_REGISTRY"), s.clone());
+    }
+    return deno_envs;
+}
+
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn handle_deno_job(
+    logs: &mut String,
+    job: &QueuedJob,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    client: &AuthedClientBackgroundTask,
+    job_dir: &str,
+    inner_content: &String,
+    base_internal_url: &str,
+    worker_name: &str,
+    envs: HashMap<String, String>,
+) -> error::Result<serde_json::Value> {
+    // let mut start = Instant::now();
+    logs.push_str("\n\n--- DENO CODE EXECUTION ---\n");
+
+    let logs_to_set = logs.clone();
+    let id = job.id.clone();
+    let db2 = db.clone();
+
+    let set_logs_f = async {
+        set_logs(&logs_to_set, &id, &db2).await;
+        Ok(()) as error::Result<()>
+    };
+
+    let write_main_f = write_file(job_dir, "main.ts", inner_content);
+
+    let write_wrapper_f = async {
+        // let mut start = Instant::now();
+        let args = windmill_parser_ts::parse_deno_signature(inner_content, true)?.args;
+        let dates = args
+            .iter()
+            .enumerate()
+            .filter_map(|(i, x)| {
+                if matches!(x.typ, Typ::Datetime) {
+                    Some(i)
+                } else {
+                    None
+                }
+            })
+            .map(|x| return format!("args[{x}] = args[{x}] ? new Date(args[{x}]) : undefined"))
+            .join("\n");
+
+        let spread = args.into_iter().map(|x| x.name).join(",");
+        // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str());
+        let wrapper_content: String = format!(
+            r#"
+import {{ main }} from "./main.ts";
+
+const args = await Deno.readTextFile("args.json")
+    .then(JSON.parse)
+    .then(({{ {spread} }}) => [ {spread} ])
+
+BigInt.prototype.toJSON = function () {{
+    return this.toString();
+}};
+
+{dates}
+async function run() {{
+    let res: any = await main(...args);
+    const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value);
+    await Deno.writeTextFile("result.json", res_json);
+    Deno.exit(0);
+}}
+run().catch(async (e) => {{
+    await Deno.writeTextFile("result.json", JSON.stringify({{ message: e.message, name: e.name, stack: e.stack }}));
+    Deno.exit(1);
+}});
+    "#,
+        );
+        write_file(job_dir, "wrapper.ts", &wrapper_content).await?;
+        Ok(()) as error::Result<()>
+    };
+
+    let write_import_map_f = async {
+        let w_id = job.workspace_id.clone();
+        let script_path_split = job.script_path().split("/");
+        let script_path_parts_len = script_path_split.clone().count();
+        let mut relative_mounts = "".to_string();
+        for c in 0..script_path_parts_len {
+            relative_mounts += ",\n          ";
+            relative_mounts += &format!(
+                "\"./{}\": \"{base_internal_url}/api/w/{w_id}/scripts/raw/p/{}{}\"",
+                (0..c).map(|_| "../").join(""),
+                &script_path_split
+                    .clone()
+                    .take(script_path_parts_len - c - 1)
+                    .join("/"),
+                if c == script_path_parts_len - 1 {
+                    ""
+                } else {
+                    "/"
+                },
+            );
+        }
+        let extra_import_map = DENO_EXTRA_IMPORT_MAP.as_str();
+        let import_map = format!(
+            r#"{{
+            "imports": {{
+              "{base_internal_url}/api/w/{w_id}/scripts/raw/p/": "{base_internal_url}/api/w/{w_id}/scripts/raw/p/",
+              "{base_internal_url}": "{base_internal_url}/api/w/{w_id}/scripts/raw/p/",
+              "/": "{base_internal_url}/api/w/{w_id}/scripts/raw/p/",
+              "./wrapper.ts": "./wrapper.ts",
+              "./main.ts": "./main.ts"{relative_mounts}
+              {extra_import_map}
+            }}
+          }}"#,
+        );
+        write_file(job_dir, "import_map.json", &import_map).await?;
+        Ok(()) as error::Result<()>
+    };
+
+    let reserved_variables_args_out_f = async {
+        let client = client.get_authed().await;
+        let args_and_out_f = async {
+            create_args_and_out_file(&client, job, job_dir).await?;
+            Ok(()) as Result<()>
+        };
+        let reserved_variables_f = async {
+            let mut vars = get_reserved_variables(job, &client.token, db).await?;
+            vars.insert("RUST_LOG".to_string(), "info".to_string());
+            Ok(vars) as Result<HashMap<String, String>>
+        };
+        let (_, reserved_variables) = tokio::try_join!(args_and_out_f, reserved_variables_f)?;
+        Ok((reserved_variables, client.token)) as error::Result<(HashMap<String, String>, String)>
+    };
+
+    let (_, (reserved_variables, token), _, _, _) = tokio::try_join!(
+        set_logs_f,
+        reserved_variables_args_out_f,
+        write_main_f,
+        write_wrapper_f,
+        write_import_map_f
+    )?;
+
+    let common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url);
+
+    //do not cache local dependencies
+    let reload = format!("--reload={base_internal_url}");
+    let child = {
+        let script_path = format!("{job_dir}/wrapper.ts");
+        let import_map_path = format!("{job_dir}/import_map.json");
+        let mut args = Vec::with_capacity(12);
+        args.push("run");
+        args.push("--no-check");
+        args.push("--import-map");
+        args.push(&import_map_path);
+        args.push(&reload);
+        args.push("--unstable");
+        if let Some(deno_flags) = DENO_FLAGS.as_ref() {
+            for flag in deno_flags {
+                args.push(flag);
+            }
+        } else if !*DISABLE_NSJAIL {
+            args.push("--allow-net");
+            args.push("--allow-read=./,/tmp/windmill/cache/deno/");
+            args.push("--allow-write=./");
+            args.push("--allow-env");
+        } else {
+            args.push("-A");
+        }
+        args.push(&script_path);
+        Command::new(DENO_PATH.as_str())
+            .current_dir(job_dir)
+            .env_clear()
+            .envs(envs)
+            .envs(reserved_variables)
+            .envs(common_deno_proc_envs)
+            .env("DENO_DIR", DENO_CACHE_DIR)
+            .args(args)
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?
+    };
+    // logs.push_str(format!("prepare: {:?}\n", start.elapsed().as_micros()).as_str());
+    // start = Instant::now();
+    handle_child(
+        &job.id,
+        db,
+        logs,
+        child,
+        false,
+        worker_name,
+        &job.workspace_id,
+        "deno run",
+        job.timeout,
+    )
+    .await?;
+    // logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str());
+    if let Err(e) = tokio::fs::remove_dir_all(format!("{DENO_CACHE_DIR}/gen/file/{job_dir}")).await
+    {
+        tracing::error!("failed to remove deno gen tmp cache dir: {}", e);
+    }
+    read_result(job_dir).await
+}
@@ -109,6 +109,7 @@ pub async fn pull_from_tar(bucket: &str, folder: String) -> error::Result<()> {
            "tar {folder_name} does not exist in bucket"
        )));
    }
+    // tracing::info!("B: {target} {folder}");

    extract_pip_tar(&target, &folder).await?;
    tokio::fs::remove_file(&target).await?;
@@ -150,9 +151,9 @@ pub async fn copy_cache_from_bucket(bucket: &str, tx: Sender<()>) -> error::Resu
            "--size-only",
            "--fast-list",
            "--filter",
-            "- deno/gen/file/**",
+            "+ deno/npm/**",
            "--filter",
-            "+ deno/**",
+            "+ deno/deps/**",
            // "--filter",
            // "+ bun/**",
            "--filter",
@@ -195,9 +196,9 @@ pub async fn copy_cache_to_bucket(bucket: &str) -> error::Result<()> {
            "--size-only",
            "--fast-list",
            "--filter",
-            "- deno/gen/file/**",
+            "+ deno/npm/**",
            "--filter",
-            "+ deno/**",
+            "+ deno/deps/**",
            // "--filter",
            // "+ bun/**",
            "--filter",
@@ -231,8 +232,8 @@ pub async fn copy_cache_to_bucket_as_tar(bucket: &str) {
            "-f",
            &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
            "go",
-            "deno",
-            "bun",
+            "deno/npm",
+            "deno/deps", // "bun",
        ],
    )
    .await
@@ -282,11 +283,9 @@ pub async fn copy_cache_to_bucket_as_tar(bucket: &str) {

 #[cfg(feature = "enterprise")]
 pub async fn copy_denogo_cache_from_bucket_as_tar(bucket: &str) {
-    use tokio::fs::metadata;
-
    tracing::info!("Copying deno,go,bun cache from bucket {bucket} as tar");

-    let start: Instant = Instant::now();
+    let mut start: Instant = Instant::now();

    if let Err(e) = execute_command(
        ROOT_TMP_CACHE_DIR,
@@ -306,6 +305,13 @@ pub async fn copy_denogo_cache_from_bucket_as_tar(bucket: &str) {
        return;
    }

+    tracing::info!(
+        "Fonished copying denogobun tar for from bucket as tar. took {}s",
+        start.elapsed().as_secs()
+    );
+
+    start = Instant::now();
+
    if let Err(e) = execute_command(
        ROOT_CACHE_DIR,
        "tar",
@@ -316,26 +322,41 @@ pub async fn copy_denogo_cache_from_bucket_as_tar(bucket: &str) {
    )
    .await
    {
-        tracing::info!("Failed to untar denogobun tar. Error: {:?}", e);
+        tracing::info!("Failed to untar denogobun tar to cache. Error: {:?}", e);
        return;
    }

-    let denogen = format!("{ROOT_TMP_CACHE_DIR}deno/gen/file");
-    if metadata(&denogen).await.is_ok() {
-        let _ = tokio::fs::remove_dir_all(denogen).await;
+    tracing::info!(
+        "Finished untaring denogobun tar to cache. took: {}s",
+        start.elapsed().as_secs()
+    );
+
+    start = Instant::now();
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "tar",
+        vec![
+            "-xpvf",
+            &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed to untar denogobun tar to tmpcache. Error: {:?}", e);
+        return;
    }

+    tracing::info!(
+        "Finished untaring denogobun tar to /tmpcache. took: {}s",
+        start.elapsed().as_secs()
+    );
+
    if let Err(e) =
        tokio::fs::remove_file(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await
    {
        tracing::info!("Failed to remove denogobuntar cache. Error: {:?}", e);
-        return;
    };
-
-    tracing::info!(
-        "Finished copying denogobuntar from bucket {bucket} as tar, took: {:?}s",
-        start.elapsed().as_secs()
-    );
 }

 #[cfg(feature = "enterprise")]
@@ -394,9 +415,9 @@ pub async fn copy_tmp_cache_to_cache() -> error::Result<()> {
            ROOT_CACHE_DIR,
            // "-l",
            "--filter",
-            "- deno/gen/file/**",
+            "+ deno/npm/**",
            "--filter",
-            "+ deno/**",
+            "+ deno/deps/**",
            // "--filter",
            // "+ bun/**",
            "--filter",
@@ -446,6 +467,7 @@ pub async fn untar_all_piptars() -> error::Result<()> {
            if metadata(&folder).await.is_ok() {
                continue;
            }
+            // tracing::info!("A: {path} {folder}");
            extract_pip_tar(&path, &folder).await?;
            Ok(()) as error::Result<()>
        } {
@@ -490,9 +512,9 @@ pub async fn copy_cache_to_tmp_cache() -> error::Result<()> {
            ROOT_TMP_CACHE_DIR,
            // "-l",
            "--filter",
-            "- deno/gen/file/**",
+            "+ deno/npm/**",
            "--filter",
-            "+ deno/**",
+            "+ deno/deps/**",
            // "--filter",
            // "+ bun/**",
            "--filter",
@@ -511,6 +533,7 @@ pub async fn copy_cache_to_tmp_cache() -> error::Result<()> {

 #[cfg(feature = "enterprise")]
 pub async fn execute_command(dir: &str, command: &str, args: Vec<&str>) -> error::Result<()> {
+    tracing::info!("Executing command: {command} {}", args.iter().join(" "));
    match Command::new(command)
        .current_dir(dir)
        .args(args.clone())
--- a/Show More
+++ b/Show More