chore(main): release 1.102.0 (#1588)

* chore(main): release 1.102.0

* Apply automatic changes

---------

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.env

@@ -3,5 +3,5 @@ DB_PASSWORD=changeme
 # this is the url that your instance is publicly exposed to
 WM_BASE_URL=http://localhost
 
-# this is the url that caddy will reverse proxy from. It might be different than WM_BASE_URL if you re using a second proxy/load balancer in front
-CADDY_REVERSE_PROXY=http://localhost
+# this is the addres caddy will listen on
+CADDY_REVERSE_PROXY=":80"

.github/uffizzi/caddy/Caddyfile

@@ -1,5 +0,0 @@
-localhost {
-        bind 0.0.0.0
-        reverse_proxy /ws/* http://0.0.0.0:3001
-        reverse_proxy /* http://0.0.0.0:8000
-}

.github/uffizzi/docker-compose.uffizzi.yml

@@ -1,53 +0,0 @@
-version: '3.7'
-
-x-uffizzi:
-  ingress:
-    service: windmill
-    port: 8000
-
-services:
-  db:
-    image: postgres:14
-    environment:
-      POSTGRES_PASSWORD: changeme
-      POSTGRES_DB: windmill
-
-  windmill:
-    image: '${WINDMILL_IMAGE}'
-    ports:
-      - 8000:8000
-    entrypoint: ['/bin/sh', '-c']
-    command: 'echo ${OAUTH_JSON_BASE64} | base64 --decode > /usr/src/app/oauth.json && ./windmill'
-    environment:
-      - DATABASE_URL=postgres://postgres:changeme@localhost/windmill?sslmode=disable
-      - BASE_URL=${EXPECTED_URL}
-      - BASE_INTERNAL_URL=http://localhost:8000
-      - RUST_LOG=info
-      - NUM_WORKERS=3
-      - KEEP_JOB_DIR=false
-      - DENO_PATH=/usr/bin/deno
-      - PYTHON_PATH=/usr/local/bin/python3
-      - METRICS_ADDR=false
-      - OAUTH_JSON_BASE64=${OAUTH_JSON_BASE64}
-    volumes:
-      - worker_dependency_cache:/tmp/windmill/cache
-    deploy:
-      resources:
-        limits:
-          memory: 250M
-
-  lsp:
-    image: '${LSP_IMAGE}'
-    ports:
-      - 3001:3001
-
-  # caddy:
-  #   image: caddy:2.5.2-alpine
-  #   restart: unless-stopped
-  #   volumes:
-  #     - ./.github/uffizzi/caddy:/etc/caddy
-  #   environment:
-  #     - BASE_URL=localhost
-
-volumes:
-  worker_dependency_cache:

.github/workflows/backend-test.yml

@@ -40,4 +40,4 @@ jobs:
             backend -> target
       - name: cargo test
         timeout-minutes: 10
-        run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill DISABLE_NSJAIL=false cargo test --all -- --nocapture
+        run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture

.github/workflows/docker-image.yml

@@ -21,7 +21,7 @@ permissions:
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
         with:
@@ -64,7 +64,7 @@ jobs:
             org.opencontainers.image.licenses=AGPLv3
 
   build_ee:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/frontend-check.yml

@@ -4,6 +4,8 @@ on:
     types: [opened,synchronize,reopened,closed]
     paths:
       - "frontend/**"
+  merge_group:
+
 jobs:
   npm_check:
     runs-on: ubuntu-latest

.github/workflows/pypi_on_release.yml

@@ -10,9 +10,15 @@ on:
       - "v*"
   workflow_dispatch:
 
+permissions:
+  contents: read
+  id-token: write
+  packages: write
+
 jobs:
   publish_pypi:
     runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v')
     container:
       image: ghcr.io/windmill-labs/python-client-builder
     steps:
@@ -24,26 +30,25 @@ jobs:
           cd python-client
           ./publish.sh
 
+
   publish_lsp:
-    needs: [publish_pypi]
-    runs-on: [self-hosted, new]
+    runs-on: ubuntu-latest
     steps:
-      - name: Sleep for 30 seconds waiting for pypi to update index
-        run: sleep 30s
+      - name: Sleep for 300 seconds waiting for pypi to update index
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: sleep 300
         shell: bash
+        
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
+      - uses: depot/setup-action@v1
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v4
         with:
           images: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=ref,event=branch
             type=ref,event=pr
@@ -57,6 +62,30 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Build and push publicly
+        uses: depot/build-push-action@v1
+        with:
+          context: "{{defaultContext}}:lsp"
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ steps.meta.outputs.tags }}
+          labels: |
+            ${{ steps.meta.outputs.labels }}
+            org.opencontainers.image.licenses=AGPLv3
+
+
+  publish_lsp_private:
+    needs: [publish_lsp]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       - name: Login to ECR
         uses: docker/login-action@v2
         with:
@@ -64,17 +93,9 @@ jobs:
           username: ${{ secrets.AWS_ACCESS_KEY_ID }}
           password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
-      - name: Build and push publicly
-        uses: docker/build-push-action@v4
-        with:
-          context: "{{defaultContext}}:lsp"
-          push: true
-          tags: |
-            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-            ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-            ${{ steps.metalocal.outputs.tags }}
-            ${{ steps.meta.outputs.tags }}
-            registry.uffizzi.com/windmill-lsp:60d
-          labels: ${{ steps.metalocal.outputs.labels }}
-          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
-          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+
+      - name: Push image to GHCR
+        run: |
+          docker buildx imagetools create \
+            --tag ${{ env.ECR_REGISTRY }}/${{ env.IMAGE_NAME }}:latest \
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest

.github/workflows/uffizzi-build.yml.archived

@@ -1,93 +0,0 @@
-name: Build PR Image
-on:
-  pull_request:
-    types: [opened,synchronize,reopened,closed]
-    paths:
-      - "backend/**"
-      - ".github/uffizzi/**"
-      - ".github/workflows/**"
-  workflow_dispatch:
-      
-jobs:
-  build-windmill:
-    name: Build and Push `windmill`
-    runs-on: ubuntu-latest
-    if: ${{ (github.event_name != 'pull_request' || github.event.action != 'closed')}}
-    outputs:
-      tags: ${{ steps.meta.outputs.tags }}
-    steps:
-      - name: Checkout git repo
-        uses: actions/checkout@v3
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Generate UUID image name
-        id: uuid
-        run: echo "UUID_TAG_APP=$(uuidgen)" >> $GITHUB_ENV
-      - name: Docker metadata
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: registry.uffizzi.com/${{ env.UUID_TAG_APP }}
-          tags: type=raw,value=60d
-      - name: Build and Push Image to registry.uffizzi.com ephemeral registry
-        uses: docker/build-push-action@v4
-        with:
-          push: true
-          context: ./
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-
-  render-compose-file:
-    name: Render Docker Compose File
-    # Pass output of this workflow to another triggered by `workflow_run` event.
-    runs-on: ubuntu-latest
-    needs:
-      - build-windmill
-    outputs:
-      compose-file-cache-key: ${{ steps.hash.outputs.hash }}
-    steps:
-      - name: Checkout git repo
-        uses: actions/checkout@v3
-      - name: Render Compose File
-        run: |
-          WINDMILL_IMAGE=${{ needs.build-windmill.outputs.tags }}
-          export WINDMILL_IMAGE
-          LSP_IMAGE=registry.uffizzi.com/windmill-lsp:60d
-          export LSP_IMAGE
-          envsubst '${WINDMILL_IMAGE} ${LSP_IMAGE}' < ./.github/uffizzi/docker-compose.uffizzi.yml > docker-compose.rendered.yml
-          cat docker-compose.rendered.yml
-      - name: Upload Rendered Compose File as Artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: preview-spec
-          path: docker-compose.rendered.yml
-          retention-days: 2
-      - name: Serialize PR Event to File
-        run:  |
-          cat << EOF > event.json
-          ${{ toJSON(github.event) }}
-          EOF
-      - name: Upload PR Event as Artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: preview-spec
-          path: event.json
-          retention-days: 2
-
-  delete-preview:
-    name: Call for Preview Deletion
-    runs-on: ubuntu-latest
-    if: ${{ github.event.action == 'closed' }}
-    steps:
-      # If this PR is closing, we will not render a compose file nor pass it to the next workflow.
-      - name: Serialize PR Event to File
-        run: echo '${{ toJSON(github.event) }}' > event.json
-      - name: Upload PR Event as Artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: preview-spec
-          path: event.json
-          retention-days: 2

.github/workflows/uffizzi-preview.yml.archived

@@ -1,115 +0,0 @@
-name: Deploy Uffizzi Preview
-
-on:
-  workflow_run:
-    workflows:
-      - "Build PR Image"
-    types:
-      - completed
-
-jobs:
-  cache-compose-file:
-    name: Cache Compose File
-    runs-on: ubuntu-latest
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
-    outputs:
-      compose-file-cache-key: ${{ env.COMPOSE_FILE_HASH }}
-      pr-number: ${{ env.PR_NUMBER }}
-    steps:
-      - name: 'Download artifacts'
-        # Fetch output (zip archive) from the workflow run that triggered this workflow.
-        uses: actions/github-script@v6
-        with:
-          script: |
-            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               run_id: context.payload.workflow_run.id,
-            });
-            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
-              return artifact.name == "preview-spec"
-            })[0];
-            if (matchArtifact === undefined) {
-              throw TypeError('Build Artifact not found!');
-            }
-            let download = await github.rest.actions.downloadArtifact({
-               owner: context.repo.owner,
-               repo: context.repo.repo,
-               artifact_id: matchArtifact.id,
-               archive_format: 'zip',
-            });
-            let fs = require('fs');
-            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/preview-spec.zip`, Buffer.from(download.data));
-      - name: 'Unzip artifact'
-        run: unzip preview-spec.zip
-      - name: Read Event into ENV
-        run: |
-          echo 'EVENT_JSON<<EOF' >> $GITHUB_ENV
-          cat event.json >> $GITHUB_ENV
-          echo 'EOF' >> $GITHUB_ENV
-      - name: Read PR Number From Event Object
-        id: pr
-        run: echo "PR_NUMBER=${{ fromJSON(env.EVENT_JSON).number }}" >> $GITHUB_ENV
-      - name: Predict Deployment URL
-        id: url
-        # Replace dots in the repo name with the plus sign
-        run: |
-          REPO=$(echo ${{ github.repository }} | sed 's/\./+/g')
-          echo "EXPECTED_URL=https://app.uffizzi.com/github.com/$REPO/pull/$PR_NUMBER" >> $GITHUB_ENV
-
-      - name: Re-Render Compose File
-        run: |
-          OAUTH_JSON_BASE64=${{ secrets.OAUTH_JSON_BASE64 }}
-          export OAUTH_JSON_BASE64
-          envsubst '${OAUTH_JSON_BASE64} ${EXPECTED_URL}' < docker-compose.rendered.yml > docker-compose.uffizzi.yml
-          # cat docker-compose.uffizzi.yml
-
-      - name: Hash Rendered Compose File
-        id: hash
-        # If the previous workflow was triggered by a PR close event, we will not have a compose file artifact.
-        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
-        run: echo "COMPOSE_FILE_HASH=$(md5sum docker-compose.uffizzi.yml | awk '{ print $1 }')" >> $GITHUB_ENV
-      - name: Cache Rendered Compose File
-        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
-        uses: actions/cache@v3
-        with:
-          path: docker-compose.uffizzi.yml
-          key: ${{ env.COMPOSE_FILE_HASH }}
-
-      - name: DEBUG - Print Job Outputs
-        if: ${{ runner.debug }}
-        run: |
-          echo "PR number: ${{ env.PR_NUMBER }}"
-          echo "Compose file hash: ${{ env.COMPOSE_FILE_HASH }}"
-          cat event.json
-
-  deploy-uffizzi-preview:
-    name: Use Remote Workflow to Preview on Uffizzi
-    needs:
-      - cache-compose-file
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
-    uses: UffizziCloud/preview-action/.github/workflows/reusable.yaml@v2
-    with:
-      # If this workflow was triggered by a PR close event, cache-key will be an empty string
-      # and this reusable workflow will delete the preview deployment.
-      compose-file-cache-key: ${{ needs.cache-compose-file.outputs.compose-file-cache-key }}
-      compose-file-cache-path: docker-compose.uffizzi.yml
-      server: https://app.uffizzi.com
-      pr-number: ${{ needs.cache-compose-file.outputs.pr-number }}
-    permissions:
-      contents: read
-      pull-requests: write
-      id-token: write
-
-  playwright:
-    runs-on: ubuntu-latest
-    needs:
-      - deploy-uffizzi-preview
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 16
-      - name: "Playwright run"
-        timeout-minutes: 2
-        run: cd frontend && npm ci @playwright/test && npx playwright install && export BASE_URL=${{ needs.deploy-uffizzi-preview.outputs.url }} && npm run test

CHANGELOG.md

@@ -1,6 +1,356 @@
 # Changelog
 
 
+## [1.102.0](https://github.com/windmill-labs/windmill/compare/v1.101.1...v1.102.0) (2023-05-19)
+
+
+### Features
+
+* add  ability to pass the full raw body ([#1611](https://github.com/windmill-labs/windmill/issues/1611)) ([b91f7d5](https://github.com/windmill-labs/windmill/commit/b91f7d501390358b01b6656297f56a9f24ef4683))
+* add GOPROXY + fix on saved inputs ([cdc4f29](https://github.com/windmill-labs/windmill/commit/cdc4f29ec1231820cfb2e0882d167b7dae3ae06e))
+* **backend:** add REQUEST_SIZE_LIMIT env variable ([1cbd704](https://github.com/windmill-labs/windmill/commit/1cbd704a257bcf2bd7b344958104e5c626c52a79))
+* **backend:** non mapped values are passed as arg 'body' ([a13d283](https://github.com/windmill-labs/windmill/commit/a13d2832d47d262f0b3ac222a8eb889fb17c75ad))
+* expose a react sdk to integrate windmill into react apps ([#1605](https://github.com/windmill-labs/windmill/issues/1605)) ([632be3b](https://github.com/windmill-labs/windmill/commit/632be3b8fb547ca4a2d976f868ee931218b653b3))
+* **frontend:** add presets components ([#1589](https://github.com/windmill-labs/windmill/issues/1589)) ([f7338c9](https://github.com/windmill-labs/windmill/commit/f7338c9c9a4cfa10d9c22d32a5ae70c4e3504ef3))
+* **lsp:** add  black + ruff + shellcheck ([#1597](https://github.com/windmill-labs/windmill/issues/1597)) ([89e55e0](https://github.com/windmill-labs/windmill/commit/89e55e0226d50951c5c99ce789af80ccaa5c1e25))
+
+
+### Bug Fixes
+
+* **frontend:** Add missing tooltips + multilpe small fix in the app builder ([#1590](https://github.com/windmill-labs/windmill/issues/1590)) ([fff2b5a](https://github.com/windmill-labs/windmill/commit/fff2b5a24abdd70371e2b8a49ff810c217c01bb1))
+* **frontend:** Fix inputValue connection to ensure done event is always sent ([#1607](https://github.com/windmill-labs/windmill/issues/1607)) ([f27abec](https://github.com/windmill-labs/windmill/commit/f27abecbaca4be46715ef15216492cb0984fe32b))
+* **frontend:** fix pdf header + icon picker ([#1586](https://github.com/windmill-labs/windmill/issues/1586)) ([a1cdf13](https://github.com/windmill-labs/windmill/commit/a1cdf13cb33494457e9f3cba24d5c7398565881f))
+* **frontend:** Fix rx ([#1609](https://github.com/windmill-labs/windmill/issues/1609)) ([c687a77](https://github.com/windmill-labs/windmill/commit/c687a775eb8efdb792c495bec72e7e547b82c068))
+* **frontend:** Fix the spinning logo position ([#1595](https://github.com/windmill-labs/windmill/issues/1595)) ([94b8bc4](https://github.com/windmill-labs/windmill/commit/94b8bc47380ea537512042ae412a3ace1ef709e7))
+* rework multiselect as app component ([#1599](https://github.com/windmill-labs/windmill/issues/1599)) ([85576b0](https://github.com/windmill-labs/windmill/commit/85576b00836225656b88c6751fdc619034b1ebca))
+
+## [1.101.1](https://github.com/windmill-labs/windmill/compare/v1.101.0...v1.101.1) (2023-05-16)
+
+
+### Bug Fixes
+
+* **backend:** make result job endpoints public ([41f2d35](https://github.com/windmill-labs/windmill/commit/41f2d35c971c42b9a4842b1411dd21603cabf084))
+* **frontend:** add temp hidden span to compute the text max length ([#1573](https://github.com/windmill-labs/windmill/issues/1573)) ([2a17d60](https://github.com/windmill-labs/windmill/commit/2a17d60caaef11f4b6cce464e1905a52095fe228))
+* **frontend:** fix app multi select ([#1574](https://github.com/windmill-labs/windmill/issues/1574)) ([45acdc8](https://github.com/windmill-labs/windmill/commit/45acdc895b6b5047a17e59dfcd9ca8cba2dd234a))
+
+## [1.101.0](https://github.com/windmill-labs/windmill/compare/v1.100.2...v1.101.0) (2023-05-15)
+
+
+### Features
+
+* **backend:** add job_id as a query arg to force set the new job_id ([b6c0018](https://github.com/windmill-labs/windmill/commit/b6c0018e2acaaed324832dfc715853ea58a4a268))
+* **frontend:** stepper standalone ([#1558](https://github.com/windmill-labs/windmill/issues/1558)) ([ad6e967](https://github.com/windmill-labs/windmill/commit/ad6e967205550b86cc8744f1ce08bb86215ce3e6))
+
+
+### Bug Fixes
+
+* **frontend:** Handle empty required in SchemaForm ([#1571](https://github.com/windmill-labs/windmill/issues/1571)) ([efc4e9c](https://github.com/windmill-labs/windmill/commit/efc4e9ce8a988aacb8e8dda264702dc08d25f7e0))
+
+## [1.100.2](https://github.com/windmill-labs/windmill/compare/v1.100.1...v1.100.2) (2023-05-14)
+
+
+### Bug Fixes
+
+* **cli:** update wmill script push ([678b574](https://github.com/windmill-labs/windmill/commit/678b574efcae66801a115d576db9d00aa9e4145d))
+* discriminate execute apps by component ([908358e](https://github.com/windmill-labs/windmill/commit/908358eb08614d07b5e846630743242b68b9e149))
+* transform_inputs now only support single line expressions ([c252b76](https://github.com/windmill-labs/windmill/commit/c252b765f1b1fd38f07cbe06548ca5cbe4047ea1))
+
+## [1.100.1](https://github.com/windmill-labs/windmill/compare/v1.100.0...v1.100.1) (2023-05-12)
+
+
+### Bug Fixes
+
+* update setup step ([178ed6f](https://github.com/windmill-labs/windmill/commit/178ed6f426020c9966380392088562e27aa77cf3))
+
+## [1.100.0](https://github.com/windmill-labs/windmill/compare/v1.99.0...v1.100.0) (2023-05-12)
+
+
+### Features
+
+* **frontend:** add download button ([9b9730d](https://github.com/windmill-labs/windmill/commit/9b9730d2b7239827fd8dfe8f46b6bd98d535e8d0))
+
+
+### Bug Fixes
+
+* **backend:** handle Date type ([5e7e46e](https://github.com/windmill-labs/windmill/commit/5e7e46e0259bfc11e92f2446858ddbe9f1b4b08e))
+* **frontend:** apps rendering should not depend on local time ([8e785d8](https://github.com/windmill-labs/windmill/commit/8e785d8ba6da16d06816d0379cadfb899be99b06))
+* **frontend:** only download result for apps ([6bbd937](https://github.com/windmill-labs/windmill/commit/6bbd9374cbd2c516dd3b56551103fcfeba01f80f))
+
+## [1.99.0](https://github.com/windmill-labs/windmill/compare/v1.98.0...v1.99.0) (2023-05-10)
+
+
+### Features
+
+* **backend:** run endpoints also support support x-www-form-urlencoded encoded payloads ([2b57418](https://github.com/windmill-labs/windmill/commit/2b57418427e9417599f9f969cb78088c5166958a))
+* **frontend:** add hide refresh button ([ef089ab](https://github.com/windmill-labs/windmill/commit/ef089ab56c5ef493118574076d9512ae3b6a42bf))
+* **frontend:** add input library to flow builder ([957fd81](https://github.com/windmill-labs/windmill/commit/957fd81576dfe65326cad2ed8487121e157e0953))
+* **frontend:** allow copy pasting nested containers ([742ee3a](https://github.com/windmill-labs/windmill/commit/742ee3a5181fdcfba1f59889a8d99347fd0c4610))
+* **frontend:** app on error ([#1556](https://github.com/windmill-labs/windmill/issues/1556)) ([6c2ba05](https://github.com/windmill-labs/windmill/commit/6c2ba053a1a023e454296a5ebf2842abf90362a8))
+* **frontend:** App select tabs ([#1557](https://github.com/windmill-labs/windmill/issues/1557)) ([4ad530f](https://github.com/windmill-labs/windmill/commit/4ad530f2f004cb33cbc95d5c3b1591a44f93bdee))
+* **frontend:** conditional rendering ([#1555](https://github.com/windmill-labs/windmill/issues/1555)) ([3d371d5](https://github.com/windmill-labs/windmill/commit/3d371d5b6524a4ec0152b15d001d8758900de457))
+* increase timeout to 900 ([018b504](https://github.com/windmill-labs/windmill/commit/018b504986a6c36c1e5ecbc5e92a763a6b6e613b))
+
+
+### Bug Fixes
+
+* **backend:** run endpoints also support support x-www-form-urlencoded encoded payloads ([5601d04](https://github.com/windmill-labs/windmill/commit/5601d047fe4736a996d064dc8ff34af5d70706a5))
+
+## [1.98.0](https://github.com/windmill-labs/windmill/compare/v1.97.0...v1.98.0) (2023-05-09)
+
+
+### Features
+
+* **frontend:** if member of a single workspace, autoset at login ([2dfb74e](https://github.com/windmill-labs/windmill/commit/2dfb74e7e45b279f5169ac89b483ed336e0bd109))
+
+
+### Bug Fixes
+
+* **backend:** grant all on raw_app ([c62670f](https://github.com/windmill-labs/windmill/commit/c62670f735da8378a896b538f3c3afeef100f7ab))
+
+## [1.97.0](https://github.com/windmill-labs/windmill/compare/v1.96.3...v1.97.0) (2023-05-09)
+
+
+### Features
+
+* **backend:** add windmill_status_code to run_wait_result ([38ec7d3](https://github.com/windmill-labs/windmill/commit/38ec7d3a857a19e474f0a1b07b73b85aa5f10f41))
+* **backend:** cache hub scripts in the worker cache ([7537f1a](https://github.com/windmill-labs/windmill/commit/7537f1a1d7162610f78a7e84a53d57f8478a5965))
+* **backend:** in python, if a value is bytes, it is encoded to base64 automaticaly ([6b5ceed](https://github.com/windmill-labs/windmill/commit/6b5ceed6525d4251517627351a15a4fe604629fc))
+
+
+### Bug Fixes
+
+* **lsp:** handle write_message errors ([9392890](https://github.com/windmill-labs/windmill/commit/939289030ba667afc9b517dfdc90f26378fa44a6))
+
+## [1.96.3](https://github.com/windmill-labs/windmill/compare/v1.96.2...v1.96.3) (2023-05-08)
+
+
+### Bug Fixes
+
+* **cli:** add folder listing ([c598083](https://github.com/windmill-labs/windmill/commit/c5980839251cdc759c5afc688e7084f5d58ad57f))
+* **cli:** show diffs only with --show-diffs ([d254088](https://github.com/windmill-labs/windmill/commit/d254088fce00091352ad95888deeaf88bc6c9d6f))
+* **cli:** show diffs only with --show-diffs ([37f08e9](https://github.com/windmill-labs/windmill/commit/37f08e9357c7a74a1c6cdc18bad2d6dc4de5d33d))
+* **cli:** variable updating ([2639250](https://github.com/windmill-labs/windmill/commit/2639250b43c3c55f5b9a43f4020fc2f0747e792b))
+
+## [1.96.2](https://github.com/windmill-labs/windmill/compare/v1.96.1...v1.96.2) (2023-05-08)
+
+
+### Bug Fixes
+
+* **cli:** add debug mode to CLI + improve error output ([8f1cdf1](https://github.com/windmill-labs/windmill/commit/8f1cdf1d61adf80bf0d7c4a5160fd3085d3814ac))
+
+## [1.96.1](https://github.com/windmill-labs/windmill/compare/v1.96.0...v1.96.1) (2023-05-08)
+
+
+### Bug Fixes
+
+* **cli:** fix cli folder sync ([239f401](https://github.com/windmill-labs/windmill/commit/239f40199955d47e4943be4c72c3d150a58f2dd9))
+* **cli:** fix cli folder sync ([a90514b](https://github.com/windmill-labs/windmill/commit/a90514b8e99a419c26512eb370895322088b6aa9))
+
+## [1.96.0](https://github.com/windmill-labs/windmill/compare/v1.95.1...v1.96.0) (2023-05-08)
+
+
+### Features
+
+* add support for full fleged apps (react, svelte, vue) ([#1536](https://github.com/windmill-labs/windmill/issues/1536)) ([13242ab](https://github.com/windmill-labs/windmill/commit/13242abff153b021cac1ecaa3cbf65ae9d87fb69))
+* **frontend:** Add a custom deepEqualWithOrderedArray to handle orde… ([#1537](https://github.com/windmill-labs/windmill/issues/1537)) ([3a291f7](https://github.com/windmill-labs/windmill/commit/3a291f7108623b5c7194f0a7f6a3774499669313))
+* **frontend:** Add label, description, input style + add displayType… ([#1540](https://github.com/windmill-labs/windmill/issues/1540)) ([bef829d](https://github.com/windmill-labs/windmill/commit/bef829d4805bb6c5330b13dc17c9a89a84ad48ca))
+* **frontend:** app modal ([#1518](https://github.com/windmill-labs/windmill/issues/1518)) ([686f5bb](https://github.com/windmill-labs/windmill/commit/686f5bbe1847cb2a92678e7cfdbb51ecf6bbe2b6))
+
+## [1.95.1](https://github.com/windmill-labs/windmill/compare/v1.95.0...v1.95.1) (2023-05-06)
+
+
+### Bug Fixes
+
+* **cli:** cli flow sync improvements ([e585e3a](https://github.com/windmill-labs/windmill/commit/e585e3aea2b18b6dc0c9fa7ffa1e6c1dfb2a3ce2))
+
+## [1.95.0](https://github.com/windmill-labs/windmill/compare/v1.94.0...v1.95.0) (2023-05-05)
+
+
+### Features
+
+* **backend:** default parameters are used in python if missing from args ([8791a86](https://github.com/windmill-labs/windmill/commit/8791a86a936301d44ae05ea09d26c9815abf8929))
+* **frontend:** App Schema Form component ([#1533](https://github.com/windmill-labs/windmill/issues/1533)) ([85c0d93](https://github.com/windmill-labs/windmill/commit/85c0d939f59411d023cd4b173ce11224d3cbc9db))
+* **frontend:** App stepper ([#1529](https://github.com/windmill-labs/windmill/issues/1529)) ([15f1c94](https://github.com/windmill-labs/windmill/commit/15f1c947bb233147f7da261fd32054a51a9c6efa))
+* **frontend:** Merge run configuration + triggers ([#1530](https://github.com/windmill-labs/windmill/issues/1530)) ([1be4658](https://github.com/windmill-labs/windmill/commit/1be4658150ef20a9f1f0fe57b5f30ba3c2d4d94e))
+
+## [1.94.0](https://github.com/windmill-labs/windmill/compare/v1.93.1...v1.94.0) (2023-05-04)
+
+
+### Features
+
+* **frontend:** add eval badge + alert ([#1522](https://github.com/windmill-labs/windmill/issues/1522)) ([32f04c7](https://github.com/windmill-labs/windmill/commit/32f04c796856fa48ddc1548752ba1e7a8802083a))
+
+
+### Bug Fixes
+
+* **backend:** fix python transformers ([a07e3e8](https://github.com/windmill-labs/windmill/commit/a07e3e84386c0895a7209fc87a4b07218271feca))
+* **frontend:** fix ArrayStaticInputEditor width ([#1528](https://github.com/windmill-labs/windmill/issues/1528)) ([b423eec](https://github.com/windmill-labs/windmill/commit/b423eec019785a62c279db01fc93eb3fe08f7f1f))
+* **frontend:** fix select width ([#1526](https://github.com/windmill-labs/windmill/issues/1526)) ([f248c09](https://github.com/windmill-labs/windmill/commit/f248c09655889ddace24f451597a56e81443be3c))
+* **frontend:** preserve customise arguments ([b4867f1](https://github.com/windmill-labs/windmill/commit/b4867f12bb4f595b5b0e8142ab5d720307ecadd3))
+
+## [1.93.1](https://github.com/windmill-labs/windmill/compare/v1.93.0...v1.93.1) (2023-05-03)
+
+
+### Bug Fixes
+
+* **cli:** add yaml support for cli ([03e6017](https://github.com/windmill-labs/windmill/commit/03e6017860526784f1a8696eceed5750b25f1c5c))
+
+## [1.93.0](https://github.com/windmill-labs/windmill/compare/v1.92.2...v1.93.0) (2023-05-03)
+
+
+### Features
+
+* **frontend:** add recompute others to background scripts ([392d0f8](https://github.com/windmill-labs/windmill/commit/392d0f8b876c9b587fe85421098f3eceb8a74dec))
+
+
+### Bug Fixes
+
+* **frontend:** deploy path for apps ([7ac9677](https://github.com/windmill-labs/windmill/commit/7ac96771a5c3d44234c790e8cea3d621d8c1d00e))
+
+## [1.93.0](https://github.com/windmill-labs/windmill/compare/v1.92.2...v1.93.0) (2023-05-03)
+
+
+### Features
+
+* **frontend:** add recompute others to background scripts ([392d0f8](https://github.com/windmill-labs/windmill/commit/392d0f8b876c9b587fe85421098f3eceb8a74dec))
+
+
+### Bug Fixes
+
+* **frontend:** deploy path for apps ([7ac9677](https://github.com/windmill-labs/windmill/commit/7ac96771a5c3d44234c790e8cea3d621d8c1d00e))
+
+## [1.92.2](https://github.com/windmill-labs/windmill/compare/v1.92.1...v1.92.2) (2023-05-02)
+
+
+### Bug Fixes
+
+* **go-client:** fix go-client gen ([82c4d66](https://github.com/windmill-labs/windmill/commit/82c4d6629e134f00389c87a948c52878e5a3f4f5))
+
+## [1.92.1](https://github.com/windmill-labs/windmill/compare/v1.92.0...v1.92.1) (2023-05-02)
+
+
+### Bug Fixes
+
+* **go-client:** fix go-client gen ([df333d9](https://github.com/windmill-labs/windmill/commit/df333d9739f601714f7a0124f47422dfb113d320))
+
+## [1.92.0](https://github.com/windmill-labs/windmill/compare/v1.91.0...v1.92.0) (2023-05-02)
+
+
+### Features
+
+* **frontend:** add labels as table action ([64065c1](https://github.com/windmill-labs/windmill/commit/64065c17f305fb4c7c078c7fa4935d5423da8f66))
+* **frontend:** add labels as table action ([2ab1714](https://github.com/windmill-labs/windmill/commit/2ab1714dfa7cffc46b4b6aa40dabdd92c5a6270f))
+* **frontend:** allow running eval in every field ([62acbb5](https://github.com/windmill-labs/windmill/commit/62acbb5ab3a0727b306e25a80b74ac8216619501))
+* **frontend:** background script can run script and flows ([#1515](https://github.com/windmill-labs/windmill/issues/1515)) ([607c803](https://github.com/windmill-labs/windmill/commit/607c803be91921b53f329a2c2c3c129ce53d6c0c))
+
+
+### Bug Fixes
+
+* **frontend:** fix small ui issues ([#1513](https://github.com/windmill-labs/windmill/issues/1513)) ([f6ff8ca](https://github.com/windmill-labs/windmill/commit/f6ff8ca232f5725f86a36379956da2731def2580))
+
+## [1.91.0](https://github.com/windmill-labs/windmill/compare/v1.90.0...v1.91.0) (2023-05-01)
+
+
+### Features
+
+* add drafts for apps ([f7374c8](https://github.com/windmill-labs/windmill/commit/f7374c8204f85b4371e61f34dcd4b66857c0f8ab))
+* introduce backend persisted draft systems for scripts ([88e37fe](https://github.com/windmill-labs/windmill/commit/88e37fe0bed58f396690622e925d5e078c60140c))
+* introduce draft for flows ([a196642](https://github.com/windmill-labs/windmill/commit/a1966427e893dc8a58c8f2862ded752884843813))
+
+## [1.90.0](https://github.com/windmill-labs/windmill/compare/v1.89.0...v1.90.0) (2023-04-28)
+
+
+### Features
+
+* **backend:** add EXIT_AFTER_NO_JOB_FOR_SECS for ephemeral workers ([de9abd1](https://github.com/windmill-labs/windmill/commit/de9abd129db13dcdf0e69e2c1e2d3aa558fb783a))
+* **backend:** add JOB_RETENTION_SECS to delete completed jobs completed after expiration period ([0b7bad3](https://github.com/windmill-labs/windmill/commit/0b7bad3816e61841ef4765db7881227274c20b23))
+* **backend:** expose tag in the job  ([#1486](https://github.com/windmill-labs/windmill/issues/1486)) ([324d4f5](https://github.com/windmill-labs/windmill/commit/324d4f5e9e89e6de600e882f23bf545c0b1dc539))
+* **frontend:** adapt style ([#1488](https://github.com/windmill-labs/windmill/issues/1488)) ([41a24ec](https://github.com/windmill-labs/windmill/commit/41a24ecd36d9cc537cbd1dd0cd1de6f689be1b8c))
+* **frontend:** add an eval input component for flow ([#1494](https://github.com/windmill-labs/windmill/issues/1494)) ([2815f1e](https://github.com/windmill-labs/windmill/commit/2815f1ec71177bb6e89d0d62a8df89030d37b1fc))
+* **frontend:** Add new integration icons ([#1479](https://github.com/windmill-labs/windmill/issues/1479)) ([7adacd4](https://github.com/windmill-labs/windmill/commit/7adacd4c9f03d17b47abc515bc391e348a7e6ec1))
+* **frontend:** refactor inline script ([#1480](https://github.com/windmill-labs/windmill/issues/1480)) ([05c837e](https://github.com/windmill-labs/windmill/commit/05c837e64f61bfb22ae1f80263deb1c879030985))
+* **frontend:** Schedules run now ([#1475](https://github.com/windmill-labs/windmill/issues/1475)) ([47f0f35](https://github.com/windmill-labs/windmill/commit/47f0f35236e02958f6bc00b5652e06e25eabeaf5))
+* **frontend:** Small style fix ([#1473](https://github.com/windmill-labs/windmill/issues/1473)) ([7ad496a](https://github.com/windmill-labs/windmill/commit/7ad496ad3f746ffd782856a35c1456999792fa94))
+* **frontend:** Support TS union type with a select field ([#1457](https://github.com/windmill-labs/windmill/issues/1457)) ([8b76324](https://github.com/windmill-labs/windmill/commit/8b763249cb1360c122cc81d50a1a95d1ad3ddd5b))
+
+
+### Bug Fixes
+
+* **frontend:** Allow 0 as select default value ([#1474](https://github.com/windmill-labs/windmill/issues/1474)) ([d8529ff](https://github.com/windmill-labs/windmill/commit/d8529ff3ed6168de60bb24626a3e36ab4beae15c))
+* **frontend:** close the modal before deleting a form modal ([#1484](https://github.com/windmill-labs/windmill/issues/1484)) ([430c733](https://github.com/windmill-labs/windmill/commit/430c73399b7e1524ee81d0ce8d7d2eaf16117f9a))
+* **frontend:** fix apply connection ([#1487](https://github.com/windmill-labs/windmill/issues/1487)) ([cf59cc0](https://github.com/windmill-labs/windmill/commit/cf59cc04efb853fe06a23042115128689b5d26ee))
+* **frontend:** Fix frontend script ([#1476](https://github.com/windmill-labs/windmill/issues/1476)) ([b60a7f6](https://github.com/windmill-labs/windmill/commit/b60a7f63d04b1fc851479060e77613c77d20198a))
+* **frontend:** fix recomputa all ([#1491](https://github.com/windmill-labs/windmill/issues/1491)) ([fb05a09](https://github.com/windmill-labs/windmill/commit/fb05a09955f937000fbba0826c03c52e65aa146e))
+* **frontend:** Flow editor design updates ([#1477](https://github.com/windmill-labs/windmill/issues/1477)) ([50d814c](https://github.com/windmill-labs/windmill/commit/50d814c3dc55841b16cd1df8ae86d021de4e880c))
+* **frontend:** Minor app editor updates ([#1458](https://github.com/windmill-labs/windmill/issues/1458)) ([8fd10b1](https://github.com/windmill-labs/windmill/commit/8fd10b1f5813b3f4200a980de4a85e7d701660a7))
+* **frontend:** register applyConnection as a callback to remove unnecessary reactivit ([#1485](https://github.com/windmill-labs/windmill/issues/1485)) ([d915f6b](https://github.com/windmill-labs/windmill/commit/d915f6b004ea7cc27a3c55b3504df902f5db1aef))
+* **frontend:** reset ui job loading state when submitting preview job triggers error ([#1483](https://github.com/windmill-labs/windmill/issues/1483)) ([6f8616f](https://github.com/windmill-labs/windmill/commit/6f8616fb273b1f4a1489878a617d239f30ecb1c0))
+* **frontend:** Update CLI login request styling ([#1454](https://github.com/windmill-labs/windmill/issues/1454)) ([c77393c](https://github.com/windmill-labs/windmill/commit/c77393c15444868308b44a72bee89e49fc23d80f))
+* **frontend:** Update direct exports ([#1456](https://github.com/windmill-labs/windmill/issues/1456)) ([4a2af13](https://github.com/windmill-labs/windmill/commit/4a2af1359ee29001236591f685cacfa9df6715df))
+
+## [1.89.0](https://github.com/windmill-labs/windmill/compare/v1.88.1...v1.89.0) (2023-04-23)
+
+
+### Features
+
+* **backend:** global cache refactor for pip using tar for each dependency ([#1443](https://github.com/windmill-labs/windmill/issues/1443)) ([369dd0d](https://github.com/windmill-labs/windmill/commit/369dd0dac61e5430856ed9abf7129bbad3b75860))
+* **backend:** only run fully deployed scripts ([3d031c7](https://github.com/windmill-labs/windmill/commit/3d031c701705459f418b11d2ca83e71943e4079b))
+* **backend:** worker groups ([#1452](https://github.com/windmill-labs/windmill/issues/1452)) ([722783f](https://github.com/windmill-labs/windmill/commit/722783f7f630e3123ffd2605deb21a915188bd20))
+* **backend:** workers are instantly ready and sync with global cache in background ([670ba51](https://github.com/windmill-labs/windmill/commit/670ba51d9bd9a9a0b07a2ed064c316234bb5819d))
+* **ee:** sync cache in background ([c919827](https://github.com/windmill-labs/windmill/commit/c919827cf8eb9437b6d9bd57b3d3ad883a66de3b))
+* **ee:** sync cache in background ([0e77e37](https://github.com/windmill-labs/windmill/commit/0e77e37fbddbb517d5e1b1f07a27a40b63371439))
+* **frontend:** Add documentation links ([#1399](https://github.com/windmill-labs/windmill/issues/1399)) ([36acbf7](https://github.com/windmill-labs/windmill/commit/36acbf793b6714dffe4dbb0e2501b9438f034858))
+* **frontend:** Add seconds input ([#1445](https://github.com/windmill-labs/windmill/issues/1445)) ([30bf7ad](https://github.com/windmill-labs/windmill/commit/30bf7ad3e9785420b4dcd814ce3c7a444d23cc9f))
+* **frontend:** add toast actions ([#1411](https://github.com/windmill-labs/windmill/issues/1411)) ([d173232](https://github.com/windmill-labs/windmill/commit/d17323286a05aa0b1680ef94d7058d2c8902782f))
+* **frontend:** reorder array items in app editor ([#1426](https://github.com/windmill-labs/windmill/issues/1426)) ([3615fb2](https://github.com/windmill-labs/windmill/commit/3615fb26fb91d64626d82ace6e6275e424ece832))
+* **frontend:** support showing metadata on script add via query param ([#1438](https://github.com/windmill-labs/windmill/issues/1438)) ([3c98452](https://github.com/windmill-labs/windmill/commit/3c98452f50913ef639eba96996f3a6c80508bd63))
+
+
+### Bug Fixes
+
+* **backend:** avoid potential conflict between pull from tar and background sync ([d76e907](https://github.com/windmill-labs/windmill/commit/d76e90757e209263da7f79fa85052969e7efd63d))
+* **backend:** global cache synco only start if all piptars have been downloaded ([5f8a730](https://github.com/windmill-labs/windmill/commit/5f8a730fdfbb9e3d518555f7272a3bb297725f28))
+* **frontend:** App color picker overflow issue ([#1449](https://github.com/windmill-labs/windmill/issues/1449)) ([32903d2](https://github.com/windmill-labs/windmill/commit/32903d2839a082d53168bb9177fc88f6ab0ec482))
+* **frontend:** fix copy content button width ([#1428](https://github.com/windmill-labs/windmill/issues/1428)) ([d96d4a5](https://github.com/windmill-labs/windmill/commit/d96d4a524edebea65bc602194c8f11f5d69e920a))
+* **frontend:** Minor update of app default codes ([#1440](https://github.com/windmill-labs/windmill/issues/1440)) ([fe75aa1](https://github.com/windmill-labs/windmill/commit/fe75aa18f2f27745db35329aa60938694640a8c6))
+* **frontend:** Update app default codes ([#1432](https://github.com/windmill-labs/windmill/issues/1432)) ([c8acfbc](https://github.com/windmill-labs/windmill/commit/c8acfbc1ff0f6c23e5a2229ca83a3b09eec826c3))
+* **frontend:** Update app mobile preview width ([#1431](https://github.com/windmill-labs/windmill/issues/1431)) ([1764613](https://github.com/windmill-labs/windmill/commit/17646130bcf8cf646a4ccdfa39f9a8791876a137))
+* **frontend:** Update flow tooltip z-indexes ([#1433](https://github.com/windmill-labs/windmill/issues/1433)) ([17cb8fc](https://github.com/windmill-labs/windmill/commit/17cb8fc3fa0b39a9750e61ca2731a15bbda690ec))
+* **frontend:** Update flow viewer styling ([#1441](https://github.com/windmill-labs/windmill/issues/1441)) ([46a29b5](https://github.com/windmill-labs/windmill/commit/46a29b5d27b8d9f7ea38c1063fc081ed5933db5d))
+
+## [1.88.1](https://github.com/windmill-labs/windmill/compare/v1.88.0...v1.88.1) (2023-04-18)
+
+
+### Bug Fixes
+
+* **frontend:** fix hub list ([1144329](https://github.com/windmill-labs/windmill/commit/1144329972fb61e2df62873ca1e485c88fabc478))
+
+## [1.88.0](https://github.com/windmill-labs/windmill/compare/v1.87.0...v1.88.0) (2023-04-17)
+
+
+### Features
+
+* **backend:** install python scripts on save ([cb7e686](https://github.com/windmill-labs/windmill/commit/cb7e686dd95397d5b37edd5aac50b6d1429c4a71))
+* **frontend:** Add runs preview popup ([#1405](https://github.com/windmill-labs/windmill/issues/1405)) ([4ab023f](https://github.com/windmill-labs/windmill/commit/4ab023f95085958ab1ad01dc249d308c7ebf423e))
+* **frontend:** cancellable inline script editor run ([e828d26](https://github.com/windmill-labs/windmill/commit/e828d2673e62e95e5e1235eeca8107ac7cfb7e45))
+* **frontend:** Remove gap when button label is empty ([#1402](https://github.com/windmill-labs/windmill/issues/1402)) ([568f59e](https://github.com/windmill-labs/windmill/commit/568f59eefb104047b8ef063f273fe238075d6407))
+* **frontend:** Unify main lists ([#1406](https://github.com/windmill-labs/windmill/issues/1406)) ([48bbbd0](https://github.com/windmill-labs/windmill/commit/48bbbd0e872a12ed1c562a6d14967a2a0f7c4735))
+* **frontend:** Update airtable instructions ([#1403](https://github.com/windmill-labs/windmill/issues/1403)) ([7dc7ece](https://github.com/windmill-labs/windmill/commit/7dc7ecef55b465fc096f71fc9de5c8b543136ff7))
+* inputs library on run page ([92a2934](https://github.com/windmill-labs/windmill/commit/92a293488e8e58350229931ab69f7924d58474be))
+
+
+### Bug Fixes
+
+* **backend:** deno uses --no-check ([a5499c2](https://github.com/windmill-labs/windmill/commit/a5499c26f3ebd8b07541a7e0cbf33a7008a8f476))
+* **backend:** do not fail on schedule not existing anymore ([a5f6d73](https://github.com/windmill-labs/windmill/commit/a5f6d73f7d53d7af9d285a85460509763263c508))
+* **frontend:** Fix app file uploads ([#1408](https://github.com/windmill-labs/windmill/issues/1408)) ([ac489ac](https://github.com/windmill-labs/windmill/commit/ac489ac2da0fbf01f5e2877612c14cfaf1ef79c2))
+* **frontend:** fix buttons width ([#1407](https://github.com/windmill-labs/windmill/issues/1407)) ([75a0482](https://github.com/windmill-labs/windmill/commit/75a0482ef046dd7e30f6d6039dbc66880182dc5e))
+* **frontend:** fix enum sync ([#1410](https://github.com/windmill-labs/windmill/issues/1410)) ([98060ce](https://github.com/windmill-labs/windmill/commit/98060ce55d5efa59a8989cf9357935976d57650b))
+* **frontend:** Handle scheduled runs in preview ([#1413](https://github.com/windmill-labs/windmill/issues/1413)) ([accdc1a](https://github.com/windmill-labs/windmill/commit/accdc1ac59ce9611f66567222a73995d3c0a3f9d))
+* **frontend:** Keep selected tab during renaming ([#1409](https://github.com/windmill-labs/windmill/issues/1409)) ([82cd048](https://github.com/windmill-labs/windmill/commit/82cd048ef4d08f31660f6f31a96940676a28996c))
+* **frontend:** Queued-running jobs preview ([#1414](https://github.com/windmill-labs/windmill/issues/1414)) ([b2a40a0](https://github.com/windmill-labs/windmill/commit/b2a40a05805344c1c34f2ba917b4cdd52dfffc3f))
+* **frontend:** Remove output when deleting a component ([#1397](https://github.com/windmill-labs/windmill/issues/1397)) ([6aa1008](https://github.com/windmill-labs/windmill/commit/6aa100893352870d5a99fdd56d7f1425a221a273))
+
 ## [1.87.0](https://github.com/windmill-labs/windmill/compare/v1.86.0...v2.0.0) (2023-04-11)
 
 

Dockerfile

@@ -39,7 +39,7 @@ WORKDIR /windmill
 ENV SQLX_OFFLINE=true
 ENV CARGO_INCREMENTAL=1
 
-FROM node:19-alpine as frontend
+FROM node:20-alpine as frontend
 
 # install dependencies
 WORKDIR /frontend

LICENSE

@@ -1,13 +1,13 @@
 
 Source code in this repository is variously licensed under the Apache License
-Version 2.0 (see file ./LICENSE-APACHE),or the AGPLv3 License (see file ./LICENSE-AGPL)
+Version 2.0 (see file ./LICENSE-APACHE), or the AGPLv3 License (see file ./LICENSE-AGPL)
 
 Every file is under copyright (c) Windmill Labs, Inc 2022 unless otherwise specified.
 Every file is under License AGPL unless otherwise specified 
 or belonging to one of the below cases:
 
-The files under backend/ are AGPL Licensed.
-The files under frontend/ are AGPL Licensed.
+The files under backend/ are AGPLv3 Licensed.
+The files under frontend/ are AGPLv3 Licensed.
 The files under python-client/ deno-client/ go-client/ are Apache 2.0 Licensed.
 
 The openapi files, including the OpenFlow spec is Apache 2.0 Licensed.

README.md

@@ -1,20 +1,32 @@
 <p align="center">
-  <a href="https://app.windmill.dev"><img src="./imgs/windmill-banner.png" alt="windmill.dev"></a>
+  <a href="https://www.windmill.dev/"><img src="./imgs/windmill-banner.png" alt="windmill.dev"></a>
 </p>
 <p align="center">
     <em>.</em>
 </p>
 <p align=center>
-Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
+Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIsm and custom UIs to trigger workflows and scripts as internal apps.
+
+<p align=center>
+Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
 </p>
 
 <p align="center">
+<a href="https://github.com/windmill-labs/windmill/blob/main/LICENSE-AGPL" target="_blank">
+    <img src="https://img.shields.io/badge/License-AGPLv3-blue.svg" alt="Package version">
+</a>
 <a href="https://github.com/windmill-labs/windmill/actions/workflows/docker-image.yml" target="_blank">
     <img src="https://github.com/windmill-labs/windmill/actions/workflows/docker-image.yml/badge.svg" alt="Docker Image CI">
 </a>
 <a href="https://pypi.org/project/wmill" target="_blank">
     <img src="https://img.shields.io/pypi/v/wmill?color=%2334D058&label=pypi%20package" alt="Package version">
 </a>
+</p>
+
+<p align="center">
+<a href="https://img.shields.io/github/commit-activity/m/windmill-labs/windmill" target="_blank">
+    <img src="https://img.shields.io/github/commit-activity/m/windmill-labs/windmill" alt="Commit activity">
+</a>
 <a href="https://discord.gg/V7PM2YHsPB" target="_blank">
   <img src="https://discordapp.com/api/guilds/930051556043276338/widget.png" alt="Discord Shield"/>
 </a>
@@ -29,9 +41,9 @@ Open-source developer infrastructure for internal tools. Self-hostable alternati
 Windmill is <b>fully open-sourced (AGPLv3)</b> and Windmill Labs offers
 dedicated instance and commercial support and licenses.
 
-![Windmill Diagram](/imgs/stacks.svg)
+![Windmill Diagram](./imgs/stacks.svg)
 
-https://user-images.githubusercontent.com/275584/218350457-bc2fdc3b-e667-4da5-a2bd-3bacc1f0ec79.mp4
+https://github.com/windmill-labs/windmill/assets/122811744/0b132cd1-ee67-4505-822f-0c7ee7104252
 
 - [Windmill - Turn scripts into workflows and UIs that you can share and run at scale](#windmill---turn-scripts-into-workflows-and-uis-that-you-can-share-and-run-at-scale)
   - [Main Concepts](#main-concepts)
@@ -61,8 +73,8 @@ https://user-images.githubusercontent.com/275584/218350457-bc2fdc3b-e667-4da5-a2
 ## Main Concepts
 
 1. Define a minimal and generic script in Python, Typescript, Go or Bash that
-   solves a specific task. Here sending an email with SMTP. The code can be
-   defined in the provided Web IDE or synchronized with your own github repo:
+   solves a specific task. Here sending a POST request. The code can be defined
+   in the provided Web IDE or synchronized with your own github repo:
    ![Step 1](./imgs/windmill-editor.png)
 
 2. Your scripts parameters are automatically parsed and generate a frontend.
@@ -116,7 +128,7 @@ export async function main(
 We have a powerful CLI to interact with the windmill platform and sync your
 scripts from local files, github repos and to run scripts and flows on the
 instance from local commands. See
-[more details](https://github.com/windmill-labs/windmill/tree/main/cli)
+[more details](https://github.com/windmill-labs/windmill/tree/main/cli).
 
 ![CLI Screencast](./cli/vhs/output/setup.gif)
 
@@ -125,7 +137,7 @@ instance from local commands. See
 You can run your script locally easily, you simply need to pass the right
 environment variables for the `wmill` client library to fetch resources and
 variables from your instance if necessary. See more:
-<https://docs.windmill.dev/docs/advanced/local_development/>
+<https://docs.windmill.dev/docs/advanced/local_development/>.
 
 ## Stack
 
@@ -152,7 +164,7 @@ variables from your instance if necessary. See more:
 
 Windmill uses [nsjail](https://github.com/google/nsjail) on top of the deno
 sandboxing. It is production multi-tenant grade secure. Do not take our word for
-it, take [fly.io's one](https://fly.io/blog/sandboxing-and-workload-isolation/)
+it, take [fly.io's one](https://fly.io/blog/sandboxing-and-workload-isolation/).
 
 ### Secrets, credentials and sensitive values
 
@@ -182,34 +194,34 @@ back to the database is ~50ms. A typical lightweight deno job will take around
 
 We only provide docker-compose setup here. For more advanced setups, like
 compiling from source or using without a postgres super user, see
-[documentation](https://docs.windmill.dev/docs/advanced/self_host)
+[documentation](https://docs.windmill.dev/docs/advanced/self_host).
 
 ### Docker compose
 
 ```
-curl https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml -o docker-compose.yml
-curl https://github.com/windmill-labs/windmill/blob/main/CaddyFile -o Caddyfile
-curl https://github.com/windmill-labs/windmill/blob/main/.env -o .env
+curl https://raw.githubusercontent.com/windmill-labs/windmill/main/docker-compose.yml -o docker-compose.yml
+curl https://raw.githubusercontent.com/windmill-labs/windmill/main/Caddyfile -o Caddyfile
+curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env
 
 docker compose up -d --pull always
 ```
 
 Go to http://localhost et voilà :)
 
-The default super-admin user is: admin@windmill.dev / changeme
+The default super-admin user is: admin@windmill.dev / changeme.
 
-From there, you can follow the setup app and creat other users.
+From there, you can follow the setup app and create other users.
 
 ### Kubernetes (k8s) and Helm charts
 
 We publish helm charts at:
-<https://github.com/windmill-labs/windmill-helm-charts>
+<https://github.com/windmill-labs/windmill-helm-charts>.
 
 ### Postgres without superuser
 
 If you do not want, or cannot (for instance, in AWS Aurora or Cloud sql) use a
 postgres superuser, you can run `./init-db-as-superuser.sql` to init the
-required users for windmill.
+required users for Windmill.
 
 ### Commercial license
 
@@ -224,9 +236,9 @@ be AGPLv3 or you must get a commercial license. Contact us at
 <ruben@windmill.dev> if you have any doubts.
 
 In addition, a commercial license grants you a dedicated engineer to transition
-your current infrastructure to Windmill, support with tight SLA, audit logs
-export features, SSO, unlimited users creation, advanced permission managing
-features such as groups and the ability to create more than one workspace.
+your current infrastructure to Windmill, support with tight SLA, and our global
+cache sync for high-performance/no dependency cache miss of cluster from 10+
+nodes to 200+ nodes.
 
 ### OAuth for self-hosting
 
@@ -248,6 +260,10 @@ and mount it at `/usr/src/app/oauth.json`.
 The redirect url for the oauth clients is:
 `<instance_url>/user/login_callback/<client>`
 
+Even if you setup oauth, you will still want to **login as admin@windmill.dev /
+changeme** to setup your instance as a super-admin and give yourself admin
+rights.
+
 [The list of all possible "connect an app" oauth clients](https://github.com/windmill-labs/windmill/blob/main/backend/oauth_connect.json)
 
 To add more "connect an app" OAuth clients to the Windmill project, read the
@@ -288,53 +304,64 @@ it being synced automatically everyday.
 
 ## Environment Variables
 
-| Environment Variable name | Default                | Description                                                                                                                                                                                        | Api Server/Worker/All |
-| ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
-| DATABASE_URL              |                        | The Postgres database url.                                                                                                                                                                         | All                   |
-| DISABLE_NSJAIL            | true                   | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
-| SERVER_BIND_ADDR          | 0.0.0.0                | IP Address on which to bind listening socket                                                                                                                                                       | Server                |
-| PORT                      | 8000                   | Exposed port                                                                                                                                                                                       | Server                |
-| NUM_WORKERS               | 3                      | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance)                                                                             | Worker                |
-| DISABLE_SERVER            | false                  | Binary would operate as a worker only instance                                                                                                                                                     | Worker                |
-| METRICS_ADDR              | None                   | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
-| JSON_FMT                  | false                  | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
-| BASE_URL                  | http://localhost:8000  | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
-| BASE_INTERNAL_URL         | http://localhost:8000  | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests.                                      | Worker                |
-| TIMEOUT                   | 300                    | The maximum time of execution of a script. When reached, the job is failed as having timedout.                                                                                                     | Worker                |
-| ZOMBIE_JOB_TIMEOUT        | 30                     | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s)                                  | Server                |
-| RESTART_ZOMBIE_JOBS       | true                   | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed                                                                              | Server                |
-| SLEEP_QUEUE               | 50                     | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker                |
-| MAX_LOG_SIZE              | 500000                 | The maximum number of characters a job can emit (log + result)                                                                                                                                     | Worker                |
-| DISABLE_NUSER             | false                  | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting                                                                                                                                 | Worker                |
-| KEEP_JOB_DIR              | false                  | Keep the job directory after the job is done. Useful for debugging.                                                                                                                                | Worker                |
-| LICENSE_KEY (EE only)     | None                   | License key checked at startup for the Enterprise Edition of Windmill                                                                                                                              | Worker                |
-| S3_CACHE_BUCKET (EE only) | None                   | The S3 bucket to sync the cache of the workers to                                                                                                                                                  | Worker                |
-| TAR_CACHE_RATE (EE only)  | 100                    | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed).                                                                          | Worker                |
-| SLACK_SIGNING_SECRET      | None                   | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack)                                                                | Server                |
-| COOKIE_DOMAIN             | None                   | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |
-| DENO_PATH                 | /usr/bin/deno          | The path to the deno binary.                                                                                                                                                                       | Worker                |
-| PYTHON_PATH               | /usr/local/bin/python3 | The path to the python binary.                                                                                                                                                                     | Worker                |
-| GO_PATH                   | /usr/bin/go            | The path to the go binary.                                                                                                                                                                         | Worker                |
-| GOPRIVATE                 |                        | The GOPRIVATE env variable to use private go modules                                                                                                                                               | Worker                |
-| NETRC                     |                        | The netrc content to use a private go registry                                                                                                                                                     | Worker                |
-| PIP_INDEX_URL             | None                   | The index url to pass for pip.                                                                                                                                                                     | Worker                |
-| PIP_EXTRA_INDEX_URL       | None                   | The extra index url to pass to pip.                                                                                                                                                                | Worker                |
-| PIP_TRUSTED_HOST          | None                   | The trusted host to pass to pip.                                                                                                                                                                   | Worker                |
-| PATH                      | None                   | The path environment variable, usually inherited                                                                                                                                                   | Worker                |
-| HOME                      | None                   | The home directory to use for Go and Bash , usually inherited                                                                                                                                      | Worker                |
-| DATABASE_CONNECTIONS      | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool                                                                                                                                      | All                   |
-| SUPERADMIN_SECRET         | None                   | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev                                                                                                              | Server                |
-| TIMEOUT_WAIT_RESULT       | 20                     | The number of seconds to wait before timeout on the 'run_wait_result' endpoint                                                                                                                     | Worker                |
-| QUEUE_LIMIT_WAIT_RESULT   | None                   | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit.           | Worker                |
-| DENO_AUTH_TOKENS          | None                   | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
-| DENO_FLAGS                | None                   | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json"                                  | Worker                |
-| NPM_CONFIG_REGISTRY       | None                   | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry                                                              | Worker                |
-| PIP_LOCAL_DEPENDENCIES    | None                   | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |                       |
-| ADDITIONAL_PYTHON_PATHS   | None                   | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
-| INCLUDE_HEADERS           | None                   | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
-| WHITELIST_WORKSPACES      | None                   | Whitelist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
-| BLACKLIST_WORKSPACES      | None                   | Blacklist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
-| INSTANCE_EVENTS_WEBHOOK   | None                   | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails                                                                                        | Server                |
+| Environment Variable name           | Default                                    | Description                                                                                                                                                                                        | Api Server/Worker/All |
+| ----------------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
+| DATABASE_URL                        |                                            | The Postgres database url.                                                                                                                                                                         | All                   |
+| DISABLE_NSJAIL                      | true                                       | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
+| SERVER_BIND_ADDR                    | 0.0.0.0                                    | IP Address on which to bind listening socket                                                                                                                                                       | Server                |
+| PORT                                | 8000                                       | Exposed port                                                                                                                                                                                       | Server                |
+| NUM_WORKERS                         | 3                                          | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance)                                                                             | Worker                |
+| DISABLE_SERVER                      | false                                      | Binary would operate as a worker only instance                                                                                                                                                     | Worker                |
+| METRICS_ADDR                        | None                                       | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
+| JSON_FMT                            | false                                      | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
+| BASE_URL                            | http://localhost:8000                      | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
+| BASE_INTERNAL_URL                   | http://localhost:8000                      | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests.                                      | Worker                |
+| TIMEOUT                             | 300                                        | The maximum time of execution of a script. When reached, the job is failed as having timedout.                                                                                                     | Worker                |
+| ZOMBIE_JOB_TIMEOUT                  | 30                                         | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s)                                  | Server                |
+| RESTART_ZOMBIE_JOBS                 | true                                       | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed                                                                              | Server                |
+| SLEEP_QUEUE                         | 50                                         | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker                |
+| MAX_LOG_SIZE                        | 500000                                     | The maximum number of characters a job can emit (log + result)                                                                                                                                     | Worker                |
+| DISABLE_NUSER                       | false                                      | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting                                                                                                                                 | Worker                |
+| KEEP_JOB_DIR                        | false                                      | Keep the job directory after the job is done. Useful for debugging.                                                                                                                                | Worker                |
+| LICENSE_KEY (EE only)               | None                                       | License key checked at startup for the Enterprise Edition of Windmill                                                                                                                              | Worker                |
+| S3_CACHE_BUCKET (EE only)           | None                                       | The S3 bucket to sync the cache of the workers to                                                                                                                                                  | Worker                |
+| TAR_CACHE_RATE (EE only)            | 100                                        | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed).                                                                          | Worker                |
+| SLACK_SIGNING_SECRET                | None                                       | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack)                                                                | Server                |
+| COOKIE_DOMAIN                       | None                                       | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |
+| DENO_PATH                           | /usr/bin/deno                              | The path to the deno binary.                                                                                                                                                                       | Worker                |
+| PYTHON_PATH                         | /usr/local/bin/python3                     | The path to the python binary.                                                                                                                                                                     | Worker                |
+| GO_PATH                             | /usr/bin/go                                | The path to the go binary.                                                                                                                                                                         | Worker                |
+| GOPRIVATE                           |                                            | The GOPRIVATE env variable to use private go modules                                                                                                                                               | Worker                |
+| GOPROXY                             |                                            | The GOPROXY env variable to use                                                                                                                                                                    | Worker                |
+| NETRC                               |                                            | The netrc content to use a private go registry                                                                                                                                                     | Worker                |
+| PIP_INDEX_URL                       | None                                       | The index url to pass for pip.                                                                                                                                                                     | Worker                |
+| PIP_EXTRA_INDEX_URL                 | None                                       | The extra index url to pass to pip.                                                                                                                                                                | Worker                |
+| PIP_TRUSTED_HOST                    | None                                       | The trusted host to pass to pip.                                                                                                                                                                   | Worker                |
+| PATH                                | None                                       | The path environment variable, usually inherited                                                                                                                                                   | Worker                |
+| HOME                                | None                                       | The home directory to use for Go and Bash , usually inherited                                                                                                                                      | Worker                |
+| DATABASE_CONNECTIONS                | 50 (Server)/3 (Worker)                     | The max number of connections in the database connection pool                                                                                                                                      | All                   |
+| SUPERADMIN_SECRET                   | None                                       | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev                                                                                                              | Server                |
+| TIMEOUT_WAIT_RESULT                 | 20                                         | The number of seconds to wait before timeout on the 'run_wait_result' endpoint                                                                                                                     | Worker                |
+| QUEUE_LIMIT_WAIT_RESULT             | None                                       | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit.           | Worker                |
+| DENO_AUTH_TOKENS                    | None                                       | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
+| DENO_FLAGS                          | None                                       | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json"                                  | Worker                |
+| NPM_CONFIG_REGISTRY                 | None                                       | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry                                                              | Worker                |
+| PIP_LOCAL_DEPENDENCIES              | None                                       | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |                       |
+| ADDITIONAL_PYTHON_PATHS             | None                                       | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
+| INCLUDE_HEADERS                     | None                                       | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
+| WHITELIST_WORKSPACES                | None                                       | Whitelist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
+| BLACKLIST_WORKSPACES                | None                                       | Blacklist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
+| INSTANCE_EVENTS_WEBHOOK             | None                                       | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails                                                                                        |
+| GLOBAL_CACHE_INTERVAL               | 10\*60                                     | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together.          | Worker                |
+| WORKER_TAGS                         | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers                                                                                                                                                         | Worker                |
+| CUSTOM_TAGS                         | None                                       | The custom tags assignable to scripts.                                                                                                                                                             | Server                |
+| JOB_RETENTION_SECS                  | 60*60*24\*60 //60 days                     | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete                                                                                                                   |
+| WAIT_RESULT_FAST_POLL_INTERVAL_MS   | 50                                         | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
+| WAIT_RESULT_SLOW_POLL_INTERVAL_MS   | 200                                        | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
+| WAIT_RESULT_FAST_POLL_DURATION_SECS | 2                                          | The duration of fast poll mode before switching to slow poll                                                                                                                                       | Server                |
+| EXIT_AFTER_NO_JOB_FOR_SECS          | None                                       | Exit worker if no job is received after duration in secs if defined                                                                                                                                | Worker                |
+| OAUTH_JSON_AS_BASE64                | None                                       | Base64 encoded JSON of the OAuth configuration. e.g `OAUTH_JSON_AS_BASE64=$(base64 oauth.json \| tr -d '\n')` to encode it                                                                         | Server                |
+| REQUEST_SIZE_LIMIT                  | 2097152 (2MB)                              | Max request size which impact the maximum size of resources and payload size of job args                                                                                                           | Server                |
 
 ## Run a local dev setup
 

backend/.vscode/settings.json

@@ -1,3 +1,4 @@
 {
-  "python.analysis.typeCheckingMode": "basic"
+  "python.analysis.typeCheckingMode": "basic",
+  "rust-analyzer.linkedProjects": ["./windmill-common/Cargo.toml"]
 }

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.87.0"
+version = "1.102.0"
 authors.workspace = true
 edition.workspace = true
 
@@ -19,7 +19,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.87.0"
+version = "1.102.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -50,6 +50,10 @@ base64.workspace = true
 sha2.workspace = true
 rsmq_async.workspace = true
 url.workspace = true
+lazy_static.workspace = true
+once_cell.workspace = true
+prometheus.workspace = true
+uuid.workspace = true
 
 
 [dev-dependencies]
@@ -100,7 +104,7 @@ rand = "0.8.5"
 rand_core = { version = "^0", features = ["std"] }
 magic-crypt = "^3"
 git-version = "^0"
-rustpython-parser = { git = "https://github.com/RustPython/RustPython" }
+rustpython-parser = "0.2.0"
 cron = "^0"
 lettre = { version = "^0", features = [
     "rustls-tls",
@@ -158,5 +162,5 @@ async_zip = { version = "0.0.11", features = ["full"] }
 once_cell = "1.17.1"
 rsmq_async = { version = "5.1.5" }
 gosyn = "0.2.2"
+bytes = "1.4.0"
 
-[patch.crates-io]

backend/README.md

@@ -1,6 +1,7 @@
 # Windmill Backend
 
-This folder holds all backend components, the [src/](./src/) folder only contains files used to build the "root" binary.
+This folder holds all backend components, the [src/](./src/) folder only
+contains files used to build the "root" binary.
 
 ## Components
 
@@ -13,3 +14,9 @@ This folder holds all backend components, the [src/](./src/) folder only contain
 | [windmill-queue](./windmill-queue/)           | Contains job & flow queuing functionality, commonly written to by the API server and read from by workers |
 | [windmill-worker](./windmill-worker/)         | The worker. Used to process and execute flows & jobs.                                                     |
 | [parsers](./parsers/)                         | Contains code to parse signatures in different langauges.                                                 |
+
+### Compile sqlx for offline ci
+
+```
+cargo sqlx prepare --merged -- --bin windmill --features enterprise
+```

backend/migrations/20230414163230_input_grant.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230414163230_input_grant.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+GRANT ALL ON input TO windmill_user;
+GRANT ALL ON input TO windmill_admin;

backend/migrations/20230421111850_add_tagged_scripts.down.sql

@@ -0,0 +1,4 @@
+-- Add down migration script here
+ALTER TABLE script DROP COLUMN tag;
+ALTER TABLE completed_job DROP COLUMN tag;
+ALTER TABLE queue DROP COLUMN tag;

backend/migrations/20230421111850_add_tagged_scripts.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+ALTER TABLE script ADD COLUMN tag VARCHAR(50);
+ALTER TABLE completed_job  ADD COLUMN tag VARCHAR(50) NOT NULL DEFAULT 'other';
+ALTER TABLE queue ADD COLUMN tag VARCHAR(50) NOT NULL DEFAULT 'other';

backend/migrations/20230429082953_add_drafts.down.sql

@@ -0,0 +1,3 @@
+-- Add down migration script here
+DROP TABLE draft;
+DROP TYPE DRAFT_TYPE;

backend/migrations/20230429082953_add_drafts.up.sql

@@ -0,0 +1,18 @@
+-- Add up migration script here
+CREATE TYPE DRAFT_TYPE AS ENUM ('script', 'flow', 'app');
+
+CREATE TABLE draft (
+    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+    path VARCHAR(255) NOT NULL,
+    typ DRAFT_TYPE NOT NULL,
+    value JSONB NOT NULL,
+    created_at TIMESTAMP NOT NULL DEFAULT now(),
+    PRIMARY KEY (workspace_id, path, typ)
+);
+
+GRANT ALL ON draft TO windmill_admin;
+GRANT ALL ON draft TO windmill_user;
+
+ALTER TABLE script ADD COLUMN draft_only BOOLEAN;
+ALTER TABLE flow ADD COLUMN draft_only BOOLEAN;
+ALTER TABLE app ADD COLUMN draft_only BOOLEAN;

backend/migrations/20230507170410_raw_app.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230507170410_raw_app.up.sql

@@ -0,0 +1,33 @@
+-- Add up migration script here
+CREATE TABLE raw_app (
+    path varchar(255) PRIMARY KEY,
+    version INTEGER NOT NULL DEFAULT 0,
+    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+    summary VARCHAR(1000) NOT NULL DEFAULT '',
+    edited_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+    data TEXT NOT NULL,
+    extra_perms JSONB NOT NULL DEFAULT '{}'
+);
+
+CREATE POLICY see_own ON raw_app FOR ALL
+USING (SPLIT_PART(raw_app.path, '/', 1) = 'u' AND SPLIT_PART(raw_app.path, '/', 2) = current_setting('session.user'));
+
+CREATE POLICY see_member ON raw_app FOR ALL
+USING (SPLIT_PART(raw_app.path, '/', 1) = 'g' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
+
+CREATE POLICY see_extra_perms_user ON raw_app FOR ALL
+USING (extra_perms ? CONCAT('u/', current_setting('session.user')))
+WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
+
+CREATE POLICY see_extra_perms_groups ON raw_app FOR ALL
+USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
+WITH CHECK (exists(
+    SELECT key, value FROM jsonb_each_text(extra_perms) 
+    WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
+    AND value::boolean));
+
+CREATE POLICY see_folder_extra_perms_user ON raw_app FOR ALL
+USING (SPLIT_PART(raw_app.path, '/', 1) = 'f' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]))
+WITH CHECK (SPLIT_PART(raw_app.path, '/', 1) = 'f' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
+
+ALTER TYPE FAVORITE_KIND ADD VALUE 'raw_app';

backend/migrations/20230509130016_raw_app_grant.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230509130016_raw_app_grant.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+GRANT ALL ON raw_app TO windmill_admin;
+GRANT ALL ON raw_app TO windmill_user;

backend/migrations/20230510101952_jsonb_to_json.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230510101952_jsonb_to_json.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE app_version
+ALTER COLUMN value TYPE json;

backend/migrations/20230510104144_jsonb_to_json_draft.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230510104144_jsonb_to_json_draft.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+ALTER TABLE draft
+ALTER COLUMN value TYPE json;

backend/migrations/20230512191209_update_app_n.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230515131104_rls_raw_app.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230515131104_rls_raw_app.up.sql

@@ -0,0 +1,2 @@
+-- Add up migration script here
+ALTER TABLE raw_app ENABLE ROW LEVEL SECURITY;

backend/migrations/20230516140626_capture_check_increase.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230516140626_capture_check_increase.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+alter table capture DROP constraint capture_payload_check;
+alter table capture add constraint caputre_payload_too_big check  (length(payload::text) < 512 * 1024);

backend/migrations/20230516141630_capture_check_increase_2.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230516141630_capture_check_increase_2.up.sql

@@ -0,0 +1,3 @@
+-- Add up migration script here
+alter table capture DROP constraint caputre_payload_too_big;
+alter table capture add constraint capture_payload_too_big check (length(payload::text) < 512 * 1024);

backend/parsers/windmill-parser-go/src/lib.rs

@@ -96,7 +96,7 @@ fn parse_go_typ(typ: &Expression) -> (Option<String>, Typ) {
                         .as_ref()
                         .and_then(|x| x.value.strip_prefix("`json:\""))
                         .and_then(|x| x.strip_suffix("\"`"))
-                        .and_then(|x| x.split(',').last().map(|x| x.to_string()));
+                        .and_then(|x| x.split(',').next().map(|x| x.to_string()));
                     let (otyp, typ) = parse_go_typ(&field.typ);
                     let name = get_name(field);
                     let key = json_tag.unwrap_or_else(|| name.to_string());

backend/parsers/windmill-parser-py/src/lib.rs

@@ -17,8 +17,8 @@ use serde_json::json;
 use windmill_common::error;
 use windmill_parser::{json_to_typ, Arg, MainArgSignature, Typ};
 
-use rustpython_parser as parser;
 use rustpython_parser::ast::{Constant, ExprKind, Located, StmtKind};
+use rustpython_parser::parser::parse_program;
 
 const DEF_MAIN: &str = "def main(";
 const FUNCTION_CALL: &str = "<function call>";
@@ -64,7 +64,7 @@ pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
             "No main function found".to_string(),
         ));
     }
-    let ast = parser::parse_program(&filtered_code, "main.py").map_err(|e| {
+    let ast = parse_program(&filtered_code, "main.py").map_err(|e| {
         error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
     })?;
     let param = ast.into_iter().find_map(|x| match x {
@@ -134,10 +134,7 @@ fn to_value(et: &ExprKind) -> Option<serde_json::Value> {
                 .into_iter()
                 .zip(values)
                 .map(|(k, v)| {
-                    let key = k
-                        .as_ref()
-                        .map(|x| x.node.clone())
-                        .and_then(|n| to_value(&n))
+                    let key = to_value(&k.node)
                         .and_then(|x| match x {
                             serde_json::Value::String(s) => Some(s),
                             _ => None,
@@ -185,6 +182,7 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
     "shopify" => "ShopifyAPI",
     "seleniumwire" => "selenium-wire",
     "openbb-terminal" => "openbb[all]",
+    "riskfolio" => "riskfolio-lib",
 };
 
 fn replace_import(x: String) -> String {
@@ -215,7 +213,7 @@ pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
         Ok(lines)
     } else {
         let code = code.split(DEF_MAIN).next().unwrap_or("");
-        let ast = parser::parse_program(code, "main.py").map_err(|e| {
+        let ast = parse_program(code, "main.py").map_err(|e| {
             error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
         })?;
         let mut imports: Vec<String> = ast

backend/parsers/windmill-parser-ts/src/lib.rs

@@ -255,6 +255,7 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
                     ),
                     false,
                 ),
+                "Date" => (Typ::Datetime, false),
                 "Base64" => (Typ::Bytes, false),
                 "Email" => (Typ::Email, false),
                 "Sql" => (Typ::Sql, false),
@@ -268,7 +269,7 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
 pub fn eval_sync(code: &str) -> Result<serde_json::Value, String> {
     let mut context = JsRuntime::new(RuntimeOptions::default());
     let code = format!("let x = {}; x", code);
-    let res = context.execute_script("<anon>", code);
+    let res = context.execute_script("<anon>", code.into());
     match res {
         Ok(global) => {
             let scope = &mut context.handle_scope();

backend/src/main.rs

@@ -6,11 +6,24 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::{
+    net::{IpAddr, Ipv4Addr, SocketAddr},
+    sync::Arc,
+};
 
 use git_version::git_version;
+use monitor::handle_zombie_jobs_periodically;
 use sqlx::{Pool, Postgres};
+use tokio::{
+    fs::{metadata, DirBuilder},
+    join,
+    sync::RwLock,
+};
 use windmill_common::{utils::rd_string, METRICS_ADDR};
+use windmill_worker::{
+    DENO_CACHE_DIR, DENO_TMP_CACHE_DIR, GO_CACHE_DIR, GO_TMP_CACHE_DIR, HUB_CACHE_DIR,
+    HUB_TMP_CACHE_DIR, PIP_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR,
+};
 
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 const DEFAULT_NUM_WORKERS: usize = 3;
@@ -18,6 +31,7 @@ const DEFAULT_PORT: u16 = 8000;
 const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0);
 
 mod ee;
+mod monitor;
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
@@ -88,7 +102,7 @@ async fn main() -> anyhow::Result<()> {
     }
 
     let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
-    let shutdown_signal = windmill_common::shutdown_signal(tx);
+    let shutdown_signal = windmill_common::shutdown_signal(tx.clone(), rx.resubscribe());
 
     #[cfg(feature = "enterprise")]
     tracing::info!(
@@ -129,6 +143,7 @@ Windmill Community Edition {GIT_VERSION}
         "DENO_PATH",
         "GO_PATH",
         "GOPRIVATE",
+        "GOPROXY",
         "NETRC",
         "PIP_INDEX_URL",
         "PIP_EXTRA_INDEX_URL",
@@ -148,6 +163,15 @@ Windmill Community Edition {GIT_VERSION}
         "BLACKLIST_WORKSPACES",
         "INSTANCE_EVENTS_WEBHOOK",
         "CLOUD_HOSTED",
+        "GLOBAL_CACHE_INTERVAL",
+        "WORKER_TAGS",
+        "CUSTOM_TAGS",
+        "JOB_RETENTION_SECS",
+        "WAIT_RESULT_FAST_POLL_DURATION_SECS",
+        "WAIT_RESULT_SLOW_POLL_INTERVAL_MS",
+        "WAIT_RESULT_FAST_POLL_INTERVAL_MS",
+        "EXIT_AFTER_NO_JOB_FOR_SECS",
+        "REQUEST_SIZE_LIMIT",
     ]);
 
     if server_mode || num_workers > 0 {
@@ -171,6 +195,8 @@ Windmill Community Edition {GIT_VERSION}
                     rsmq.clone(),
                 )
                 .await?;
+                tracing::info!("All workers exited.");
+                tx.send(())?; // signal server to shutdown
             }
             Ok(()) as anyhow::Result<()>
         };
@@ -230,9 +256,11 @@ pub fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>
     let rx2 = rx.resubscribe();
     let base_internal_url = base_internal_url.to_string();
     tokio::spawn(async move {
-        windmill_worker::handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq).await
+        join!(
+            handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq),
+            windmill_api::delete_expired_items_perdiodically(&db2, rx2)
+        );
     });
-    tokio::spawn(async move { windmill_api::delete_expired_items_perdiodically(&db2, rx2).await });
 }
 
 pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
@@ -263,6 +291,30 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
 
     let mut handles = Vec::with_capacity(num_workers as usize);
 
+    if metadata(&ROOT_TMP_CACHE_DIR).await.is_ok() {
+        if let Err(e) = tokio::fs::remove_dir_all(&ROOT_TMP_CACHE_DIR).await {
+            tracing::info!(error = %e, "Could not remove root tmp cache dir");
+        }
+    }
+
+    for x in [
+        PIP_CACHE_DIR,
+        DENO_CACHE_DIR,
+        GO_CACHE_DIR,
+        HUB_CACHE_DIR,
+        TAR_PIP_TMP_CACHE_DIR,
+        DENO_TMP_CACHE_DIR,
+        GO_TMP_CACHE_DIR,
+        HUB_TMP_CACHE_DIR,
+    ] {
+        DirBuilder::new()
+            .recursive(true)
+            .create(x)
+            .await
+            .expect("could not create initial worker dir");
+    }
+
+    let sync_barrier = Arc::new(RwLock::new(None));
     for i in 1..(num_workers + 1) {
         let db1 = db.clone();
         let instance_name = instance_name.clone();
@@ -271,6 +323,7 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
         let rx = rx.resubscribe();
         let base_internal_url = base_internal_url.clone();
         let rsmq2 = rsmq.clone();
+        let sync_barrier = sync_barrier.clone();
         handles.push(tokio::spawn(monitor.instrument(async move {
             tracing::info!(worker = %worker_name, "starting worker");
             windmill_worker::run_worker(
@@ -278,10 +331,12 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
                 &instance_name,
                 worker_name,
                 i as u64,
+                num_workers as u32,
                 &ip,
                 rx,
                 &base_internal_url,
                 rsmq2,
+                sync_barrier,
             )
             .await
         })));

backend/src/monitor.rs

@@ -0,0 +1,148 @@
+use std::time::Duration;
+
+use once_cell::sync::OnceCell;
+use sqlx::{Pool, Postgres};
+use tokio::sync::mpsc;
+use uuid::Uuid;
+use windmill_common::{error, jobs::{JobKind, QueuedJob}, METRICS_ENABLED};
+use windmill_worker::{
+    create_token_for_owner, handle_job_error, AuthedClient, SESSION_TOKEN_EXPIRY,
+};
+
+lazy_static::lazy_static! {
+    static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
+    .ok()
+    .and_then(|x| x.parse::<String>().ok())
+    .unwrap_or_else(|| "30".to_string());
+
+
+    pub static ref RESTART_ZOMBIE_JOBS: bool = std::env::var("RESTART_ZOMBIE_JOBS")
+    .ok()
+    .and_then(|x| x.parse::<bool>().ok())
+    .unwrap_or(true);
+
+    static ref QUEUE_ZOMBIE_RESTART_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
+        "queue_zombie_restart_count",
+        "Total number of jobs restarted due to ping timeout."
+    )
+    .unwrap();
+    static ref QUEUE_ZOMBIE_DELETE_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
+        "queue_zombie_delete_count",
+        "Total number of jobs deleted due to their ping timing out in an unrecoverable state."
+    )
+    .unwrap();
+}
+
+pub async fn handle_zombie_jobs_periodically<
+    R: rsmq_async::RsmqConnection + Send + Sync + Clone,
+>(
+    db: &Pool<Postgres>,
+    mut rx: tokio::sync::broadcast::Receiver<()>,
+    base_internal_url: &str,
+    rsmq: Option<R>,
+) {
+    loop {
+        handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
+
+        tokio::select! {
+            _ = tokio::time::sleep(Duration::from_secs(30))    => (),
+            _ = rx.recv() => {
+                    println!("received killpill for monitor job");
+                    break;
+            }
+        }
+    }
+}
+
+async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
+    db: &Pool<Postgres>,
+    base_internal_url: &str,
+    rsmq: Option<R>,
+) {
+    if *RESTART_ZOMBIE_JOBS {
+        let restarted = sqlx::query!(
+                "UPDATE queue SET running = false, started_at = null, logs = logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND job_kind != $3 AND same_worker = false RETURNING id, workspace_id, last_ping",
+                *ZOMBIE_JOB_TIMEOUT,
+                JobKind::Flow: JobKind,
+                JobKind::FlowPreview: JobKind,
+            )
+            .fetch_all(db)
+            .await
+            .ok()
+            .unwrap_or_else(|| vec![]);
+
+        if *METRICS_ENABLED {
+            QUEUE_ZOMBIE_RESTART_COUNT.inc_by(restarted.len() as _);
+        }
+        for r in restarted {
+            tracing::info!(
+                "restarted zombie job {} {} {}",
+                r.id,
+                r.workspace_id,
+                r.last_ping
+            );
+        }
+    }
+
+    let mut timeout_query = "SELECT * FROM queue WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2".to_string();
+    if *RESTART_ZOMBIE_JOBS {
+        timeout_query.push_str(" AND same_worker = true");
+    };
+    let timeouts = sqlx::query_as::<_, QueuedJob>(&timeout_query)
+        .bind(ZOMBIE_JOB_TIMEOUT.as_str())
+        .bind(JobKind::Flow)
+        .fetch_all(db)
+        .await
+        .ok()
+        .unwrap_or_else(|| vec![]);
+
+    if *METRICS_ENABLED {
+        QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _);
+    }
+
+    for job in timeouts {
+        tracing::info!("timedout zombie job {} {}", job.id, job.workspace_id,);
+
+        // since the job is unrecoverable, the same worker queue should never be sent anything
+        let (same_worker_tx_never_used, _same_worker_rx_never_used) = mpsc::channel::<Uuid>(1);
+
+        let token = create_token_for_owner(
+            &db,
+            &job.workspace_id,
+            &job.permissioned_as,
+            "ephemeral-zombie-jobs",
+            *SESSION_TOKEN_EXPIRY,
+            &job.email,
+        )
+        .await
+        .expect("could not create job token");
+
+        let client = AuthedClient {
+            base_internal_url: base_internal_url.to_string(),
+            token,
+            workspace: job.workspace_id.to_string(),
+            client: OnceCell::new(),
+        };
+
+        let last_ping = job.last_ping.clone();
+        let _ = handle_job_error(
+            db,
+            &client,
+            job,
+            error::Error::ExecutionErr(format!(
+                "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {})",
+                last_ping
+                    .map(|x| x.to_string())
+                    .unwrap_or_else(|| "no ping".to_string()),
+                *ZOMBIE_JOB_TIMEOUT
+            )),
+            None,
+            true,
+            same_worker_tx_never_used,
+            "",
+            base_internal_url,
+            rsmq.clone(),
+        )
+        .await;
+    }
+}

backend/tests/worker.rs

@@ -1,14 +1,19 @@
+use std::sync::Arc;
+
 use futures::{stream, Stream};
 use serde::Deserialize;
 use serde_json::json;
 use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, Transaction};
+use tokio::sync::RwLock;
 use windmill_api::jobs::{CompletedJob, Job};
+use windmill_api_client::types::{CreateFlowBody, RawScript};
 use windmill_common::{
     flow_status::{FlowStatus, FlowStatusModule},
     flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
+    jobs::{JobPayload, RawCode},
     scripts::ScriptLang,
 };
-use windmill_queue::{get_queued_job, JobPayload, RawCode};
+use windmill_queue::get_queued_job;
 
 async fn initialize_tracing() {
     use std::sync::Once;
@@ -128,8 +133,7 @@ mod suspend_resume {
     use futures::{Stream, StreamExt};
     use serde_json::json;
     use sqlx::{query_scalar, types::Uuid};
-    use windmill_common::flows::FlowValue;
-    use windmill_queue::JobPayload;
+    use windmill_common::{flows::FlowValue, jobs::JobPayload};
 
     use super::*;
 
@@ -393,7 +397,6 @@ mod retry {
     use serde_json::json;
     use tokio::io::{AsyncReadExt, AsyncWriteExt};
     use windmill_common::flows::FlowValue;
-    use windmill_queue::JobPayload;
 
     use super::*;
 
@@ -839,10 +842,12 @@ impl RunJob {
             /* schedule_path */ None,
             /* parent_job */ None,
             /* root job  */ None,
+            /* job_id */ None,
             /* is_flow_step */ false,
             /* running */ false,
             None,
             true,
+            None,
         )
         .await
         .expect("push has to succeed");
@@ -877,7 +882,7 @@ async fn in_test_worker<Fut: std::future::Future>(
     port: u16,
 ) -> <Fut as std::future::Future>::Output {
     let (quit, worker) = spawn_test_worker(db, port);
-    let worker = tokio::time::timeout(std::time::Duration::from_secs(19), worker);
+    let worker = tokio::time::timeout(std::time::Duration::from_secs(45), worker);
     tokio::pin!(worker);
 
     let res = tokio::select! {
@@ -912,7 +917,6 @@ fn spawn_test_worker(
     let db = db.to_owned();
     let worker_instance: &str = "test worker instance";
     let worker_name: String = next_worker_name();
-    let i_worker: u64 = Default::default();
     let ip: &str = Default::default();
     let future = async move {
         let base_internal_url = format!("http://localhost:{}", port);
@@ -920,11 +924,13 @@ fn spawn_test_worker(
             &db,
             worker_instance,
             worker_name,
-            i_worker,
+            1,
+            1,
             ip,
             rx,
             &base_internal_url,
             None,
+            Arc::new(RwLock::new(None)),
         )
         .await
     };
@@ -1002,6 +1008,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                         content: numbers.to_string(),
                         path: None,
                         lock: None,
+                        tag: None,
                     },
                     stop_after_if: Default::default(),
                     summary: Default::default(),
@@ -1029,6 +1036,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                                 content: doubles.to_string(),
                                 path: None,
                                 lock: None,
+                                tag: None,
                             },
                             stop_after_if: Default::default(),
                             summary: Default::default(),
@@ -1129,6 +1137,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                         content: write_file.clone(),
                         path: None,
                         lock: None,
+                        tag: None,
                     },
                     stop_after_if: Default::default(),
                     summary: Default::default(),
@@ -1167,6 +1176,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                     content: write_file,
                                     path: None,
                                     lock: None,
+                                    tag: None,
                                 },
                                 stop_after_if: Default::default(),
                                 summary: Default::default(),
@@ -1192,6 +1202,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                     .to_string(),
                                     path: None,
                                     lock: None,
+                                    tag: None,
                                 },
                                 stop_after_if: Default::default(),
                                 summary: Default::default(),
@@ -1234,6 +1245,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                         .to_string(),
                         path: None,
                         lock: None,
+                        tag: None,
                     },
                     stop_after_if: Default::default(),
                     summary: Default::default(),
@@ -2432,9 +2444,12 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
     client
         .create_flow(
             "test-workspace",
-            &windmill_api_client::types::OpenFlowWPath {
-                open_flow: flow,
-                path: "g/all/flow_lock_all".to_owned(),
+            &CreateFlowBody {
+                open_flow_w_path: windmill_api_client::types::OpenFlowWPath {
+                    open_flow: flow,
+                    path: "g/all/flow_lock_all".to_owned(),
+                },
+                draft_only: None,
             },
         )
         .await
@@ -2453,20 +2468,21 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
         .modules
         .into_iter()
         .for_each(|m| {
+            tracing::error!("Module: {:?}", m.value);
             assert!(matches!(
                 m.value,
-                windmill_api_client::types::FlowModuleValue::Rawscript {
+                windmill_api_client::types::FlowModuleValue::RawScript(RawScript {
                     language: windmill_api_client::types::RawScriptLanguage::Deno | windmill_api_client::types::RawScriptLanguage::Bash,
                     lock: Some(ref lock),
                     ..
-                } if lock == "")
+                }) if lock == "")
                 || matches!(
                 m.value,
-                windmill_api_client::types::FlowModuleValue::Rawscript {
+                windmill_api_client::types::FlowModuleValue::RawScript(RawScript{
                     language: windmill_api_client::types::RawScriptLanguage::Go | windmill_api_client::types::RawScriptLanguage::Python3,
                     lock: Some(ref lock),
                     ..
-                } if lock.len() > 0)
+                }) if lock.len() > 0)
             );
         });
 }

backend/windmill-api-client/Cargo.toml

@@ -11,7 +11,7 @@ path = "./src/lib.rs"
 
 
 [dependencies]
-progenitor-client = { git = "https://github.com/oxidecomputer/progenitor" }
+progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", rev = "3d96016ae8d422e90513b2d34fb5b63eeab30b01" }
 reqwest = { version = "0.11", features = ["json", "stream"] }
 serde = { version = "1.0", features = ["derive"] }
 chrono.workspace = true
@@ -21,5 +21,7 @@ rand.workspace = true
 base64.workspace = true
 
 [build-dependencies]
-progenitor = { git = "https://github.com/oxidecomputer/progenitor" }
+prettyplease = "0.1.25"
+progenitor = { git = "https://github.com/oxidecomputer/progenitor", rev = "3d96016ae8d422e90513b2d34fb5b63eeab30b01" }
 serde_json = "1.0"
+syn = "1.0"

backend/windmill-api-client/build.rs

@@ -13,7 +13,9 @@ fn main() {
     let spec = serde_json::from_reader(file).unwrap();
     let mut generator = progenitor::Generator::default();
 
-    let content = generator.generate_text(&spec).unwrap();
+    let tokens = generator.generate_tokens(&spec).unwrap();
+    let ast = syn::parse2(tokens).unwrap();
+    let content = prettyplease::unparse(&ast);
 
     let mut out_file = Path::new(&env::var("OUT_DIR").unwrap()).to_path_buf();
     out_file.push("codegen.rs");

backend/windmill-api/Cargo.toml

@@ -72,3 +72,4 @@ prometheus.workspace = true
 async_zip.workspace = true
 rsmq_async.workspace = true
 regex.workspace = true
+bytes.workspace = true

backend/windmill-api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.87.0
+  version: 1.102.0
   title: Windmill API
 
   contact:
@@ -2125,6 +2125,41 @@ paths:
                 items:
                   type: string
 
+
+  /w/{workspace}/drafts/create:
+    post:
+      summary: create draft
+      operationId: createDraft
+      tags:
+        - draft
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                path:
+                  type: string
+                typ:
+                  type: string
+                  enum: ["flow", "script", "app"]
+                value: {}
+              required:
+                - path
+                - typ
+                - enum
+      responses:
+        "201":
+          description: draft created
+          content:
+            text/plain:
+              schema:
+                type: string
+
+
   /w/{workspace}/scripts/create:
     post:
       summary: create script
@@ -2139,38 +2174,8 @@ paths:
         content:
           application/json:
             schema:
-              type: object
-              properties:
-                path:
-                  type: string
-                parent_hash:
-                  type: string
-                summary:
-                  type: string
-                description:
-                  type: string
-                content:
-                  type: string
-                schema:
-                  type: object
-                is_template:
-                  type: boolean
-                lock:
-                  type: array
-                  items:
-                    type: string
-                language:
-                  type: string
-                  enum: [python3, deno, go, bash]
-                kind:
-                  type: string
-                  enum: [script, failure, trigger, command, approval]
-              required:
-                - path
-                - summary
-                - description
-                - content
-                - language
+              $ref: "#/components/schemas/NewScript"
+
       responses:
         "201":
           description: script created
@@ -2179,6 +2184,22 @@ paths:
               schema:
                 type: string
 
+  /workers/custom_tags:
+    get:
+      summary: get all instance custom tags (tags are used to dispatch jobs to different worker groups)
+      operationId: getCustomTags
+      tags:
+        - worker
+      responses:
+        "200":
+          description: list of custom tags
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+
   /scripts/python/tojsonschema:
     post:
       summary: inspect python code to infer jsonschema of arguments
@@ -2348,6 +2369,25 @@ paths:
               schema:
                 $ref: "#/components/schemas/Script"
 
+  /w/{workspace}/scripts/get/draft/{path}:
+    get:
+      summary: get script by path with draft
+      operationId: getScriptByPathWithDraft
+      tags:
+        - script
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: script details
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/NewScriptWithDraft"
+
+
+
   /w/{workspace}/scripts/raw/p/{path}:
     get:
       summary: raw script by path
@@ -2477,6 +2517,7 @@ paths:
           schema:
             type: integer
         - $ref: "#/components/parameters/ParentJob"
+        - $ref: "#/components/parameters/NewJobId"
         - name: invisible_to_owner
           description: make the run invisible to the the script owner (default false)
           in: query
@@ -2509,6 +2550,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/ScriptPath"
         - $ref: "#/components/parameters/ParentJob"
+        - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - $ref: "#/components/parameters/QueueLimit"
 
@@ -2536,6 +2578,7 @@ paths:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/ScriptPath"
         - $ref: "#/components/parameters/ParentJob"
+        - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - $ref: "#/components/parameters/QueueLimit"
         - $ref: "#/components/parameters/Payload"
@@ -2558,6 +2601,7 @@ paths:
         - $ref: "#/components/parameters/ScriptPath"
         - $ref: "#/components/parameters/IncludeHeader"
         - $ref: "#/components/parameters/QueueLimit"
+        - $ref: "#/components/parameters/NewJobId"
 
       requestBody:
         description: script args
@@ -2663,7 +2707,14 @@ paths:
               schema:
                 type: array
                 items:
-                  $ref: "#/components/schemas/Flow"
+                  allOf:
+                    - $ref: "#/components/schemas/Flow"
+                    - type: object
+                      properties:
+                        has_draft:
+                          type: boolean
+                        draft_only:
+                          type: boolean
 
   /w/{workspace}/flows/get/{path}:
     get:
@@ -2682,6 +2733,29 @@ paths:
               schema:
                 $ref: "#/components/schemas/Flow"
 
+  /w/{workspace}/flows/get/draft/{path}:
+    get:
+      summary: get flow by path with draft
+      operationId: getFlowByPathWithDraft
+      tags:
+        - flow
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: flow details with draft
+          content:
+            application/json:
+              schema:
+                allOf:
+                  - $ref: "#/components/schemas/Flow"
+                  - type: object
+                    properties:
+                      draft:
+                        $ref: "#/components/schemas/Flow"
+
+
   /w/{workspace}/flows/exists/{path}:
     get:
       summary: exists flow by path
@@ -2713,7 +2787,12 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/OpenFlowWPath"
+              allOf:
+                - $ref: "#/components/schemas/OpenFlowWPath"
+                - type: object
+                  properties:
+                    draft_only:
+                      type: boolean
 
       responses:
         "201":
@@ -2757,6 +2836,16 @@ paths:
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
         - $ref: "#/components/parameters/ScriptPath"
+      requestBody:
+        description: archiveFlow
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                archived:
+                  type: boolean
       responses:
         "200":
           description: flow archived
@@ -2803,6 +2892,80 @@ paths:
                 items:
                   $ref: "#/components/schemas/Input"
 
+  /w/{workspace}/raw_apps/list:
+    get:
+      summary: list all available raw apps
+      operationId: listRawApps
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Page"
+        - $ref: "#/components/parameters/PerPage"
+        - $ref: "#/components/parameters/OrderDesc"
+        - $ref: "#/components/parameters/CreatedBy"
+        - name: path_start
+          description: mask to filter matching starting path
+          in: query
+          schema:
+            type: string
+        - name: path_exact
+          description: mask to filter exact matching path
+          in: query
+          schema:
+            type: string
+        - name: starred_only
+          description: |
+            (default false)
+            show only the starred items
+          in: query
+          schema:
+            type: boolean
+      responses:
+        "200":
+          description: All available raw apps
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/ListableRawApp"
+
+  /w/{workspace}/raw_apps/exists/{path}:
+    get:
+      summary: does an app exisst at path
+      operationId: existsRawApp
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: app exists
+          content:
+            application/json:
+              schema:
+                type: boolean
+
+  /w/{workspace}/apps/get_data/{version}/{path}:
+    get:
+      summary: get app by path
+      operationId: getRawAppData
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/VersionId"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: app details
+          content:
+            text/javascript:
+              schema:
+                type: string
+
   /w/{workspace}/apps/list:
     get:
       summary: list all available apps
@@ -2842,6 +3005,44 @@ paths:
                 items:
                   $ref: "#/components/schemas/ListableApp"
 
+  /w/{workspace}/apps/create:
+    post:
+      summary: create app
+      operationId: createApp
+      tags:
+        - app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: new app
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                path:
+                  type: string
+                value: {}
+                summary:
+                  type: string
+                policy:
+                  $ref: "#/components/schemas/Policy"
+                draft_only:
+                  type: boolean
+              required:
+                - path
+                - value
+                - summary
+                - policy
+      responses:
+        "201":
+          description: app created
+          content:
+            text/plain:
+              schema:
+                type: string
+
   /w/{workspace}/apps/exists/{path}:
     get:
       summary: does an app exisst at path
@@ -2876,6 +3077,24 @@ paths:
               schema:
                 $ref: "#/components/schemas/AppWithLastVersion"
 
+  /w/{workspace}/apps/get/draft/{path}:
+    get:
+      summary: get app by path with draft
+      operationId: getAppByPathWithDraft
+      tags:
+        - app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: app details with draft
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/AppWithLastVersionWDraft"
+
+
   /w/{workspace}/apps_u/public_app/{path}:
     get:
       summary: get public app by secret
@@ -2927,16 +3146,16 @@ paths:
               schema:
                 $ref: "#/components/schemas/AppWithLastVersion"
 
-  /w/{workspace}/apps/create:
+  /w/{workspace}/raw_apps/create:
     post:
-      summary: create app
-      operationId: createApp
+      summary: create raw app
+      operationId: createRawApp
       tags:
-        - app
+        - raw_app
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
       requestBody:
-        description: new app
+        description: new raw app
         required: true
         content:
           application/json:
@@ -2945,19 +3164,65 @@ paths:
               properties:
                 path:
                   type: string
-                value: {}
+                value:
+                  type: string
                 summary:
                   type: string
-                policy:
-                  $ref: "#/components/schemas/Policy"
               required:
                 - path
                 - value
                 - summary
-                - policy
       responses:
         "201":
-          description: app created
+          description: raw app created
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /w/{workspace}/raw_apps/update/{path}:
+    post:
+      summary: update app
+      operationId: updateRawApp
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+      requestBody:
+        description: updateraw  app
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                path:
+                  type: string
+                summary:
+                  type: string
+                value:
+                  type: string
+      responses:
+        "200":
+          description: app updated
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /w/{workspace}/raw_apps/delete/{path}:
+    delete:
+      summary: delete raw app
+      operationId: deleteRawApp
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: app deleted
           content:
             text/plain:
               schema:
@@ -3029,6 +3294,8 @@ paths:
             schema:
               type: object
               properties:
+                component:
+                  type: string
                 #script: script/<path>
                 #flow: flow/<path>
                 path:
@@ -3050,6 +3317,7 @@ paths:
                   type: object
               required:
                 - args
+                - component
 
       responses:
         "200":
@@ -3080,6 +3348,7 @@ paths:
           schema:
             type: integer
         - $ref: "#/components/parameters/ParentJob"
+        - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
           description: make the run invisible to the the flow owner (default false)
@@ -3125,6 +3394,7 @@ paths:
           schema:
             type: integer
         - $ref: "#/components/parameters/ParentJob"
+        - $ref: "#/components/parameters/NewJobId"
         - $ref: "#/components/parameters/IncludeHeader"
         - name: invisible_to_owner
           description: make the run invisible to the the script owner (default false)
@@ -3162,6 +3432,8 @@ paths:
           in: query
           schema:
             type: boolean
+        - $ref: "#/components/parameters/NewJobId"
+
       requestBody:
         description: preview
         required: true
@@ -3193,6 +3465,7 @@ paths:
           in: query
           schema:
             type: boolean
+        - $ref: "#/components/parameters/NewJobId"
 
       requestBody:
         description: preview
@@ -3233,6 +3506,7 @@ paths:
         - $ref: "#/components/parameters/Running"
         - $ref: "#/components/parameters/ArgsFilter"
         - $ref: "#/components/parameters/ResultFilter"
+        - $ref: "#/components/parameters/Tag"
       responses:
         "200":
           description: All available queued jobs
@@ -3263,6 +3537,7 @@ paths:
         - $ref: "#/components/parameters/JobKinds"
         - $ref: "#/components/parameters/ArgsFilter"
         - $ref: "#/components/parameters/ResultFilter"
+        - $ref: "#/components/parameters/Tag"
         - name: is_skipped
           description: is the job skipped
           in: query
@@ -3300,6 +3575,7 @@ paths:
         - $ref: "#/components/parameters/StartedAfter"
         - $ref: "#/components/parameters/JobKinds"
         - $ref: "#/components/parameters/ArgsFilter"
+        - $ref: "#/components/parameters/Tag"
         - $ref: "#/components/parameters/ResultFilter"
         - name: is_skipped
           description: is the job skipped
@@ -3378,7 +3654,7 @@ paths:
                   mem_peak:
                     type: integer
 
-  /w/{workspace}/jobs/completed/get/{id}:
+  /w/{workspace}/jobs_u/completed/get/{id}:
     get:
       summary: get completed job
       operationId: getCompletedJob
@@ -3395,7 +3671,7 @@ paths:
               schema:
                 $ref: "#/components/schemas/CompletedJob"
 
-  /w/{workspace}/jobs/completed/get_result/{id}:
+  /w/{workspace}/jobs_u/completed/get_result/{id}:
     get:
       summary: get completed job result
       operationId: getCompletedJobResult
@@ -3411,6 +3687,30 @@ paths:
             application/json:
               schema: {}
 
+  /w/{workspace}/jobs_u/completed/get_result_maybe/{id}:
+    get:
+      summary: get completed job result if job is completed
+      operationId: getCompletedJobResultMaybe
+      tags:
+        - job
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/JobId"
+      responses:
+        "200":
+          description: result
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  completed:
+                    type: boolean
+                  result: {}
+                required:
+                  - completed
+                  - result
+
   /w/{workspace}/jobs/completed/delete/{id}:
     post:
       summary: delete completed job (erase content but keep run id)
@@ -4391,7 +4691,7 @@ paths:
           schema:
             type: string
             enum:
-              [script, group_, resource, schedule, variable, flow, folder, app]
+              [script, group_, resource, schedule, variable, flow, folder, app, raw_app]
       responses:
         "200":
           description: acls
@@ -4417,7 +4717,7 @@ paths:
           schema:
             type: string
             enum:
-              [script, group_, resource, schedule, variable, flow, folder, app]
+              [script, group_, resource, schedule, variable, flow, folder, app, raw_app]
       requestBody:
         description: acl to add
         required: true
@@ -4454,7 +4754,7 @@ paths:
           schema:
             type: string
             enum:
-              [script, group_, resource, schedule, variable, flow, folder, app]
+              [script, group_, resource, schedule, variable, flow, folder, app, raw_app]
       requestBody:
         description: acl to add
         required: true
@@ -4534,7 +4834,7 @@ paths:
                   type: string
                 favorite_kind:
                   type: string
-                  enum: [flow, app, script]
+                  enum: [flow, app, script, raw_app]
       responses:
         "200":
           description: star item
@@ -4557,7 +4857,7 @@ paths:
                   type: string
                 favorite_kind:
                   type: string
-                  enum: [flow, app, script]
+                  enum: [flow, app, script, raw_app]
       responses:
         "200":
           description: unstar item
@@ -4691,6 +4991,12 @@ components:
       required: true
       schema:
         type: string
+    VersionId:
+      name: version
+      in: path
+      required: true
+      schema:
+        type: number
     Token:
       name: token
       in: path
@@ -4779,6 +5085,15 @@ components:
       schema:
         type: string
         format: uuid
+    NewJobId:
+      name: job_id
+      description:
+        The job id to assign to the created job. if missing, job is chosen randomly using the ULID scheme.
+        If a job id already exists in the queue or as a completed job, the request to create one will fail (Bad Request)
+      in: query
+      schema:
+        type: string
+        format: uuid
     IncludeHeader:
       name: include_header
       description: |
@@ -4858,6 +5173,12 @@ components:
       in: query
       schema:
         type: string
+    Tag:
+      name: tag
+      description: filter on jobs with a given tag/worker group
+      in: query
+      schema:
+        type: string
     ResultFilter:
       name: result
       description: filter on jobs containing those result as a json subset (@> in postgres)
@@ -4987,6 +5308,12 @@ components:
           enum: [script, failure, trigger, command, approval]
         starred:
           type: boolean
+        tag:
+          type: string
+        has_draft:
+          type: boolean
+        draft_only:
+          type: boolean
       required:
         - hash
         - path
@@ -5003,6 +5330,57 @@ components:
         - kind
         - starred
 
+    NewScript:
+      type: object
+      properties:
+        path:
+          type: string
+        parent_hash:
+          type: string
+        summary:
+          type: string
+        description:
+          type: string
+        content:
+          type: string
+        schema:
+          type: object
+        is_template:
+          type: boolean
+        lock:
+          type: array
+          items:
+            type: string
+        language:
+          type: string
+          enum: [python3, deno, go, bash]
+        kind:
+          type: string
+          enum: [script, failure, trigger, command, approval]
+        tag:
+          type: string
+        draft_only:
+          type: boolean
+      required:
+        - path
+        - summary
+        - description
+        - content
+        - language
+
+    NewScriptWithDraft:
+      allOf:
+        - $ref: "#/components/schemas/NewScript"
+        - type: object
+          properties:
+            draft:
+              $ref: "#/components/schemas/NewScript"
+            hash:
+              type: string
+          required:
+            - hash
+
+
     ScriptArgs:
       type: object
       additionalProperties: {}
@@ -5138,6 +5516,8 @@ components:
           type: boolean
         mem_peak:
           type: integer
+        tag:
+          type: string
       required:
         - id
         - running
@@ -5147,6 +5527,7 @@ components:
         - is_flow_step
         - email
         - visible_to_owner
+        - tag
 
     CompletedJob:
       type: object
@@ -5226,6 +5607,8 @@ components:
           type: boolean
         mem_peak:
           type: integer
+        tag:
+          type: string
       required:
         - id
         - created_by
@@ -5240,6 +5623,7 @@ components:
         - is_skipped
         - email
         - visible_to_owner
+        - tag
 
     Job:
       allOf:
@@ -5280,6 +5664,10 @@ components:
           type: array
           items:
             type: string
+        folders_owners:
+          type: array
+          items:
+            type: string
         usage:
           $ref: "#/components/schemas/Usage"
       required:
@@ -5291,6 +5679,7 @@ components:
         - operator
         - disabled
         - folders
+        - folders_owners
 
     Usage:
       type: object
@@ -5630,6 +6019,8 @@ components:
         language:
           type: string
           enum: [python3, deno, go, bash]
+        tag:
+          type: string
 
       required:
         - content
@@ -6001,6 +6392,8 @@ components:
           type: boolean
         starred:
           type: boolean
+        draft_only:
+          type: boolean
       required:
         - path
         - edited_by
@@ -6083,6 +6476,34 @@ components:
         - edited_at
         - execution_mode
 
+    ListableRawApp:
+      type: object
+      properties:
+        workspace_id:
+          type: string
+        path:
+          type: string
+        summary:
+          type: string
+        extra_perms:
+          type: object
+          additionalProperties:
+            type: boolean
+        starred:
+          type: boolean
+        version:
+          type: number
+        edited_at:
+          type: string
+          format: date-time
+      required:
+        - workspace_id
+        - path
+        - summary
+        - extra_perms
+        - version
+        - edited_at
+
     AppWithLastVersion:
       type: object
       properties:
@@ -6126,6 +6547,15 @@ components:
         - execution_mode
         - extra_perms
 
+    AppWithLastVersionWDraft:
+      allOf:
+        - $ref: "#/components/schemas/AppWithLastVersion"
+        - type: object
+          properties:
+            draft_only:
+              type: boolean
+            draft: {}
+
     SlackToken:
       type: object
       properties:

backend/windmill-api/src/apps.rs

@@ -9,7 +9,6 @@ use std::collections::HashMap;
  */
 use crate::{
     db::{UserDB, DB},
-    jobs::script_path_to_payload,
     users::{require_owner_of_path, Authed, OptAuthed},
     variables::build_crypt,
     webhook_util::{WebhookMessage, WebhookShared},
@@ -32,17 +31,19 @@ use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     apps::ListAppQuery,
     error::{to_anyhow, Error, JsonResult, Result},
+    jobs::{script_path_to_payload, JobPayload, RawCode},
     users::username_to_permissioned_as,
     utils::{
         http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
     },
 };
-use windmill_queue::{push, JobPayload, QueueTransaction, RawCode};
+use windmill_queue::{push, QueueTransaction};
 
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/list", get(list_apps))
         .route("/get/p/*path", get(get_app))
+        .route("/get/draft/*path", get(get_app_w_draft))
         .route("/secret_of/*path", get(get_secret_id))
         .route("/get/v/*id", get(get_app_by_id))
         .route("/exists/*path", get(exists_app))
@@ -74,6 +75,9 @@ pub struct ListableApp {
     pub execution_mode: String,
     pub starred: bool,
     pub edited_at: chrono::DateTime<chrono::Utc>,
+    pub has_draft: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
 }
 
 #[derive(FromRow, Serialize, Deserialize)]
@@ -98,6 +102,23 @@ pub struct AppWithLastVersion {
     pub extra_perms: serde_json::Value,
 }
 
+#[derive(Serialize, Deserialize)]
+pub struct AppWithLastVersionAndDraft {
+    pub id: i64,
+    pub path: String,
+    pub summary: String,
+    pub policy: serde_json::Value,
+    pub versions: Vec<i64>,
+    pub value: serde_json::Value,
+    pub created_by: String,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    pub extra_perms: serde_json::Value,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
+}
+
 pub type StaticFields = Map<String, Value>;
 
 #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
@@ -126,6 +147,7 @@ pub struct CreateApp {
     pub summary: String,
     pub value: serde_json::Value,
     pub policy: Policy,
+    pub draft_only: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -156,6 +178,8 @@ async fn list_apps(
             "app_version.created_at as edited_at",
             "app.extra_perms",
             "favorite.path IS NOT NULL as starred",
+            "draft.path IS NOT NULL as has_draft",
+            "draft_only"
         ])
         .left()
         .join("favorite")
@@ -168,6 +192,11 @@ async fn list_apps(
         .on(
             "app_version.id = versions[array_upper(versions, 1)]"
         )
+        .left()
+        .join("draft")
+        .on(
+            "draft.path = app.path AND draft.workspace_id = app.workspace_id AND draft.typ = 'app'"
+        )
         .order_desc("favorite.path IS NOT NULL")
         .order_by("app_version.created_at", true)
         .and_where("app.workspace_id = ?".bind(&w_id))
@@ -215,6 +244,37 @@ async fn get_app(
     Ok(Json(app))
 }
 
+async fn get_app_w_draft(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<AppWithLastVersionAndDraft> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    let app_o = sqlx::query_as!(
+        AppWithLastVersionAndDraft,
+        r#"SELECT app.id, app.path, app.summary, app.versions, app.policy,
+        app.extra_perms, app_version.value, 
+        app_version.created_at, app_version.created_by,
+        app.draft_only, draft.value as "draft?"
+        from app
+        INNER JOIN app_version ON
+        app_version.id = app.versions[array_upper(app.versions, 1)]
+        LEFT JOIN draft ON 
+        app.path = draft.path AND app.workspace_id = draft.workspace_id AND draft.typ = 'app' 
+        WHERE app.path = $1 AND app.workspace_id = $2"#,
+        path.to_owned(),
+        &w_id
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    let app = not_found_if_none(app_o, "App", path)?;
+    Ok(Json(app))
+}
+
 async fn get_app_by_id(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -324,14 +384,39 @@ async fn create_app(
         return Err(Error::BadRequest("App path cannot be empty".to_string()));
     }
 
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)",
+        &app.path,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(false);
+
+    if exists {
+        return Err(Error::BadRequest(format!(
+            "App with path {} already exists",
+            &app.path
+        )));
+    }
+
+    sqlx::query!(
+        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
+        &app.path,
+        &w_id
+    )
+    .execute(&mut tx)
+    .await?;
+
     let id = sqlx::query_scalar!(
         "INSERT INTO app
-            (workspace_id, path, summary, policy, versions)
-            VALUES ($1, $2, $3, $4, '{}') RETURNING id",
+            (workspace_id, path, summary, policy, versions, draft_only)
+            VALUES ($1, $2, $3, $4, '{}', $5) RETURNING id",
         w_id,
         app.path,
         app.summary,
         json!(app.policy),
+        app.draft_only,
     )
     .fetch_one(&mut tx)
     .await?;
@@ -339,9 +424,10 @@ async fn create_app(
     let v_id = sqlx::query_scalar!(
         "INSERT INTO app_version
             (app_id, value, created_by)
-            VALUES ($1, $2, $3) RETURNING id",
+            VALUES ($1, $2::text::json, $3) RETURNING id",
         id,
-        app.value,
+        //to preserve key orders
+        serde_json::to_string(&app.value).unwrap(),
         authed.username,
     )
     .fetch_one(&mut tx)
@@ -411,6 +497,14 @@ async fn delete_app(
     let path = path.to_path();
     let mut tx = user_db.begin(&authed).await?;
 
+    sqlx::query!(
+        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
+        path,
+        &w_id
+    )
+    .execute(&mut tx)
+    .await?;
+
     sqlx::query!(
         "DELETE FROM app WHERE path = $1 AND workspace_id = $2",
         path,
@@ -441,7 +535,6 @@ async fn update_app(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
-    Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
     Json(ns): Json<EditApp>,
 ) -> Result<String> {
@@ -456,11 +549,25 @@ async fn update_app(
         sqlb.and_where_eq("path", "?".bind(&path));
         sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
 
+        sqlb.set("draft_only", "NULL");
         if let Some(npath) = &ns.path {
             if npath != path {
-                if !authed.is_admin {
-                    require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db)
-                        .await?;
+                require_owner_of_path(&authed, path)?;
+
+                let exists = sqlx::query_scalar!(
+                    "SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)",
+                    npath,
+                    w_id
+                )
+                .fetch_one(&mut tx)
+                .await?
+                .unwrap_or(false);
+
+                if exists {
+                    return Err(Error::BadRequest(format!(
+                        "App with path {} already exists",
+                        npath
+                    )));
                 }
             }
             sqlb.set_str("path", npath);
@@ -504,9 +611,10 @@ async fn update_app(
         let v_id = sqlx::query_scalar!(
             "INSERT INTO app_version
                 (app_id, value, created_by)
-                VALUES ($1, $2, $3) RETURNING id",
+                VALUES ($1, $2::text::json, $3) RETURNING id",
             app_id,
-            nvalue,
+            //to preserve key orders
+            serde_json::to_string(&nvalue).unwrap(),
             authed.username,
         )
         .fetch_one(&mut tx)
@@ -521,6 +629,15 @@ async fn update_app(
         .execute(&mut tx)
         .await?;
     }
+
+    sqlx::query!(
+        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
+        path,
+        &w_id
+    )
+    .execute(&mut tx)
+    .await?;
+
     audit_log(
         &mut tx,
         &authed.username,
@@ -550,6 +667,7 @@ pub struct ExecuteApp {
     // - script: script/<path>
     // - flow: flow/<path>
     pub path: Option<String>,
+    pub component: String,
     pub raw_code: Option<RawCode>,
     // if set, the app is executed as viewer with the given static fields
     pub force_viewer_static_fields: Option<StaticFields>,
@@ -588,11 +706,16 @@ async fn execute_component(
 
     let policy = if let Some(static_fields) = payload.clone().force_viewer_static_fields {
         let mut hm = HashMap::new();
+
         if let Some(path) = payload.path.clone() {
-            hm.insert(path, static_fields);
+            hm.insert(format!("{}:{path}", payload.component), static_fields);
         } else {
             hm.insert(
-                digest(payload.raw_code.clone().unwrap().content.as_str()),
+                format!(
+                    "{}:{}",
+                    payload.component,
+                    digest(payload.raw_code.clone().unwrap().content.as_str())
+                ),
                 static_fields,
             );
         }
@@ -643,16 +766,16 @@ async fn execute_component(
         }
     };
 
-    let (job_payload, args) = match &payload {
-        ExecuteApp { args, raw_code: Some(raw_code), path: None, .. } => {
+    let (job_payload, args, tag) = match &payload {
+        ExecuteApp { args, component, raw_code: Some(raw_code), path: None, .. } => {
             let content = &raw_code.content;
             let payload = JobPayload::Code(raw_code.clone());
             let path = digest(content);
-            let args = build_args(policy, path, args)?;
-            (payload, args)
+            let args = build_args(policy, component, path, args)?;
+            (payload, args, None)
         }
-        ExecuteApp { args, raw_code: None, path: Some(path), .. } => {
-            let payload = if path.starts_with("script/") {
+        ExecuteApp { args, component, raw_code: None, path: Some(path), .. } => {
+            let (payload, tag) = if path.starts_with("script/") {
                 script_path_to_payload(
                     path.strip_prefix("script/").unwrap(),
                     tx.transaction_mut(),
@@ -660,15 +783,18 @@ async fn execute_component(
                 )
                 .await?
             } else if path.starts_with("flow/") {
-                JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string())
+                (
+                    JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string()),
+                    None,
+                )
             } else {
                 return Err(Error::BadRequest(format!(
                     "path must start with script/ or flow/ (got {})",
                     path
                 )));
             };
-            let args = build_args(policy, path.to_string(), args)?;
-            (payload, args)
+            let args = build_args(policy, component, path.to_string(), args)?;
+            (payload, args, tag)
         }
         _ => unreachable!(),
     };
@@ -685,10 +811,12 @@ async fn execute_component(
         None,
         None,
         None,
+        None,
         false,
         false,
         None,
         true,
+        tag,
     )
     .await?;
     tx.commit().await?;
@@ -739,15 +867,18 @@ async fn exists_app(
 
 fn build_args(
     policy: Policy,
+    component: &str,
     path: String,
     args: &Map<String, Value>,
 ) -> Result<Map<String, Value>> {
     // disallow var and res access in args coming from the user for security reasons
     args.into_iter()
         .try_for_each(|x| disallow_var_res_access(x.1))?;
+    let key = format!("{}:{}", component, &path);
     let static_args = policy
         .triggerables
-        .get(&path)
+        .get(&key)
+        .or_else(|| policy.triggerables.get(&path))
         .map(|x| x.clone())
         .or_else(|| {
             if matches!(policy.execution_mode, ExecutionMode::Viewer) {

backend/windmill-api/src/drafts.rs

@@ -0,0 +1,95 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2042
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use crate::{
+    db::{UserDB, DB},
+    users::{maybe_refresh_folders, require_owner_of_path, Authed},
+};
+
+use axum::{
+    extract::{Extension, Path},
+    routing::post,
+    Json, Router,
+};
+use hyper::StatusCode;
+use serde::{Deserialize, Serialize};
+use windmill_common::error::Result;
+
+pub fn workspaced_service() -> Router {
+    Router::new().route("/create", post(create_draft))
+}
+
+#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
+#[sqlx(type_name = "DRAFT_TYPE", rename_all = "lowercase")]
+#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
+pub enum DraftType {
+    Script,
+    Flow,
+    App,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct Draft {
+    pub path: String,
+    pub value: serde_json::Value,
+    pub typ: DraftType,
+}
+
+async fn create_draft(
+    authed: Authed,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Json(draft): Json<Draft>,
+) -> Result<(StatusCode, String)> {
+    let authed = maybe_refresh_folders(&draft.path, &w_id, authed, &db).await;
+
+    let mut tx = user_db.begin(&authed).await?;
+
+    require_owner_of_path(&authed, &draft.path)?;
+
+    sqlx::query!(
+        "INSERT INTO draft
+            (workspace_id, path, value, typ)
+            VALUES ($1, $2, $3::text::json, $4)
+            ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = $3::text::json",
+        &w_id,
+        draft.path,
+        //to preserve key orders
+        serde_json::to_string(&draft.value).unwrap(),
+        draft.typ: DraftType,
+    )
+    .execute(&mut tx)
+    .await?;
+
+    tx.commit().await?;
+
+    Ok((StatusCode::CREATED, format!("draft {} created", draft.path)))
+}
+
+// async fn get_draft(
+//     authed: Authed,
+//     Extension(user_db): Extension<UserDB>,
+//     Path((w_id, path)): Path<(String, StripPath)>,
+// ) -> JsonResult<Draft> {
+//     let path = path.to_path();
+//     let mut tx = user_db.begin(&authed).await?;
+
+//     let script_o = sqlx::query_as!(
+//         Draft,
+//         r#"SELECT path, value, typ as "typ: DraftType" FROM draft WHERE path = $1 AND workspace_id = $2"#,
+//         path,
+//         w_id
+//     )
+//     .fetch_optional(&mut tx)
+//     .await?;
+//     tx.commit().await?;
+
+//     let draft = not_found_if_none(script_o, "draft", path)?;
+//     Ok(Json(draft))
+// }

backend/windmill-api/src/favorite.rs

@@ -29,6 +29,8 @@ pub enum FavoriteKind {
     Script,
     Flow,
     App,
+    #[allow(non_camel_case_types)]
+    Raw_App,
 }
 #[derive(Deserialize)]
 pub struct Favorite {

backend/windmill-api/src/flows.rs

@@ -19,6 +19,7 @@ use axum::{
     Json, Router,
 };
 use hyper::StatusCode;
+use serde::{Deserialize, Serialize};
 use sql_builder::prelude::*;
 use sql_builder::SqlBuilder;
 use sqlx::{Postgres, Transaction};
@@ -26,12 +27,14 @@ use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{self, to_anyhow, Error, JsonResult, Result},
     flows::{Flow, ListFlowQuery, ListableFlow, NewFlow},
+    jobs::JobPayload,
     schedule::Schedule,
+    scripts::Schema,
     utils::{
         http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
     },
 };
-use windmill_queue::{push, schedule::push_scheduled_job, JobPayload, QueueTransaction};
+use windmill_queue::{push, schedule::push_scheduled_job, QueueTransaction};
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -41,6 +44,7 @@ pub fn workspaced_service() -> Router {
         .route("/archive/*path", post(archive_flow_by_path))
         .route("/delete/*path", delete(delete_flow_by_path))
         .route("/get/*path", get(get_flow_by_path))
+        .route("/get/draft/*path", get(get_flow_by_path_w_draft))
         .route("/exists/*path", get(exists_flow_by_path))
         .route("/list_paths", get(list_paths))
 }
@@ -71,6 +75,8 @@ async fn list_flows(
             "archived",
             "extra_perms",
             "favorite.path IS NOT NULL as starred",
+            "draft.path IS NOT NULL as has_draft",
+            "draft_only"
         ])
         .left()
         .join("favorite")
@@ -78,6 +84,11 @@ async fn list_flows(
             "favorite.favorite_kind = 'flow' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?"
                 .bind(&authed.username),
         )
+        .left()
+        .join("draft")
+        .on(
+            "draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'flow'"
+        )
         .order_desc("favorite.path IS NOT NULL")
         .order_by("edited_at", lq.order_desc.unwrap_or(true))
         .and_where("o.workspace_id = ?".bind(&w_id))
@@ -85,9 +96,8 @@ async fn list_flows(
         .limit(per_page)
         .clone();
 
-    if !lq.show_archived.unwrap_or(false) {
-        sqlb.and_where_eq("archived", false);
-    }
+    sqlb.and_where_eq("archived", lq.show_archived.unwrap_or(false));
+
     if let Some(ps) = &lq.path_start {
         sqlb.and_where_like_left("path", "?".bind(ps));
     }
@@ -193,7 +203,7 @@ async fn create_flow(
 
     sqlx::query!(
         "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, \
-         schema, dependency_job) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL)",
+         schema, dependency_job, draft_only) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL, $8)",
         w_id,
         nf.path,
         nf.summary,
@@ -201,6 +211,15 @@ async fn create_flow(
         nf.value,
         &authed.username,
         nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
+        nf.draft_only
+    )
+    .execute(&mut tx)
+    .await?;
+
+    sqlx::query!(
+        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
+        nf.path,
+        &w_id
     )
     .execute(&mut tx)
     .await?;
@@ -238,10 +257,12 @@ async fn create_flow(
         None,
         None,
         None,
+        None,
         false,
         false,
         None,
         true,
+        None,
     )
     .await?;
 
@@ -308,7 +329,7 @@ async fn update_flow(
     let old_dep_job = not_found_if_none(old_dep_job, "Flow", flow_path)?;
     sqlx::query!(
         "UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, \
-         edited_at = now(), schema = $6::text::json, dependency_job = NULL WHERE path = $7 AND workspace_id = $8",
+         edited_at = now(), schema = $6::text::json, dependency_job = NULL, draft_only = NULL WHERE path = $7 AND workspace_id = $8",
         nf.path,
         nf.summary,
         nf.description,
@@ -325,58 +346,65 @@ async fn update_flow(
         check_schedule_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;
 
         if !authed.is_admin {
-            require_owner_of_path(&w_id, &authed.username, &authed.groups, &flow_path, &db).await?;
+            require_owner_of_path(&authed, flow_path)?;
         }
+    }
 
-        let mut schedulables: Vec<Schedule> = sqlx::query_as!(
-            Schedule,
-                "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
-                nf.path,
-                flow_path,
-                w_id,
-            )
-            .fetch_all(&mut tx)
-            .await?;
-
-        let schedule = sqlx::query_as!(Schedule,
-            "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
+    let mut schedulables: Vec<Schedule> = sqlx::query_as!(
+        Schedule,
+            "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
             nf.path,
             flow_path,
             w_id,
         )
-        .fetch_optional(&mut tx)
+        .fetch_all(&mut tx)
         .await?;
 
-        if let Some(schedule) = schedule {
-            schedulables.push(schedule);
-        }
+    let schedule = sqlx::query_as!(Schedule,
+        "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
+        nf.path,
+        flow_path,
+        w_id,
+    )
+    .fetch_optional(&mut tx)
+    .await?;
 
-        for schedule in schedulables.into_iter() {
-            // TODO: Why is this in the loop in the first place? Seems like it's just doing nothing after the first iteration? Should this use schedule.path?
-            clear_schedule(tx.transaction_mut(), flow_path, true).await?;
-
-            if schedule.enabled {
-                tx = push_scheduled_job(tx, schedule).await?;
-            }
-        }
-
-        audit_log(
-            &mut tx,
-            &authed.username,
-            "flows.update",
-            ActionKind::Create,
-            &w_id,
-            Some(&nf.path.to_string()),
-            Some(
-                [Some(("flow", nf.path.as_str()))]
-                    .into_iter()
-                    .flatten()
-                    .collect(),
-            ),
-        )
-        .await?;
+    if let Some(schedule) = schedule {
+        schedulables.push(schedule);
     }
 
+    for schedule in schedulables.into_iter() {
+        clear_schedule(tx.transaction_mut(), &schedule.path, true).await?;
+
+        if schedule.enabled {
+            tx = push_scheduled_job(tx, schedule).await?;
+        }
+    }
+
+    sqlx::query!(
+        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
+        flow_path,
+        &w_id
+    )
+    .execute(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "flows.update",
+        ActionKind::Create,
+        &w_id,
+        Some(&nf.path.to_string()),
+        Some(
+            [Some(("flow", nf.path.as_str()))]
+                .into_iter()
+                .flatten()
+                .collect(),
+        ),
+    )
+    .await?;
+
     webhook.send_message(
         w_id.clone(),
         WebhookMessage::UpdateFlow {
@@ -398,10 +426,12 @@ async fn update_flow(
         None,
         None,
         None,
+        None,
         false,
         false,
         None,
         true,
+        None,
     )
     .await?;
     sqlx::query!(
@@ -444,6 +474,44 @@ async fn get_flow_by_path(
     Ok(Json(flow))
 }
 
+#[derive(Serialize, sqlx::FromRow)]
+pub struct FlowWDraft {
+    pub path: String,
+    pub summary: String,
+    pub description: String,
+    pub schema: Option<Schema>,
+    pub value: serde_json::Value,
+    pub extra_perms: serde_json::Value,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
+}
+
+async fn get_flow_by_path_w_draft(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<FlowWDraft> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    let flow_o = sqlx::query_as::<_, FlowWDraft>(
+        "SELECT flow.path, flow.summary, flow,description, flow.schema, flow.value, flow.extra_perms, flow.draft_only, draft.value as draft FROM flow
+        LEFT JOIN draft ON 
+        flow.path = draft.path AND flow.workspace_id = draft.workspace_id AND draft.typ = 'flow' 
+        WHERE flow.path = $1 AND flow.workspace_id = $2",
+    )
+    .bind(path)
+    .bind(w_id)
+    .fetch_optional(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    let flow = not_found_if_none(flow_o, "Flow", path)?;
+    Ok(Json(flow))
+}
+
 async fn exists_flow_by_path(
     Extension(db): Extension<DB>,
     Path((w_id, path)): Path<(String, StripPath)>,
@@ -462,17 +530,24 @@ async fn exists_flow_by_path(
     Ok(Json(exists))
 }
 
+#[derive(Deserialize)]
+struct Archived {
+    archived: Option<bool>,
+}
+
 async fn archive_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, path)): Path<(String, StripPath)>,
+    Json(archived): Json<Archived>,
 ) -> Result<String> {
     let path = path.to_path();
     let mut tx = user_db.begin(&authed).await?;
 
     sqlx::query!(
-        "UPDATE flow SET archived = true WHERE path = $1 AND workspace_id = $2",
+        "UPDATE flow SET archived = $1 WHERE path = $2 AND workspace_id = $3",
+        archived.archived.unwrap_or(true),
         path,
         &w_id
     )
@@ -507,6 +582,14 @@ async fn delete_flow_by_path(
     let path = path.to_path();
     let mut tx = user_db.begin(&authed).await?;
 
+    sqlx::query!(
+        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
+        path,
+        &w_id
+    )
+    .execute(&mut tx)
+    .await?;
+
     sqlx::query!(
         "DELETE FROM flow WHERE path = $1 AND workspace_id = $2",
         path,
@@ -578,6 +661,7 @@ mod tests {
                         language: scripts::ScriptLang::Deno,
                         path: None,
                         lock: None,
+                        tag: None,
                     },
                     stop_after_if: Some(StopAfterIf {
                         expr: "foo = 'bar'".to_string(),

backend/windmill-api/src/folders.rs

@@ -22,13 +22,13 @@ use lazy_static::lazy_static;
 use regex::Regex;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
-    error::{self, to_anyhow, Error, JsonResult, Result},
+    error::{self, to_anyhow, JsonResult, Result},
     users::username_to_permissioned_as,
     utils::{not_found_if_none, paginate, Pagination},
 };
 
 use serde::{Deserialize, Serialize};
-use sqlx::{query_scalar, FromRow, Postgres, Transaction};
+use sqlx::{FromRow, Postgres, Transaction};
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -41,7 +41,7 @@ pub fn workspaced_service() -> Router {
         .route("/delete/:name", delete(delete_folder))
         .route("/addowner/:name", post(add_owner))
         .route("/removeowner/:name", post(remove_owner))
-        .route("/is_owner/*path", get(is_owner))
+        .route("/is_owner/*path", get(is_owner_api))
 }
 
 #[derive(FromRow, Serialize, Deserialize, Clone)]
@@ -219,47 +219,29 @@ async fn create_folder(
     Ok(format!("Created folder {}", ng.name))
 }
 
-pub async fn is_owner(
-    Authed { username, is_admin, groups, .. }: Authed,
-    Extension(db): Extension<DB>,
-    Path((w_id, name)): Path<(String, String)>,
+pub async fn is_owner_api(
+    authed: Authed,
+    Path((_w_id, name)): Path<(String, String)>,
 ) -> JsonResult<bool> {
-    if is_admin {
-        Ok(Json(true))
+    Ok(Json(is_owner(&authed, &name)))
+}
+
+pub fn is_owner(Authed { is_admin, folders, .. }: &Authed, name: &str) -> bool {
+    if *is_admin {
+        true
     } else {
-        Ok(Json(
-            require_is_owner(&name, &username, &groups, &w_id, &db)
-                .await
-                .is_ok(),
-        ))
+        folders.into_iter().any(|x| x.0 == name && x.2)
     }
 }
 
-pub async fn require_is_owner(
-    folder_name: &str,
-    username: &str,
-    groups: &Vec<String>,
-    w_id: &str,
-    db: &DB,
-) -> Result<()> {
-    let is_owner = query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM folder WHERE CONCAT('u/', $1::text) = ANY(owners) AND name = $2 AND workspace_id = $4) OR exists(
-            SELECT 1 FROM folder, unnest(folder.owners) as o
-            WHERE o = ANY($3::text[]) AND folder.name = $2 AND folder.workspace_id = $4)",
-        username,
-        folder_name,
-        groups,
-        w_id,
-    ).fetch_one(db)
-    .await?
-    .unwrap_or(false);
-    if !is_owner {
-        Err(Error::BadRequest(format!(
-            "{} is not an owner of {} and hence is not authorized to perform this operation",
-            username, folder_name
-        )))
-    } else {
+pub fn require_is_owner(authed: &Authed, name: &str) -> Result<()> {
+    if is_owner(authed, name) {
         Ok(())
+    } else {
+        Err(windmill_common::error::Error::NotAuthorized(format!(
+            "You are not owner of the folder {}",
+            name
+        )))
     }
 }
 
@@ -423,6 +405,15 @@ async fn get_folder_usage(
     .await?
     .unwrap_or(0);
 
+    let raw_apps = sqlx::query_scalar!(
+        "SELECT count(path) FROM raw_app WHERE path LIKE 'f/' || $1 || '%'  AND workspace_id = $2",
+        name,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(0);
+
     let resources = sqlx::query_scalar!(
         "SELECT count(path) FROM resource WHERE path LIKE 'f/' || $1 || '%'  AND workspace_id = $2",
         name,
@@ -446,7 +437,7 @@ async fn get_folder_usage(
         scripts,
         flows,
         schedules,
-        apps,
+        apps: apps + raw_apps,
         resources,
         variables,
     }))
@@ -491,7 +482,6 @@ async fn delete_folder(
 
 async fn add_owner(
     authed: Authed,
-    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, name)): Path<(String, String)>,
@@ -500,9 +490,7 @@ async fn add_owner(
     let mut tx = user_db.begin(&authed).await?;
 
     not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
-    if !authed.is_admin {
-        require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
-    }
+    require_is_owner(&authed, &name)?;
 
     sqlx::query!(
         "UPDATE folder SET owners = array_append(owners, $1) WHERE name = $2 AND workspace_id = $3 AND NOT $1 = ANY(owners) RETURNING name",
@@ -538,14 +526,14 @@ pub async fn get_folders_for_user(
     username: &str,
     groups: &[String],
     db: &DB,
-) -> Result<Vec<(String, bool)>> {
+) -> Result<Vec<(String, bool, bool)>> {
     let mut perms = groups
         .into_iter()
         .map(|x| format!("g/{}", x))
         .collect::<Vec<_>>();
     perms.insert(0, format!("u/{}", username));
     let folders = sqlx::query!(
-        "SELECT name, (EXISTS (SELECT 1 FROM (SELECT key, value FROM jsonb_each_text(extra_perms) WHERE key = ANY($1)) t  WHERE value::boolean IS true)) as write  FROM folder
+        "SELECT name, (EXISTS (SELECT 1 FROM (SELECT key, value FROM jsonb_each_text(extra_perms) WHERE key = ANY($1)) t  WHERE value::boolean IS true)) as write, $1 && owners::text[] as owner  FROM folder
         WHERE extra_perms ?| $1  AND workspace_id = $2",
         &perms[..],
         w_id,
@@ -553,7 +541,7 @@ pub async fn get_folders_for_user(
     .fetch_all(db)
     .await?
     .into_iter()
-    .map(|x| (x.name, x.write.unwrap_or(false)))
+    .map(|x| (x.name, x.write.unwrap_or(false), x.owner.unwrap_or(false)))
     .collect();
 
     Ok(folders)
@@ -561,7 +549,6 @@ pub async fn get_folders_for_user(
 
 async fn remove_owner(
     authed: Authed,
-    Extension(db): Extension<DB>,
     Extension(user_db): Extension<UserDB>,
     Extension(webhook): Extension<WebhookShared>,
     Path((w_id, name)): Path<(String, String)>,
@@ -570,9 +557,7 @@ async fn remove_owner(
     let mut tx = user_db.begin(&authed).await?;
 
     not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
-    if !authed.is_admin {
-        require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
-    }
+    require_is_owner(&authed, &name)?;
 
     sqlx::query!(
         "UPDATE folder SET owners = array_remove(owners, $1) WHERE name = $2 AND workspace_id = $3 RETURNING name",

backend/windmill-api/src/granular_acls.rs

@@ -43,25 +43,29 @@ async fn add_granular_acl(
     Json(GranularAcl { owner, write }): Json<GranularAcl>,
 ) -> Result<String> {
     let path = path.to_path();
+
     let (kind, path) = path
         .split_once('/')
         .ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
     let mut tx = user_db.begin(&authed).await?;
 
-    if !authed.is_admin {
-        if kind == "folder" {
-            crate::folders::require_is_owner(&path, &authed.username, &authed.groups, &w_id, &db)
-                .await?;
-        } else if kind == "group_" {
-        } else {
-            require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
-        }
-    }
     let identifier = if kind == "group_" || kind == "folder" {
         "name"
     } else {
         "path"
     };
+
+    if !authed.is_admin {
+        if kind == "folder" {
+            crate::folders::require_is_owner(&authed, path)?;
+        } else if kind == "group_" {
+            crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db)
+                .await?;
+        } else {
+            require_owner_of_path(&authed, path)?;
+        }
+    }
+
     let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
         "UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), \
          true) WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
@@ -86,12 +90,22 @@ async fn remove_granular_acl(
     Json(GranularAcl { owner, write: _ }): Json<GranularAcl>,
 ) -> Result<String> {
     let path = path.to_path();
-    if !authed.is_admin {
-        require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
-    }
+
     let (kind, path) = path
         .split_once('/')
         .ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
+
+    if !authed.is_admin {
+        if kind == "folder" {
+            crate::folders::require_is_owner(&authed, path)?;
+        } else if kind == "group_" {
+            crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db)
+                .await?;
+        } else {
+            require_owner_of_path(&authed, path)?;
+        }
+    }
+
     let mut tx = user_db.begin(&authed).await?;
 
     let identifier = if kind == "group_" || kind == "folder" {
@@ -99,6 +113,11 @@ async fn remove_granular_acl(
     } else {
         "path"
     };
+
+    if identifier == "path" {
+        require_owner_of_path(&authed, path)?;
+    }
+
     let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
         "UPDATE {kind} SET extra_perms = extra_perms - $1 WHERE {identifier} = $2 AND \
          workspace_id = $3 RETURNING extra_perms"

backend/windmill-api/src/groups.rs

@@ -292,6 +292,12 @@ async fn delete_group(
 ) -> Result<String> {
     let mut tx = user_db.begin(&authed).await?;
 
+    if name == "all" {
+        return Err(Error::BadRequest(
+            "The group 'all' is a special group that contains all users and cannot be deleted".to_string(),
+        ));
+    }
+
     if !authed.is_admin {
         require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
     }

backend/windmill-api/src/inputs.rs

@@ -22,11 +22,10 @@ use std::{
 };
 use windmill_common::{
     error::JsonResult,
+    jobs::JobKind,
     scripts::to_i64,
     utils::{paginate, Pagination},
 };
-use windmill_queue::JobKind;
-
 pub fn workspaced_service() -> Router {
     Router::new()
         .route("/history", get(get_input_history))
@@ -113,9 +112,9 @@ async fn get_input_history(
     let mut tx = user_db.begin(&authed).await?;
 
     let sql = &format!(
-        "select distinct on (args) * from completed_job \
+        "select * from (select distinct on (args) * from completed_job \
         where {} = $1 and job_kind = $2 and workspace_id = $3 \
-        order by args, started_at desc limit $4 offset $5",
+        order by args, started_at desc) t ORDER BY started_at desc limit $4 offset $5",
         r.runnable_type.column_name()
     );
 
@@ -170,7 +169,7 @@ async fn list_saved_inputs(
     let rows = sqlx::query_as::<_, InputRow>(
         "select * from input \
          where runnable_id = $1 and runnable_type = $2 and workspace_id = $3 \
-         and is_public IS true OR created_by = $4 \
+         and (is_public IS true OR created_by = $4) \
          order by created_at desc limit $5 offset $6",
     )
     .bind(&r.runnable_id)

backend/windmill-api/src/jobs.rs

@@ -17,11 +17,12 @@ use axum::{
     extract::{FromRequest, Json, Path, Query},
     response::{IntoResponse, Response},
     routing::{get, post},
-    Extension, Router,
+    Extension, Form, RequestExt, Router,
 };
 use base64::Engine;
+use bytes::Bytes;
 use hmac::Mac;
-use hyper::{HeaderMap, Request, StatusCode};
+use hyper::{header::CONTENT_TYPE, http, HeaderMap, Request, StatusCode};
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use sql_builder::{prelude::*, quote, SqlBuilder};
 use sqlx::{query_scalar, types::Uuid, FromRow, Postgres, Transaction};
@@ -31,22 +32,27 @@ use windmill_common::{
     error::{self, to_anyhow, Error},
     flow_status::{Approval, FlowStatus, FlowStatusModule},
     flows::FlowValue,
+    jobs::{script_path_to_payload, JobKind, JobPayload, QueuedJob, RawCode},
     oauth2::HmacSha256,
     scripts::{ScriptHash, ScriptLang},
     users::username_to_permissioned_as,
     utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
 };
-use windmill_queue::{
-    get_queued_job, push, JobKind, JobPayload, QueueTransaction, QueuedJob, RawCode,
-};
+use windmill_queue::{get_queued_job, push, QueueTransaction};
 
 pub fn workspaced_service() -> Router {
     Router::new()
-        .route("/run/f/*script_path", post(run_flow_by_path))
-        .route("/run/p/*script_path", post(run_job_by_path))
+        .route(
+            "/run/f/*script_path",
+            post(run_flow_by_path).head(|| async { "" }),
+        )
+        .route(
+            "/run/p/*script_path",
+            post(run_job_by_path).head(|| async { "" }),
+        )
         .route(
             "/run_wait_result/p/*script_path",
-            post(run_wait_result_job_by_path),
+            post(run_wait_result_job_by_path).head(|| async { "" }),
         )
         .route(
             "/run_wait_result/p/*script_path",
@@ -54,13 +60,13 @@ pub fn workspaced_service() -> Router {
         )
         .route(
             "/run_wait_result/h/:hash",
-            post(run_wait_result_job_by_hash),
+            post(run_wait_result_job_by_hash).head(|| async { "" }),
         )
         .route(
             "/run_wait_result/f/*script_path",
-            post(run_wait_result_flow_by_path),
+            post(run_wait_result_flow_by_path).head(|| async { "" }),
         )
-        .route("/run/h/:hash", post(run_job_by_hash))
+        .route("/run/h/:hash", post(run_job_by_hash).head(|| async { "" }))
         .route("/run/preview", post(run_preview_job))
         .route("/run/preview_flow", post(run_preview_flow_job))
         .route("/list", get(list_jobs))
@@ -69,6 +75,10 @@ pub fn workspaced_service() -> Router {
         .route("/completed/list", get(list_completed_jobs))
         .route("/completed/get/:id", get(get_completed_job))
         .route("/completed/get_result/:id", get(get_completed_job_result))
+        .route(
+            "/completed/get_result_maybe/:id",
+            get(get_completed_job_result_maybe),
+        )
         .route("/completed/delete/:id", post(delete_completed_job))
         .route("/flow/resume/:id", post(resume_suspended_flow_as_owner))
         .route(
@@ -102,6 +112,12 @@ pub fn global_service() -> Router {
             get(get_suspended_job_flow),
         )
         .route("/get/:id", get(get_job))
+        .route("/completed/get/:id", get(get_completed_job))
+        .route("/completed/get_result/:id", get(get_completed_job_result))
+        .route(
+            "/completed/get_result_maybe/:id",
+            get(get_completed_job_result_maybe),
+        )
         .route("/getupdate/:id", get(get_job_update))
         .route("/queue/cancel/:id", post(cancel_job_api))
         .route("/queue/force_cancel/:id", post(force_cancel))
@@ -221,6 +237,26 @@ pub async fn get_path_for_hash<'c>(
     Ok(path)
 }
 
+pub async fn get_path_and_tag_for_hash<'c>(
+    db: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    hash: i64,
+) -> error::Result<(String, Option<String>)> {
+    let script = sqlx::query!(
+        "select path, tag from script where hash = $1 AND workspace_id = $2",
+        hash,
+        w_id
+    )
+    .fetch_one(db)
+    .await
+    .map_err(|e| {
+        Error::InternalErr(format!(
+            "querying getting path for hash {hash} in {w_id}: {e}"
+        ))
+    })?;
+    Ok((script.path, script.tag))
+}
+
 async fn get_job(
     Extension(db): Extension<DB>,
     Path((w_id, id)): Path<(String, Uuid)>,
@@ -307,6 +343,7 @@ pub struct CompletedJob {
     pub visible_to_owner: bool,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub mem_peak: Option<i32>,
+    pub tag: String,
 }
 
 #[derive(Deserialize, Clone)]
@@ -318,6 +355,7 @@ pub struct RunJobQuery {
     invisible_to_owner: Option<bool>,
     queue_limit: Option<i64>,
     payload: Option<String>,
+    job_id: Option<Uuid>,
 }
 
 lazy_static::lazy_static! {
@@ -391,6 +429,7 @@ pub struct ListQueueQuery {
     pub suspended: Option<bool>,
     // filter by matching a subset of the args using base64 encoded json subset
     pub args: Option<String>,
+    pub tag: Option<String>,
 }
 
 fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> SqlBuilder {
@@ -413,6 +452,9 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
     if let Some(cb) = &lq.created_by {
         sqlb.and_where_eq("created_by", "?".bind(cb));
     }
+    if let Some(t) = &lq.tag {
+        sqlb.and_where_eq("tag", "?".bind(t));
+    }
     if let Some(r) = &lq.running {
         sqlb.and_where_eq("running", &r);
     }
@@ -545,6 +587,7 @@ async fn list_jobs(
             job_kinds: lq.job_kinds,
             suspended: lq.suspended,
             args: lq.args,
+            tag: lq.tag,
         },
         &[
             "'QueuedJob' as typ",
@@ -574,6 +617,7 @@ async fn list_jobs(
             "visible_to_owner",
             "suspend",
             "mem_peak",
+            "tag",
         ],
     );
     let sqlc = list_completed_jobs_query(
@@ -609,6 +653,7 @@ async fn list_jobs(
             "visible_to_owner",
             "null as suspend",
             "mem_peak",
+            "tag",
         ],
     );
     let sql = format!(
@@ -627,7 +672,7 @@ async fn list_jobs(
 pub async fn resume_suspended_flow_as_owner(
     authed: Authed,
     Extension(db): Extension<DB>,
-    Path((w_id, flow_id)): Path<(String, Uuid)>,
+    Path((_w_id, flow_id)): Path<(String, Uuid)>,
     QueryOrBody(value): QueryOrBody<serde_json::Value>,
 ) -> error::Result<StatusCode> {
     let value = value.unwrap_or(serde_json::Value::Null);
@@ -635,16 +680,11 @@ pub async fn resume_suspended_flow_as_owner(
 
     let (flow, job_id) = get_suspended_flow_info(flow_id, &mut tx).await?;
 
-    if !authed.is_admin {
-        require_owner_of_path(
-            &w_id,
-            &authed.username,
-            &authed.groups,
-            &flow.script_path.clone().unwrap_or_else(|| String::new()),
-            &db,
-        )
-        .await?;
-    }
+    require_owner_of_path(
+        &authed,
+        &flow.script_path.clone().unwrap_or_else(|| String::new()),
+    )?;
+
     insert_resume_job(0, job_id, &flow, value, Some(authed.username), &mut tx).await?;
 
     resume_immediately_if_relevant(flow, job_id, &mut tx).await?;
@@ -671,7 +711,7 @@ pub async fn resume_suspended_job(
     }
     mac.verify_slice(hex::decode(secret)?.as_ref())
         .map_err(|_| anyhow::anyhow!("Invalid signature"))?;
-    let flow = get_suspended_prent_flow_info(job_id, &mut tx).await?;
+    let flow = get_suspended_parent_flow_info(job_id, &mut tx).await?;
 
     let exists = sqlx::query_scalar!(
         r#"
@@ -762,7 +802,7 @@ struct FlowInfo {
     script_path: Option<String>,
 }
 
-async fn get_suspended_prent_flow_info<'c>(
+async fn get_suspended_parent_flow_info<'c>(
     job_id: Uuid,
     tx: &mut Transaction<'c, Postgres>,
 ) -> error::Result<FlowInfo> {
@@ -833,8 +873,9 @@ pub async fn cancel_suspended_job(
         .map_err(|_| anyhow::anyhow!("Invalid signature"))?;
 
     let whom = approver.approver.unwrap_or_else(|| "unknown".to_string());
-    let parent_flow = get_root_job(db, &w_id, job).await?;
-    let (mut tx, job) = windmill_queue::cancel_job(
+    let parent_flow = get_suspended_parent_flow_info(job, &mut tx).await?.id;
+
+    let (mut tx, cjob) = windmill_queue::cancel_job(
         &whom,
         Some("approval request disapproved".to_string()),
         parent_flow,
@@ -844,7 +885,7 @@ pub async fn cancel_suspended_job(
         false,
     )
     .await?;
-    if job.is_some() {
+    if cjob.is_some() {
         audit_log(
             &mut tx,
             &whom,
@@ -855,26 +896,13 @@ pub async fn cancel_suspended_job(
             None,
         )
         .await?;
-    }
-    tx.commit().await?;
-    Ok("Flow of job cancelled".to_string())
-}
+        tx.commit().await?;
 
-pub async fn get_root_job(db: DB, w_id: &str, job: Uuid) -> error::Result<Uuid> {
-    let mut tx = db.begin().await?;
-    let mut job_id = job;
-    loop {
-        let (job, ntx) = get_job_by_id(tx, w_id, job_id).await?;
-        tx = ntx;
-        let p_job = job.and_then(|x| match x {
-            Job::QueuedJob(job) => job.parent_job,
-            Job::CompletedJob(job) => job.parent_job,
-        });
-        if let Some(p_job) = p_job {
-            job_id = p_job;
-        } else {
-            return Ok(job_id);
-        }
+        Ok(format!("Flow {parent_flow} of job {job} cancelled"))
+    } else {
+        Ok(format!(
+            "Flow {parent_flow} of job {job} was not cancellable"
+        ))
     }
 }
 
@@ -1090,6 +1118,7 @@ struct UnifiedJob {
     visible_to_owner: bool,
     suspend: Option<i32>,
     mem_peak: Option<i32>,
+    tag: String,
 }
 
 impl From<UnifiedJob> for Job {
@@ -1125,6 +1154,7 @@ impl From<UnifiedJob> for Job {
                 email: uj.email,
                 visible_to_owner: uj.visible_to_owner,
                 mem_peak: uj.mem_peak,
+                tag: uj.tag,
             }),
             "QueuedJob" => Job::QueuedJob(QueuedJob {
                 workspace_id: uj.workspace_id,
@@ -1160,6 +1190,7 @@ impl From<UnifiedJob> for Job {
                 mem_peak: uj.mem_peak,
                 root_job: None,
                 leaf_jobs: None,
+                tag: uj.tag,
             }),
             t => panic!("job type {} not valid", t),
         }
@@ -1176,6 +1207,7 @@ struct Preview {
     path: Option<String>,
     args: Option<serde_json::Map<String, serde_json::Value>>,
     language: ScriptLang,
+    tag: Option<String>,
 }
 
 #[derive(Deserialize)]
@@ -1185,6 +1217,75 @@ struct PreviewFlow {
     args: Option<serde_json::Map<String, serde_json::Value>>,
 }
 
+pub struct JsonOrForm(
+    Option<serde_json::Map<String, serde_json::Value>>,
+    Option<String>,
+);
+
+#[axum::async_trait]
+impl<S> FromRequest<S, axum::body::Body> for JsonOrForm
+where
+    S: Send + Sync,
+{
+    type Rejection = Response;
+
+    async fn from_request(
+        req: Request<axum::body::Body>,
+        _state: &S,
+    ) -> Result<Self, Self::Rejection> {
+        let content_type_header = req.headers().get(CONTENT_TYPE);
+        let content_type = content_type_header.and_then(|value| value.to_str().ok());
+        if let Some(content_type) = content_type {
+            if content_type.starts_with("application/json") {
+                if req
+                    .uri()
+                    .query()
+                    .map(|x| x.contains("raw=true"))
+                    .unwrap_or(false)
+                {
+                    let bytes = Bytes::from_request(req, _state)
+                        .await
+                        .map_err(IntoResponse::into_response)?;
+                    let str = String::from_utf8(bytes.to_vec()).map_err(|e| {
+                        Error::BadRequest(format!("invalid utf8: {}", e)).into_response()
+                    })?;
+                    let payload =
+                        serde_json::from_str::<Option<serde_json::Value>>(&str).map_err(|e| {
+                            Error::BadRequest(format!("invalid json: {}", e)).into_response()
+                        })?;
+                    return match payload {
+                        Some(serde_json::Value::Object(map)) => Ok(Self(Some(map), Some(str))),
+                        None => Ok(Self(None, Some(str))),
+                        Some(x) => {
+                            let mut map = serde_json::Map::new();
+                            map.insert("body".to_string(), x);
+                            Ok(Self(Some(map), Some(str)))
+                        }
+                    };
+                } else {
+                    let Json(payload): Json<Option<serde_json::Value>> =
+                        req.extract().await.map_err(IntoResponse::into_response)?;
+                    return match payload {
+                        Some(serde_json::Value::Object(map)) => Ok(Self(Some(map), None)),
+                        None => Ok(Self(None, None)),
+                        Some(x) => {
+                            let mut map = serde_json::Map::new();
+                            map.insert("body".to_string(), x);
+                            Ok(Self(Some(map), None))
+                        }
+                    };
+                }
+            }
+
+            if content_type.starts_with("application/x-www-form-urlencoded") {
+                let Form(payload) = req.extract().await.map_err(IntoResponse::into_response)?;
+                return Ok(Self(Some(payload), None));
+            }
+        }
+
+        Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response())
+    }
+}
 pub struct QueryOrBody<D>(pub Option<D>);
 
 #[axum::async_trait]
@@ -1231,6 +1332,19 @@ fn decode_payload<D: DeserializeOwned>(t: String) -> anyhow::Result<D> {
         .context("invalid base64")?;
     serde_json::from_slice(vec.as_slice()).context("invalid json")
 }
+
+fn add_raw_string(
+    raw_string: Option<String>,
+    mut args: serde_json::Map<String, serde_json::Value>,
+) -> serde_json::Map<String, serde_json::Value> {
+    if let Some(raw_string) = raw_string {
+        args.insert(
+            "raw_string".to_string(),
+            serde_json::Value::String(raw_string),
+        );
+    }
+    return args;
+}
 pub async fn run_flow_by_path(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -1238,13 +1352,13 @@ pub async fn run_flow_by_path(
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::Result<(StatusCode, String)> {
     let flow_path = flow_path.to_path();
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
     let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
-
+    let args = add_raw_string(raw_string, args);
     let (uuid, tx) = push(
         tx,
         &w_id,
@@ -1257,10 +1371,12 @@ pub async fn run_flow_by_path(
         None,
         run_query.parent_job,
         run_query.parent_job,
+        run_query.job_id,
         false,
         false,
         None,
         !run_query.invisible_to_owner.unwrap_or(false),
+        None,
     )
     .await?;
     tx.commit().await?;
@@ -1274,13 +1390,15 @@ pub async fn run_job_by_path(
     Path((w_id, script_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::Result<(StatusCode, String)> {
     let script_path = script_path.to_path();
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
-    let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
+    let (job_payload, tag) =
+        script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
     let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
+    let args = add_raw_string(raw_string, args);
 
     let (uuid, tx) = push(
         tx,
@@ -1293,11 +1411,13 @@ pub async fn run_job_by_path(
         scheduled_for,
         None,
         run_query.parent_job,
+        run_query.job_id,
         run_query.parent_job,
         false,
         false,
         None,
         !run_query.invisible_to_owner.unwrap_or(false),
+        tag,
     )
     .await?;
     tx.commit().await?;
@@ -1347,14 +1467,13 @@ async fn run_wait_result<T>(
     uuid: Uuid,
     Path((w_id, _)): Path<(String, T)>,
 ) -> error::JsonResult<serde_json::Value> {
-    let mut result = None;
-    let iters = if timeout <= 0 {
-        20
-    } else if timeout <= 1 {
-        timeout * 10
+    let mut result;
+    let timeout_ms = if timeout <= 0 {
+        2000
     } else {
-        10 + ((timeout - 1) * 2)
+        (timeout * 1000) as u64
     };
+
     let mut g = Guard {
         done: false,
         id: uuid,
@@ -1362,7 +1481,10 @@ async fn run_wait_result<T>(
         db: user_db.clone(),
         authed: authed.clone(),
     };
-    for i in 0..iters {
+
+    let fast_poll_duration = *WAIT_RESULT_FAST_POLL_DURATION_SECS as u64 * 1000;
+    let mut accumulated_delay = 0 as u64;
+    loop {
         let mut tx = user_db.clone().begin(&authed).await?;
         result = sqlx::query_scalar!(
             "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
@@ -1377,11 +1499,27 @@ async fn run_wait_result<T>(
         if result.is_some() {
             break;
         }
-        let delay = if i < 10 { 100 } else { 500 };
+
+        let delay = if accumulated_delay <= fast_poll_duration {
+            *WAIT_RESULT_FAST_POLL_INTERVAL_MS
+        } else {
+            *WAIT_RESULT_SLOW_POLL_INTERVAL_MS
+        };
+        accumulated_delay += delay;
+        if accumulated_delay > timeout_ms {
+            break;
+        };
         tokio::time::sleep(core::time::Duration::from_millis(delay)).await;
     }
     if let Some(result) = result {
         g.done = true;
+        let status_code = result
+            .get("windmill_status_code")
+            .and_then(|x| x.as_i64())
+            .and_then(|x| StatusCode::from_u16(x as u16).ok());
+        if let Some(status_code) = status_code {
+            return Err(Error::CustomStatusCode(status_code, result));
+        }
         Ok(Json(result))
     } else {
         Err(Error::ExecutionErr(format!("timeout after {}s", timeout)))
@@ -1417,9 +1555,22 @@ lazy_static::lazy_static! {
         .ok()
         .and_then(|x| x.parse().ok())
         .unwrap_or(20);
+    pub static ref WAIT_RESULT_FAST_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_FAST_POLL_INTERVAL_MS")
+        .ok()
+        .and_then(|x| x.parse().ok())
+        .unwrap_or(50);
+    pub static ref WAIT_RESULT_FAST_POLL_DURATION_SECS: u16 = std::env::var("WAIT_RESULT_FAST_POLL_DURATION_SECS")
+        .ok()
+        .and_then(|x| x.parse().ok())
+        .unwrap_or(2);
+    pub static ref WAIT_RESULT_SLOW_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_SLOW_POLL_INTERVAL_MS")
+        .ok()
+        .and_then(|x| x.parse().ok())
+        .unwrap_or(200);
 }
 
 pub async fn run_wait_result_job_by_path_get(
+    method: hyper::http::Method,
     authed: Authed,
     Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
     Extension(user_db): Extension<UserDB>,
@@ -1427,6 +1578,9 @@ pub async fn run_wait_result_job_by_path_get(
     Path((w_id, script_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
 ) -> error::JsonResult<serde_json::Value> {
+    if method == http::Method::HEAD {
+        return Ok(Json(serde_json::json!("")));
+    }
     let payload_r = run_query
         .payload
         .map(decode_payload)
@@ -1441,7 +1595,8 @@ pub async fn run_wait_result_job_by_path_get(
     check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
     let script_path = script_path.to_path();
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
-    let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
+    let (job_payload, tag) =
+        script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
 
     let (uuid, tx) = push(
         tx,
@@ -1455,10 +1610,12 @@ pub async fn run_wait_result_job_by_path_get(
         None,
         run_query.parent_job,
         run_query.parent_job,
+        run_query.job_id,
         false,
         false,
         None,
         !run_query.invisible_to_owner.unwrap_or(false),
+        tag,
     )
     .await?;
     tx.commit().await?;
@@ -1481,14 +1638,16 @@ pub async fn run_wait_result_job_by_path(
     Path((w_id, script_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::JsonResult<serde_json::Value> {
     check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
     let script_path = script_path.to_path();
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
-    let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
+    let (job_payload, tag) =
+        script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
 
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
+    let args = add_raw_string(raw_string, args);
 
     let (uuid, tx) = push(
         tx,
@@ -1502,10 +1661,12 @@ pub async fn run_wait_result_job_by_path(
         None,
         run_query.parent_job,
         run_query.parent_job,
+        run_query.job_id,
         false,
         false,
         None,
         !run_query.invisible_to_owner.unwrap_or(false),
+        tag,
     )
     .await?;
     tx.commit().await?;
@@ -1528,15 +1689,16 @@ pub async fn run_wait_result_job_by_hash(
     Path((w_id, script_hash)): Path<(String, ScriptHash)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::JsonResult<serde_json::Value> {
     check_queue_too_long(db, run_query.queue_limit).await?;
 
     let hash = script_hash.0;
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
-    let path = get_path_for_hash(tx.transaction_mut(), &w_id, hash).await?;
+    let (path, tag) = get_path_and_tag_for_hash(tx.transaction_mut(), &w_id, hash).await?;
 
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
+    let args = add_raw_string(raw_string, args);
 
     let (uuid, tx) = push(
         tx,
@@ -1550,10 +1712,12 @@ pub async fn run_wait_result_job_by_hash(
         None,
         run_query.parent_job,
         run_query.parent_job,
+        run_query.job_id,
         false,
         false,
         None,
         !run_query.invisible_to_owner.unwrap_or(false),
+        tag,
     )
     .await?;
     tx.commit().await?;
@@ -1576,7 +1740,7 @@ pub async fn run_wait_result_flow_by_path(
     Path((w_id, flow_path)): Path<(String, StripPath)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::JsonResult<serde_json::Value> {
     check_queue_too_long(db, run_query.queue_limit).await?;
 
@@ -1584,6 +1748,7 @@ pub async fn run_wait_result_flow_by_path(
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
     let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
+    let args = add_raw_string(raw_string, args);
 
     let (uuid, tx) = push(
         tx,
@@ -1597,10 +1762,12 @@ pub async fn run_wait_result_flow_by_path(
         None,
         run_query.parent_job,
         run_query.parent_job,
+        run_query.job_id,
         false,
         false,
         None,
         !run_query.invisible_to_owner.unwrap_or(false),
+        None,
     )
     .await?;
     tx.commit().await?;
@@ -1615,21 +1782,6 @@ pub async fn run_wait_result_flow_by_path(
     .await
 }
 
-// a similar function exists on the worker
-pub async fn script_path_to_payload<'c>(
-    script_path: &str,
-    db: &mut Transaction<'c, Postgres>,
-    w_id: &String,
-) -> std::result::Result<JobPayload, Error> {
-    let job_payload = if script_path.starts_with("hub/") {
-        JobPayload::ScriptHub { path: script_path.to_owned() }
-    } else {
-        let script_hash = windmill_common::get_latest_hash_for_path(db, w_id, script_path).await?;
-        JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() }
-    };
-    Ok(job_payload)
-}
-
 async fn run_preview_job(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -1660,10 +1812,12 @@ async fn run_preview_job(
         None,
         None,
         None,
+        run_query.job_id,
         false,
         false,
         None,
         true,
+        preview.tag,
     )
     .await?;
     tx.commit().await?;
@@ -1696,10 +1850,12 @@ async fn run_preview_flow_job(
         None,
         None,
         None,
+        run_query.job_id,
         false,
         false,
         None,
         true,
+        None,
     )
     .await?;
     tx.commit().await?;
@@ -1714,13 +1870,14 @@ pub async fn run_job_by_hash(
     Path((w_id, script_hash)): Path<(String, ScriptHash)>,
     Query(run_query): Query<RunJobQuery>,
     headers: HeaderMap,
-    Json(args): Json<Option<serde_json::Map<String, serde_json::Value>>>,
+    JsonOrForm(args, raw_string): JsonOrForm,
 ) -> error::Result<(StatusCode, String)> {
     let hash = script_hash.0;
     let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
-    let path = get_path_for_hash(tx.transaction_mut(), &w_id, hash).await?;
+    let (path, tag) = get_path_and_tag_for_hash(tx.transaction_mut(), &w_id, hash).await?;
     let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
     let args = run_query.add_include_headers(headers, args.unwrap_or_default());
+    let args = add_raw_string(raw_string, args);
 
     let (uuid, tx) = push(
         tx,
@@ -1734,10 +1891,12 @@ pub async fn run_job_by_hash(
         None,
         run_query.parent_job,
         run_query.parent_job,
+        run_query.job_id,
         false,
         false,
         None,
         !run_query.invisible_to_owner.unwrap_or(false),
+        tag,
     )
     .await?;
     tx.commit().await?;
@@ -1828,6 +1987,9 @@ fn list_completed_jobs_query(
     if let Some(h) = &lq.script_hash {
         sqlb.and_where_eq("script_hash", "?".bind(h));
     }
+    if let Some(t) = &lq.tag {
+        sqlb.and_where_eq("tag", "?".bind(t));
+    }
     if let Some(cb) = &lq.created_by {
         sqlb.and_where_eq("created_by", "?".bind(cb));
     }
@@ -1885,6 +2047,7 @@ pub struct ListCompletedQuery {
     pub args: Option<String>,
     // filter by matching a subset of the result using base64 encoded json subset
     pub result: Option<String>,
+    pub tag: Option<String>,
 }
 
 async fn list_completed_jobs(
@@ -1930,6 +2093,7 @@ async fn list_completed_jobs(
             "email",
             "visible_to_owner",
             "mem_peak",
+            "tag",
         ],
     )
     .sql()?;
@@ -1972,6 +2136,31 @@ async fn get_completed_job_result(
     Ok(Json(result))
 }
 
+#[derive(Serialize)]
+struct CompletedJobResult {
+    completed: bool,
+    result: Option<serde_json::Value>,
+}
+
+async fn get_completed_job_result_maybe(
+    Extension(db): Extension<DB>,
+    Path((w_id, id)): Path<(String, Uuid)>,
+) -> error::JsonResult<CompletedJobResult> {
+    let result_o = sqlx::query_scalar!(
+        "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
+        id,
+        w_id,
+    )
+    .fetch_optional(&db)
+    .await?;
+
+    if let Some(result) = result_o {
+        Ok(Json(CompletedJobResult { completed: true, result }))
+    } else {
+        Ok(Json(CompletedJobResult { completed: false, result: None }))
+    }
+}
+
 async fn delete_completed_job(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,

backend/windmill-api/src/lib.rs

@@ -15,6 +15,7 @@ use crate::{
     webhook_util::WebhookShared,
 };
 use argon2::Argon2;
+use axum::extract::DefaultBodyLimit;
 use axum::{middleware::from_extractor, routing::get, Extension, Router};
 use db::DB;
 use git_version::git_version;
@@ -33,6 +34,7 @@ mod apps;
 mod audit;
 mod capture;
 mod db;
+mod drafts;
 mod favorite;
 mod flows;
 mod folders;
@@ -41,6 +43,7 @@ mod groups;
 mod inputs;
 pub mod jobs;
 mod oauth2;
+mod raw_apps;
 mod resources;
 mod schedule;
 mod scripts;
@@ -50,7 +53,7 @@ mod users;
 mod utils;
 mod variables;
 mod webhook_util;
-mod worker_ping;
+mod workers;
 mod workspaces;
 
 pub const GIT_VERSION: &str =
@@ -58,9 +61,15 @@ pub const GIT_VERSION: &str =
 
 pub use users::delete_expired_items_perdiodically;
 
+pub const DEFAULT_BODY_LIMIT: usize = 2097152; // 2MB
 lazy_static::lazy_static! {
     pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
 
+    pub static ref REQUEST_SIZE_LIMIT: usize = std::env::var("REQUEST_SIZE_LIMIT")
+    .ok()
+    .and_then(|x| x.parse::<usize>().ok())
+    .unwrap_or(DEFAULT_BODY_LIMIT);
+
 
     pub static ref COOKIE_DOMAIN: Option<String> = std::env::var("COOKIE_DOMAIN").ok();
 
@@ -105,7 +114,8 @@ pub async fn run_server(
         .layer(Extension(user_db))
         .layer(Extension(auth_cache.clone()))
         .layer(CookieManagerLayer::new())
-        .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())));
+        .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())))
+        .layer(DefaultBodyLimit::max(*REQUEST_SIZE_LIMIT));
 
     let cors = CorsLayer::new()
         .allow_methods([Method::GET, Method::POST])
@@ -122,6 +132,7 @@ pub async fn run_server(
                         // Reordered alphabetically
                         .nest("/acls", granular_acls::workspaced_service())
                         .nest("/apps", apps::workspaced_service())
+                        .nest("/raw_apps", raw_apps::workspaced_service())
                         .nest("/audit", audit::workspaced_service())
                         .nest("/capture", capture::workspaced_service())
                         .nest("/favorites", favorite::workspaced_service())
@@ -134,6 +145,7 @@ pub async fn run_server(
                         .nest("/resources", resources::workspaced_service())
                         .nest("/schedules", schedule::workspaced_service())
                         .nest("/scripts", scripts::workspaced_service())
+                        .nest("/drafts", drafts::workspaced_service())
                         .nest(
                             "/users",
                             users::workspaced_service().layer(Extension(argon2.clone())),
@@ -146,7 +158,7 @@ pub async fn run_server(
                     "/users",
                     users::global_service().layer(Extension(argon2.clone())),
                 )
-                .nest("/workers", worker_ping::global_service())
+                .nest("/workers", workers::global_service())
                 .nest("/scripts", scripts::global_service())
                 .nest("/flows", flows::global_service())
                 .nest("/apps", apps::global_service().layer(cors.clone()))

backend/windmill-api/src/oauth2.rs

@@ -19,6 +19,7 @@ use axum::{
     routing::{get, post},
     Json, Router,
 };
+use base64::Engine;
 use hmac::Mac;
 use hyper::StatusCode;
 use itertools::Itertools;
@@ -29,6 +30,7 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use sqlx::{Postgres, Transaction};
 use tower_cookies::{Cookie, Cookies};
 use windmill_audit::{audit_log, ActionKind};
+use windmill_common::jobs::JobPayload;
 use windmill_common::users::username_to_permissioned_as;
 use windmill_common::utils::{not_found_if_none, now_from_db};
 
@@ -44,7 +46,7 @@ use crate::{BASE_URL, HTTP_CLIENT, IS_SECURE, OAUTH_CLIENTS, SLACK_SIGNING_SECRE
 use windmill_common::error::{self, to_anyhow, Error};
 use windmill_common::oauth2::*;
 
-use windmill_queue::{JobPayload, QueueTransaction};
+use windmill_queue::QueueTransaction;
 
 use std::{fs, str};
 
@@ -118,7 +120,14 @@ pub fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
     ))?;
 
     let path = "./oauth.json";
-    let content = if std::path::Path::new(path).exists() {
+    let content: String = if let Ok(e) = std::env::var("OAUTH_JSON_AS_BASE64") {
+        str::from_utf8(
+            &base64::engine::general_purpose::STANDARD
+                .decode(e)
+                .map_err(to_anyhow)?,
+        )?
+        .to_string()
+    } else if std::path::Path::new(path).exists() {
         fs::read_to_string(path).map_err(to_anyhow)?
     } else {
         "{}".to_string()
@@ -753,17 +762,20 @@ async fn slack_command(
 
     if let Some(settings) = settings {
         if let Some(path) = &settings.slack_command_script {
-            let payload = if let Some(path) = path.strip_prefix("flow/") {
-                JobPayload::Flow(path.to_string())
+            let (payload, tag) = if let Some(path) = path.strip_prefix("flow/") {
+                (JobPayload::Flow(path.to_string()), None)
             } else {
                 let path = path.strip_prefix("script/").unwrap_or_else(|| path);
-                let script_hash = windmill_common::get_latest_hash_for_path(
+                let (script_hash, tag) = windmill_common::get_latest_deployed_hash_for_path(
                     tx.transaction_mut(),
                     &settings.workspace_id,
                     path,
                 )
                 .await?;
-                JobPayload::ScriptHash { hash: script_hash, path: path.to_owned() }
+                (
+                    JobPayload::ScriptHash { hash: script_hash, path: path.to_owned() },
+                    tag,
+                )
             };
             let mut map = serde_json::Map::new();
             map.insert("text".to_string(), serde_json::Value::String(form.text));
@@ -784,10 +796,12 @@ async fn slack_command(
                 None,
                 None,
                 None,
+                None,
                 false,
                 false,
                 None,
                 true,
+                tag,
             )
             .await?;
             let url = BASE_URL.to_owned();

backend/windmill-api/src/raw_apps.rs

@@ -0,0 +1,333 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+use crate::{
+    db::{UserDB, DB},
+    users::{require_owner_of_path, Authed},
+    webhook_util::{WebhookMessage, WebhookShared},
+};
+use axum::{
+    body,
+    extract::{Extension, Json, Path, Query},
+    response::Response,
+    routing::{delete, get, post},
+    Router,
+};
+use bytes::Bytes;
+use hyper::{header, StatusCode};
+use serde::{Deserialize, Serialize};
+use sql_builder::{bind::Bind, SqlBuilder};
+use sqlx::FromRow;
+use std::str;
+use windmill_audit::{audit_log, ActionKind};
+use windmill_common::{
+    apps::ListAppQuery,
+    error::{Error, JsonResult, Result},
+    utils::{not_found_if_none, paginate, Pagination, StripPath},
+};
+
+pub fn workspaced_service() -> Router {
+    Router::new()
+        .route("/list", get(list_apps))
+        .route("/get_data/:version/*path", get(get_data))
+        .route("/exists/*path", get(exists_app))
+        .route("/update/*path", post(update_app))
+        .route("/delete/*path", delete(delete_app))
+        .route("/create", post(create_app))
+}
+
+#[derive(FromRow, Deserialize, Serialize)]
+pub struct ListableApp {
+    pub path: String,
+    pub workspace_id: String,
+    pub summary: String,
+    pub edited_at: chrono::DateTime<chrono::Utc>,
+    pub extra_perms: serde_json::Value,
+    pub starred: bool,
+    pub version: i32,
+}
+
+#[derive(Deserialize)]
+pub struct CreateApp {
+    pub path: String,
+    pub summary: String,
+    pub value: String,
+}
+
+#[derive(Deserialize)]
+pub struct EditApp {
+    pub path: Option<String>,
+    pub summary: Option<String>,
+    pub value: Option<String>,
+}
+
+async fn list_apps(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Query(pagination): Query<Pagination>,
+    Query(lq): Query<ListAppQuery>,
+) -> JsonResult<Vec<ListableApp>> {
+    let (per_page, offset) = paginate(pagination);
+
+    let mut sqlb = SqlBuilder::select_from("raw_app as app")
+        .fields(&[
+            "app.workspace_id",
+            "app.path",
+            "app.edited_at",
+            "app.summary",
+            "app.extra_perms",
+            "app.version",
+            "favorite.path IS NOT NULL as starred",
+        ])
+        .left()
+        .join("favorite")
+        .on(
+            "favorite.favorite_kind = 'raw_app' AND favorite.workspace_id = app.workspace_id AND favorite.path = app.path AND favorite.usr = ?"
+                .bind(&authed.username),
+        )
+        .order_desc("favorite.path IS NOT NULL")
+        .order_by("app.edited_at", true)
+        .and_where("app.workspace_id = ?".bind(&w_id))
+        .offset(offset)
+        .limit(per_page)
+        .clone();
+
+    if lq.starred_only.unwrap_or(false) {
+        sqlb.and_where_is_not_null("favorite.path");
+    }
+
+    let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
+    let mut tx = user_db.begin(&authed).await?;
+    let rows = sqlx::query_as::<_, ListableApp>(&sql)
+        .fetch_all(&mut tx)
+        .await?;
+
+    tx.commit().await?;
+
+    Ok(Json(rows))
+}
+
+async fn get_data(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, _version, path)): Path<(String, u16, StripPath)>,
+) -> Result<Response> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    let app_o = sqlx::query_scalar!(
+        "SELECT data FROM raw_app
+        WHERE path = $1 AND workspace_id = $2",
+        path.to_owned(),
+        &w_id
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    let app = not_found_if_none(app_o, "App", path)?;
+    let res = Response::builder().header(header::CONTENT_TYPE, "text/javascript");
+
+    Ok(res
+        .body(body::boxed(body::Full::from(Bytes::from(app))))
+        .unwrap())
+}
+
+async fn create_app(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(webhook): Extension<WebhookShared>,
+    Path(w_id): Path<String>,
+    Json(app): Json<CreateApp>,
+) -> Result<(StatusCode, String)> {
+    let mut tx = user_db.begin(&authed).await?;
+    if &app.path == "" {
+        return Err(Error::BadRequest("App path cannot be empty".to_string()));
+    }
+
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)",
+        app.path,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(false);
+
+    if exists {
+        return Err(Error::BadRequest(format!(
+            "App with path {} already exists",
+            &app.path
+        )));
+    }
+
+    sqlx::query!(
+        "INSERT INTO raw_app
+            (workspace_id, path, summary, extra_perms, data)
+            VALUES ($1, $2, $3, '{}', $4)",
+        w_id,
+        app.path,
+        app.summary,
+        app.value,
+    )
+    .execute(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "apps.create",
+        ActionKind::Create,
+        &w_id,
+        Some(&app.path),
+        None,
+    )
+    .await?;
+
+    tx.commit().await?;
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::CreateApp { workspace: w_id, path: app.path.clone() },
+    );
+
+    Ok((StatusCode::CREATED, app.path))
+}
+
+async fn delete_app(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(webhook): Extension<WebhookShared>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> Result<String> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    sqlx::query!(
+        "DELETE FROM raw_app WHERE path = $1 AND workspace_id = $2",
+        path,
+        w_id
+    )
+    .execute(&mut tx)
+    .await?;
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "apps.delete",
+        ActionKind::Delete,
+        &w_id,
+        Some(path),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+    webhook.send_message(
+        w_id.clone().clone(),
+        WebhookMessage::DeleteApp { workspace: w_id, path: path.to_owned() },
+    );
+
+    Ok(format!("app {} deleted", path))
+}
+
+async fn update_app(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(webhook): Extension<WebhookShared>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+    Json(app): Json<EditApp>,
+) -> Result<String> {
+    use sql_builder::prelude::*;
+
+    let path = path.to_path();
+
+    let mut tx = user_db.begin(&authed).await?;
+    let npath = &app.path;
+    let npath = if npath.is_some() || app.summary.is_some() {
+        let mut sqlb = SqlBuilder::update_table("raw_app");
+        sqlb.and_where_eq("path", "?".bind(&path));
+        sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
+
+        if let Some(npath) = &app.path {
+            if npath != path {
+                require_owner_of_path(&authed, path)?;
+
+                let exists = sqlx::query_scalar!(
+                    "SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)",
+                    npath,
+                    w_id
+                )
+                .fetch_one(&mut tx)
+                .await?
+                .unwrap_or(false);
+
+                if exists {
+                    return Err(Error::BadRequest(format!(
+                        "App with path {} already exists",
+                        npath
+                    )));
+                }
+            }
+            sqlb.set_str("path", npath);
+        }
+
+        if let Some(nsummary) = &app.summary {
+            sqlb.set_str("summary", nsummary);
+        }
+
+        if let Some(value) = &app.value {
+            sqlb.set_str("data", value);
+            sqlb.set("version", "version + 1");
+        }
+
+        sqlb.returning("path");
+
+        let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
+        let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
+        not_found_if_none(npath_o, "Raw App", path)?
+    } else {
+        "".to_string()
+    };
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "apps.update",
+        ActionKind::Update,
+        &w_id,
+        Some(&npath),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::UpdateApp {
+            workspace: w_id,
+            old_path: path.to_owned(),
+            new_path: npath.clone(),
+        },
+    );
+
+    Ok(format!("app {} updated (npath: {:?})", path, npath))
+}
+
+async fn exists_app(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}

backend/windmill-api/src/resources.rs

@@ -386,9 +386,8 @@ async fn update_resource(
         if npath != path {
             check_path_conflict(&mut tx, &w_id, &npath).await?;
 
-            if !authed.is_admin {
-                require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
-            }
+            require_owner_of_path(&authed, path)?;
+
             sqlx::query!(
                 "UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3",
                 npath,

backend/windmill-api/src/schedule.rs

@@ -23,9 +23,9 @@ use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{Error, JsonResult, Result},
     schedule::Schedule,
-    utils::{not_found_if_none, paginate, Pagination, StripPath},
+    utils::{not_found_if_none, paginate, Pagination, StripPath}, jobs::JobKind,
 };
-use windmill_queue::{self, schedule::push_scheduled_job, JobKind, QueueTransaction};
+use windmill_queue::{self, schedule::push_scheduled_job, QueueTransaction};
 
 pub fn workspaced_service() -> Router {
     Router::new()

backend/windmill-api/src/scripts.rs

@@ -32,9 +32,10 @@ use std::{
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
     error::{Error, JsonResult, Result},
+    jobs::JobPayload,
     schedule::Schedule,
     scripts::{
-        to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Script, ScriptHash,
+        to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Schema, Script, ScriptHash,
         ScriptKind, ScriptLang,
     },
     users::username_to_permissioned_as,
@@ -45,8 +46,32 @@ use windmill_common::{
 use windmill_parser::MainArgSignature;
 use windmill_queue::{self, schedule::push_scheduled_job, QueueTransaction};
 
+lazy_static::lazy_static! {
+    pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
+        .ok()
+        .map(|x| x.split(',').map(|x| x.to_string()).filter(|x| !x.is_empty()).collect::<Vec<_>>()).unwrap_or_default();
+
+}
+
 const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;
 
+#[derive(Serialize, sqlx::FromRow)]
+pub struct ScriptWDraft {
+    pub hash: ScriptHash,
+    pub path: String,
+    pub summary: String,
+    pub description: String,
+    pub content: String,
+    pub language: ScriptLang,
+    pub kind: ScriptKind,
+    pub tag: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft: Option<serde_json::Value>,
+    pub schema: Option<Schema>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
+}
+
 pub fn global_service() -> Router {
     Router::new()
         .route(
@@ -73,6 +98,7 @@ pub fn workspaced_service() -> Router {
         .route("/list", get(list_scripts))
         .route("/create", post(create_script))
         .route("/archive/p/*path", post(archive_script_by_path))
+        .route("/get/draft/*path", get(get_script_by_path_w_draft))
         .route("/get/p/*path", get(get_script_by_path))
         .route("/raw/p/*path", get(raw_script_by_path))
         .route("/exists/p/*path", get(exists_script_by_path))
@@ -96,22 +122,18 @@ async fn list_scripts(
 
     let mut sqlb = SqlBuilder::select_from("script as o")
         .fields(&[
-            "o.workspace_id",
             "hash",
             "o.path",
-            "array_remove(array[parent_hashes[1]], NULL) as parent_hashes",
             "summary",
-            "description",
-            "created_by",
-            "created_at",
+            "COALESCE(draft.created_at, o.created_at) as created_at",
             "archived",
-            "deleted",
-            "is_template",
             "extra_perms",
-            "CASE WHEN lock_error_logs IS NOT NULL THEN 'error' ELSE null END as lock_error_logs",
+            "CASE WHEN lock_error_logs IS NOT NULL THEN true ELSE false END as has_deploy_errors",
             "language",
-            "kind",
             "favorite.path IS NOT NULL as starred",
+            "tag",
+            "draft.path IS NOT NULL as has_draft",
+            "draft_only"
         ])
         .left()
         .join("favorite")
@@ -119,6 +141,11 @@ async fn list_scripts(
             "favorite.favorite_kind = 'script' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?"
                 .bind(&authed.username),
         )
+        .left()
+        .join("draft")
+        .on(
+            "draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'script'"
+        )
         .order_desc("favorite.path IS NOT NULL")
         .order_by("created_at", lq.order_desc.unwrap_or(true))
         .and_where("o.workspace_id = ?".bind(&w_id))
@@ -128,7 +155,7 @@ async fn list_scripts(
 
     if lq.show_archived.unwrap_or(false) {
         sqlb.and_where_eq(
-            "created_at",
+            "o.created_at",
             "(select max(created_at) from script where o.path = path 
             AND workspace_id = ?)"
                 .bind(&w_id),
@@ -234,10 +261,20 @@ async fn create_script(
     }
     let parent_hashes_and_perms: Option<ParentInfo> = match (&ns.parent_hash, clashing_script) {
         (None, None) => Ok(None),
-        (None, Some(s)) => Err(Error::BadRequest(format!(
+        (None, Some(s)) if !s.draft_only.unwrap_or(false) => Err(Error::BadRequest(format!(
             "Path conflict for {} with non-archived hash {}",
             &ns.path, &s.hash
         ))),
+        (None, Some(s)) => {
+            sqlx::query!(
+                "DELETE FROM script WHERE hash = $1 AND workspace_id = $2",
+                s.hash.0,
+                &w_id
+            )
+            .execute(&mut tx)
+            .await?;
+            Ok(None)
+        }
         (Some(p_hash), o) => {
             if sqlx::query_scalar!(
                 "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
@@ -272,10 +309,7 @@ async fn create_script(
             let ps = get_script_by_hash_internal(tx.transaction_mut(), &w_id, p_hash).await?;
 
             if ps.path != ns.path {
-                if !authed.is_admin {
-                    require_owner_of_path(&w_id, &authed.username, &authed.groups, &ps.path, &db)
-                        .await?;
-                }
+                require_owner_of_path(&authed, &ps.path)?;
             }
 
             let ph = {
@@ -332,8 +366,8 @@ async fn create_script(
     //::text::json is to ensure we use serde_json with preserve order
     sqlx::query!(
         "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \
-         content, created_by, schema, is_template, extra_perms, lock, language, kind) \
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14)",
+         content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only) \
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16)",
         &w_id,
         &hash.0,
         ns.path,
@@ -348,11 +382,21 @@ async fn create_script(
         lock,
         ns.language: ScriptLang,
         ns.kind.unwrap_or(ScriptKind::Script): ScriptKind,
+        ns.tag,
+        ns.draft_only
     )
     .execute(&mut tx)
     .await?;
 
     if let Some(p_path) = parent_hashes_and_perms.as_ref().map(|x| x.p_path.clone()) {
+        sqlx::query!(
+            "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'",
+            p_path,
+            &w_id
+        )
+        .execute(&mut tx)
+        .await?;
+
         let schedulables = sqlx::query_as!(
         Schedule,
             "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND workspace_id = $3 AND is_flow IS false RETURNING *",
@@ -370,6 +414,14 @@ async fn create_script(
                 tx = push_scheduled_job(tx, schedule).await?;
             }
         }
+    } else {
+        sqlx::query!(
+            "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'",
+            ns.path,
+            &w_id
+        )
+        .execute(&mut tx)
+        .await?;
     }
 
     if p_hashes.is_some() && !p_hashes.unwrap().is_empty() {
@@ -428,7 +480,7 @@ async fn create_script(
         let (_, new_tx) = windmill_queue::push(
             tx,
             &w_id,
-            windmill_queue::JobPayload::Dependencies { hash, dependencies, language: ns.language },
+            JobPayload::Dependencies { hash, dependencies, language: ns.language },
             serde_json::Map::new(),
             &authed.username,
             &authed.email,
@@ -437,10 +489,12 @@ async fn create_script(
             None,
             None,
             None,
+            None,
             false,
             false,
             None,
             true,
+            None,
         )
         .await?;
         tx = new_tx;
@@ -487,6 +541,31 @@ async fn get_script_by_path(
     Ok(Json(script))
 }
 
+async fn get_script_by_path_w_draft(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<ScriptWDraft> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    let script_o = sqlx::query_as::<_, ScriptWDraft>(
+        "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, draft.value as draft FROM script LEFT JOIN draft ON 
+         script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
+         WHERE script.path = $1 AND script.workspace_id = $2 \
+         AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \
+         workspace_id = $2)",
+    )
+    .bind(path)
+    .bind(w_id)
+    .fetch_optional(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    let script = not_found_if_none(script_o, "Script", path)?;
+    Ok(Json(script))
+}
+
 async fn list_paths(
     authed: Authed,
     Extension(user_db): Extension<UserDB>,
@@ -522,7 +601,22 @@ async fn raw_script_by_path(
     Extension(user_db): Extension<UserDB>,
     Path((w_id, path)): Path<(String, StripPath)>,
 ) -> Result<String> {
-    let path = path.to_path().split(".").next().unwrap_or_default();
+    let path = path.to_path();
+    if !path.ends_with(".py")
+        && !path.ends_with(".ts")
+        && !path.ends_with(".go")
+        && !path.ends_with(".sh")
+    {
+        return Err(Error::BadRequest(format!(
+            "Path must ends with a .py, .ts, .go. or .sh extension: {}",
+            path
+        )));
+    }
+    let path = path
+        .trim_end_matches(".py")
+        .trim_end_matches(".ts")
+        .trim_end_matches(".go")
+        .trim_end_matches(".sh");
     let mut tx = user_db.begin(&authed).await?;
 
     let content_o = sqlx::query_scalar!(
@@ -636,6 +730,8 @@ async fn archive_script_by_path(
     let path = path.to_path();
     let mut tx = user_db.begin(&authed).await?;
 
+    require_owner_of_path(&authed, path)?;
+
     let hash: i64 = sqlx::query_scalar!(
         "UPDATE script SET archived = true WHERE path = $1 AND workspace_id = $2 RETURNING hash",
         path,
@@ -749,7 +845,19 @@ async fn delete_script_by_path(
     let mut tx = user_db.begin(&authed).await?;
     let path = path.to_path();
 
-    require_admin(authed.is_admin, &authed.username)?;
+    let draft_only = sqlx::query_scalar!(
+        "SELECT draft_only FROM script WHERE path = $1 AND workspace_id = $2",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    if !draft_only {
+        require_admin(authed.is_admin, &authed.username)?;
+    }
+
     let script = sqlx::query_scalar!(
         "DELETE FROM script WHERE path = $1 AND workspace_id = $2 RETURNING path",
         path,

backend/windmill-api/src/users.rs

@@ -359,7 +359,8 @@ pub struct Authed {
     pub username: String,
     pub is_admin: bool,
     pub groups: Vec<String>,
-    pub folders: Vec<(String, bool)>,
+    // (folder name, can write, is owner)
+    pub folders: Vec<(String, bool, bool)>,
 }
 
 pub async fn maybe_refresh_folders(path: &str, w_id: &str, authed: Authed, db: &DB) -> Authed {
@@ -369,7 +370,7 @@ pub async fn maybe_refresh_folders(path: &str, w_id: &str, authed: Authed, db: &
     let splitted = path.split('/').collect::<Vec<_>>();
     if splitted.len() >= 2
         && splitted[0] == "f"
-        && !authed.folders.iter().any(|(f, _)| f == splitted[1])
+        && !authed.folders.iter().any(|(f, _, _)| f == splitted[1])
     {
         let name = &authed.username;
         let groups = get_groups_for_user(w_id, name, db)
@@ -509,6 +510,7 @@ pub struct UserInfo {
     pub disabled: bool,
     pub role: Option<String>,
     pub folders: Vec<String>,
+    pub folders_owners: Vec<String>,
 }
 
 #[derive(FromRow, Serialize)]
@@ -808,9 +810,14 @@ async fn whoami(
             disabled: false,
             role: Some("superadmin".to_string()),
             folders: folders
+                .clone()
                 .into_iter()
                 .filter_map(|x| if x.1 { Some(x.0) } else { None })
                 .collect(),
+            folders_owners: folders
+                .into_iter()
+                .filter_map(|x| if x.2 { Some(x.0) } else { None })
+                .collect(),
         }))
     }
 }
@@ -896,9 +903,14 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo
         disabled: usr.disabled,
         role: usr.role,
         folders: folders
+            .clone()
             .into_iter()
             .filter_map(|x| if x.1 { Some(x.0) } else { None })
             .collect(),
+        folders_owners: folders
+            .into_iter()
+            .filter_map(|x| if x.2 { Some(x.0) } else { None })
+            .collect(),
     }))
 }
 
@@ -913,47 +925,45 @@ pub async fn get_groups_for_user(w_id: &str, username: &str, db: &DB) -> Result<
     Ok(groups)
 }
 pub async fn is_owner_of_path(
-    Authed { username, is_admin, groups, .. }: Authed,
-    Extension(db): Extension<DB>,
-    Path((w_id, path)): Path<(String, StripPath)>,
+    authed: Authed,
+    Path((_w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<bool> {
     let path = path.to_path();
-    if is_admin {
+    if authed.is_admin {
         Ok(Json(true))
     } else {
-        Ok(Json(
-            require_owner_of_path(&w_id, &username, &groups, path, &db)
-                .await
-                .is_ok(),
-        ))
+        Ok(Json(require_owner_of_path(&authed, path).is_ok()))
     }
 }
 
-pub async fn require_owner_of_path(
-    w_id: &str,
-    username: &str,
-    groups: &Vec<String>,
-    path: &str,
-    db: &DB,
-) -> Result<()> {
+pub fn require_owner_of_path(authed: &Authed, path: &str) -> Result<()> {
+    if authed.is_admin {
+        return Ok(());
+    }
     if !path.is_empty() {
         let splitted = path.split("/").collect::<Vec<&str>>();
         if splitted[0] == "u" {
-            if splitted[1] == username {
-                return Ok(());
+            if splitted[1] == authed.username {
+                Ok(())
             } else {
-                return Err(Error::BadRequest(format!(
+                Err(Error::BadRequest(format!(
                     "only the owner {} is authorized to perform this operation",
                     splitted[1]
-                )));
+                )))
             }
-        } else if splitted[0] == "g" {
-            return crate::groups::require_is_owner(splitted[1], username, groups, w_id, db).await;
         } else if splitted[0] == "f" {
-            return crate::folders::require_is_owner(splitted[1], username, groups, w_id, db).await;
+            crate::folders::require_is_owner(authed, splitted[1])
+        } else {
+            Err(Error::BadRequest(format!(
+                "Not recognized path kind: {}",
+                path
+            )))
         }
+    } else {
+        Err(Error::BadRequest(format!(
+            "Cannot be owner of an empty path"
+        )))
     }
-    Err(Error::BadRequest(format!("not recognized owner kind")))
 }
 
 async fn whois(
@@ -1034,6 +1044,11 @@ async fn decline_invite(
 
 lazy_static! {
     pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z_0-9]+$"#).unwrap();
+
+    pub static ref JOB_RETENTION_SECS: u32 = std::env::var("JOB_RETENTION_SECS")
+    .ok()
+    .and_then(|x| x.parse::<u32>().ok())
+    .unwrap_or(60 * 60 * 24 * 60); // 60 days
 }
 
 async fn accept_invite(
@@ -1903,16 +1918,24 @@ pub async fn delete_expired_items_perdiodically(
             Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
         }
 
-        let magic_links_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
-            "DELETE FROM magic_link WHERE expiration <= now()
-        RETURNING concat(substring(token for 10), '*****')",
-        )
-        .fetch_all(db)
-        .await;
+        if *JOB_RETENTION_SECS > 0 {
+            let deleted_jobs = sqlx::query_scalar!(
+                "DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id",
+                *JOB_RETENTION_SECS as i64
+            )
+            .fetch_all(db)
+            .await;
 
-        match magic_links_deleted_r {
-            Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens),
-            Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
+            match deleted_jobs {
+                Ok(deleted_jobs) => {
+                    tracing::info!(
+                        "deleted {} jobs completed JOB_RETENTION_SECS ago: {:?}",
+                        deleted_jobs.len(),
+                        deleted_jobs
+                    )
+                }
+                Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
+            }
         }
 
         tokio::select! {

backend/windmill-api/src/variables.rs

@@ -12,13 +12,6 @@ use crate::{
     users::{maybe_refresh_folders, require_owner_of_path, Authed},
     webhook_util::{WebhookMessage, WebhookShared},
 };
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
 
 use axum::{
     extract::{Extension, Path, Query},
@@ -344,15 +337,20 @@ async fn update_variable(
     if let Some(npath) = &ns.path {
         sqlb.set_str("path", npath);
     }
+    let ns_value_is_none = ns.value.is_none();
     if let Some(nvalue) = ns.value {
-        let is_secret = sqlx::query_scalar!(
-            "SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2",
-            &path,
-            &w_id
-        )
-        .fetch_optional(&mut tx)
-        .await?
-        .unwrap_or(false);
+        let is_secret = if ns.is_secret.is_some() {
+            ns.is_secret.unwrap()
+        } else {
+            sqlx::query_scalar!(
+                "SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2",
+                &path,
+                &w_id
+            )
+            .fetch_optional(&mut tx)
+            .await?
+            .unwrap_or(false)
+        };
 
         let value = if is_secret && !already_encrypted.unwrap_or(false) {
             let mc = build_crypt(&mut tx, &w_id).await?;
@@ -368,9 +366,17 @@ async fn update_variable(
     }
 
     if let Some(nbool) = ns.is_secret {
-        if !nbool {
+        let old_secret = sqlx::query_scalar!(
+            "SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2",
+            &path,
+            &w_id
+        )
+        .fetch_optional(&mut tx)
+        .await?
+        .unwrap_or(false);
+        if old_secret != nbool && ns_value_is_none {
             return Err(Error::BadRequest(
-                "A variable can not be updated to be non secret".to_owned(),
+                "cannot change is_secret without updating value too".to_string(),
             ));
         }
         sqlb.set_str("is_secret", nbool);
@@ -380,9 +386,8 @@ async fn update_variable(
     if let Some(npath) = ns.path {
         if npath != path {
             check_path_conflict(&mut tx, &w_id, &npath).await?;
-            if !authed.is_admin {
-                require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
-            }
+            require_owner_of_path(&authed, path)?;
+
             let mut v = sqlx::query_scalar!(
                 "SELECT value FROM resource  WHERE path = $1 AND workspace_id = $2",
                 path,

backend/windmill-api/src/workers.rs

@@ -21,7 +21,16 @@ use windmill_common::{
 };
 
 pub fn global_service() -> Router {
-    Router::new().route("/list", get(list_worker_pings))
+    Router::new()
+        .route("/list", get(list_worker_pings))
+        .route("/custom_tags", get(get_custom_tags))
+}
+
+lazy_static::lazy_static! {
+    pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
+        .ok()
+        .map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<_>>()).unwrap_or_default();
+
 }
 
 #[derive(FromRow, Serialize, Deserialize)]
@@ -54,3 +63,7 @@ async fn list_worker_pings(
     tx.commit().await?;
     Ok(Json(rows))
 }
+
+async fn get_custom_tags() -> Json<Vec<String>> {
+    Json(CUSTOM_TAGS.clone())
+}

backend/windmill-api/src/workspaces.rs

@@ -1165,6 +1165,8 @@ where
                 "edited_at",
                 "edited_by",
                 "archived",
+                "has_draft",
+                "draft_only",
             ] {
                 if obj.contains_key(key) {
                     obj.remove(key);

backend/windmill-common/src/error.rs

@@ -13,6 +13,7 @@ use axum::{
     response::Json,
 };
 
+use hyper::StatusCode;
 #[cfg(feature = "sqlx")]
 use sqlx::migrate::MigrateError;
 use thiserror::Error;
@@ -60,6 +61,8 @@ pub enum Error {
     Anyhow(#[from] anyhow::Error),
     #[error("Error: {0:#?}")]
     JsonErr(serde_json::Value),
+    #[error("Custom Status Code: {0:#?}")]
+    CustomStatusCode(StatusCode, serde_json::Value),
 }
 
 impl Error {
@@ -82,6 +85,7 @@ impl IntoResponse for Error {
             Self::NotFound(_) => axum::http::StatusCode::NOT_FOUND,
             Self::NotAuthorized(_) => axum::http::StatusCode::UNAUTHORIZED,
             Self::RequireAdmin(_) => axum::http::StatusCode::FORBIDDEN,
+            Self::CustomStatusCode(code, _) => code,
             Self::SqlErr(_) | Self::BadRequest(_) => axum::http::StatusCode::BAD_REQUEST,
             _ => axum::http::StatusCode::INTERNAL_SERVER_ERROR,
         };

backend/windmill-common/src/external_ip.rs

@@ -11,10 +11,16 @@
 //! For users writing scripts to access their infrastructure with firewalls requiring incoming
 //! connections to be from whitelisted IP addresses.
 
+use std::time::Duration;
+
 use reqwest::Result;
 
 pub async fn get_ip() -> Result<String> {
-    reqwest::get("https://hub.windmill.dev/getip")
+    reqwest::ClientBuilder::new()
+        .timeout(Duration::from_secs(3))
+        .build()?
+        .get("https://hub.windmill.dev/getip")
+        .send()
         .await?
         .error_for_status()?
         .text()

backend/windmill-common/src/flows.rs

@@ -33,6 +33,8 @@ pub struct Flow {
     pub archived: bool,
     pub schema: Option<Schema>,
     pub extra_perms: serde_json::Value,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
 }
 
 #[derive(Serialize)]
@@ -47,6 +49,9 @@ pub struct ListableFlow {
     pub archived: bool,
     pub extra_perms: serde_json::Value,
     pub starred: bool,
+    pub has_draft: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -57,6 +62,7 @@ pub struct NewFlow {
     pub description: String,
     pub value: serde_json::Value,
     pub schema: Option<Schema>,
+    pub draft_only: Option<bool>,
 }
 
 #[derive(Deserialize, Serialize, Debug, Clone, Default)]
@@ -252,6 +258,8 @@ pub enum FlowModuleValue {
         lock: Option<String>,
         #[serde(skip_serializing_if = "Option::is_none")]
         path: Option<String>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        tag: Option<String>,
         language: ScriptLang,
     },
     Identity,

backend/windmill-common/src/jobs.rs

@@ -1 +1,201 @@
+use serde::{Deserialize, Serialize};
+use sqlx::{Postgres, Transaction};
+use uuid::Uuid;
 
+use crate::{
+    error,
+    flow_status::FlowStatus,
+    flows::FlowValue,
+    get_latest_deployed_hash_for_path,
+    scripts::{ScriptHash, ScriptLang},
+};
+
+#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
+#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")]
+#[serde(rename_all(serialize = "lowercase"))]
+pub enum JobKind {
+    Script,
+    #[allow(non_camel_case_types)]
+    Script_Hub,
+    Preview,
+    Dependencies,
+    Flow,
+    FlowPreview,
+    Identity,
+    FlowDependencies,
+}
+
+#[derive(Debug, sqlx::FromRow, Serialize, Clone)]
+pub struct QueuedJob {
+    pub workspace_id: String,
+    pub id: Uuid,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub parent_job: Option<Uuid>,
+    pub created_by: String,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub started_at: Option<chrono::DateTime<chrono::Utc>>,
+    pub scheduled_for: chrono::DateTime<chrono::Utc>,
+    pub running: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub script_hash: Option<ScriptHash>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub script_path: Option<String>,
+    pub args: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub logs: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub raw_code: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub raw_lock: Option<String>,
+    pub canceled: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub canceled_by: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub canceled_reason: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub last_ping: Option<chrono::DateTime<chrono::Utc>>,
+    pub job_kind: JobKind,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub schedule_path: Option<String>,
+    pub permissioned_as: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub flow_status: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub raw_flow: Option<serde_json::Value>,
+    pub is_flow_step: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub language: Option<ScriptLang>,
+    pub same_worker: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub pre_run_error: Option<String>,
+    pub email: String,
+    pub visible_to_owner: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub suspend: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub mem_peak: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub root_job: Option<Uuid>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub leaf_jobs: Option<serde_json::Value>,
+    pub tag: String,
+}
+
+impl QueuedJob {
+    pub fn script_path(&self) -> &str {
+        self.script_path
+            .as_ref()
+            .map(String::as_str)
+            .unwrap_or("tmp/main")
+    }
+}
+
+impl QueuedJob {
+    pub fn parse_raw_flow(&self) -> Option<FlowValue> {
+        self.raw_flow
+            .as_ref()
+            .and_then(|v| serde_json::from_value::<FlowValue>(v.clone()).ok())
+    }
+
+    pub fn parse_flow_status(&self) -> Option<FlowStatus> {
+        self.flow_status
+            .as_ref()
+            .and_then(|v| serde_json::from_value::<FlowStatus>(v.clone()).ok())
+    }
+}
+
+impl Default for QueuedJob {
+    fn default() -> Self {
+        Self {
+            workspace_id: "".to_string(),
+            id: Uuid::default(),
+            parent_job: None,
+            created_by: "".to_string(),
+            created_at: chrono::Utc::now(),
+            started_at: None,
+            scheduled_for: chrono::Utc::now(),
+            running: false,
+            script_hash: None,
+            script_path: None,
+            args: None,
+            logs: None,
+            raw_code: None,
+            raw_lock: None,
+            canceled: false,
+            canceled_by: None,
+            canceled_reason: None,
+            last_ping: None,
+            job_kind: JobKind::Identity,
+            schedule_path: None,
+            permissioned_as: "".to_string(),
+            flow_status: None,
+            raw_flow: None,
+            is_flow_step: false,
+            language: None,
+            same_worker: false,
+            pre_run_error: None,
+            email: "".to_string(),
+            visible_to_owner: false,
+            suspend: None,
+            mem_peak: None,
+            root_job: None,
+            leaf_jobs: None,
+            tag: "deno".to_string(),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum JobPayload {
+    ScriptHub { path: String },
+    ScriptHash { hash: ScriptHash, path: String },
+    Code(RawCode),
+    Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang },
+    FlowDependencies { path: String },
+    Flow(String),
+    RawFlow { value: FlowValue, path: Option<String> },
+    Identity,
+}
+
+#[derive(Clone, Serialize, Deserialize, Debug)]
+pub struct RawCode {
+    pub content: String,
+    pub path: Option<String>,
+    pub language: ScriptLang,
+    pub lock: Option<String>,
+}
+
+type Tag = String;
+
+pub async fn script_path_to_payload<'c>(
+    script_path: &str,
+    db: &mut Transaction<'c, Postgres>,
+    w_id: &String,
+) -> error::Result<(JobPayload, Option<Tag>)> {
+    let (job_payload, tag) = if script_path.starts_with("hub/") {
+        (JobPayload::ScriptHub { path: script_path.to_owned() }, None)
+    } else {
+        let (script_hash, tag) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?;
+        (
+            JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() },
+            tag,
+        )
+    };
+    Ok((job_payload, tag))
+}
+
+pub async fn script_hash_to_tag<'c>(
+    script_hash: &ScriptHash,
+    db: &mut Transaction<'c, Postgres>,
+    w_id: &String,
+) -> error::Result<Option<Tag>> {
+    Ok(sqlx::query_scalar!(
+        "select tag from script where hash = $1 AND workspace_id = $2",
+        script_hash.0,
+        w_id
+    )
+    .fetch_optional(db)
+    .await?
+    .flatten())
+}

backend/windmill-common/src/lib.rs

@@ -6,7 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::{net::SocketAddr, sync::Arc};
+use std::net::SocketAddr;
 
 use error::Error;
 
@@ -15,6 +15,7 @@ pub mod error;
 pub mod external_ip;
 pub mod flow_status;
 pub mod flows;
+pub mod jobs;
 pub mod more_serde;
 pub mod oauth2;
 pub mod schedule;
@@ -42,11 +43,14 @@ lazy_static::lazy_static! {
     .flatten();
     pub static ref METRICS_ENABLED: bool = METRICS_ADDR.is_some();
     pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
-    pub static ref IS_READY: Arc<std::sync::atomic::AtomicBool> = Arc::new(std::sync::atomic::AtomicBool::new(false));
+    pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
 }
 
 #[cfg(feature = "tokio")]
-pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::Result<()> {
+pub async fn shutdown_signal(
+    tx: tokio::sync::broadcast::Sender<()>,
+    mut rx: tokio::sync::broadcast::Receiver<()>,
+) -> anyhow::Result<()> {
     use std::io;
     use tokio::signal::unix::SignalKind;
 
@@ -60,6 +64,9 @@ pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::
     tokio::select! {
         _ = terminate() => {},
         _ = tokio::signal::ctrl_c() => {},
+        _ = rx.recv() => {
+            tracing::info!("shutdown monitor received killpill");
+        },
     }
     println!("signal received, starting graceful shutdown");
     let _ = tx.send(());
@@ -146,23 +153,44 @@ pub async fn connect(
         .map_err(|err| Error::ConnectingToDatabase(err.to_string()))
 }
 
-// TODO: Move this elsewhere
-pub async fn get_latest_hash_for_path<'c>(
+type Tag = String;
+
+pub async fn get_latest_deployed_hash_for_path<'c>(
     db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
     w_id: &str,
     script_path: &str,
-) -> error::Result<scripts::ScriptHash> {
-    let script_hash_o = sqlx::query_scalar!(
-        "select hash from script where path = $1 AND workspace_id = $2 AND
-    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2) AND
-    deleted = false",
+) -> error::Result<(scripts::ScriptHash, Option<Tag>)> {
+    let r_o = sqlx::query!(
+        "select hash, tag from script where path = $1 AND workspace_id = $2 AND
+    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
+    deleted = false AND archived = false AND lock IS not NULL AND lock_error_logs IS NULL)",
         script_path,
         w_id
     )
     .fetch_optional(db)
     .await?;
 
-    let script_hash = utils::not_found_if_none(script_hash_o, "ScriptHash", script_path)?;
+    let script = utils::not_found_if_none(r_o, "script", script_path)?;
 
-    Ok(scripts::ScriptHash(script_hash))
+    Ok((scripts::ScriptHash(script.hash), script.tag))
+}
+
+pub async fn get_latest_hash_for_path<'c>(
+    db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
+    w_id: &str,
+    script_path: &str,
+) -> error::Result<(scripts::ScriptHash, Option<Tag>)> {
+    let r_o = sqlx::query!(
+        "select hash, tag from script where path = $1 AND workspace_id = $2 AND
+    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
+    deleted = false AND archived = false)",
+        script_path,
+        w_id
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let script = utils::not_found_if_none(r_o, "script", script_path)?;
+
+    Ok((scripts::ScriptHash(script.hash), script.tag))
 }

backend/windmill-common/src/scripts.rs

@@ -136,27 +136,28 @@ pub struct Script {
     pub lock_error_logs: Option<String>,
     pub language: ScriptLang,
     pub kind: ScriptKind,
+    pub tag: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
 }
 
 #[derive(Serialize)]
 #[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
 pub struct ListableScript {
-    pub workspace_id: String,
     pub hash: ScriptHash,
     pub path: String,
-    pub parent_hashes: Option<ScriptHashes>,
     pub summary: String,
-    pub description: String,
-    pub created_by: String,
     pub created_at: chrono::DateTime<chrono::Utc>,
     pub archived: bool,
-    pub deleted: bool,
-    pub is_template: bool,
     pub extra_perms: serde_json::Value,
-    pub lock_error_logs: Option<String>,
     pub language: ScriptLang,
-    pub kind: ScriptKind,
     pub starred: bool,
+    pub tag: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub has_draft: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
+    pub has_deploy_errors: bool,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -186,6 +187,8 @@ pub struct NewScript {
     pub lock: Option<Vec<String>>,
     pub language: ScriptLang,
     pub kind: Option<ScriptKind>,
+    pub tag: Option<String>,
+    pub draft_only: Option<bool>,
 }
 
 #[derive(Deserialize)]

backend/windmill-queue/Cargo.toml

@@ -34,3 +34,4 @@ cron.workspace = true
 rsmq_async.workspace = true
 tokio.workspace = true
 futures-core.workspace = true
+itertools.workspace = true

backend/windmill-queue/src/jobs.rs

@@ -8,9 +8,8 @@
 
 use std::collections::HashMap;
 
-use anyhow::Context;
+use itertools::Itertools;
 use reqwest::Client;
-use serde::{Deserialize, Serialize};
 use sqlx::{Pool, Postgres, Transaction};
 use tracing::{instrument, Instrument};
 use ulid::Ulid;
@@ -20,8 +19,8 @@ use windmill_common::{
     error::{self, Error},
     flow_status::{FlowStatus, JobResult, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL},
     flows::{FlowModule, FlowModuleValue, FlowValue},
-    scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang},
-    utils::StripPath,
+    jobs::{JobKind, JobPayload, QueuedJob, RawCode},
+    scripts::{ScriptHash, ScriptLang},
     METRICS_ENABLED,
 };
 
@@ -50,6 +49,21 @@ lazy_static::lazy_static! {
     .unwrap();
     pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();
 
+    pub static ref ACCEPTED_TAGS: Vec<String> = std::env::var("WORKER_TAGS")
+        .ok()
+        .map(|x| x.split(',').map(|x| x.to_string()).collect())
+        .unwrap_or_else(|| vec![
+            "deno".to_string(),
+            "python3".to_string(),
+            "go".to_string(),
+            "bash".to_string(),
+            "dependency".to_string(),
+            "flow".to_string(),
+            "hub".to_string(),
+            "other".to_string()]);
+
+    pub static ref ACCEPTED_TAGS_FILTER: String = format!(" AND ({})",
+        ACCEPTED_TAGS.clone().into_iter().map(|x| format!("(tag = '{x}')")).join(" OR "));
 }
 
 const MAX_FREE_EXECS: i32 = 1000;
@@ -165,6 +179,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
             None
         }
     } else {
+        let accepted_tags_filter = &*ACCEPTED_TAGS_FILTER;
         /* Jobs can be started if they:
          * - haven't been started before,
          *   running = false
@@ -185,7 +200,9 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
                        AND scheduled_for <= now())
                    OR (suspend_until IS NOT NULL
                        AND (   suspend <= 0
-                            OR suspend_until <= now()))) {workspaces_filter}
+                            OR suspend_until <= now()))) 
+                    {workspaces_filter}
+                    {accepted_tags_filter}
                 ORDER BY scheduled_for
                 FOR UPDATE SKIP LOCKED
                 LIMIT 1
@@ -307,13 +324,32 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
     schedule_path: Option<String>,
     parent_job: Option<Uuid>,
     root_job: Option<Uuid>,
+    job_id: Option<Uuid>,
     is_flow_step: bool,
     mut same_worker: bool,
     pre_run_error: Option<&windmill_common::error::Error>,
     visible_to_owner: bool,
+    mut tag: Option<String>,
 ) -> Result<(Uuid, QueueTransaction<'c, R>), Error> {
     let args_json = serde_json::Value::Object(args);
-    let job_id: Uuid = Ulid::new().into();
+    let job_id: Uuid = if let Some(job_id) = job_id {
+        let conflicting_id = sqlx::query_scalar!(
+            "SELECT 1 FROM queue WHERE id = $1 UNION ALL select 1 FROM completed_job WHERE id = $1",
+            job_id
+        )
+        .fetch_optional(&mut tx)
+        .await?;
+
+        if conflicting_id.is_some() {
+            return Err(Error::BadRequest(format!(
+                "Job with id {job_id} already exists"
+            )));
+        }
+
+        job_id
+    } else {
+        Ulid::new().into()
+    };
 
     if cfg!(feature = "enterprise") {
         let premium_workspace = *CLOUD_HOSTED
@@ -437,16 +473,14 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
                 )
             }
             JobPayload::ScriptHub { path } => {
-                let script = get_hub_script(&HTTP_CLIENT, path.clone(), email)
-                    .await
-                    .context("error fetching hub script")?;
                 (
                     None,
                     Some(path),
-                    Some((script.content, script.lockfile)),
+                    None,
+                    // Some((script.content, script.lockfile)),
                     JobKind::Script_Hub,
                     None,
-                    Some(script.language),
+                    None,
                 )
             }
             JobPayload::Code(RawCode { content, path, language, lock }) => (
@@ -558,12 +592,34 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
 
     let flow_status = raw_flow.as_ref().map(FlowStatus::new);
 
+    let tag = if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies {
+        "dependency".to_string()
+    } else if job_kind == JobKind::Flow || job_kind == JobKind::FlowPreview {
+        "flow".to_string()
+    } else if job_kind == JobKind::Identity {
+        // identity is a light script, deno is too
+        "deno".to_string()
+    } else if job_kind == JobKind::Script_Hub {
+        "hub".to_string()
+    } else {
+        if tag == Some("".to_string()) {
+            tag = None;
+        }
+        tag.unwrap_or_else(|| {
+            language
+                .as_ref()
+                .map(|x| x.as_str())
+                .unwrap_or_else(|| "deno")
+                .to_string()
+        })
+    };
+
     let uuid = sqlx::query_scalar!(
         "INSERT INTO queue
             (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, 
                 script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, \
-         flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job)
-            VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23) \
+         flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job, tag)
+            VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23, $24) \
          RETURNING id",
         workspace_id,
         job_id,
@@ -587,7 +643,8 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
         pre_run_error.map(|e| e.to_string()),
         email,
         visible_to_owner,
-        root_job
+        root_job,
+        tag
     )
     .fetch_one(&mut tx)
     .await
@@ -645,127 +702,3 @@ pub fn canceled_job_to_result(job: &QueuedJob) -> serde_json::Value {
     let canceler = job.canceled_by.as_deref().unwrap_or_else(|| "unknown");
     serde_json::json!({"message": format!("Job canceled: {reason} by {canceler}"), "name": "Canceled", "reason": reason, "canceler": canceler})
 }
-
-pub async fn get_hub_script(
-    client: &reqwest::Client,
-    path: String,
-    email: &str,
-) -> error::Result<HubScript> {
-    get_full_hub_script_by_path(email, StripPath(path), client)
-        .await
-        .map(|e| e)
-}
-
-#[derive(Debug, sqlx::FromRow, Serialize, Clone)]
-pub struct QueuedJob {
-    pub workspace_id: String,
-    pub id: Uuid,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub parent_job: Option<Uuid>,
-    pub created_by: String,
-    pub created_at: chrono::DateTime<chrono::Utc>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub started_at: Option<chrono::DateTime<chrono::Utc>>,
-    pub scheduled_for: chrono::DateTime<chrono::Utc>,
-    pub running: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub script_hash: Option<ScriptHash>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub script_path: Option<String>,
-    pub args: Option<serde_json::Value>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub logs: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub raw_code: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub raw_lock: Option<String>,
-    pub canceled: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub canceled_by: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub canceled_reason: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub last_ping: Option<chrono::DateTime<chrono::Utc>>,
-    pub job_kind: JobKind,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub schedule_path: Option<String>,
-    pub permissioned_as: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub flow_status: Option<serde_json::Value>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub raw_flow: Option<serde_json::Value>,
-    pub is_flow_step: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub language: Option<ScriptLang>,
-    pub same_worker: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub pre_run_error: Option<String>,
-    pub email: String,
-    pub visible_to_owner: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub suspend: Option<i32>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub mem_peak: Option<i32>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub root_job: Option<Uuid>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub leaf_jobs: Option<serde_json::Value>,
-}
-
-impl QueuedJob {
-    pub fn script_path(&self) -> &str {
-        self.script_path
-            .as_ref()
-            .map(String::as_str)
-            .unwrap_or("tmp/main")
-    }
-}
-
-impl QueuedJob {
-    pub fn parse_raw_flow(&self) -> Option<FlowValue> {
-        self.raw_flow
-            .as_ref()
-            .and_then(|v| serde_json::from_value::<FlowValue>(v.clone()).ok())
-    }
-
-    pub fn parse_flow_status(&self) -> Option<FlowStatus> {
-        self.flow_status
-            .as_ref()
-            .and_then(|v| serde_json::from_value::<FlowStatus>(v.clone()).ok())
-    }
-}
-
-#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
-#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")]
-#[serde(rename_all(serialize = "lowercase"))]
-pub enum JobKind {
-    Script,
-    #[allow(non_camel_case_types)]
-    Script_Hub,
-    Preview,
-    Dependencies,
-    Flow,
-    FlowPreview,
-    Identity,
-    FlowDependencies,
-}
-
-#[derive(Debug, Clone)]
-pub enum JobPayload {
-    ScriptHub { path: String },
-    ScriptHash { hash: ScriptHash, path: String },
-    Code(RawCode),
-    Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang },
-    FlowDependencies { path: String },
-    Flow(String),
-    RawFlow { value: FlowValue, path: Option<String> },
-    Identity,
-}
-
-#[derive(Clone, Serialize, Deserialize, Debug)]
-pub struct RawCode {
-    pub content: String,
-    pub path: Option<String>,
-    pub language: ScriptLang,
-    pub lock: Option<String>,
-}

backend/windmill-queue/src/schedule.rs

@@ -6,16 +6,17 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use crate::{push, JobPayload};
+use crate::push;
+use crate::QueueTransaction;
 use sqlx::{query_scalar, Postgres, Transaction};
 use std::str::FromStr;
+use windmill_common::jobs::JobPayload;
 use windmill_common::{
     error::{self, Result},
     schedule::Schedule,
     users::username_to_permissioned_as,
     utils::{now_from_db, StripPath},
 };
-use crate::{QueueTransaction};
 
 pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
     mut tx: QueueTransaction<'c, R>,
@@ -66,18 +67,19 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
         }
     }
 
-    let payload = if schedule.is_flow {
-        JobPayload::Flow(schedule.script_path)
+    let (payload, tag) = if schedule.is_flow {
+        (JobPayload::Flow(schedule.script_path), None)
     } else {
-        JobPayload::ScriptHash {
-            hash: windmill_common::get_latest_hash_for_path(
-                tx.transaction_mut(),
-                &schedule.workspace_id,
-                &schedule.script_path,
-            )
-            .await?,
-            path: schedule.script_path,
-        }
+        let (hash, tag) = windmill_common::get_latest_hash_for_path(
+            tx.transaction_mut(),
+            &schedule.workspace_id,
+            &schedule.script_path,
+        )
+        .await?;
+        (
+            JobPayload::ScriptHash { hash, path: schedule.script_path },
+            tag,
+        )
     };
 
     sqlx::query!(
@@ -100,10 +102,12 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
         Some(schedule.path.clone()),
         None,
         None,
+        None,
         false,
         false,
         None,
         true,
+        tag,
     )
     .await?;
     Ok(tx) // TODO: Bubble up pushed UUID from here

backend/windmill-worker/loader.py

@@ -30,7 +30,7 @@ class WindmillFinder(MetaPathFinder):
             script_path = "/".join(splitted)
             import requests
 
-            url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}"
+            url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}.py"
 
             r = requests.get(
                 url, headers={"Authorization": f"Bearer {os.environ.get('WM_TOKEN')}"}

backend/windmill-worker/nsjail/download.py.config.proto

@@ -3,7 +3,7 @@ name: "python download pip"
 mode: ONCE
 hostname: "python"
 log_level: ERROR
-time_limit: 300
+time_limit: 900
 
 rlimit_as: 2048
 rlimit_cpu: 1000

backend/windmill-worker/src/common.rs

@@ -0,0 +1,41 @@
+use sqlx::{Pool, Postgres};
+use tokio::{fs::File, io::AsyncReadExt};
+use windmill_common::error::{self, Error};
+use windmill_queue::CLOUD_HOSTED;
+
+use crate::MAX_RESULT_SIZE;
+
+pub async fn read_result(job_dir: &str) -> error::Result<serde_json::Value> {
+    let mut file = File::open(format!("{job_dir}/result.json")).await?;
+    let mut content = "".to_string();
+    file.read_to_string(&mut content).await?;
+    if *CLOUD_HOSTED && content.len() > MAX_RESULT_SIZE {
+        return Err(Error::ExecutionErr("Result is too large for the cloud app (limit 2MB). 
+        If using this script as part of the flow, use the shared folder to pass heavy data between steps.".to_owned()));
+    }
+    serde_json::from_str(&content)
+        .map_err(|e| Error::ExecutionErr(format!("Error parsing result: {e}")))
+}
+
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn set_logs(logs: &str, id: &uuid::Uuid, db: &Pool<Postgres>) {
+    if sqlx::query!(
+        "UPDATE queue SET logs = $1 WHERE id = $2",
+        logs.to_owned(),
+        id
+    )
+    .execute(db)
+    .await
+    .is_err()
+    {
+        tracing::error!(%id, "error updating logs for id {id}")
+    };
+}
+
+pub fn capitalize(s: &str) -> String {
+    let mut c = s.chars();
+    match c.next() {
+        None => String::new(),
+        Some(f) => f.to_uppercase().collect::<String>() + c.as_str(),
+    }
+}

backend/windmill-worker/src/global_cache.rs

@@ -0,0 +1,521 @@
+#[cfg(feature = "enterprise")]
+use crate::{ROOT_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_CACHE_RATE, TAR_PIP_TMP_CACHE_DIR, TMP_DIR};
+#[cfg(feature = "enterprise")]
+use itertools::Itertools;
+#[cfg(feature = "enterprise")]
+use rand::Rng;
+#[cfg(feature = "enterprise")]
+use std::process::Stdio;
+
+#[cfg(feature = "enterprise")]
+use tokio::{process::Command, sync::mpsc::Sender, time::Instant};
+
+#[cfg(feature = "enterprise")]
+use windmill_common::error;
+
+#[cfg(feature = "enterprise")]
+const TAR_CACHE_FILENAME: &str = "denogocache.tar";
+
+#[cfg(feature = "enterprise")]
+pub async fn build_tar_and_push(bucket: &str, folder: String) -> error::Result<()> {
+    tracing::info!("Started building and pushing piptar {folder}");
+    let start = Instant::now();
+    let folder_name = folder.split("/").last().unwrap();
+    let tar_path = format!("{TAR_PIP_TMP_CACHE_DIR}/{folder_name}.tar",);
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "tar",
+        vec!["-c", "-f", &tar_path, "-C", &folder, "."],
+    )
+    .await
+    {
+        tracing::info!("Failed to tar cache. Error: {:?}", e);
+        return Err(e);
+    }
+
+    let tar_metadata = tokio::fs::metadata(&tar_path).await;
+    if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 {
+        tracing::info!("Failed to tar cache: {folder}");
+        return Err(error::Error::ExecutionErr(format!(
+            "Failed to tar cache: {folder}"
+        )));
+    }
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "rclone",
+        vec![
+            "copyto",
+            &tar_path,
+            &format!(":s3,env_auth=true:{bucket}/tar/pip/{folder_name}.tar"),
+            "-v",
+            "--size-only",
+            "--fast-list",
+            "--s3-no-check-bucket",
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed to copy piptar {folder} to bucket. Error: {:?}", e);
+        return Err(e);
+    }
+
+    tracing::info!(
+        "Finished copying piptar {folder} to bucket {bucket} as tar, took: {:?}s. Size of tar: {}",
+        start.elapsed().as_secs(),
+        tar_metadata.unwrap().len()
+    );
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn pull_from_tar(bucket: &str, folder: String) -> error::Result<()> {
+    use tokio::fs::metadata;
+    let folder_name = folder.split("/").last().unwrap();
+
+    tracing::info!("Attempting to pull piptar {folder_name} from bucket");
+
+    let start = Instant::now();
+    let tar_path = format!("tar/pip/{folder_name}.tar");
+    let target = format!("{ROOT_TMP_CACHE_DIR}/{tar_path}.single");
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "rclone",
+        vec![
+            "copyto",
+            &format!(":s3,env_auth=true:{bucket}/{tar_path}"),
+            &target,
+            "-v",
+            "--size-only",
+            "--fast-list",
+        ],
+    )
+    .await
+    {
+        tracing::info!(
+            "Failed to copy tar {folder_name} from bucket. Error: {:?}",
+            e
+        );
+        return Err(e);
+    }
+
+    if metadata(&target).await.is_err() {
+        tracing::info!(
+            "piptar {folder_name} not found in bucket. Took {:?}ms",
+            start.elapsed().as_millis()
+        );
+        return Err(error::Error::ExecutionErr(format!(
+            "tar {folder_name} does not exist in bucket"
+        )));
+    }
+
+    extract_pip_tar(&target, &folder).await?;
+    tokio::fs::remove_file(&target).await?;
+    tracing::info!(
+        "Finished pulling and extracting {folder_name}. Took {:?}ms",
+        start.elapsed().as_millis()
+    );
+
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn cache_global(bucket: &str, tx: Sender<()>) -> error::Result<()> {
+    copy_cache_from_bucket(bucket, tx).await?;
+    copy_cache_to_bucket(bucket).await?;
+
+    // this is to prevent excessive tar upload. 1/100*15min = each worker sync its tar once per day on average
+    if rand::thread_rng().gen_range(0..*TAR_CACHE_RATE) == 0 {
+        copy_cache_to_bucket_as_tar(bucket).await;
+    }
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn copy_cache_from_bucket(bucket: &str, tx: Sender<()>) -> error::Result<()> {
+    tracing::info!("Copying cache from bucket in the background {bucket}");
+    let bucket = bucket.to_string();
+
+    let start = Instant::now();
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "rclone",
+        vec![
+            "copy",
+            &format!(":s3,env_auth=true:{bucket}"),
+            &ROOT_TMP_CACHE_DIR,
+            "--size-only",
+            "--fast-list",
+            "--filter",
+            "- deno/gen/file/**",
+            "--filter",
+            "+ deno/**",
+            "--filter",
+            "+ go/**",
+            "--filter",
+            "+ tar/**",
+            "--filter",
+            "- *",
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed to copy cache from bucket. Error: {:?}", e);
+        return Err(e);
+    }
+
+    tracing::info!(
+        "Finished copying cache from bucket {bucket}, took {:?}s",
+        start.elapsed().as_secs()
+    );
+
+    tx.send(()).await.expect("can send copy cache signal");
+
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn copy_cache_to_bucket(bucket: &str) -> error::Result<()> {
+    tracing::info!("Copying cache to bucket {bucket}");
+    let start = Instant::now();
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "rclone",
+        vec![
+            "copy",
+            &ROOT_TMP_CACHE_DIR,
+            &format!(":s3,env_auth=true:{bucket}"),
+            "--size-only",
+            "--fast-list",
+            "--filter",
+            "- deno/gen/file/**",
+            "--filter",
+            "+ deno/**",
+            "--filter",
+            "+ go/**",
+            "--filter",
+            "- *",
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed to copy cache to bucket. Error: {:?}", e);
+        return Err(e);
+    }
+    tracing::info!(
+        "Finished copying cache to bucket {bucket}, took: {:?}s",
+        start.elapsed().as_secs()
+    );
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn copy_cache_to_bucket_as_tar(bucket: &str) {
+    tracing::info!("Copying cache to bucket {bucket} as tar");
+    let start = Instant::now();
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "tar",
+        vec![
+            "-c",
+            "-f",
+            &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
+            "go",
+            "deno",
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed to tar cache. Error: {:?}", e);
+        return;
+    }
+
+    let tar_metadata =
+        tokio::fs::metadata(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await;
+    if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 {
+        tracing::info!("Failed to tar cache");
+        return;
+    }
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "rclone",
+        vec![
+            "copyto",
+            &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
+            &format!(":s3,env_auth=true:{bucket}/{TAR_CACHE_FILENAME}"),
+            "-v",
+            "--size-only",
+            "--fast-list",
+            "--s3-no-check-bucket",
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed to copy tar to bucket. Error: {:?}", e);
+        return;
+    }
+
+    if let Err(e) =
+        tokio::fs::remove_file(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await
+    {
+        tracing::info!("Failed to remove tar cache. Error: {:?}", e);
+    };
+
+    tracing::info!(
+        "Finished copying cache to bucket {bucket} as tar, took: {:?}s. Size of new tar: {}",
+        start.elapsed().as_secs(),
+        tar_metadata.unwrap().len()
+    );
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn copy_denogo_cache_from_bucket_as_tar(bucket: &str) {
+    use tokio::fs::metadata;
+
+    tracing::info!("Copying denogo cache from bucket {bucket} as tar");
+
+    let start: Instant = Instant::now();
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "rclone",
+        vec![
+            "copyto",
+            &format!(":s3,env_auth=true:{bucket}/{TAR_CACHE_FILENAME}"),
+            &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
+            "-v",
+            "--size-only",
+            "--fast-list",
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed copying denogo tar from cache. Error: {:?}", e);
+        return;
+    }
+
+    if let Err(e) = execute_command(
+        ROOT_CACHE_DIR,
+        "tar",
+        vec![
+            "-xpvf",
+            &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed to untar denogo. Error: {:?}", e);
+        return;
+    }
+
+    let denogen = format!("{ROOT_TMP_CACHE_DIR}deno/gen/file");
+    if metadata(&denogen).await.is_ok() {
+        let _ = tokio::fs::remove_dir_all(denogen).await;
+    }
+
+    if let Err(e) =
+        tokio::fs::remove_file(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await
+    {
+        tracing::info!("Failed to remove denotar cache. Error: {:?}", e);
+        return;
+    };
+
+    tracing::info!(
+        "Finished copying denogotar from bucket {bucket} as tar, took: {:?}s",
+        start.elapsed().as_secs()
+    );
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn copy_all_piptars_from_bucket(bucket: &str) {
+    tracing::info!("Copying all piptars cache from bucket {bucket}");
+
+    let start = Instant::now();
+
+    if let Err(e) = execute_command(
+        ROOT_CACHE_DIR,
+        "rclone",
+        vec![
+            "copy",
+            &format!(":s3,env_auth=true:{bucket}/tar/pip/"),
+            &TAR_PIP_TMP_CACHE_DIR,
+            "-v",
+            "--size-only",
+            "--fast-list",
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed transferring all piptars from cache. Error: {:?}", e);
+        return;
+    }
+
+    tracing::info!(
+        "Finished transferring piptars from bucket {bucket} as tar, took: {:?}s",
+        start.elapsed().as_secs()
+    );
+}
+
+// async fn check_if_bucket_syncable(bucket: &str) -> bool {
+//     match Command::new("rclone")
+//         .arg("lsf")
+//         .arg(format!(":s3,env_auth=true:{bucket}/NOSYNC"))
+
+//         .arg("-vv")
+//         .arg("--fast-list")
+//         .stdin(Stdio::null())
+//         .stdout(Stdio::null())
+//         .output()
+//         .await;
+//     return true;
+// }
+
+#[cfg(feature = "enterprise")]
+pub async fn copy_tmp_cache_to_cache() -> error::Result<()> {
+    let start: Instant = Instant::now();
+    execute_command(
+        TMP_DIR,
+        "rclone",
+        vec![
+            "sync",
+            ROOT_TMP_CACHE_DIR,
+            ROOT_CACHE_DIR,
+            "--filter",
+            "- deno/gen/file/**",
+            "--filter",
+            "+ deno/**",
+            "--filter",
+            "+ go/**",
+            "--filter",
+            "- *",
+        ],
+    )
+    .await?;
+
+    tracing::info!(
+        "Finished copying local tmp cache to local cache. Took {}ms",
+        start.elapsed().as_millis(),
+    );
+
+    if let Err(e) = untar_all_piptars().await {
+        tracing::info!("Failed to untar piptars. Error: {:?}", e);
+    }
+
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn untar_all_piptars() -> error::Result<()> {
+    use tokio::fs::{self, metadata};
+
+    use crate::PIP_CACHE_DIR;
+
+    let start: Instant = Instant::now();
+
+    let mut entries = fs::read_dir(TAR_PIP_TMP_CACHE_DIR).await?;
+    while let Some(entry) = entries.next_entry().await? {
+        if let Err(e) = {
+            let entry_path = entry.path();
+            let path = entry_path.to_str().expect("Could not convert path to str");
+            let folder = format!(
+                "{PIP_CACHE_DIR}/{}",
+                path.split('/')
+                    .last()
+                    .unwrap()
+                    .strip_suffix(".tar")
+                    .unwrap()
+            );
+            if metadata(&folder).await.is_ok() {
+                continue;
+            }
+            extract_pip_tar(&path, &folder).await?;
+            Ok(()) as error::Result<()>
+        } {
+            tracing::info!("Failed to extract pip tar. Error: {:?}", e);
+        }
+    }
+
+    tracing::info!(
+        "Finished untarring all piptars. Took {}ms",
+        start.elapsed().as_millis(),
+    );
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn extract_pip_tar(tar: &str, folder: &str) -> error::Result<()> {
+    use tokio::fs;
+
+    let start: Instant = Instant::now();
+    fs::create_dir_all(&folder).await?;
+    if let Err(e) = execute_command(&folder, "tar", vec!["-xpvf", tar]).await {
+        tracing::info!("Failed to untar piptar. Error: {:?}", e);
+        fs::remove_dir_all(&folder).await?;
+        return Err(e);
+    }
+    tracing::info!(
+        "Finished extracting pip tar {folder}. Took {}ms",
+        start.elapsed().as_millis(),
+    );
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn copy_cache_to_tmp_cache() -> error::Result<()> {
+    let start: Instant = Instant::now();
+    execute_command(
+        TMP_DIR,
+        "rclone",
+        vec![
+            "sync",
+            ROOT_CACHE_DIR,
+            ROOT_TMP_CACHE_DIR,
+            "--filter",
+            "- deno/gen/file/**",
+            "--filter",
+            "+ deno/**",
+            "--filter",
+            "+ go/**",
+            "--filter",
+            "- *",
+        ],
+    )
+    .await?;
+    tracing::info!(
+        "Finished copying local cache to local tmp cache. Took {}ms",
+        start.elapsed().as_millis()
+    );
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn execute_command(dir: &str, command: &str, args: Vec<&str>) -> error::Result<()> {
+    match Command::new(command)
+        .current_dir(dir)
+        .args(args.clone())
+        .stdin(Stdio::null())
+        .stdout(Stdio::null())
+        .spawn()
+    {
+        Ok(mut h) => {
+            if !h.wait().await.unwrap().success() {
+                return Err(error::Error::ExecutionErr(format!(
+                    "Failed to apply {command} with args: {}",
+                    args.iter().join(" ")
+                )));
+            }
+        }
+        Err(e) => {
+            return Err(error::Error::ExecutionErr(format!(
+                "Failed to apply {command} with args: {}. Error: {e:?}",
+                args.iter().join(" ")
+            )));
+        }
+    }
+    Ok(())
+}

backend/windmill-worker/src/go_executor.rs

@@ -0,0 +1,380 @@
+use std::process::Stdio;
+
+use itertools::Itertools;
+use tokio::{
+    fs::{DirBuilder, File},
+    io::AsyncReadExt,
+    process::Command,
+};
+use uuid::Uuid;
+use windmill_common::{
+    error::{self, Error},
+    jobs::QueuedJob,
+    utils::calculate_hash,
+};
+use windmill_parser_go::parse_go_imports;
+
+use crate::{
+    common::{capitalize, read_result, set_logs},
+    create_args_and_out_file, get_reserved_variables, handle_child, write_file,
+    AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_CACHE_DIR,
+    HOME_ENV, NETRC, NSJAIL_PATH, PATH_ENV,
+};
+
+const GO_REQ_SPLITTER: &str = "//go.sum\n";
+const NSJAIL_CONFIG_RUN_GO_CONTENT: &str = include_str!("../nsjail/run.go.config.proto");
+
+lazy_static::lazy_static! {
+    static ref GO_PATH: String = std::env::var("GO_PATH").unwrap_or_else(|_| "/usr/bin/go".to_string());
+}
+
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn handle_go_job(
+    logs: &mut String,
+    job: &QueuedJob,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    client: &AuthedClientBackgroundTask,
+    inner_content: &str,
+    job_dir: &str,
+    requirements_o: Option<String>,
+    shared_mount: &str,
+    base_internal_url: &str,
+    worker_name: &str,
+) -> Result<serde_json::Value, Error> {
+    //go does not like executing modules at temp root
+    let job_dir = &format!("{job_dir}/go");
+    let (skip_go_mod, skip_tidy) = if let Some(requirements) = requirements_o {
+        gen_go_mod(inner_content, job_dir, &requirements).await?
+    } else {
+        (false, false)
+    };
+
+    logs.push_str("\n\n--- GO DEPENDENCIES SETUP ---\n");
+    set_logs(logs, &job.id, db).await;
+
+    install_go_dependencies(
+        &job.id,
+        inner_content,
+        logs,
+        job_dir,
+        db,
+        true,
+        skip_go_mod,
+        skip_tidy,
+        worker_name,
+        &job.workspace_id,
+    )
+    .await?;
+
+    logs.push_str("\n\n--- GO CODE EXECUTION ---\n");
+    set_logs(logs, &job.id, db).await;
+    let client = &client.get_authed().await;
+    create_args_and_out_file(client, job, job_dir).await?;
+    {
+        let sig = windmill_parser_go::parse_go_sig(&inner_content)?;
+        drop(inner_content);
+
+        const WRAPPER_CONTENT: &str = r#"package main
+
+import (
+    "encoding/json"
+    "os"
+    "fmt"
+    "mymod/inner"
+)
+
+func main() {{
+
+    dat, err := os.ReadFile("args.json")
+    if err != nil {{
+        fmt.Println(err)
+        os.Exit(1)
+    }}
+
+    var req inner.Req
+
+    if err := json.Unmarshal(dat, &req); err != nil {{
+        fmt.Println(err)
+        os.Exit(1)
+    }}
+
+    res, err := inner.Run(req)
+    if err != nil {{
+        fmt.Println(err)
+        os.Exit(1)
+    }}
+    res_json, err := json.Marshal(res)
+    if err != nil {{
+        fmt.Println(err)
+        os.Exit(1)
+    }}
+    f, err := os.OpenFile("result.json", os.O_APPEND|os.O_WRONLY, os.ModeAppend)
+    if err != nil {{
+        fmt.Println(err)
+        os.Exit(1)
+    }}
+    _, err = f.WriteString(string(res_json))
+    if err != nil {{
+        fmt.Println(err)
+        os.Exit(1)
+    }}
+}}"#;
+
+        write_file(job_dir, "main.go", WRAPPER_CONTENT).await?;
+
+        {
+            let spread = &sig
+                .args
+                .clone()
+                .into_iter()
+                .map(|x| format!("req.{}", capitalize(&x.name)))
+                .join(", ");
+            let req_body = &sig
+                .args
+                .into_iter()
+                .map(|x| {
+                    format!(
+                        "{} {} `json:\"{}\"`",
+                        capitalize(&x.name),
+                        windmill_parser_go::otyp_to_string(x.otyp),
+                        x.name
+                    )
+                })
+                .join("\n");
+            let runner_content: String = format!(
+                r#"package inner
+type Req struct {{
+    {req_body}
+}}
+
+func Run(req Req) (interface{{}}, error){{
+    return main({spread})
+}}
+
+"#,
+            );
+            write_file(&format!("{job_dir}/inner"), "runner.go", &runner_content).await?;
+        }
+    }
+    let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
+    reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
+
+    let child = if !*DISABLE_NSJAIL {
+        let _ = write_file(
+            job_dir,
+            "run.config.proto",
+            &NSJAIL_CONFIG_RUN_GO_CONTENT
+                .replace("{JOB_DIR}", job_dir)
+                .replace("{CACHE_DIR}", GO_CACHE_DIR)
+                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
+                .replace("{SHARED_MOUNT}", shared_mount),
+        )
+        .await?;
+        let build_go = Command::new(GO_PATH.as_str())
+            .current_dir(job_dir)
+            .env_clear()
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .env("GOPATH", GO_CACHE_DIR)
+            .env("HOME", HOME_ENV.as_str())
+            .args(vec!["build", "main.go"])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?;
+        handle_child(
+            &job.id,
+            db,
+            logs,
+            build_go,
+            false,
+            worker_name,
+            &job.workspace_id,
+            "go build",
+        )
+        .await?;
+
+        Command::new(NSJAIL_PATH.as_str())
+            .current_dir(job_dir)
+            .env_clear()
+            .envs(reserved_variables)
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .args(vec!["--config", "run.config.proto", "--", "/tmp/go/main"])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?
+    } else {
+        if let Some(ref netrc) = *NETRC {
+            write_file(&HOME_ENV, ".netrc", netrc).await?;
+        }
+        let mut cmd = Command::new(GO_PATH.as_str());
+        cmd.current_dir(job_dir)
+            .env_clear()
+            .envs(reserved_variables)
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .env("GOPATH", GO_CACHE_DIR)
+            .env("HOME", HOME_ENV.as_str());
+
+        if let Some(ref goprivate) = *GOPRIVATE {
+            cmd.env("GOPRIVATE", goprivate);
+        }
+        if let Some(ref goproxy) = *GOPROXY {
+            cmd.env("GOPROXY", goproxy);
+        }
+
+        cmd.args(vec!["run", "main.go"])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?
+    };
+    handle_child(
+        &job.id,
+        db,
+        logs,
+        child,
+        !*DISABLE_NSJAIL,
+        worker_name,
+        &job.workspace_id,
+        "go run",
+    )
+    .await?;
+    read_result(job_dir).await
+}
+
+async fn gen_go_mod(
+    inner_content: &str,
+    job_dir: &str,
+    requirements: &str,
+) -> error::Result<(bool, bool)> {
+    gen_go_mymod(inner_content, job_dir).await?;
+
+    let md = requirements.split_once(GO_REQ_SPLITTER);
+    if let Some((req, sum)) = md {
+        write_file(job_dir, "go.mod", &req).await?;
+        write_file(job_dir, "go.sum", &sum).await?;
+        Ok((true, true))
+    } else {
+        write_file(job_dir, "go.mod", &requirements).await?;
+        Ok((true, false))
+    }
+}
+
+pub async fn install_go_dependencies(
+    job_id: &Uuid,
+    code: &str,
+    logs: &mut String,
+    job_dir: &str,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    non_dep_job: bool,
+    skip_go_mod: bool,
+    has_sum: bool,
+    worker_name: &str,
+    w_id: &str,
+) -> error::Result<String> {
+    if !skip_go_mod {
+        gen_go_mymod(code, job_dir).await?;
+        let child = Command::new("go")
+            .current_dir(job_dir)
+            .args(vec!["mod", "init", "mymod"])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?;
+
+        handle_child(job_id, db, logs, child, false, worker_name, w_id, "go init").await?;
+    }
+
+    let mut new_lockfile = false;
+
+    let hash = if !has_sum {
+        calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str())
+    } else {
+        "".to_string()
+    };
+
+    let mut skip_tidy = has_sum;
+
+    if !has_sum {
+        if let Some(cached) = sqlx::query_scalar!(
+            "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
+            hash
+        )
+        .fetch_optional(db)
+        .await?
+        {
+            logs.push_str(&format!("\nfound cached resolution"));
+            gen_go_mod(code, job_dir, &cached).await?;
+            skip_tidy = true;
+            new_lockfile = false;
+        } else {
+            new_lockfile = true;
+        }
+    }
+
+    let mod_command = if skip_tidy { "download" } else { "tidy" };
+    let child = Command::new(GO_PATH.as_str())
+        .current_dir(job_dir)
+        .env("GOPATH", GO_CACHE_DIR)
+        .args(vec!["mod", mod_command])
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()?;
+    handle_child(
+        job_id,
+        db,
+        logs,
+        child,
+        false,
+        worker_name,
+        &w_id,
+        &format!("go {mod_command}"),
+    )
+    .await
+    .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
+
+    if (!new_lockfile || has_sum) && non_dep_job {
+        return Ok("".to_string());
+    }
+
+    let mut req_content = "".to_string();
+
+    let mut file = File::open(format!("{job_dir}/go.mod")).await?;
+    file.read_to_string(&mut req_content).await?;
+    req_content.push_str(GO_REQ_SPLITTER);
+    let sum_path = format!("{job_dir}/go.sum");
+    if tokio::fs::metadata(&sum_path).await.is_ok() {
+        let mut file = File::open(sum_path).await?;
+        file.read_to_string(&mut req_content).await?;
+    }
+
+    if non_dep_job {
+        sqlx::query!(
+            "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
+            hash,
+            req_content
+        ).fetch_optional(db).await?;
+
+        return Ok(String::new());
+    } else {
+        Ok(req_content)
+    }
+}
+
+async fn gen_go_mymod(code: &str, job_dir: &str) -> error::Result<()> {
+    let code = if code.trim_start().starts_with("package") {
+        code.to_string()
+    } else {
+        format!("package inner; {code}")
+    };
+
+    let mymod_dir = format!("{job_dir}/inner");
+    DirBuilder::new()
+        .recursive(true)
+        .create(&mymod_dir)
+        .await
+        .expect("could not create go's mymod dir");
+
+    write_file(&mymod_dir, "inner_main.go", &code).await?;
+
+    Ok(())
+}

backend/windmill-worker/src/jobs.rs

@@ -10,11 +10,13 @@ use sqlx::{Pool, Postgres};
 use tracing::instrument;
 use uuid::Uuid;
 use windmill_common::{
-    error::Error, flow_status::FlowStatusModule, schedule::Schedule, METRICS_ENABLED,
-};
-use windmill_queue::{
-    delete_job, schedule::get_schedule_opt, JobKind, QueueTransaction, QueuedJob, CLOUD_HOSTED,
+    error::Error,
+    flow_status::FlowStatusModule,
+    jobs::{JobKind, QueuedJob},
+    schedule::Schedule,
+    METRICS_ENABLED,
 };
+use windmill_queue::{delete_job, schedule::get_schedule_opt, QueueTransaction, CLOUD_HOSTED};
 
 #[instrument(level = "trace", skip_all)]
 pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Send>(
@@ -124,9 +126,10 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
                    , email
                    , visible_to_owner
                    , mem_peak
+                   , tag
                 )
             VALUES ($1, $2, $3, $4, $5, $6, COALESCE($26, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,\
-                    $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29)
+                    $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29, $30)
          ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)",
         queued_job.workspace_id,
         queued_job.id,
@@ -156,7 +159,8 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
         duration: Option<i64>,
         queued_job.email,
         queued_job.visible_to_owner,
-        mem_peak
+        mem_peak,
+        queued_job.tag,
     )
     .execute(&mut tx)
     .await

backend/windmill-worker/src/js_eval.rs

@@ -143,7 +143,6 @@ fn add_closing_bracket(s: &str) -> String {
     s
 }
 
-const SPLIT_PAT: &str = ";";
 async fn eval(
     context: &mut JsRuntime,
     expr: &str,
@@ -151,21 +150,6 @@ async fn eval(
     by_id: Option<IdContext>,
     has_client: bool,
 ) -> anyhow::Result<serde_json::Value> {
-    let exprs = expr
-        .trim()
-        .split(SPLIT_PAT)
-        .map(|x| x.trim())
-        .filter(|x| !x.is_empty())
-        .collect::<Vec<&str>>();
-    let expr = if exprs.is_empty() {
-        "return undefined;".to_string()
-    } else {
-        format!(
-            "{};\n    return {};",
-            exprs.iter().take(exprs.len() - 1).join(";\n"),
-            exprs.last().unwrap()
-        )
-    };
     let (api_code, by_id_code) = if has_client {
         let by_id_code = if let Some(by_id) = by_id {
             format!(
@@ -232,13 +216,18 @@ async function resource(path) {{
         (String::new(), String::new())
     };
 
+    let f = if expr.contains("return ") {
+        expr.to_string()
+    } else {
+        format!("return {expr}")
+    };
     let code = format!(
         r#"
 {api_code}
 {}
 {by_id_code}
 (async () => {{ 
-    {expr} 
+    {f};
 }})()
         "#,
         env.into_iter()
@@ -251,8 +240,7 @@ async function resource(path) {{
             })
             .join(""),
     );
-    tracing::debug!("{}", code);
-    let global = context.execute_script("<anon>", code)?;
+    let global = context.execute_script("<anon>", code.into())?;
     let global = context.resolve_value(global).await?;
 
     let scope = &mut context.handle_scope();

backend/windmill-worker/src/lib.rs

@@ -1,5 +1,9 @@
+mod common;
+mod global_cache;
+mod go_executor;
 mod jobs;
 mod js_eval;
+mod python_executor;
 mod worker;
 mod worker_flow;
 

backend/windmill-worker/src/main2.rs

@@ -1,92 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-
-// use std::{net::SocketAddr, time::Duration};
-
-// use anyhow::Context;
-// use sqlx::{postgres::PgPoolOptions, Pool, Postgres};
-// use windmill_common::{
-//     error::{self, Error},
-//     utils::rd_string,
-// };
-
-// #[tokio::main]
-// async fn main() -> anyhow::Result<()> {
-//     // dotenv().ok();
-
-//     windmill_common::tracing_init::initialize_tracing();
-
-//     let db = async {
-//         let database_url = std::env::var("DATABASE_URL")
-//             .map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
-
-//         let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
-//             Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
-//             Err(_) => 10,
-//         };
-
-//         Ok::<Pool<Postgres>, error::Error>(
-//             PgPoolOptions::new()
-//                 .max_connections(max_connections)
-//                 .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
-//                 .connect(&database_url)
-//                 .await
-//                 .map_err(|err| Error::ConnectingToDatabase(err.to_string()))?,
-//         )
-//     }
-//     .await?;
-
-//     let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
-//         .ok()
-//         .map(|s| {
-//             s.parse::<bool>()
-//                 .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
-//                 .or_else(|_| s.parse::<SocketAddr>().map(Some))
-//         })
-//         .transpose()?
-//         .flatten();
-
-//     let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
-//     let shutdown_signal = windmill_common::shutdown_signal(tx);
-
-//     let workers_f = async {
-//         let instance_name = rd_string(5);
-
-//         let ip = windmill_common::external_ip::get_ip()
-//             .await
-//             .unwrap_or_else(|e| {
-//                 tracing::warn!(error = e.to_string(), "failed to get external IP");
-//                 "unretrievable IP".to_string()
-//             });
-//         let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
-//         windmill_worker::run_worker(
-//             &db.clone(),
-//             &instance_name,
-//             worker_name,
-//             1,
-//             1,
-//             &ip,
-//             rx.resubscribe(),
-//         )
-//         .await;
-//         Ok(()) as anyhow::Result<()>
-//     };
-
-//     let metrics_f = async {
-//         match metrics_addr {
-//             Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
-//                 .await
-//                 .map_err(anyhow::Error::from),
-//             None => Ok(()),
-//         }
-//     };
-
-//     futures::try_join!(shutdown_signal, workers_f, metrics_f)?;
-
-//     Ok(())
-// }

backend/windmill-worker/src/python_executor.rs

@@ -0,0 +1,603 @@
+use std::process::Stdio;
+
+use itertools::Itertools;
+use regex::Regex;
+use sqlx::{Pool, Postgres};
+use tokio::{
+    fs::{metadata, DirBuilder, File},
+    io::AsyncReadExt,
+    process::Command,
+};
+use uuid::Uuid;
+use windmill_common::{
+    error::{self, Error},
+    jobs::QueuedJob,
+    utils::calculate_hash,
+};
+
+lazy_static::lazy_static! {
+    static ref PYTHON_PATH: String =
+    std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string());
+
+    static ref PIP_INDEX_URL: Option<String> = std::env::var("PIP_INDEX_URL").ok();
+    static ref PIP_EXTRA_INDEX_URL: Option<String> = std::env::var("PIP_EXTRA_INDEX_URL").ok();
+    static ref PIP_TRUSTED_HOST: Option<String> = std::env::var("PIP_TRUSTED_HOST").ok();
+    static ref PIP_LOCAL_DEPENDENCIES: Option<Vec<String>> = {
+        let pip_local_dependencies = std::env::var("PIP_LOCAL_DEPENDENCIES")
+            .ok()
+            .map(|x| x.split(',').map(|x| x.to_string()).collect());
+        if pip_local_dependencies == Some(vec!["".to_string()]) {
+            None
+        } else {
+            pip_local_dependencies
+        }
+    };
+
+    static ref ADDITIONAL_PYTHON_PATHS: Option<Vec<String>> = std::env::var("ADDITIONAL_PYTHON_PATHS")
+        .ok()
+        .map(|x| x.split(':').map(|x| x.to_string()).collect());
+
+    static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap();
+
+}
+
+const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto");
+const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto");
+const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
+
+#[cfg(feature = "enterprise")]
+use crate::global_cache::{build_tar_and_push, pull_from_tar};
+
+use crate::{
+    common::{read_result, set_logs},
+    create_args_and_out_file, get_reserved_variables, handle_child, write_file,
+    AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV,
+    PIP_CACHE_DIR, S3_CACHE_BUCKET,
+};
+
+pub async fn create_dependencies_dir(job_dir: &str) {
+    DirBuilder::new()
+        .recursive(true)
+        .create(&format!("{job_dir}/dependencies"))
+        .await
+        .expect("could not create dependencies dir");
+}
+
+pub async fn pip_compile(
+    job_id: &Uuid,
+    requirements: &str,
+    logs: &mut String,
+    job_dir: &str,
+    db: &Pool<Postgres>,
+    worker_name: &str,
+    w_id: &str,
+) -> error::Result<String> {
+    logs.push_str(&format!("\nresolving dependencies..."));
+    set_logs(logs, job_id, db).await;
+    logs.push_str(&format!("\ncontent of requirements:\n{}\n", requirements));
+    let requirements = if let Some(pip_local_dependencies) = PIP_LOCAL_DEPENDENCIES.as_ref() {
+        let deps = pip_local_dependencies.clone();
+        requirements
+            .lines()
+            .filter(|s| {
+                if !deps.contains(&s.to_string()) {
+                    return true;
+                } else {
+                    logs.push_str(&format!("\nignoring local dependency: {}", s));
+                    return false;
+                }
+            })
+            .join("\n")
+    } else {
+        requirements.to_string()
+    };
+    let req_hash = calculate_hash(&requirements);
+    if let Some(cached) = sqlx::query_scalar!(
+        "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
+        req_hash
+    )
+    .fetch_optional(db)
+    .await?
+    {
+        logs.push_str(&format!("\nfound cached resolution"));
+        return Ok(cached);
+    }
+    let file = "requirements.in";
+
+    write_file(job_dir, file, &requirements).await?;
+
+    let mut args = vec!["-q", "--no-header", file, "--resolver=backtracking"];
+    if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
+        args.extend(["--extra-index-url", url]);
+    }
+    if let Some(url) = PIP_INDEX_URL.as_ref() {
+        args.extend(["--index-url", url]);
+    }
+    if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
+        args.extend(["--trusted-host", host]);
+    }
+    let child = Command::new("pip-compile")
+        .current_dir(job_dir)
+        .args(args)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()?;
+    handle_child(
+        job_id,
+        db,
+        logs,
+        child,
+        false,
+        worker_name,
+        &w_id,
+        "pip-compile",
+    )
+    .await
+    .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
+    let path_lock = format!("{job_dir}/requirements.txt");
+    let mut file = File::open(path_lock).await?;
+    let mut req_content = "".to_string();
+    file.read_to_string(&mut req_content).await?;
+    let lockfile = req_content
+        .lines()
+        .filter(|x| !x.trim_start().starts_with('#'))
+        .map(|x| x.to_string())
+        .collect::<Vec<String>>()
+        .join("\n");
+    sqlx::query!(
+        "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
+        req_hash,
+        lockfile
+    ).fetch_optional(db).await?;
+    Ok(lockfile)
+}
+
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn handle_python_job(
+    requirements_o: Option<String>,
+    job_dir: &str,
+    worker_dir: &str,
+    worker_name: &str,
+    job: &QueuedJob,
+    logs: &mut String,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    client: &AuthedClientBackgroundTask,
+    inner_content: &String,
+    shared_mount: &str,
+    base_internal_url: &str,
+) -> windmill_common::error::Result<serde_json::Value> {
+    create_dependencies_dir(job_dir).await;
+
+    let mut additional_python_paths: Vec<String> =
+        ADDITIONAL_PYTHON_PATHS.to_owned().unwrap_or_else(|| vec![]);
+
+    let requirements = match requirements_o {
+        Some(r) => r,
+        None => {
+            let requirements = windmill_parser_py::parse_python_imports(&inner_content)?.join("\n");
+            if requirements.is_empty() {
+                "".to_string()
+            } else {
+                pip_compile(
+                    &job.id,
+                    &requirements,
+                    logs,
+                    job_dir,
+                    db,
+                    worker_name,
+                    &job.workspace_id,
+                )
+                .await
+                .map_err(|e| {
+                    Error::ExecutionErr(format!("pip compile failed: {}", e.to_string()))
+                })?
+            }
+        }
+    };
+
+    if requirements.len() > 0 {
+        additional_python_paths = handle_python_reqs(
+            requirements
+                .split("\n")
+                .filter(|x| !x.starts_with("--"))
+                .collect(),
+            &job.id,
+            &job.workspace_id,
+            logs,
+            db,
+            worker_name,
+            job_dir,
+            worker_dir,
+        )
+        .await?;
+    }
+    logs.push_str("\n\n--- PYTHON CODE EXECUTION ---\n");
+
+    set_logs(logs, &job.id, db).await;
+
+    let relative_imports = RELATIVE_IMPORT_REGEX.is_match(&inner_content);
+
+    let script_path_splitted = &job.script_path().split("/");
+    let dirs_full = script_path_splitted
+        .clone()
+        .take(script_path_splitted.clone().count() - 1)
+        .join("/")
+        .replace("-", "_");
+    let dirs = if dirs_full.len() > 0 {
+        dirs_full
+    } else {
+        "tmp".to_string()
+    };
+    let last = script_path_splitted
+        .clone()
+        .last()
+        .unwrap()
+        .replace("-", "_")
+        .replace(" ", "_")
+        .to_lowercase();
+    let module_dir = format!("{}/{}", job_dir, dirs);
+    tokio::fs::create_dir_all(format!("{module_dir}/")).await?;
+    let _ = write_file(&module_dir, &format!("{last}.py"), inner_content).await?;
+    if relative_imports {
+        let _ = write_file(&job_dir, "loader.py", RELATIVE_PYTHON_LOADER).await?;
+    }
+
+    let sig = windmill_parser_py::parse_python_signature(inner_content)?;
+    let transforms = sig
+        .args
+        .iter()
+        .map(|x| match x.typ {
+            windmill_parser::Typ::Bytes => {
+                let name = &x.name;
+                format!(
+                    "if \"{name}\" in kwargs and kwargs[\"{name}\"] is not None:\n    \
+                                     kwargs[\"{name}\"] = base64.b64decode(kwargs[\"{name}\"])\n",
+                )
+            }
+            windmill_parser::Typ::Datetime => {
+                let name = &x.name;
+                format!(
+                    "if \"{name}\" in kwargs and kwargs[\"{name}\"] is not None:\n    \
+                                     kwargs[\"{name}\"] = datetime.strptime(kwargs[\"{name}\"], \
+                                     '%Y-%m-%dT%H:%M')\n",
+                )
+            }
+            _ => "".to_string(),
+        })
+        .collect::<Vec<String>>()
+        .join("");
+    let client = client.get_authed().await;
+    create_args_and_out_file(&client, job, job_dir).await?;
+
+    let import_loader = if relative_imports {
+        "import loader"
+    } else {
+        ""
+    };
+    let import_base64 = if sig
+        .args
+        .iter()
+        .any(|x| x.typ == windmill_parser::Typ::Bytes)
+    {
+        "import base64"
+    } else {
+        ""
+    };
+    let import_datetime = if sig
+        .args
+        .iter()
+        .any(|x| x.typ == windmill_parser::Typ::Datetime)
+    {
+        "from datetime import datetime"
+    } else {
+        ""
+    };
+    let spread = if sig.star_kwargs {
+        "args = kwargs".to_string()
+    } else {
+        sig.args
+            .into_iter()
+            .map(|x| {
+                let name = &x.name;
+                if x.default.is_none() {
+                    format!("args[\"{name}\"] = kwargs.get(\"{name}\")")
+                } else {
+                    format!(
+                        r#"args["{name}"] = kwargs.get("{name}")
+if args["{name}"] is None:
+    del args["{name}"]"#
+                    )
+                }
+            })
+            .join("\n")
+    };
+
+    let module_dir_dot = dirs.replace("/", ".").replace("-", "_");
+    let wrapper_content: String = format!(
+        r#"
+import json
+{import_loader}
+{import_base64}
+{import_datetime}
+import traceback
+import sys
+from {module_dir_dot} import {last} as inner_script
+
+
+with open("args.json") as f:
+    kwargs = json.load(f, strict=False)
+args = {{}}
+{transforms}
+{spread}
+for k, v in list(args.items()):
+    if v == '<function call>':
+        del args[k]
+
+def to_b_64(v: bytes):
+    import base64
+    b64 = base64.b64encode(v)
+    return b64.decode('ascii')
+    
+try:
+    res = inner_script.main(**args)
+    typ = type(res)
+    if typ.__name__ == 'DataFrame':
+        if typ.__module__ == 'pandas.core.frame':
+            res = res.values.tolist()
+        elif typ.__module__ == 'polars.dataframe.frame':
+            res = res.rows()
+    elif typ.__name__ == 'bytes':
+        res = to_b_64(res)
+    elif typ.__name__ == 'dict':
+        for k, v in res.items():
+            if type(v).__name__ == 'bytes':
+                res[k] = to_b_64(v)
+    res_json = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '')
+    with open("result.json", 'w') as f:
+        f.write(res_json)
+except Exception as e:
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    tb = traceback.format_tb(exc_traceback)
+    with open("result.json", 'w') as f:
+        err_json = json.dumps({{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:])  }}, separators=(',', ':'), default=str).replace('\n', '')
+        f.write(err_json)
+        sys.exit(1)
+"#,
+    );
+    write_file(job_dir, "wrapper.py", &wrapper_content).await?;
+
+    let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
+    let additional_python_paths_folders = additional_python_paths.iter().join(":");
+    if !*DISABLE_NSJAIL {
+        let shared_deps = additional_python_paths
+            .into_iter()
+            .map(|pp| {
+                format!(
+                    r#"
+mount {{
+    src: "{pp}"
+    dst: "{pp}"
+    is_bind: true
+    rw: false
+}}
+        "#
+                )
+            })
+            .join("\n");
+        let _ = write_file(
+            job_dir,
+            "run.config.proto",
+            &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT
+                .replace("{JOB_DIR}", job_dir)
+                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
+                .replace("{SHARED_MOUNT}", shared_mount)
+                .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str())
+                .replace("{MAIN}", format!("{dirs}/{last}").as_str())
+                .replace(
+                    "{ADDITIONAL_PYTHON_PATHS}",
+                    additional_python_paths_folders.as_str(),
+                ),
+        )
+        .await?;
+    } else {
+        reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
+    }
+
+    tracing::info!(
+        worker_name = %worker_name,
+        job_id = %job.id,
+        workspace_id = %job.workspace_id,
+        "started python code execution {}",
+        job.id
+    );
+    let child = if !*DISABLE_NSJAIL {
+        Command::new(NSJAIL_PATH.as_str())
+            .current_dir(job_dir)
+            .env_clear()
+            // inject PYTHONPATH here - for some reason I had to do it in nsjail conf
+            .envs(reserved_variables)
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .args(vec![
+                "--config",
+                "run.config.proto",
+                "--",
+                PYTHON_PATH.as_str(),
+                "-u",
+                "-m",
+                "wrapper",
+            ])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?
+    } else {
+        Command::new(PYTHON_PATH.as_str())
+            .current_dir(job_dir)
+            .env_clear()
+            .envs(reserved_variables)
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .args(vec!["-u", "-m", "wrapper"])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?
+    };
+
+    handle_child(
+        &job.id,
+        db,
+        logs,
+        child,
+        !*DISABLE_NSJAIL,
+        worker_name,
+        &job.workspace_id,
+        "python run",
+    )
+    .await?;
+    read_result(job_dir).await
+}
+
+pub async fn handle_python_reqs(
+    requirements: Vec<&str>,
+    job_id: &Uuid,
+    w_id: &str,
+    logs: &mut String,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    worker_name: &str,
+    job_dir: &str,
+    worker_dir: &str,
+) -> error::Result<Vec<String>> {
+    let mut req_paths: Vec<String> = vec![];
+    let mut vars = vec![("PATH", PATH_ENV.as_str())];
+    if !*DISABLE_NSJAIL {
+        if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
+            vars.push(("EXTRA_INDEX_URL", url));
+        }
+        if let Some(url) = PIP_INDEX_URL.as_ref() {
+            vars.push(("INDEX_URL", url));
+        }
+        if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
+            vars.push(("TRUSTED_HOST", host));
+        }
+        let _ = write_file(
+            job_dir,
+            "download.config.proto",
+            &NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT
+                .replace("{WORKER_DIR}", &worker_dir)
+                .replace("{CACHE_DIR}", PIP_CACHE_DIR)
+                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
+        )
+        .await?;
+    };
+
+    for req in requirements {
+        // todo: handle many reqs
+        let venv_p = format!("{PIP_CACHE_DIR}/{req}");
+        if metadata(&venv_p).await.is_ok() {
+            req_paths.push(venv_p);
+            continue;
+        }
+
+        #[cfg(feature = "enterprise")]
+        if let Some(ref bucket) = *S3_CACHE_BUCKET {
+            sqlx::query_scalar!("UPDATE queue SET last_ping = now() WHERE id = $1", job_id)
+                .execute(db)
+                .await?;
+            if pull_from_tar(bucket, venv_p.clone()).await.is_ok() {
+                req_paths.push(venv_p.clone());
+                continue;
+            }
+        }
+
+        logs.push_str("\n--- PIP INSTALL ---\n");
+        logs.push_str(&format!("\n{req} is being installed for the first time.\n It will be cached for all ulterior uses."));
+
+        tracing::info!(
+            worker_name = %worker_name,
+            job_id = %job_id,
+            workspace_id = %w_id,
+            "started setup python dependencies"
+        );
+
+        let child = if !*DISABLE_NSJAIL {
+            tracing::info!(
+                worker_name = %worker_name,
+                job_id = %job_id,
+                workspace_id = %w_id,
+                "starting nsjail"
+            );
+            let mut vars = vars.clone();
+            let req = req.to_string();
+            vars.push(("REQ", &req));
+            vars.push(("TARGET", &venv_p));
+            Command::new(NSJAIL_PATH.as_str())
+                .current_dir(job_dir)
+                .env_clear()
+                .envs(vars)
+                .args(vec!["--config", "download.config.proto"])
+                .stdout(Stdio::piped())
+                .stderr(Stdio::piped())
+                .spawn()?
+        } else {
+            let mut args = vec![
+                "-m",
+                "pip",
+                "install",
+                &req,
+                "-I",
+                "--no-deps",
+                "--no-color",
+                "--isolated",
+                "--no-warn-conflicts",
+                "--disable-pip-version-check",
+                "-t",
+                venv_p.as_str(),
+            ];
+            if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
+                args.extend(["--extra-index-url", url]);
+            }
+            if let Some(url) = PIP_INDEX_URL.as_ref() {
+                args.extend(["--index-url", url]);
+            }
+            if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
+                args.extend(["--trusted-host", &host]);
+            }
+            Command::new(PYTHON_PATH.as_str())
+                .env_clear()
+                .env("PATH", PATH_ENV.as_str())
+                .args(args)
+                .stdout(Stdio::piped())
+                .stderr(Stdio::piped())
+                .spawn()?
+        };
+
+        let child = handle_child(
+            &job_id,
+            db,
+            logs,
+            child,
+            false,
+            worker_name,
+            &w_id,
+            &format!("pip install {req}"),
+        )
+        .await;
+        tracing::info!(
+            worker_name = %worker_name,
+            job_id = %job_id,
+            workspace_id = %w_id,
+            is_ok = child.is_ok(),
+            "finished setting up python dependencies {}",
+            job_id
+        );
+        child?;
+
+        #[cfg(feature = "enterprise")]
+        if let Some(ref bucket) = *S3_CACHE_BUCKET {
+            let venv_p = venv_p.clone();
+            tokio::spawn(build_tar_and_push(bucket, venv_p));
+        }
+        req_paths.push(venv_p);
+    }
+    Ok(req_paths)
+}

backend/windmill-worker/src/worker_flow.rs

@@ -19,6 +19,9 @@ use tokio::sync::mpsc::Sender;
 use tracing::instrument;
 use uuid::Uuid;
 use windmill_common::flow_status::{FlowStatusModuleWParent, Iterator, JobResult};
+use windmill_common::jobs::{
+    script_hash_to_tag, script_path_to_payload, JobPayload, QueuedJob, RawCode,
+};
 use windmill_common::{
     error::{self, to_anyhow, Error},
     flow_status::{
@@ -30,9 +33,7 @@ use windmill_common::{
 
 type DB = sqlx::Pool<sqlx::Postgres>;
 
-use windmill_queue::{
-    canceled_job_to_result, get_queued_job, push, JobPayload, QueueTransaction, QueuedJob, RawCode,
-};
+use windmill_queue::{canceled_job_to_result, get_queued_job, push, QueueTransaction};
 
 // #[instrument(level = "trace", skip_all)]
 pub async fn update_flow_status_after_job_completion<
@@ -1315,7 +1316,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         ContinuePayload::ForloopJobs { n, .. } => *n,
     };
     for i in (0..len).into_iter() {
-        let payload = match &job_payloads {
+        let payload_tag = match &job_payloads {
             ContinuePayload::SingleJob(payload) => payload.clone(),
             ContinuePayload::BranchAllJobs(payloads) => payloads[i].clone(),
             ContinuePayload::ForloopJobs { modules, .. } => {
@@ -1324,13 +1325,16 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                     failure_module.id_append(&format!("{}/{}", status.step, i));
                     fm = Some(failure_module);
                 }
-                JobPayload::RawFlow {
-                    value: FlowValue {
-                        modules: (*modules).clone(),
-                        failure_module: fm.clone(),
-                        same_worker: flow.same_worker,
+                JobPayloadWithTag {
+                    payload: JobPayload::RawFlow {
+                        value: FlowValue {
+                            modules: (*modules).clone(),
+                            failure_module: fm.clone(),
+                            same_worker: flow.same_worker,
+                        },
+                        path: Some(format!("{}/loop-{}", flow_job.script_path(), i)),
                     },
-                    path: Some(format!("{}/loop-{}", flow_job.script_path(), i)),
+                    tag: None,
                 }
             }
         };
@@ -1370,10 +1374,11 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
         } else {
             flow_job.root_job.or_else(|| Some(flow_job.id))
         };
+
         let (uuid, inner_tx) = push(
             tx,
             &flow_job.workspace_id,
-            payload,
+            payload_tag.payload,
             ok.unwrap_or_else(|| Map::new()),
             &flow_job.created_by,
             &flow_job.email,
@@ -1382,10 +1387,12 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
             flow_job.schedule_path.clone(),
             Some(flow_job.id),
             root_job,
+            None,
             true,
             continue_on_same_worker,
             err,
             flow_job.visible_to_owner,
+            payload_tag.tag,
         )
         .await?;
         tx = inner_tx;
@@ -1604,10 +1611,15 @@ enum NextStatus {
     },
 }
 
+#[derive(Clone)]
+struct JobPayloadWithTag {
+    payload: JobPayload,
+    tag: Option<String>,
+}
 enum ContinuePayload {
-    SingleJob(JobPayload),
+    SingleJob(JobPayloadWithTag),
     ForloopJobs { n: usize, modules: Vec<FlowModule> },
-    BranchAllJobs(Vec<JobPayload>),
+    BranchAllJobs(Vec<JobPayloadWithTag>),
 }
 
 enum NextFlowTransform {
@@ -1615,22 +1627,6 @@ enum NextFlowTransform {
     Continue(ContinuePayload, NextStatus),
 }
 
-// a similar function exists on the backend
-// TODO: rewrite this to use an endpoint in the backend directly, instead of checking for hub itself, and then using the API
-async fn script_path_to_payload<'c>(
-    script_path: &str,
-    db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
-    w_id: &String,
-) -> Result<JobPayload, Error> {
-    let job_payload = if script_path.starts_with("hub/") {
-        JobPayload::ScriptHub { path: script_path.to_owned() }
-    } else {
-        let script_hash = windmill_common::get_latest_hash_for_path(db, w_id, script_path).await?;
-        JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() }
-    };
-    Ok(job_payload)
-}
-
 async fn compute_next_flow_transform<'c>(
     flow_job: &QueuedJob,
     flow: &FlowValue,
@@ -1647,7 +1643,10 @@ async fn compute_next_flow_transform<'c>(
         FlowModuleValue::Identity => Ok((
             tx,
             NextFlowTransform::Continue(
-                ContinuePayload::SingleJob(JobPayload::Identity),
+                ContinuePayload::SingleJob(JobPayloadWithTag {
+                    payload: JobPayload::Identity,
+                    tag: None,
+                }),
                 NextStatus::NextStep,
             ),
         )),
@@ -1656,41 +1655,46 @@ async fn compute_next_flow_transform<'c>(
             Ok((
                 tx,
                 NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(payload),
+                    ContinuePayload::SingleJob(JobPayloadWithTag { payload, tag: None }),
                     NextStatus::NextStep,
                 ),
             ))
         }
         FlowModuleValue::Script { path: script_path, hash: script_hash, .. } => {
-            let payload = if script_hash.is_none() {
+            let (payload, tag) = if script_hash.is_none() {
                 script_path_to_payload(script_path, &mut tx, &flow_job.workspace_id).await?
             } else {
-                JobPayload::ScriptHash {
-                    hash: script_hash.clone().unwrap(),
-                    path: script_path.to_owned(),
-                }
+                let hash = script_hash.clone().unwrap();
+                let tag = script_hash_to_tag(&hash, &mut tx, &flow_job.workspace_id).await?;
+                (
+                    JobPayload::ScriptHash { hash, path: script_path.to_owned() },
+                    tag,
+                )
             };
             Ok((
                 tx,
                 NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(payload),
+                    ContinuePayload::SingleJob(JobPayloadWithTag { payload, tag }),
                     NextStatus::NextStep,
                 ),
             ))
         }
-        FlowModuleValue::RawScript { path, content, language, lock, .. } => {
+        FlowModuleValue::RawScript { path, content, language, lock, tag, .. } => {
             let path = path
                 .clone()
                 .or_else(|| Some(format!("{}/step-{}", flow_job.script_path(), status.step)));
             Ok((
                 tx,
                 NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(JobPayload::Code(RawCode {
-                        path,
-                        content: content.clone(),
-                        language: language.clone(),
-                        lock: lock.clone(),
-                    })),
+                    ContinuePayload::SingleJob(JobPayloadWithTag {
+                        payload: JobPayload::Code(RawCode {
+                            path,
+                            content: content.clone(),
+                            language: language.clone(),
+                            lock: lock.clone(),
+                        }),
+                        tag: tag.clone(),
+                    }),
                     NextStatus::NextStep,
                 ),
             ))
@@ -1788,13 +1792,20 @@ async fn compute_next_flow_transform<'c>(
                     Ok((
                         tx,
                         NextFlowTransform::Continue(
-                            ContinuePayload::SingleJob(JobPayload::RawFlow {
-                                value: FlowValue {
-                                    modules: (*modules).clone(),
-                                    failure_module: fm,
-                                    same_worker: flow.same_worker,
+                            ContinuePayload::SingleJob(JobPayloadWithTag {
+                                payload: JobPayload::RawFlow {
+                                    value: FlowValue {
+                                        modules: (*modules).clone(),
+                                        failure_module: fm,
+                                        same_worker: flow.same_worker,
+                                    },
+                                    path: Some(format!(
+                                        "{}/loop-{}",
+                                        flow_job.script_path(),
+                                        ns.index
+                                    )),
                                 },
-                                path: Some(format!("{}/loop-{}", flow_job.script_path(), ns.index)),
+                                tag: None,
                             }),
                             NextStatus::NextLoopIteration(ns),
                         ),
@@ -1867,17 +1878,20 @@ async fn compute_next_flow_transform<'c>(
             Ok((
                 tx,
                 NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(JobPayload::RawFlow {
-                        value: FlowValue {
-                            modules,
-                            failure_module: fm,
-                            same_worker: flow.same_worker,
+                    ContinuePayload::SingleJob(JobPayloadWithTag {
+                        payload: JobPayload::RawFlow {
+                            value: FlowValue {
+                                modules,
+                                failure_module: fm,
+                                same_worker: flow.same_worker,
+                            },
+                            path: Some(format!(
+                                "{}/branchone-{}",
+                                flow_job.script_path(),
+                                status.step
+                            )),
                         },
-                        path: Some(format!(
-                            "{}/branchone-{}",
-                            flow_job.script_path(),
-                            status.step
-                        )),
+                        tag: None,
                     }),
                     NextStatus::BranchChosen(branch),
                 ),
@@ -1907,17 +1921,20 @@ async fn compute_next_flow_transform<'c>(
                                                     .id_append(&format!("{}/{i}", status.step));
                                                 fm = Some(failure_module);
                                             }
-                                            JobPayload::RawFlow {
-                                                value: FlowValue {
-                                                    modules: b.modules.clone(),
-                                                    failure_module: fm.clone(),
-                                                    same_worker: flow.same_worker,
+                                            JobPayloadWithTag {
+                                                payload: JobPayload::RawFlow {
+                                                    value: FlowValue {
+                                                        modules: b.modules.clone(),
+                                                        failure_module: fm.clone(),
+                                                        same_worker: flow.same_worker,
+                                                    },
+                                                    path: Some(format!(
+                                                        "{}/branchall-{}",
+                                                        flow_job.script_path(),
+                                                        i
+                                                    )),
                                                 },
-                                                path: Some(format!(
-                                                    "{}/branchall-{}",
-                                                    flow_job.script_path(),
-                                                    i
-                                                )),
+                                                tag: None,
                                             }
                                         })
                                         .collect(),
@@ -1978,17 +1995,20 @@ async fn compute_next_flow_transform<'c>(
             Ok((
                 tx,
                 NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(JobPayload::RawFlow {
-                        value: FlowValue {
-                            modules,
-                            failure_module: fm.clone(),
-                            same_worker: flow.same_worker,
+                    ContinuePayload::SingleJob(JobPayloadWithTag {
+                        payload: JobPayload::RawFlow {
+                            value: FlowValue {
+                                modules,
+                                failure_module: fm.clone(),
+                                same_worker: flow.same_worker,
+                            },
+                            path: Some(format!(
+                                "{}/branchall-{}",
+                                flow_job.script_path(),
+                                branch_status.branch
+                            )),
                         },
-                        path: Some(format!(
-                            "{}/branchall-{}",
-                            flow_job.script_path(),
-                            branch_status.branch
-                        )),
+                        tag: None,
                     }),
                     NextStatus::NextBranchStep(NextBranch { status: branch_status, flow_jobs }),
                 ),

cli/README.md

@@ -3,6 +3,8 @@
 A simple CLI allowing interactions with windmill from the command line.
 ![](./vhs/output/setup.gif)
 
+You can find more information in [Windmill Docs](https://docs.windmill.dev/docs/advanced/cli)
+
 ## Installation
 
 Install the `wmill` CLI tool using
@@ -18,11 +20,14 @@ workspace settings.
 ## Running Flows & Scripts
 
 Run a script or flow using `wmill flow/script run u/username/path/to/script` and
-pass any inputs using `--input/-i <name>=<value>` curl-style syntax using
-`-i @-` for stdin or `-i @<filename>` is also supported
+pass any inputs using `--data` + Inputs specified as a JSON string or a file using `@ <filename>` or stdin using @-.
+
+Curl-style syntax using `-d @-` for stdin or `-d @<filename>` is also supported.
 
 Flow Steps and Logs will be streamed during execution automatically.
 
+![CLI input example](./vhs/output/cli_inputs_example.png)
+
 ## Pushing Resources, Scripts & More
 
 The CLI can push specifications to a windmill instance. See the
@@ -48,6 +53,9 @@ wmill sync pull
 wmill sync push
 ```
 
+We recommend using the --yaml option to use yaml instead of json as the encoding
+format. Yaml will be made the default soon.
+
 ### Pushing individual files
 
 You can push individual resources using

cli/apps.ts

@@ -1,120 +1,65 @@
+// deno-lint-ignore-file no-explicit-any
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
-import { Any, decoverto, model, property } from "./decoverto.ts";
 import {
   AppService,
   AppWithLastVersion,
   colors,
   Command,
   ListableApp,
-  microdiff,
   Policy,
   Table,
 } from "./deps.ts";
 import {
-  Difference,
   GlobalOptions,
-  PushDiffs,
-  Resource,
-  setValueByPath,
+  isSuperset,
+  parseFromFile,
+  removeType,
 } from "./types.ts";
 
-@model()
-export class AppFile implements Resource, PushDiffs {
-  @property(Any)
+export interface AppFile {
   value: any;
-  @property(() => String)
   summary: string;
-  @property(Any)
   policy: Policy;
+}
 
-  constructor(value: string, summary: string, policy: Policy) {
-    this.value = value;
-    this.summary = summary;
-    this.policy = policy;
-  }
-  async pushDiffs(
-    workspace: string,
-    remotePath: string,
-    diffs: Difference[]
-  ): Promise<void> {
-    let app: AppWithLastVersion | undefined = undefined;
+export async function pushApp(
+  workspace: string,
+  remotePath: string,
+  app: AppFile | AppWithLastVersion | undefined,
+  newApp: AppFile,
+  raw: boolean
+): Promise<void> {
+  remotePath = removeType(remotePath, "app");
+  if (raw) {
+    // deleting old app if it exists in raw mode
     try {
       app = await AppService.getAppByPath({ workspace, path: remotePath });
-    } catch (e) {}
-
-    if (app) {
-      console.log(
-        colors.bold.yellow(
-          `Applying ${diffs.length} diffs to existing app... ${remotePath}`
-        )
-      );
-      const changeset: {
-        summary?: string | undefined;
-        value?: any;
-        policy?: Policy | undefined;
-      } = {};
-      for (const diff of diffs) {
-        if (
-          diff.type !== "REMOVE" &&
-          diff.path[0] !== "value" &&
-          diff.path[0] !== "policy" &&
-          (diff.path.length !== 1 ||
-            !["summary"].includes(diff.path[0] as string))
-        ) {
-          throw new Error("Invalid app diff with path " + diff.path);
-        }
-        if (diff.type === "CREATE" || diff.type === "CHANGE") {
-          setValueByPath(changeset, diff.path, diff.value);
-        } else if (diff.type === "REMOVE") {
-          setValueByPath(changeset, diff.path, null);
-        }
-      }
-
-      if (
-        (!changeset?.policy ||
-          JSON.stringify(changeset?.policy) == JSON.stringify(app.policy)) &&
-        (!changeset?.value ||
-          JSON.stringify(changeset?.value) == JSON.stringify(app.value)) &&
-        (!changeset?.summary || changeset.summary == app.summary)
-      ) {
-        console.log(
-          colors.yellow(`No changes to push for app ${remotePath}, skipping`)
-        );
-        return;
-      }
-
-      const hasChanges = Object.values(changeset).some(
-        (v) => v !== null && typeof v !== "undefined"
-      );
-      if (!hasChanges) {
-        return;
-      }
-
-      await AppService.updateApp({
-        workspace,
-        path: remotePath,
-        requestBody: changeset,
-      });
-    } else {
-      console.log(colors.yellow.bold("Creating new app..."));
-
-      await AppService.createApp({
-        workspace,
-        requestBody: {
-          path: remotePath,
-          policy: this.policy,
-          summary: this.summary,
-          value: this.value,
-        },
-      });
+    } catch {
+      //ignore
     }
   }
-  async push(workspace: string, remotePath: string): Promise<void> {
-    await this.pushDiffs(
+
+  if (app) {
+    if (isSuperset(newApp, app)) {
+      return;
+    }
+    await AppService.updateApp({
       workspace,
-      remotePath,
-      microdiff({}, this, { cyclesFix: false })
-    );
+      path: remotePath,
+      requestBody: {
+        ...newApp,
+      },
+    });
+  } else {
+    console.log(colors.yellow.bold("Creating new app..."));
+
+    await AppService.createApp({
+      workspace,
+      requestBody: {
+        path: remotePath,
+        ...newApp,
+      },
+    });
   }
 }
 
@@ -154,19 +99,24 @@ async function push(opts: GlobalOptions, filePath: string) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  await pushApp(filePath, workspace.workspaceId, remotePath);
-  console.log(colors.bold.underline.green("App pushed"));
-}
+  let app: AppWithLastVersion | undefined = undefined;
+  try {
+    app = await AppService.getAppByPath({
+      workspace: workspace.workspaceId,
+      path: remotePath,
+    });
+  } catch {
+    // app doesn't exist
+  }
 
-export async function pushApp(
-  filePath: string,
-  workspace: string,
-  remotePath: string
-) {
-  const data = decoverto
-    .type(AppFile)
-    .rawToInstance(await Deno.readTextFile(filePath));
-  await data.push(workspace, remotePath);
+  await pushApp(
+    workspace.workspaceId,
+    remotePath,
+    app,
+    parseFromFile(filePath),
+    false
+  );
+  console.log(colors.bold.underline.green("App pushed"));
 }
 
 const command = new Command()

cli/context.ts

@@ -1,5 +1,5 @@
 // deno-lint-ignore-file no-explicit-any
-import { colors, GlobalUserInfo, setClient, UserService } from "./deps.ts";
+import { colors, GlobalUserInfo, log, setClient, UserService } from "./deps.ts";
 import { loginInteractive, tryGetLoginInfo } from "./login.ts";
 import { GlobalOptions } from "./types.ts";
 import {
@@ -53,7 +53,7 @@ export async function resolveWorkspace(
 ): Promise<Workspace> {
   const res = await tryResolveWorkspace(opts);
   if (res.isError) {
-    console.log(colors.red.bold(res.error));
+    log.info(colors.red.bold(res.error));
     return Deno.exit(-1);
   } else {
     return res.value;
@@ -75,7 +75,7 @@ export async function requireLogin(
   try {
     return await UserService.globalWhoami();
   } catch {
-    console.log(
+    log.info(
       "! Could not reach API given existing credentials. Attempting to reauth..."
     );
     const newToken = await loginInteractive(workspace.remote);
@@ -119,7 +119,7 @@ export async function tryResolveVersion(
 
 export function validatePath(path: string): boolean {
   if (!(path.startsWith("g") || path.startsWith("u") || path.startsWith("f"))) {
-    console.log(
+    log.info(
       colors.red(
         "Given remote path looks invalid. Remote paths are typically of the form <u|g|f>/<username|group|folder>/..."
       )

cli/decoverto.ts

@@ -1,8 +0,0 @@
-// globally shared decoverto instance
-import { Decoverto } from "npm:decoverto";
-const decoverto = new Decoverto();
-
-// TODO: Properly type FlowModule
-
-export { Any, array, map, MapShape, model, property } from "npm:decoverto";
-export { decoverto };

cli/deps.ts

@@ -1,6 +1,6 @@
 // windmill
-export { setClient } from "https://deno.land/x/windmill@v1.69.3/mod.ts";
-export * from "https://deno.land/x/windmill@v1.69.3/windmill-api/index.ts";
+export { setClient } from "https://deno.land/x/windmill@v1.95.1/mod.ts";
+export * from "https://deno.land/x/windmill@v1.95.1/windmill-api/index.ts";
 
 // cliffy
 export { Command } from "https://deno.land/x/cliffy@v0.25.7/command/command.ts";
@@ -37,3 +37,4 @@ export { default as microdiff } from "https://deno.land/x/microdiff@v1.3.1/index
 export { default as objectHash } from "https://deno.land/x/object_hash@2.0.3.1/mod.ts";
 export { default as gitignore_parser } from "npm:gitignore-parser";
 export { default as JSZip } from "npm:jszip@3.7.1";
+export * as log from "https://deno.land/std@0.186.0/log/mod.ts";

cli/flow.ts

@@ -1,121 +1,94 @@
 // deno-lint-ignore-file no-explicit-any
-import {
-  Difference,
-  GlobalOptions,
-  PushDiffs,
-  Resource,
-  setValueByPath,
-} from "./types.ts";
+import { GlobalOptions, isSuperset } from "./types.ts";
+import { parse as yamlParse } from "https://deno.land/std@0.184.0/yaml/mod.ts";
+
 import {
   colors,
   Command,
   Flow,
+  FlowModule,
   FlowService,
   JobService,
-  microdiff,
-  OpenFlowWPath,
   Table,
 } from "./deps.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import { resolve, track_job } from "./script.ts";
-import { Any, decoverto, model, property } from "./decoverto.ts";
 
-// this is effectively "OpenFlow" but a copy as it is accepted by the CLI
-@model()
-export class FlowFile implements Resource, PushDiffs {
-  @property(() => String)
+export interface FlowFile {
   summary: string;
-  @property(() => String)
   description?: string;
-  @property(Any)
   value: any;
-  @property(Any)
   schema?: any;
+}
 
-  constructor(value: any, summary?: string) {
-    this.summary = summary ?? "";
-    this.value = value;
+const alreadySynced: string[] = [];
+
+export async function pushFlow(
+  workspace: string,
+  remotePath: string,
+  localFlowPath: string
+): Promise<void> {
+  if (alreadySynced.includes(localFlowPath)) {
+    return;
+  }
+  alreadySynced.push(localFlowPath);
+  let flow: Flow | undefined = undefined;
+  try {
+    flow = await FlowService.getFlowByPath({
+      workspace: workspace,
+      path: remotePath,
+    });
+  } catch {
+    // flow doesn't exist
   }
-  async pushDiffs(
-    workspace: string,
-    remotePath: string,
-    diffs: Difference[]
-  ): Promise<void> {
-    if (
-      await FlowService.existsFlowByPath({
-        workspace: workspace,
-        path: remotePath,
-      })
-    ) {
-      console.log(
-        colors.bold.yellow(
-          `Applying ${diffs.length} diffs to existing flow... ${remotePath}`
-        )
-      );
 
-      // TODO: Make these optional in backend (not path ofc)
-      const changeset: OpenFlowWPath = {
-        path: remotePath,
-        summary: this.summary,
-        value: this.value,
-        description: this.description, // This is OpenAPIed as optional, but isn't
-        schema: this.schema, // Same
-      };
-      const base_changeset = { ...changeset };
-      for (const diff of diffs) {
-        if (
-          diff.type !== "REMOVE" &&
-          diff.path[0] !== "value" &&
-          (diff.path.length !== 1 ||
-            !["summary", "description", "schema"].includes(
-              diff.path[0] as string
-            ))
-        ) {
-          throw new Error("Invalid flow diff with path " + diff.path);
-        }
-        if (diff.type === "CREATE" || diff.type === "CHANGE") {
-          setValueByPath(changeset, diff.path, diff.value);
-        } else if (diff.type === "REMOVE") {
-          setValueByPath(changeset, diff.path, null);
-        }
-      }
-      const hasChanges = Object.values(changeset).some(
-        (v) => v !== null && typeof v !== "undefined"
-      );
-      if (!hasChanges) {
-        return;
+  if (!localFlowPath.endsWith("/")) {
+    localFlowPath += "/";
+  }
+  const localFlowRaw = await Deno.readTextFile(localFlowPath + "flow.yaml");
+  const localFlow = yamlParse(localFlowRaw) as FlowFile;
+
+  function replaceInlineScripts(modules: FlowModule[]) {
+    modules.forEach((m) => {
+      if (m.value.type == "rawscript") {
+        const path = m.value.content.split(" ")[1];
+        m.value.content = Deno.readTextFileSync(localFlowPath + path);
+      } else if (m.value.type == "forloopflow") {
+        replaceInlineScripts(m.value.modules);
+      } else if (m.value.type == "branchall") {
+        m.value.branches.forEach((b) => replaceInlineScripts(b.modules));
+      } else if (m.value.type == "branchone") {
+        m.value.branches.forEach((b) => replaceInlineScripts(b.modules));
+        replaceInlineScripts(m.value.default);
       }
+    });
+  }
 
-      const update = {
-        ...changeset,
-        ...base_changeset,
-      };
+  replaceInlineScripts(localFlow.value.modules);
 
-      await FlowService.updateFlow({
-        workspace: workspace,
-        path: remotePath,
-        requestBody: update,
-      });
-    } else {
-      console.log(colors.bold.yellow("Creating new flow..."));
-      await FlowService.createFlow({
-        workspace: workspace,
-        requestBody: {
-          path: remotePath,
-          summary: this.summary,
-          value: this.value,
-          schema: this.schema,
-          description: this.description,
-        },
-      });
+  if (flow) {
+    if (isSuperset(localFlow, flow)) {
+      console.log(colors.bold.green("Flow is up to date"));
+      return;
     }
-  }
-  async push(workspace: string, remotePath: string): Promise<void> {
-    await this.pushDiffs(
-      workspace,
-      remotePath,
-      microdiff({}, this, { cyclesFix: false })
-    );
+    console.log(colors.bold.yellow(`Updating flow ${remotePath}...`));
+    await FlowService.updateFlow({
+      workspace: workspace,
+      path: remotePath,
+      requestBody: {
+        path: remotePath,
+        ...localFlow,
+      },
+    });
+  } else {
+    console.log(colors.bold.yellow("Creating new flow..."));
+    await FlowService.createFlow({
+      workspace: workspace,
+      requestBody: {
+        path: remotePath,
+        ...localFlow,
+      },
+    });
   }
 }
 
@@ -128,21 +101,10 @@ async function push(opts: Options, filePath: string, remotePath: string) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
 
-  await pushFlow(filePath, workspace.workspaceId, remotePath);
+  await pushFlow(workspace.workspaceId, remotePath, filePath);
   console.log(colors.bold.underline.green("Flow pushed"));
 }
 
-export async function pushFlow(
-  filePath: string,
-  workspace: string,
-  remotePath: string
-) {
-  const data = decoverto
-    .type(FlowFile)
-    .rawToInstance(await Deno.readTextFile(filePath));
-  await data.push(workspace, remotePath);
-}
-
 async function list(opts: GlobalOptions & { showArchived?: boolean }) {
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);

cli/folder.ts

@@ -1,135 +1,85 @@
-import { colors, Command, FolderService, microdiff } from "./deps.ts";
+// deno-lint-ignore-file no-explicit-any
+import { colors, Command, Folder, FolderService, log, Table } from "./deps.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
-import {
-  Difference,
-  GlobalOptions,
-  PushDiffs,
-  Resource,
-  setValueByPath,
-} from "./types.ts";
-import {
-  array,
-  decoverto,
-  map,
-  MapShape,
-  model,
-  property,
-} from "./decoverto.ts";
+import { GlobalOptions, isSuperset, parseFromFile } from "./types.ts";
 
-@model()
-export class FolderFile implements Resource, PushDiffs {
-  @property(array(() => String))
+export interface FolderFile {
   owners: Array<string> | undefined;
-  @property(
-    map(
-      () => String,
-      () => Boolean,
-      { shape: MapShape.Object }
-    )
-  )
   extra_perms: Map<string, boolean> | undefined;
-  @property(() => String)
   display_name: string | undefined;
+}
 
-  async push(workspace: string, remotePath: string): Promise<void> {
-    if (remotePath.startsWith("/")) {
-      remotePath = remotePath.substring(1);
-    }
-    if (remotePath.startsWith("f/")) {
-      remotePath = remotePath.substring(2);
-    }
+async function list(opts: GlobalOptions) {
+  const workspace = await resolveWorkspace(opts);
+  await requireLogin(opts);
 
-    await this.pushDiffs(
-      workspace,
-      remotePath,
-      microdiff({}, this, { cyclesFix: false })
-    );
+  const folders = await FolderService.listFolders({
+    workspace: workspace.workspaceId,
+  });
+
+  new Table()
+    .header(["Name", "Owners", "Extra Perms"])
+    .padding(2)
+    .border(true)
+    .body(
+      folders.map((x) => [
+        x.name,
+        x.owners?.join(",") ?? "-",
+        JSON.stringify(x.extra_perms ?? {}),
+      ])
+    )
+    .render();
+}
+
+export async function pushFolder(
+  workspace: string,
+  name: string,
+  folder: Folder | FolderFile | undefined,
+  localFolder: FolderFile,
+  raw: boolean
+): Promise<void> {
+  if (name.startsWith("/")) {
+    name = name.substring(1);
+  }
+  if (name.startsWith("f/")) {
+    name = name.substring(2);
+  }
+  name = name.split("/")[0];
+  log.debug(`Processing local folder ${name}`);
+
+  if (raw) {
+    // deleting old app if it exists in raw mode
+    try {
+      folder = await FolderService.getFolder({ workspace, name });
+      log.debug(`Folder ${name} exists on remote`);
+    } catch {
+      log.debug(`Folder ${name} does not exist on remote`);
+      //ignore
+    }
   }
 
-  async pushDiffs(
-    workspace: string,
-    remotePath: string,
-    diffs: Difference[]
-  ): Promise<void> {
-    if (remotePath.startsWith("/")) {
-      remotePath = remotePath.substring(1);
-    }
-    if (remotePath.startsWith("f/")) {
-      remotePath = remotePath.substring(2);
-    }
-
-    // TODO: Support this in backend
-    let exists: boolean;
-    try {
-      exists = !!(await FolderService.getFolder({
-        workspace,
-        name: remotePath,
-      }));
-    } catch {
-      exists = false;
-    }
-    if (exists) {
-      console.log(
-        colors.bold.yellow(
-          `Applying ${diffs.length} diffs to existing folder... ${remotePath}`
-        )
-      );
-
-      const changeset: {
-        owners?: string[] | undefined;
-        extra_perms?: any;
-        display_name?: string | undefined;
-      } = {};
-      for (const diff of diffs) {
-        if (
-          diff.type !== "REMOVE" &&
-          (diff.path.length !== 1 ||
-            !["owners", "extra_perms", "display_name"].includes(
-              diff.path[0] as string
-            ))
-        ) {
-          console.log(diff.path);
-          throw new Error("Invalid folder diff with path " + diff.path);
-        }
-        if (diff.type === "CREATE" || diff.type === "CHANGE") {
-          setValueByPath(changeset, diff.path, diff.value);
-        } else if (diff.type === "REMOVE") {
-          setValueByPath(changeset, diff.path, null);
-        }
-      }
-
-      const hasChanges = Object.values(changeset).some(
-        (v) => v !== null && typeof v !== "undefined"
-      );
-      if (!hasChanges) {
-        return;
-      }
-      try {
-        await FolderService.updateFolder({
-          workspace: workspace,
-          name: remotePath,
-          requestBody: {
-            ...changeset,
-            extra_perms: changeset.extra_perms
-              ? Object.fromEntries(this.extra_perms?.entries() ?? [])
-              : undefined,
-          },
-        });
-      } catch (e) {
-        console.error(colors.red.bold(e.body));
-        throw e;
-      }
-    } else {
-      console.log(colors.bold.yellow("Creating new folder: " + remotePath));
-      await FolderService.createFolder({
-        workspace: workspace,
-        requestBody: {
-          name: remotePath,
-          extra_perms: Object.fromEntries(this.extra_perms?.entries() ?? []),
-          owners: this.owners,
-        },
-      });
+  if (folder) {
+    if (isSuperset(localFolder, folder)) {
+      log.debug(`Folder ${name} is up to date`);
+      return;
     }
+    log.debug(`Folder ${name} is not up-to-date, updating...`);
+    await FolderService.updateFolder({
+      workspace: workspace,
+      name: name,
+      requestBody: {
+        ...localFolder,
+      },
+    });
+  } else {
+    console.log(colors.bold.yellow("Creating new folder: " + name));
+    await FolderService.createFolder({
+      workspace: workspace,
+      requestBody: {
+        name: name,
+        ...localFolder,
+      },
+    });
   }
 }
 
@@ -146,25 +96,21 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
     throw new Error("file path must refer to a file.");
   }
 
-  console.log(colors.bold.yellow("Pushing resource..."));
+  console.log(colors.bold.yellow("Pushing folder..."));
 
-  await pushFolder(workspace.workspaceId, filePath, remotePath);
-  console.log(colors.bold.underline.green("Resource pushed"));
-}
-
-export async function pushFolder(
-  workspace: string,
-  filePath: string,
-  remotePath: string
-) {
-  const data = decoverto
-    .type(FolderFile)
-    .rawToInstance(await Deno.readTextFile(filePath));
-  data.push(workspace, remotePath);
+  await pushFolder(
+    workspace.workspaceId,
+    remotePath,
+    undefined,
+    parseFromFile(filePath),
+    false
+  );
+  console.log(colors.bold.underline.green("Folder pushed"));
 }
 
 const command = new Command()
-  .description("resource related commands")
+  .description("folder related commands")
+  .action(list as any)
   .command(
     "push",
     "push a local folder spec. This overrides any remote versions."

cli/hub.ts

@@ -1,14 +1,15 @@
-import { Command } from "./deps.ts";
+// deno-lint-ignore-file no-explicit-any
+import { Command, ResourceService, log } from "./deps.ts";
 import { requireLogin, resolveWorkspace } from "./context.ts";
-import { ResourceTypeFile } from "./resource-type.ts";
+import { pushResourceType } from "./resource-type.ts";
 import { GlobalOptions } from "./types.ts";
 
 async function pull(opts: GlobalOptions) {
   const workspace = await resolveWorkspace(opts);
 
   if (workspace.workspaceId !== "admins") {
-    console.log(
-      "Should only sync to admins workspace, but current is not admins.",
+    log.info(
+      "Should only sync to admins workspace, but current is not admins."
     );
     return;
   }
@@ -24,15 +25,12 @@ async function pull(opts: GlobalOptions) {
     created_by: string;
     created_at: Date;
     comments: never[];
-  }[] = await fetch(
-    "https://hub.windmill.dev/resource_types/list",
-    {
-      headers: {
-        "Accept": "application/json",
-        "X-email": userInfo.email,
-      },
+  }[] = await fetch("https://hub.windmill.dev/resource_types/list", {
+    headers: {
+      Accept: "application/json",
+      "X-email": userInfo.email,
     },
-  )
+  })
     .then((r) => r.json())
     .then((list: { id: number; name: string }[]) =>
       list.map((x) =>
@@ -40,9 +38,9 @@ async function pull(opts: GlobalOptions) {
           "https://hub.windmill.dev/resource_types/" + x.id + "/" + x.name,
           {
             headers: {
-              "Accept": "application/json",
+              Accept: "application/json",
             },
-          },
+          }
         )
       )
     )
@@ -50,7 +48,7 @@ async function pull(opts: GlobalOptions) {
     .then((x) =>
       x.map((x) =>
         x.json().catch((e) => {
-          console.log(e);
+          log.info(e);
           return undefined;
         })
       )
@@ -58,14 +56,23 @@ async function pull(opts: GlobalOptions) {
     .then((x) => Promise.all(x))
     .then((x) => x.filter((x) => x).map((x) => x.resource_type));
 
-  for (
-    const x of list
-  ) {
-    console.log("syncing " + x.name);
-    const f = new ResourceTypeFile();
-    f.description = x.description;
-    f.schema = JSON.parse(x.schema);
-    await f.push(workspace.workspaceId, x.name);
+  const resourceTypes = await ResourceService.listResourceType({
+    workspace: workspace.workspaceId,
+  });
+
+  for (const x of list) {
+    if (resourceTypes.find((y) => y.name === x.name)) {
+      log.info("skipping " + x.name);
+      continue;
+    }
+    log.info("syncing " + x.name);
+    await pushResourceType(
+      workspace.workspaceId,
+      x.name + ".resource-type.json",
+      undefined,
+      x,
+      false
+    );
   }
 }
 

cli/login.ts

@@ -1,11 +1,11 @@
 import { GlobalOptions } from "./types.ts";
-import { colors, getAvailablePort, Secret, Select } from "./deps.ts";
-import { open } from 'https://deno.land/x/open/index.ts';
+import { colors, getAvailablePort, log, Secret, Select } from "./deps.ts";
+import { open } from "https://deno.land/x/open/index.ts";
 
 export async function loginInteractive(remote: string) {
   let token: string | undefined;
   if (
-    await Select.prompt({
+    (await Select.prompt({
       message: "How do you want to login",
       options: [
         {
@@ -17,7 +17,7 @@ export async function loginInteractive(remote: string) {
           value: "t",
         },
       ],
-    }) === "b"
+    })) === "b"
   ) {
     token = await browserLogin(remote);
   } else {
@@ -29,7 +29,7 @@ export async function loginInteractive(remote: string) {
 
 // deno-lint-ignore require-await
 export async function tryGetLoginInfo(
-  opts: GlobalOptions,
+  opts: GlobalOptions
 ): Promise<string | undefined> {
   if (opts.token) {
     return opts.token;
@@ -39,22 +39,22 @@ export async function tryGetLoginInfo(
 }
 
 export async function browserLogin(
-  baseUrl: string,
+  baseUrl: string
 ): Promise<string | undefined> {
   const port = await getAvailablePort();
   if (port == undefined) {
-    console.log(colors.red.underline("failed to aquire port"));
+    log.info(colors.red.underline("failed to aquire port"));
     return undefined;
   }
 
   const server = Deno.listen({ transport: "tcp", port });
-  const url = `${baseUrl}user/cli?port=${port}`
-  console.log(`Login by going to ${url}`);
+  const url = `${baseUrl}user/cli?port=${port}`;
+  log.info(`Login by going to ${url}`);
   try {
-    await open(url)
-    console.log("Opened browser for you");
+    await open(url);
+    log.info("Opened browser for you");
   } catch {
-    console.error(`Failed to open browser, please navigate to ${url}`)
+    console.error(`Failed to open browser, please navigate to ${url}`);
   }
   const firstConnection = await server.accept();
   const httpFirstConnection = Deno.serveHttp(firstConnection);
@@ -63,7 +63,7 @@ export async function browserLogin(
   const token = params.get("token");
   // const _workspace = params.get("workspace");
   await firstRequest?.respondWith(
-    Response.redirect(baseUrl + "user/cli-success", 302),
+    Response.redirect(baseUrl + "user/cli-success", 302)
   );
 
   setTimeout(() => {

cli/main.ts

@@ -18,8 +18,16 @@ import folder from "./folder.ts";
 import sync from "./sync.ts";
 import { tryResolveVersion } from "./context.ts";
 import { GlobalOptions } from "./types.ts";
+import * as log from "https://deno.land/std@0.186.0/log/mod.ts";
 
-export const VERSION = "v1.87.0";
+addEventListener("error", (event) => {
+  if (event.error) {
+    console.error("Error details of: " + event.error.message);
+    console.error(JSON.stringify(event.error, null, 4));
+  }
+});
+
+export const VERSION = "v1.102.0";
 
 let command: any = new Command()
   .name("wmill")
@@ -28,6 +36,11 @@ let command: any = new Command()
     "--workspace <workspace:string>",
     "Specify the target workspace. This overrides the default workspace."
   )
+  .globalOption("--debug", "Show debug logs")
+  .globalOption(
+    "--show-diffs",
+    "Show diff informations when syncing (may show sensitive informations)"
+  )
   .globalOption(
     "--token <token:string>",
     "Specify an API token. This will override any stored token."
@@ -73,7 +86,26 @@ if (Number.parseInt(VERSION.replace("v", "").replace(".", "")) > 1700) {
   command = command.command("push", push).command("pull", pull);
 }
 
+export let showDiffs = false;
 try {
+  const LOG_LEVEL = Deno.args.includes("--debug") ? "DEBUG" : "INFO";
+  // const NO_COLORS = Deno.args.includes("--no-colors");
+  showDiffs = Deno.args.includes("--show-diffs");
+
+  log.setup({
+    handlers: {
+      console: new log.handlers.ConsoleHandler(LOG_LEVEL, {
+        formatter: "{msg}",
+      }),
+    },
+    loggers: {
+      default: {
+        level: LOG_LEVEL,
+        handlers: ["console"],
+      },
+    },
+  });
+  log.debug("Debug logging enabled. CLI build against " + VERSION);
   await command.parse(Deno.args);
 } catch (e) {
   if (e.name === "ApiError") {