chore(main): release 1.30.0 (#399)

* chore(main): release 1.30.0

* Apply automatic changes

Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>

.github/CODEOWNERS

@@ -1 +1,4 @@
 *       @rubenfiszel
+
+/community/ @fatonramadani @rubenfiszel
+/frontend/ @fatonramadani @rubenfiszel

.github/DockerfileBackendTests

@@ -0,0 +1,62 @@
+FROM python:3.10-slim-buster as nsjail
+
+WORKDIR /nsjail
+
+RUN apt-get -y update \
+    && apt-get install -y \
+    bison=2:3.3.* \
+    flex=2.6.* \
+    g++=4:8.3.* \
+    gcc=4:8.3.* \
+    git=1:2.20.* \
+    libprotobuf-dev=3.6.* \
+    libnl-route-3-dev=3.4.* \
+    make=4.2.* \
+    pkg-config=0.29-6 \
+    protobuf-compiler=3.6.*
+
+RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
+    && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
+RUN make
+
+
+FROM rust:slim-buster as builder
+
+RUN apt-get update && apt-get install -y git libssl-dev pkg-config
+
+RUN apt-get -y update \
+    && apt-get install -y \
+    curl lld
+
+ENV SQLX_OFFLINE=true
+
+COPY ./nsjail /nsjail
+
+RUN mkdir -p /frontend/build
+
+RUN apt-get update \
+    && apt-get install -y ca-certificates tzdata libpq5 \
+    make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
+    libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
+    libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
+    libv8-dev tesseract-ocr \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV TZ=Etc/UTC
+
+ENV PYTHON_VERSION 3.10.4
+
+RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \
+    && tar -xf Python-${PYTHON_VERSION}.tgz && cd Python-${PYTHON_VERSION}/ && ./configure --enable-optimizations \
+    && make -j 4 && make install
+
+RUN /usr/local/bin/python3 -m pip install pip-tools
+RUN /usr/local/bin/python3 -m pip install nltk
+RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon
+
+COPY --from=nsjail /nsjail/nsjail /bin/nsjail
+
+COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
+
+RUN apt-get update \
+    && apt-get install -y postgresql-client

.github/pull_hub_items.sh

@@ -8,6 +8,7 @@ for item in ${RT[@]}; do
     body=$(curl -s -H "accept: application/json" https://hub.windmill.dev/resource_types/${id}/${name})
     jq -r '.resource_type.schema' <<< "$body" > ./tmp
     description=$(jq -r '.resource_type.description' <<< "$body")
+    description=$(echo -E $description)
     echo "{\"workspace_id\": \"starter\", \"name\": \"$name\", \"schema\": $(cat ./tmp), \"description\": \"$description\"} " | jq . > community/resource_types/${name}.json
     rm ./tmp
 done

.github/workflows/backend-test.yml

@@ -0,0 +1,43 @@
+name: Backend only integration tests
+
+on:
+  push:
+    branches:
+      - "main"
+    paths:
+      - "backend/**"
+      - ".github/workflows/backend-test.yml"
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "backend/**"
+      - ".github/workflows/backend-test.yml"
+
+jobs:
+  cargo_test:
+    runs-on: [self-hosted, new]
+    container:
+      image: ghcr.io/windmill-labs/backend-tests
+      options: --privileged
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: windmill
+          POSTGRES_USER: admin
+          POSTGRES_PASSWORD: changeme
+
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v3
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: backend -> target
+      - run: psql postgres://admin:changeme@postgres:5432/windmill -c "CREATE ROLE app LOGIN PASSWORD 'changeme';"
+      - name: cargo test
+        timeout-minutes: 5
+        run: mkdir frontend/build && cd backend && DATABASE_URL=postgres://admin:changeme@postgres:5432/windmill cargo test

.github/workflows/docker-image.yml

@@ -42,7 +42,6 @@ jobs:
           context: .
           push: true
           tags: |
-            ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:latest
             ${{ steps.metalocal.outputs.tags }}
 
           labels: ${{ steps.metalocal.outputs.labels }}
@@ -76,12 +75,12 @@ jobs:
           context: .
           push: true
           tags: |
-            ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
             ${{ steps.metalocal.outputs.tags }}
             ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.metalocal.outputs.labels }}
-          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
-          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
   playwright:
     runs-on: [self-hosted, new]
     needs: [build]
@@ -105,7 +104,7 @@ jobs:
         run: echo "::set-output name=id::$(docker run --network=host --rm -d -p 8000:8000 --privileged -it -e DATABASE_URL=postgres://admin:changeme@localhost:5432/windmill -e BASE_INTERNAL_URL=http://localhost:8000 ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}:latest)"
         id: docker-container
       - name: "Playwright run"
-        timeout-minutes: 10
+        timeout-minutes: 2
         run: cd frontend && npm ci @playwright/test && npx playwright install && npm run test
       - name: "Clean up"
         run: docker kill ${{ steps.docker-container.outputs.id }}

.github/workflows/pull-hub.yml

@@ -17,3 +17,4 @@ jobs:
         uses: peter-evans/create-pull-request@v4
         with:
           title: sync hub items with community
+          commit-message: sync hub items with community

.github/workflows/pypi_on_release.yml

@@ -1,12 +1,13 @@
 env:
-  LOCAL_REGISTRY: registry.wimill.xyz
-  IMAGE_NAME: ${{ github.repository }}
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}-lsp
 
 name: Publish python-client
 on:
   push:
     tags:
       - "v*"
+  workflow_dispatch:
 
 jobs:
   publish_pypi:
@@ -31,22 +32,34 @@ jobs:
           fetch-depth: 0
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
-      - name: Docker meta local
-        id: metalocal
+
+      - name: Docker meta
+        id: meta
         uses: docker/metadata-action@v4
         with:
-          images: ${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-lsp
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=ref,event=branch
+            type=ref,event=pr
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
 
-      - name: Build and push
+      - name: Login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push publicly
         uses: docker/build-push-action@v3
         with:
           context: "{{defaultContext}}:lsp"
           push: true
-          tags: ${{ steps.metalocal.outputs.tags }}
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ steps.metalocal.outputs.tags }}
+            ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.metalocal.outputs.labels }}
-          cache-from: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-lsp:buildcache
-          cache-to: type=registry,ref=${{ env.LOCAL_REGISTRY }}/${{ env.IMAGE_NAME }}-lsp:buildcache,mode=max
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max

.github/workflows/sign-cla.yml

@@ -12,7 +12,7 @@ jobs:
       - name: "CLA Assistant"
         if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
         # Beta Release
-        uses: cla-assistant/github-action@v2.2.0
+        uses: cla-assistant/github-action@v2.2.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           PERSONAL_ACCESS_TOKEN: ${{ secrets.CLA_PAT }}

CHANGELOG.md

@@ -1,6 +1,96 @@
 # Changelog
 
 
+## [1.30.0](https://github.com/windmill-labs/windmill/compare/v1.29.0...v1.30.0) (2022-08-13)
+
+
+### Features
+
+* add literal object type support ([#401](https://github.com/windmill-labs/windmill/issues/401)) ([845de82](https://github.com/windmill-labs/windmill/commit/845de8206214ed265aef895f0d13636e6e0e26ce))
+* support union type will null | undefined ([#400](https://github.com/windmill-labs/windmill/issues/400)) ([0384727](https://github.com/windmill-labs/windmill/commit/0384727a56347aa01a5fee06c82bd49eab2522fa))
+* support union types ([#398](https://github.com/windmill-labs/windmill/issues/398)) ([e68ea1b](https://github.com/windmill-labs/windmill/commit/e68ea1b8fc4f88e587121387ecac6858d04ebae2))
+
+## [1.29.0](https://github.com/windmill-labs/windmill/compare/v1.28.1...v1.29.0) (2022-08-10)
+
+
+### Features
+
+* _value, _index => iter.value, iter.index ([07f4a21](https://github.com/windmill-labs/windmill/commit/07f4a217d0c6b46fd3defaa0242d229a60c69463))
+* remove res1 wrapping ([e76a981](https://github.com/windmill-labs/windmill/commit/e76a9816ee09e59d5c38bf0c19231bac8347148c))
+
+
+### Bug Fixes
+
+* do not skip undefined values ([8b68a87](https://github.com/windmill-labs/windmill/commit/8b68a87c523fe13a9f45786ee0fbb57b10efda13))
+* **python:** not filled field with default <function_call> now call the default function ([33962c4](https://github.com/windmill-labs/windmill/commit/33962c44660fd20173a0ae14b00a66a985dd4fc7))
+* surface new _iterator value ([13b1904](https://github.com/windmill-labs/windmill/commit/13b1904a7ab5a6e7a7c82d2a2806648441759756))
+* update logs even if last new log was < 500ms ([c69621f](https://github.com/windmill-labs/windmill/commit/c69621fa7a9a18223e7854f0824bd6fbcabdfe10))
+
+## [1.28.1](https://github.com/windmill-labs/windmill/compare/v1.28.0...v1.28.1) (2022-08-05)
+
+
+### Bug Fixes
+
+* **frontend:** add toggl connect ([#341](https://github.com/windmill-labs/windmill/issues/341)) ([b94895f](https://github.com/windmill-labs/windmill/commit/b94895f24eb4ba1b67f499a98c6e6e8d9d006b14))
+* **frontend:** schedule args in flow ([#343](https://github.com/windmill-labs/windmill/issues/343)) ([350a25c](https://github.com/windmill-labs/windmill/commit/350a25c837b1367fa5568dd1de0196202d632bd0))
+* improve flow viewer with retrieving hub script ([80e28db](https://github.com/windmill-labs/windmill/commit/80e28dbba3e77154c0017bd8e74d144e6aae13fb))
+
+## [1.28.0](https://github.com/windmill-labs/windmill/compare/v1.27.2...v1.28.0) (2022-08-04)
+
+
+### Features
+
+* **frontend:** global flow preview ([#329](https://github.com/windmill-labs/windmill/issues/329)) ([615f69e](https://github.com/windmill-labs/windmill/commit/615f69e935e9c9c0b60edfb6dc2e82aebba623b9))
+
+
+### Bug Fixes
+
+* **api:** add discord webhook manual instructions ([a9a4b9b](https://github.com/windmill-labs/windmill/commit/a9a4b9b21d7b68a3e46c28ce13986d7a9ebd2cac))
+* **backend:** generalize oauth clients to take in extra params ([6332910](https://github.com/windmill-labs/windmill/commit/6332910dd27f78d555f0ab040545e98dedbea89d))
+* **backend:** handle better some flow edge-cases ([3bcd542](https://github.com/windmill-labs/windmill/commit/3bcd542130bc0cb45dfb1fa7681dd4b7beb95c7e))
+* **backend:** handle better some flow edge-cases ([9885361](https://github.com/windmill-labs/windmill/commit/988536128bd04dab94cc686bc2db547e57894587))
+* **backend:** handle better some flow edge-cases ([70de6e3](https://github.com/windmill-labs/windmill/commit/70de6e3972af81aec68b706dca93e16182a584bb))
+* **backend:** prometheus histogram for worker job timer ([#312](https://github.com/windmill-labs/windmill/issues/312)) ([4055586](https://github.com/windmill-labs/windmill/commit/40555868e6221620beca85ebafad2da67e56ec08))
+* **frontend:** add jpeg support ([0e8552b](https://github.com/windmill-labs/windmill/commit/0e8552ba800f13add6b25a83a765dace8d4369e7))
+* **frontend:** loading template pick the language as well ([82c7ddc](https://github.com/windmill-labs/windmill/commit/82c7ddc00e79a1cc5336a0a219f46d705c2c8d88))
+* **frontend:** Use the bracket notation when an identifier is not a valid JS expression ([#327](https://github.com/windmill-labs/windmill/issues/327)) ([05324bd](https://github.com/windmill-labs/windmill/commit/05324bd3562f6066cdc12d74c87033325d1c7ef1))
+* **oauth2:** remove discord oauth integration ([986e76d](https://github.com/windmill-labs/windmill/commit/986e76dc8729a53d09cd83531d474f9b5fe88f35))
+
+## [1.27.2](https://github.com/windmill-labs/windmill/compare/v1.27.1...v1.27.2) (2022-08-02)
+
+
+### Bug Fixes
+
+* **deno-client:** getResource can now fetch non-object values ([b128388](https://github.com/windmill-labs/windmill/commit/b128388cc652d4cd369a88b93985a2c051003abd))
+
+## [1.27.1](https://github.com/windmill-labs/windmill/compare/v1.27.0...v1.27.1) (2022-08-02)
+
+
+### Bug Fixes
+
+* migrate to new style radio button ([893ee94](https://github.com/windmill-labs/windmill/commit/893ee941d72a7036f0ea272c49bbe5cd3eee64d5))
+
+## [1.27.0](https://github.com/windmill-labs/windmill/compare/v1.26.3...v1.27.0) (2022-08-02)
+
+
+### Features
+
+* add primitive sql format ([#320](https://github.com/windmill-labs/windmill/issues/320)) ([9daff2a](https://github.com/windmill-labs/windmill/commit/9daff2a228791234a3dd70c0ee829e284daf1592))
+
+
+### Bug Fixes
+
+* prefer `COPY` over `ADD` ([#319](https://github.com/windmill-labs/windmill/issues/319)) ([24a7e46](https://github.com/windmill-labs/windmill/commit/24a7e46fe99d5a1f7d5b22334fa5f6ce76e82d94))
+* typos ([#301](https://github.com/windmill-labs/windmill/issues/301)) ([9e84b45](https://github.com/windmill-labs/windmill/commit/9e84b458b139e86eb51dba9c5b228f141ca649b3))
+
+## [1.26.3](https://github.com/windmill-labs/windmill/compare/v1.26.2...v1.26.3) (2022-08-01)
+
+
+### Bug Fixes
+
+* displaying which group you are a member of that gave you access to item ([1bd0269](https://github.com/windmill-labs/windmill/commit/1bd026924b8a3b01f7729b627f939d8af872a483))
+* refresh jobs result when hopping from flow to flow ([c86abe6](https://github.com/windmill-labs/windmill/commit/c86abe6ae01efd519f67ead233ebddf39f1539c0))
+
 ## [1.26.2](https://github.com/windmill-labs/windmill/compare/v1.26.1...v1.26.2) (2022-07-31)
 
 

Dockerfile

@@ -32,6 +32,7 @@ RUN mkdir /backend
 COPY /backend/openapi.yaml /backend/openapi.yaml
 COPY /openflow.openapi.yaml /openflow.openapi.yaml
 RUN npm run generate-backend-client
+ENV NODE_OPTIONS "--max-old-space-size=8192"
 RUN npm run build
 RUN npm run check
 
@@ -48,7 +49,7 @@ COPY ./backend/.cargo/ .cargo/
 
 RUN apt-get -y update \
     && apt-get install -y \
-    curl
+    curl lld
 
 ENV CARGO_INCREMENTAL=1
 
@@ -58,11 +59,11 @@ RUN rm src/*.rs
 RUN rm ./target/release/deps/windmill*
 ENV SQLX_OFFLINE=true
 
-ADD ./backend ./
-ADD ./nsjail /nsjail
+COPY ./backend ./
+COPY ./nsjail /nsjail
 
-COPY --from=1 /frontend /frontend
-ADD .git/ .git/
+COPY --from=frontend /frontend /frontend
+COPY .git/ .git/
 
 RUN cargo build --release
 
@@ -87,6 +88,8 @@ RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VER
     && make -j 4 && make install
 
 RUN /usr/local/bin/python3 -m pip install pip-tools
+RUN /usr/local/bin/python3 -m pip install nltk
+RUN mkdir -p /nsjail_data/python && HOME=/nsjail_data/python /usr/local/bin/python3 -m nltk.downloader vader_lexicon
 
 COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
 

README.md

@@ -2,7 +2,11 @@
   <a href="https://app.windmill.dev"><img src="./imgs/windmill.svg" alt="windmill.dev"></a>
 </p>
 <p align="center">
-    <em>Windmill is an open-source developer platform to quickly build production-grade multi-steps automations and internal apps from minimal Python and Typescript scripts.</em>
+    <em>.</em>
+</p>
+<p align=center>
+Open-source and self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger flows and scripts as internal apps. Convert code to no-code modules and, if the auto-generated UI is not sufficient, use it solely as an highly scalable backend layer. Add automation to your product or build your own no-code tool and delegate the core layer to Windmill
+.
 </p>
 <p align="center">
 <a href="https://github.com/windmill-labs/windmill/actions/workflows/docker-image.yml" target="_blank">
@@ -27,24 +31,27 @@
 
 **Hub**: <https://hub.windmill.dev>
 
+**Contributor's guide**: <https://docs.windmill.dev/docs/contributors_guide>
+
+**Roadmap**: <https://github.com/orgs/windmill-labs/projects/2>
+
 You can show your support for the project by starring this repo.
 
 ---
 
-If you would like to run this in production self-hosted, or know someone that
-would, Windmill would gladly help you to achieve it and more. If interested,
-send an email to ruben@windmill.dev (founder and creator of Windmill).
+Windmill Labs offers commercial licenses and support to convert your existing
+automation and help you scale it in production. If interested, contact
+ruben@windmill.dev (founder of Windmill).
 
 ---
 
 # Windmill
 
 <p align="center">
-<b>Disclaimer: </b>Windmill is in <b>BETA</b>. It is secure to run in production but the API might change,
-especially concerning flows.
+<b>Disclaimer: </b>Windmill is in <b>BETA</b>. It is secure to run in production but we are still <a href="https://github.com/orgs/windmill-labs/projects/2">improving the product fast<a/>.
 </p>
 
-![Windmill Screenshot](./imgs/windmill.webp)
+![Windmill Screenshot](./imgs/windmill.png)
 
 Windmill is <b>fully open-sourced</b>:
 
@@ -56,26 +63,41 @@ Windmill is <b>fully open-sourced</b>:
 1. Define a minimal and generic script in Python or Typescript that solve a
    specific task. Here sending an email with SMTP. The code can be defined in
    the provided Web IDE or synchronized with your own github repo:
-   ![Step 1](./imgs/step1.png)
+   ![Step 1](./imgs/python-script.png)
 
 2. Your scripts parameters are automatically parsed and generate a frontend. You
    can narrow down the types during task definition to specify regex for string,
    an enum or a specific format for objects. Each script correspond to an app by
-   itself: ![Step 2](./imgs/step2.png)
+   itself: ![Step 2](./imgs/arguments.png)
 
 3. Make it flow! You can chain your scripts or scripts made by the community
-   inside flow by piping output to input using "Dynamic" fields that are just
-   plain Javascript. You can also refer to external variables, output from any
-   steps or inputs of the flow itself. The flow parameters then generate
-   automatically an intuitive forms that can be triggered by anyone, like for
-   scripts. ![Step 3](./imgs/step3.png)
+   shared on [WindmillHub](https://hub.windmill.dev). There is tight integration
+   between Windmill and the hub to make it easy to build flows from a soon-to-be
+   exhaustive library of generic modules. In flows, one can pipe output to input
+   using "Dynamic" expressions that are just plain Javascript underneath. Flows
+   can contain for-loops, branching (coming soon). As such and coupled with
+   inputs being able to refer to any step's output, they are actual DAG rather
+   than just linear sequences. They are backed by an open JSON spec we call
+   [OpenFlow](https://docs.windmill.dev/docs/openflow)
+   ![Step 3](./imgs/flow.png)
+
+Both scripts and flows are not restricted to be triggered by the UI. They can be
+triggered by a schedule, watch for changes (using
+[internal states](https://docs.windmill.dev/docs/reference#internal-state)) or
+triggered through API with either an async or sync webhook. The latter kind of
+endpoints make Windmill akin to a self-hostable AWS Lambda. Windmill can be the
+central place to host, build and run all of your integrations, automation and
+internal apps. We include credentials management and OAuth integration, groups
+and much more!
 
 ## Layout
 
 - `backend/`: The whole Rust backend
 - `frontend`: The whole Svelte frontend
-- `community/`: Scripts and resource types created and curated by the community,
-  included in every workspace
+- `community/`: Scripts and resource types included in every workspace. It is
+  useful for Python scripts since the [WindmillHub](https://hub.windmill.dev)
+  only allow deno scripts and for sharing resource types that will be included
+  in every workspace.
 - `lsp/`: The lsp asssistant for the monaco editor
 - `nsjail/`: The nsjail configuration files for sandboxing of the scripts'
   execution
@@ -86,26 +108,41 @@ Windmill is <b>fully open-sourced</b>:
 
 ## Stack
 
-- postgres as the database
+- Postgres as the database
 - backend in Rust with the following highly-available and horizontally scalable
   architecture:
   - stateless API backend
-  - workers that pull jobs from a queue
+  - workers that pull jobs from a queue in Postgres (and later, Kafka or Redis.
+    Upvote [#173](#https://github.com/windmill-labs/windmill/issues/173) if
+    interested )
 - frontend in Svelte
-- scripts executions are sandboxed using google's nsjail
-- javascript runtime is deno_core rust library (which itself uses the rusty_v8
-  and hence V8 underneath)
+- scripts executions are sandboxed using google's
+  [nsjail](https://github.com/google/nsjail)
+- javascript runtime is the
+  [deno_core rust library](https://denolib.gitbook.io/guide/) (which itself uses
+  the [rusty_v8](https://github.com/denoland/rusty_v8) and hence V8 underneath)
 - typescript runtime is deno
 - python runtime is python3
 
-### Development stack
+## Sandboxing and workload isolation
 
-- caddy is the reverse proxy used for local development, see frontend's
-  Caddyfile and CaddyfileRemote
+Windmill uses nsjail on top of the deno sandboxing. It is pretty secure. Do not
+take our word for it, take
+[fly.io's one](https://fly.io/blog/sandboxing-and-workload-isolation/)
+
+## Performance
+
+The performances are great, as long as you do not exceed the parrallelism of the
+workers, we are
+[worse than AWS Lambda for small workloads but not by that much](https://docs.windmill.dev/docs/benchmark)
 
 ## Architecture
 
-![Architecture](./imgs/architecture.svg)
+<p align="center">
+<img src="./imgs/architecture.svg">
+<img src="./imgs/diagram.svg">
+
+</p>
 
 ## How to self-host
 

backend/.cargo/config

@@ -1,3 +1,3 @@
 [build]
-rustflags = ["--cfg", "tokio_unstable"]
+rustflags = ["--cfg", "tokio_unstable", "-C", "link-arg=-fuse-ld=lld"]
 incremental = true

backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.26.2"
+version = "1.30.0"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"
 
@@ -24,6 +24,8 @@ chrono = { version = "^0", features = ["serde"]}
 tracing = "^0"
 tracing-subscriber = { version = "^0", features = ["env-filter", "json"]}
 console-subscriber = "^0"
+prometheus = { version = "^0", default-features = false }
+phf = { version = "0.11", features = ["macros"] }
 
 rust-embed = "^6"
 mime_guess = "^2"
@@ -37,7 +39,6 @@ magic-crypt = "^3"
 git-version = "^0"
 rustpython-parser = "^0"
 cron = "^0"
-external-ip = "^4"
 lettre = { version = "^0", features = ["rustls-tls", "tokio1", "tokio1-rustls-tls", "builder", "smtp-transport"],  default-features = false}
 urlencoding = "^2"
 url = "^2"
@@ -60,5 +61,5 @@ swc_ecma_ast = "^0"
 
 sqlx = { version = "^0", features = ["macros", "offline", "migrate", "uuid", "json", "chrono", "postgres", "runtime-tokio-rustls"]}
 dotenv = "^0"
-ulid = { version = "^0", features = ["uuid"] }
+ulid = { version = "^1", features = ["uuid"] }
 futures = "^0"

backend/NOTICE

@@ -1,5 +1,5 @@
-Ruben Fiszel
+Windmill Labs, Inc
 
-Copyright (c) 2021  Ruben Fiszel
+Copyright (c) 2021  Windmill Labs, Inc
 
 Source code in this directory is licensed the GNU Affero General Public License. 

backend/oauth_connect.json

@@ -33,34 +33,62 @@
         "token_url": "https://oauth2.googleapis.com/token",
         "scopes": [
             "https://www.googleapis.com/auth/spreadsheets"
-        ]
+        ],
+        "extra_params": {
+            "access_type": "offline",
+            "consent": "prompt"
+        }
     },
     "gdrive": {
         "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
         "token_url": "https://oauth2.googleapis.com/token",
         "scopes": [
             "https://www.googleapis.com/auth/drive"
-        ]
+        ],
+        "extra_params": {
+            "access_type": "offline",
+            "consent": "prompt"
+        }
     },
     "gmail": {
         "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
         "token_url": "https://oauth2.googleapis.com/token",
         "scopes": [
             "https://mail.google.com/"
-        ]
+        ],
+        "extra_params": {
+            "access_type": "offline",
+            "consent": "prompt"
+        }
     },
     "gcal": {
         "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
         "token_url": "https://oauth2.googleapis.com/token",
         "scopes": [
             "https://www.googleapis.com/auth/calendar.events"
-        ]
+        ],
+        "extra_params": {
+            "access_type": "offline",
+            "consent": "prompt"
+        }
     },
     "gcloud": {
         "auth_url": "https://accounts.google.com/o/oauth2/v2/auth",
         "token_url": "https://oauth2.googleapis.com/token",
         "scopes": [
             "https://www.googleapis.com/auth/cloud-platform"
-        ]
+        ],
+        "extra_params": {
+            "access_type": "offline",
+            "consent": "prompt"
+        }
+    },
+    "basecamp": {
+        "auth_url": "https://launchpad.37signals.com/authorization/new",
+        "token_url": "https://launchpad.37signals.com/authorization/token",
+        "scopes": [],
+        "extra_params": {
+            "type": "web_server"
+        }
     }
 }

backend/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.3"
 
 info:
-  version: 1.26.2
+  version: 1.30.0
   title: Windmill server API
   contact:
     name: Windmill contact
@@ -341,7 +341,7 @@ paths:
 
   /workspaces/list_as_superadmin:
     get:
-      summary: list all workspaces as super admin (require to be super amdin)
+      summary: list all workspaces as super admin (require to be super admin)
       operationId: listWorkspacesAsSuperAdmin
       tags:
         - workspace
@@ -1236,9 +1236,15 @@ paths:
             application/json:
               schema:
                 additionalProperties:
-                  type: array
-                  items:
-                    type: string
+                  type: object
+                  properties:
+                    extra_params:
+                      additionalProperties:
+                        type: string
+                    scopes:
+                      type: array
+                      items:
+                        type: string
 
   /w/{workspace}/resources/create:
     post:
@@ -1632,7 +1638,7 @@ paths:
         - $ref: "#/components/parameters/OrderDesc"
         - $ref: "#/components/parameters/CreatedBy"
         - name: path_start
-          description: mask to filter matching starting parh
+          description: mask to filter matching starting path
           in: query
           schema:
             type: string
@@ -2011,7 +2017,7 @@ paths:
         - $ref: "#/components/parameters/OrderDesc"
         - $ref: "#/components/parameters/CreatedBy"
         - name: path_start
-          description: mask to filter matching starting parh
+          description: mask to filter matching starting path
           in: query
           schema:
             type: string
@@ -2087,14 +2093,7 @@ paths:
         content:
           application/json:
             schema:
-              allOf:
-                - $ref: "../openflow.openapi.yaml#/components/schemas/OpenFlow"
-                - type: object
-                  properties:
-                    path:
-                      type: string
-                  required:
-                    - path
+              $ref: "#/components/schemas/OpenFlowWPath"
 
       responses:
         "201":
@@ -2119,14 +2118,7 @@ paths:
         content:
           application/json:
             schema:
-              allOf:
-                - $ref: "../openflow.openapi.yaml#/components/schemas/OpenFlow"
-                - type: object
-                  properties:
-                    path:
-                      type: string
-                  required:
-                    - path
+              $ref: "#/components/schemas/OpenFlowWPath"
 
       responses:
         "200":
@@ -2242,7 +2234,7 @@ paths:
       parameters:
         - $ref: "#/components/parameters/WorkspaceId"
       requestBody:
-        description: previw
+        description: preview
         required: true
         content:
           application/json:
@@ -3637,18 +3629,89 @@ components:
               typ:
                 oneOf:
                   - type: string
-                    enum: ["str", "float", "int", "bool", "email", "unknown"]
+                    enum:
+                      [
+                        "float",
+                        "int",
+                        "bool",
+                        "email",
+                        "unknown",
+                        "bytes",
+                        "dict",
+                        "datetime",
+                        "sql",
+                      ]
                   - type: object
                     properties:
                       resource:
                         type: string
+                        nullable: true
                     required:
                       - resource
+                  - type: object
+                    properties:
+                      str:
+                        type: array
+                        items:
+                          type: string
+                        nullable: true
+                    required:
+                      - str
+                  - type: object
+                    properties:
+                      object:
+                        type: array
+                        items:
+                          type: object
+                          properties:
+                            key:
+                              type: string
+                            typ:
+                              oneOf:
+                                - type: string
+                                  enum:
+                                    [
+                                      "float",
+                                      "int",
+                                      "bool",
+                                      "email",
+                                      "unknown",
+                                      "bytes",
+                                      "dict",
+                                      "datetime",
+                                      "sql",
+                                    ]
+                                - type: object
+                                  properties:
+                                    str: {}
+                                  required: [str]
+                          required:
+                            - key
+                            - typ
+                    required:
+                      - object
                   - type: object
                     properties:
                       list:
-                        type: string
-                        enum: ["str", "float", "int", "email"]
+                        oneOf:
+                          - type: string
+                            enum:
+                              [
+                                "float",
+                                "int",
+                                "bool",
+                                "email",
+                                "unknown",
+                                "bytes",
+                                "dict",
+                                "datetime",
+                                "sql",
+                              ]
+                          - type: object
+                            properties:
+                              str: {}
+                            required: [str]
+                        nullable: true
                     required:
                       - list
               has_default:
@@ -3999,6 +4062,16 @@ components:
         - archived
         - extra_perms
 
+    OpenFlowWPath:
+      allOf:
+        - $ref: "../openflow.openapi.yaml#/components/schemas/OpenFlow"
+        - type: object
+          properties:
+            path:
+              type: string
+          required:
+            - path
+
     FlowPreview:
       type: object
       properties:

backend/sqlx-data.json

@@ -21,6 +21,32 @@
     },
     "query": "DELETE FROM queue WHERE workspace_id = $1 AND id = $2 RETURNING 1"
   },
+  "04207ceb4e5e520e3a759ed0b0b513c6fc9afda4f0e34a8fbbfce227e984e053": {
+    "describe": {
+      "columns": [
+        {
+          "name": "id",
+          "ordinal": 0,
+          "type_info": "Uuid"
+        },
+        {
+          "name": "workspace_id",
+          "ordinal": 1,
+          "type_info": "Varchar"
+        }
+      ],
+      "nullable": [
+        false,
+        false
+      ],
+      "parameters": {
+        "Left": [
+          "Timestamptz"
+        ]
+      }
+    },
+    "query": "UPDATE queue SET running = false WHERE last_ping < $1 and running = true RETURNING id, workspace_id"
+  },
   "04a15674cf66f2822085e65ca33ee42b8e7f3f9d63ffac0585a12368d0322252": {
     "describe": {
       "columns": [
@@ -723,26 +749,6 @@
     },
     "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE name = $1 AND workspace_id = $2)"
   },
-  "37d3ee8009055e869941e548a6d5a352053a5d7782f662c34b94706488abccb6": {
-    "describe": {
-      "columns": [
-        {
-          "name": "id",
-          "ordinal": 0,
-          "type_info": "Uuid"
-        }
-      ],
-      "nullable": [
-        false
-      ],
-      "parameters": {
-        "Left": [
-          "Timestamptz"
-        ]
-      }
-    },
-    "query": "UPDATE queue SET running = false WHERE last_ping < $1 RETURNING id"
-  },
   "3d363466d79075df3f74f946eff43ca89faefca3bcdf2c533425ca3868b0369a": {
     "describe": {
       "columns": [
@@ -1887,27 +1893,6 @@
     },
     "query": "SELECT value FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter')"
   },
-  "8dbab3cc7d25a38301c54756a26827a0957c4edb5e68b788c19d3e60b5f038ea": {
-    "describe": {
-      "columns": [
-        {
-          "name": "count",
-          "ordinal": 0,
-          "type_info": "Int8"
-        }
-      ],
-      "nullable": [
-        null
-      ],
-      "parameters": {
-        "Left": [
-          "Text",
-          "Text"
-        ]
-      }
-    },
-    "query": "SELECT COUNT(id) FROM queue WHERE created_by = $1 AND workspace_id = $2"
-  },
   "8ede6fb740b145b3a8320adb789500870c7a8ec807a7b156b9ff7a15791b78f8": {
     "describe": {
       "columns": [],
@@ -2041,6 +2026,27 @@
     },
     "query": "SELECT email, login_type::TEXT, super_admin, verified, name, company FROM password WHERE email = $1"
   },
+  "92ee6274b2287bc4d368bfdb47d6452df75bd265cdd891f55112df2795294dac": {
+    "describe": {
+      "columns": [
+        {
+          "name": "count",
+          "ordinal": 0,
+          "type_info": "Int8"
+        }
+      ],
+      "nullable": [
+        null
+      ],
+      "parameters": {
+        "Left": [
+          "Text",
+          "Text"
+        ]
+      }
+    },
+    "query": "SELECT COUNT(id) FROM queue WHERE permissioned_as = $1 AND workspace_id = $2"
+  },
   "9490a4388f43e45e32911f1129e623f2d73ce2da7a948fe134bea1c87cdbefd1": {
     "describe": {
       "columns": [
@@ -2789,27 +2795,6 @@
     },
     "query": "SELECT premium FROM workspace WHERE id = $1"
   },
-  "d9beef36b05f9889794d0eb9bae309974abc85240305ca154de25a54bc7d60f9": {
-    "describe": {
-      "columns": [
-        {
-          "name": "sum",
-          "ordinal": 0,
-          "type_info": "Int8"
-        }
-      ],
-      "nullable": [
-        null
-      ],
-      "parameters": {
-        "Left": [
-          "Text",
-          "Text"
-        ]
-      }
-    },
-    "query": "\n           SELECT SUM(duration_ms)\n             FROM completed_job\n            WHERE created_by = $1\n              AND created_at > NOW() - INTERVAL '1200 seconds'\n              AND workspace_id = $2"
-  },
   "d9c8f6ec7bd10e533876526255c15e376ccb4f898b9c0ab8840b2930bda24fdc": {
     "describe": {
       "columns": [
@@ -2957,6 +2942,27 @@
     },
     "query": "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter') AND\n         created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter')))"
   },
+  "e69142610bd94f6c65022cca5ccf7458b7c5bb77b3ed0278715222dd920d8644": {
+    "describe": {
+      "columns": [
+        {
+          "name": "sum",
+          "ordinal": 0,
+          "type_info": "Int8"
+        }
+      ],
+      "nullable": [
+        null
+      ],
+      "parameters": {
+        "Left": [
+          "Text",
+          "Text"
+        ]
+      }
+    },
+    "query": "\n           SELECT SUM(duration_ms)\n             FROM completed_job\n            WHERE permissioned_as = $1\n              AND created_at > NOW() - INTERVAL '1200 seconds'\n              AND workspace_id = $2"
+  },
   "e94abd39ec51b7e0c48c190d47ed766fd4f401187c3b60b3e599426c95232f7f": {
     "describe": {
       "columns": [],

backend/src/client.rs

@@ -42,6 +42,6 @@ pub async fn get_resource(
         let value = res.json::<Option<serde_json::Value>>().await?;
         Ok(value)
     } else {
-        Err(Error::NotFound(format!("Variable not found at {path}")))?
+        Err(Error::NotFound(format!("Resource not found at {path}")))?
     }
 }

backend/src/db.rs

@@ -12,9 +12,9 @@ use std::time::Duration;
 
 pub type DB = Pool<Postgres>;
 
-pub async fn connect(database_url: &str) -> Result<DB, Error> {
+pub async fn connect(database_url: &str, max_connections: u32) -> Result<DB, Error> {
     PgPoolOptions::new()
-        .max_connections(100)
+        .max_connections(max_connections)
         .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
         .connect(database_url)
         .await

backend/src/error.rs

@@ -45,7 +45,7 @@ pub enum Error {
     HexErr(#[from] hex::FromHexError),
     #[error("Migrating database: {0}")]
     DatabaseMigration(#[from] MigrateError),
-    #[error("{0}")]
+    #[error(transparent)]
     Anyhow(#[from] anyhow::Error),
 }
 

backend/src/external_ip.rs

@@ -0,0 +1,14 @@
+//! Used to determine the internet address that connections from workers will appear to come from.
+//!
+//! For users writing scripts to access their infrastructure with firewalls requiring incoming
+//! connections to be from whitelisted IP addresses.
+
+use reqwest::Result;
+
+pub async fn get_ip() -> Result<String> {
+    reqwest::get("https://hub.windmill.dev/getip")
+        .await?
+        .error_for_status()?
+        .text()
+        .await
+}

backend/src/fixtures/base.sql

@@ -0,0 +1,19 @@
+-- used for backend automated testing
+-- https://docs.rs/sqlx/latest/sqlx/attr.test.html
+
+INSERT INTO workspace
+            (id,               name,             owner,       domain)
+     VALUES ('test-workspace', 'test-workspace', 'test-user', null);
+
+CREATE FUNCTION "notify_insert_on_completed_job" ()
+RETURNS TRIGGER AS $$
+BEGIN
+    PERFORM pg_notify('insert on completed_job', NEW.id::text);
+    RETURN new;
+END;
+$$ LANGUAGE PLPGSQL;
+
+  CREATE TRIGGER "notify_insert_on_completed_job"
+ AFTER INSERT ON "completed_job"
+    FOR EACH ROW
+EXECUTE FUNCTION "notify_insert_on_completed_job" ();

backend/src/flows.rs

@@ -77,10 +77,12 @@ pub struct FlowValue {
 
 #[derive(Deserialize, Serialize, Debug, Clone)]
 pub struct FlowModule {
+    #[serde(default)]
     pub input_transform: HashMap<String, InputTransform>,
     pub value: FlowModuleValue,
     pub stop_after_if_expr: Option<String>,
     pub skip_if_stopped: Option<bool>,
+    pub summary: Option<String>,
 }
 
 #[derive(Deserialize, Serialize, Debug, Clone)]
@@ -422,6 +424,7 @@ mod tests {
                     value: FlowModuleValue::Script { path: "test".to_string() },
                     stop_after_if_expr: None,
                     skip_if_stopped: Some(false),
+                    summary: None,
                 },
                 FlowModule {
                     input_transform: HashMap::new(),
@@ -432,6 +435,7 @@ mod tests {
                     }),
                     stop_after_if_expr: Some("foo = 'bar'".to_string()),
                     skip_if_stopped: None,
+                    summary: None,
                 },
                 FlowModule {
                     input_transform: [(
@@ -444,15 +448,17 @@ mod tests {
                         value: Box::new(FlowValue { modules: vec![], failure_module: None }),
                         skip_failures: true,
                     },
-                    stop_after_if_expr: Some("previous.res1.isEmpty()".to_string()),
+                    stop_after_if_expr: Some("previous.isEmpty()".to_string()),
                     skip_if_stopped: None,
+                    summary: None,
                 },
             ],
             failure_module: Some(FlowModule {
                 input_transform: HashMap::new(),
                 value: FlowModuleValue::Flow { path: "test".to_string() },
-                stop_after_if_expr: Some("previous.res1.isEmpty()".to_string()),
+                stop_after_if_expr: Some("previous.isEmpty()".to_string()),
                 skip_if_stopped: None,
+                summary: None,
             }),
         };
         println!("{}", serde_json::json!(fv).to_string());

backend/src/jobs.rs

@@ -24,6 +24,7 @@ use crate::{
     scripts::{get_hub_script_by_path, ScriptHash, ScriptLang},
     users::{owner_to_token_owner, Authed},
     utils::{require_admin, Pagination, StripPath},
+    worker,
     worker_flow::init_flow_status,
 };
 use axum::{
@@ -40,7 +41,7 @@ use ulid::Ulid;
 use uuid::Uuid;
 
 const MAX_NB_OF_JOBS_IN_Q_PER_USER: i64 = 10;
-const MAX_DURATION_LAST_1200: std::time::Duration = std::time::Duration::from_secs(400);
+const MAX_DURATION_LAST_1200: std::time::Duration = std::time::Duration::from_secs(900);
 
 pub fn workspaced_service() -> Router {
     Router::new()
@@ -318,7 +319,12 @@ pub async fn get_path_for_hash<'c>(
         w_id
     )
     .fetch_one(db)
-    .await?;
+    .await
+    .map_err(|e| {
+        Error::InternalErr(format!(
+            "querying getting path for hash {hash} in {w_id}: {e}"
+        ))
+    })?;
     Ok(path)
 }
 
@@ -1036,12 +1042,15 @@ pub async fn push<'c>(
     let premium_workspace =
         sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", workspace_id)
             .fetch_one(&mut tx)
-            .await?;
+            .await
+            .map_err(|e| {
+                Error::InternalErr(format!("fetching if {workspace_id} is premium: {e}"))
+            })?;
 
     if !premium_workspace && std::env::var("CLOUD_HOSTED").is_ok() {
         let rate_limiting_queue = sqlx::query_scalar!(
-            "SELECT COUNT(id) FROM queue WHERE created_by = $1 AND workspace_id = $2",
-            user,
+            "SELECT COUNT(id) FROM queue WHERE permissioned_as = $1 AND workspace_id = $2",
+            permissioned_as,
             workspace_id
         )
         .fetch_one(&mut tx)
@@ -1061,10 +1070,10 @@ pub async fn push<'c>(
             "
            SELECT SUM(duration_ms)
              FROM completed_job
-            WHERE created_by = $1
+            WHERE permissioned_as = $1
               AND created_at > NOW() - INTERVAL '1200 seconds'
               AND workspace_id = $2",
-            user,
+            permissioned_as,
             workspace_id
         )
         .fetch_one(&mut tx)
@@ -1090,7 +1099,12 @@ pub async fn push<'c>(
                 workspace_id
             )
             .fetch_one(&mut tx)
-            .await?;
+            .await
+            .map_err(|e| {
+                Error::InternalErr(format!(
+                    "fetching language for hash {hash} in {workspace_id}: {e}"
+                ))
+            })?;
             (
                 Some(hash.0),
                 Some(path),
@@ -1202,38 +1216,35 @@ pub async fn push<'c>(
     .await
     .map_err(|e| Error::InternalErr(format!("Could not insert into queue {job_id}: {e}")))?;
 
-    let uuid_string = job_id.to_string();
-    let uuid_str = uuid_string.as_str();
-    let mut hm = HashMap::from([("uuid", uuid_str), ("permissioned_as", &permissioned_as)]);
     {
+        let uuid_string = job_id.to_string();
+        let uuid_str = uuid_string.as_str();
+        let mut hm = HashMap::from([("uuid", uuid_str), ("permissioned_as", &permissioned_as)]);
+
         let s: String;
-        let audit_o = match job_kind {
-            JobKind::Preview => Some((
-                "jobs.run.preview",
-                Some(format!("{}", script_path.unwrap_or_else(String::new))),
-            )),
+        let operation_name = match job_kind {
+            JobKind::Preview => "jobs.run.preview",
             JobKind::Script => {
                 s = ScriptHash(script_hash.unwrap()).to_string();
                 hm.insert("hash", s.as_str());
-                Some(("jobs.run.script", script_path))
+                "jobs.run.script"
             }
-            JobKind::Flow => Some(("jobs.run.flow", script_path)),
-            JobKind::FlowPreview => Some(("jobs.run.flow_preview", script_path)),
-            _ => None,
+            JobKind::Flow => "jobs.run.flow",
+            JobKind::FlowPreview => "jobs.run.flow_preview",
+            JobKind::Script_Hub => "jobs.run.script_hub",
+            JobKind::Dependencies => "jobs.run.dependencies",
         };
 
-        if let Some((operation_name, resource)) = audit_o {
-            audit_log(
-                &mut tx,
-                &user,
-                operation_name,
-                ActionKind::Execute,
-                workspace_id,
-                resource.as_ref().map(|x| x.as_str()),
-                Some(hm),
-            )
-            .await?;
-        }
+        audit_log(
+            &mut tx,
+            &user,
+            operation_name,
+            ActionKind::Execute,
+            workspace_id,
+            script_path.as_ref().map(|x| x.as_str()),
+            Some(hm),
+        )
+        .await?;
     }
     Ok((uuid, tx))
 }
@@ -1244,7 +1255,9 @@ pub async fn add_completed_job_error<E: ToString + std::fmt::Debug>(
     queued_job: &QueuedJob,
     logs: String,
     e: E,
+    metrics: &worker::Metrics,
 ) -> Result<(Uuid, Map<String, Value>), Error> {
+    metrics.jobs_failed.inc();
     let mut output_map = serde_json::Map::new();
     output_map.insert(
         "error".to_string(),
@@ -1255,7 +1268,7 @@ pub async fn add_completed_job_error<E: ToString + std::fmt::Debug>(
         &queued_job,
         false,
         false,
-        Some(output_map.clone()),
+        serde_json::Value::Object(output_map.clone()),
         format!("\n{}\n{}", logs, e.to_string()),
     )
     .await?;
@@ -1268,10 +1281,9 @@ pub async fn add_completed_job(
     queued_job: &QueuedJob,
     success: bool,
     skipped: bool,
-    result: Option<Map<String, Value>>,
+    result: serde_json::Value,
     logs: String,
 ) -> Result<Uuid, Error> {
-    let result_json = result.map(serde_json::Value::Object);
     let job_id = queued_job.id.clone();
     let duration_ms = (chrono::Utc::now() - queued_job.started_at.unwrap_or(queued_job.created_at))
         .num_milliseconds() as i32;
@@ -1295,7 +1307,7 @@ pub async fn add_completed_job(
         queued_job.script_hash.map(|x| x.0),
         queued_job.script_path,
         queued_job.args,
-        result_json,
+        result,
         logs,
         queued_job.raw_code,
         queued_job.canceled,
@@ -1318,6 +1330,7 @@ pub async fn add_completed_job(
 
 #[instrument(level = "trace", skip_all)]
 pub async fn postprocess_queued_job(
+    is_flow_step: bool,
     schedule_path: Option<String>,
     script_path: Option<String>,
     w_id: &str,
@@ -1325,7 +1338,9 @@ pub async fn postprocess_queued_job(
     db: &DB,
 ) -> crate::error::Result<()> {
     let _ = delete_job(db, w_id, job_id).await?;
-    schedule_again_if_scheduled(schedule_path, script_path, &w_id, db).await?;
+    if !is_flow_step {
+        schedule_again_if_scheduled(schedule_path, script_path, &w_id, db).await?;
+    }
     Ok(())
 }
 

backend/src/js_eval.rs

@@ -294,10 +294,10 @@ multiline template`";
             ("params".to_string(), json!({"test": 2})),
             ("value".to_string(), json!({"test": 2})),
         ];
-        let code = r#"variable("test")"#;
+        let code = r#"params.test"#;
 
         let res = eval_timeout(code.to_string(), env, None, vec![]).await?;
-        assert_eq!(res, json!("test"));
+        assert_eq!(res, json!(2));
         Ok(())
     }
 }

backend/src/lib.rs

@@ -6,13 +6,14 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use anyhow::Context;
 use argon2::Argon2;
 use axum::{handler::Handler, middleware::from_extractor, routing::get, Extension, Router};
 use db::DB;
+use futures::FutureExt;
 use git_version::git_version;
 use slack_http_verifier::SlackVerifier;
 use std::{net::SocketAddr, sync::Arc};
-use tokio::sync::Mutex;
 use tower::ServiceBuilder;
 use tower_cookies::CookieManagerLayer;
 use tower_http::trace::TraceLayer;
@@ -26,6 +27,7 @@ mod client;
 mod db;
 mod email;
 mod error;
+mod external_ip;
 mod flows;
 mod granular_acls;
 mod groups;
@@ -33,6 +35,8 @@ mod jobs;
 mod js_eval;
 mod oauth2;
 mod parser;
+mod parser_py;
+mod parser_ts;
 mod resources;
 mod schedule;
 mod scripts;
@@ -57,10 +61,13 @@ use crate::{
     utils::rd_string,
 };
 
+pub use crate::tracing_init::initialize_tracing;
+
 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 pub const DEFAULT_NUM_WORKERS: usize = 3;
 pub const DEFAULT_TIMEOUT: i32 = 300;
 pub const DEFAULT_SLEEP_QUEUE: u64 = 50;
+pub const DEFAULT_MAX_CONNECTIONS: u32 = 100;
 
 pub async fn migrate_db(db: &DB) -> anyhow::Result<()> {
     let app_password = std::env::var("APP_USER_PASSWORD").unwrap_or_else(|_| "changeme".to_owned());
@@ -73,11 +80,13 @@ pub async fn migrate_db(db: &DB) -> anyhow::Result<()> {
 pub async fn connect_db() -> anyhow::Result<DB> {
     let database_url = std::env::var("DATABASE_URL")
         .map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
-    Ok(db::connect(&database_url).await?)
-}
 
-pub async fn initialize_tracing() -> anyhow::Result<()> {
-    tracing_init::initialize_tracing().await
+    let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
+        Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
+        Err(_) => DEFAULT_MAX_CONNECTIONS,
+    };
+
+    Ok(db::connect(&database_url, max_connections).await?)
 }
 
 struct BaseUrl(String);
@@ -190,14 +199,13 @@ pub async fn run_server(
     Ok(())
 }
 
-pub fn monitor_db(db: &DB, timeout: i32, tx: tokio::sync::broadcast::Sender<()>) {
+pub fn monitor_db(db: &DB, timeout: i32, rx: tokio::sync::broadcast::Receiver<()>) {
     let db1 = db.clone();
     let db2 = db.clone();
 
-    let rx1 = tx.subscribe();
-    let rx2 = tx.subscribe();
+    let rx2 = rx.resubscribe();
 
-    tokio::spawn(async move { worker::restart_zombie_jobs_periodically(&db1, timeout, rx1).await });
+    tokio::spawn(async move { worker::restart_zombie_jobs_periodically(&db1, timeout, rx).await });
     tokio::spawn(async move { users::delete_expired_items_perdiodically(&db2, rx2).await });
 }
 
@@ -210,32 +218,23 @@ pub async fn run_workers(
     base_url: String,
     disable_nuser: bool,
     disable_nsjail: bool,
-    tx: tokio::sync::broadcast::Sender<()>,
+    rx: tokio::sync::broadcast::Receiver<()>,
 ) -> anyhow::Result<()> {
     let instance_name = rd_string(5);
 
-    let mutex = Arc::new(Mutex::new(0));
-
-    let sources: external_ip::Sources = external_ip::get_http_sources();
-    let consensus = external_ip::ConsensusBuilder::new()
-        .add_sources(sources)
-        .build();
-
-    let ip = consensus
-        .get_consensus()
-        .await
-        .map(|x| x.to_string())
-        .unwrap_or_else(|| "Unretrievable ip".to_string());
+    let ip = external_ip::get_ip().await.unwrap_or_else(|e| {
+        tracing::warn!(error = e.to_string(), "failed to get external IP");
+        "unretrievable IP".to_string()
+    });
 
     let mut handles = Vec::new();
     for i in 1..(num_workers + 1) {
         let db1 = db.clone();
         let instance_name = instance_name.clone();
         let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
-        let m1 = mutex.clone();
         let ip = ip.clone();
-        let tx = tx.clone();
         let base_url = base_url.clone();
+        let rx = rx.resubscribe();
         handles.push(tokio::spawn(async move {
             tracing::info!(addr = %addr.to_string(), worker = %worker_name, "starting worker");
             worker::run_worker(
@@ -245,13 +244,12 @@ pub async fn run_workers(
                 worker_name,
                 i as u64,
                 num_workers as u64,
-                m1,
                 &ip,
                 sleep_queue,
                 &base_url,
                 disable_nuser,
                 disable_nsjail,
-                tx,
+                rx,
             )
             .await
         }));
@@ -287,3 +285,24 @@ pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::
     let _ = tx.send(());
     Ok(())
 }
+
+pub async fn serve_metrics(
+    addr: SocketAddr,
+    mut rx: tokio::sync::broadcast::Receiver<()>,
+) -> Result<(), hyper::Error> {
+    axum::Server::bind(&addr)
+        .serve(
+            Router::new()
+                .route("/metrics", get(metrics))
+                .into_make_service(),
+        )
+        .with_graceful_shutdown(rx.recv().map(drop))
+        .await
+}
+
+async fn metrics() -> Result<String, Error> {
+    let metric_families = prometheus::gather();
+    Ok(prometheus::TextEncoder::new()
+        .encode_to_string(&metric_families)
+        .map_err(anyhow::Error::from)?)
+}

backend/src/main.rs

@@ -14,7 +14,7 @@ use dotenv::dotenv;
 async fn main() -> anyhow::Result<()> {
     dotenv().ok();
 
-    windmill::initialize_tracing().await?;
+    windmill::initialize_tracing();
 
     let db = windmill::connect_db().await?;
 
@@ -23,6 +23,16 @@ async fn main() -> anyhow::Result<()> {
         .and_then(|x| x.parse::<i32>().ok())
         .unwrap_or(windmill::DEFAULT_NUM_WORKERS as i32);
 
+    let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
+        .ok()
+        .map(|s| {
+            s.parse::<bool>()
+                .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
+                .or_else(|_| s.parse::<SocketAddr>().map(Some))
+        })
+        .transpose()?
+        .flatten();
+
     let (server_mode, monitor_mode, migrate_db) = (true, true, true);
 
     if migrate_db {
@@ -30,7 +40,7 @@ async fn main() -> anyhow::Result<()> {
     }
 
     let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
-    let shutdown_signal = windmill::shutdown_signal(tx.clone());
+    let shutdown_signal = windmill::shutdown_signal(tx);
 
     if server_mode || monitor_mode || num_workers > 0 {
         let addr = SocketAddr::from(([0, 0, 0, 0], 8000));
@@ -51,7 +61,7 @@ async fn main() -> anyhow::Result<()> {
                         server: "smtp.gmail.com".to_string(),
                         password: std::env::var("SMTP_PASSWORD").unwrap_or("NOPASS".to_string()),
                     },
-                    rx,
+                    rx.resubscribe(),
                 )
                 .await?;
             }
@@ -90,7 +100,7 @@ async fn main() -> anyhow::Result<()> {
                     base_url,
                     disable_nuser,
                     disable_nsjail,
-                    tx.clone(),
+                    rx.resubscribe(),
                 )
                 .await?;
             }
@@ -99,12 +109,21 @@ async fn main() -> anyhow::Result<()> {
 
         let monitor_f = async {
             if monitor_mode {
-                windmill::monitor_db(&db, timeout, tx.clone());
+                windmill::monitor_db(&db, timeout, rx.resubscribe());
             }
             Ok(()) as anyhow::Result<()>
         };
 
-        futures::try_join!(shutdown_signal, server_f, workers_f, monitor_f)?;
+        let metrics_f = async {
+            match metrics_addr {
+                Some(addr) => windmill::serve_metrics(addr, rx.resubscribe())
+                    .await
+                    .map_err(anyhow::Error::from),
+                None => Ok(()),
+            }
+        };
+
+        futures::try_join!(shutdown_signal, server_f, workers_f, monitor_f, metrics_f)?;
     }
 
     Ok(())

backend/src/oauth2.rs

@@ -64,6 +64,7 @@ pub fn workspaced_service() -> Router {
 pub struct ClientWithScopes {
     client: OClient,
     scopes: Vec<String>,
+    extra_params: Option<HashMap<String, String>>,
 }
 
 pub type BasicClientsMap = HashMap<String, ClientWithScopes>;
@@ -73,6 +74,7 @@ pub struct OAuthConfig {
     auth_url: String,
     token_url: String,
     scopes: Option<Vec<String>>,
+    extra_params: Option<HashMap<String, String>>,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -106,7 +108,7 @@ pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
         match serde_json::from_str::<HashMap<String, OAuthClient>>(&content) {
             Ok(clients) => clients,
             Err(e) => {
-                tracing::error!("Error while deserializing oauth.json: {e}");
+                tracing::error!("deserializing oauth.json: {e}");
                 HashMap::new()
             }
         }
@@ -120,6 +122,7 @@ pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
         .filter(|x| oauths.contains_key(&x.0))
         .map(|(k, v)| {
             let scopes = v.scopes.clone();
+            let extra_params = v.extra_params.clone();
             let named_client = build_basic_client(
                 k.clone(),
                 v,
@@ -129,7 +132,11 @@ pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
             );
             (
                 named_client.0,
-                ClientWithScopes { client: named_client.1, scopes: scopes.unwrap_or(vec![]) },
+                ClientWithScopes {
+                    client: named_client.1,
+                    scopes: scopes.unwrap_or(vec![]),
+                    extra_params,
+                },
             )
         })
         .collect();
@@ -139,6 +146,8 @@ pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
         .filter(|x| oauths.contains_key(&x.0))
         .map(|(k, v)| {
             let scopes = v.scopes.clone();
+            let extra_params = v.extra_params.clone();
+
             let named_client = build_basic_client(
                 k.clone(),
                 v,
@@ -148,7 +157,11 @@ pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
             );
             (
                 named_client.0,
-                ClientWithScopes { client: named_client.1, scopes: scopes.unwrap_or(vec![]) },
+                ClientWithScopes {
+                    client: named_client.1,
+                    scopes: scopes.unwrap_or(vec![]),
+                    extra_params,
+                },
             )
         })
         .collect();
@@ -160,6 +173,7 @@ pub async fn build_oauth_clients(base_url: &str) -> anyhow::Result<AllClients> {
                 auth_url: "https://slack.com/oauth/authorize".to_string(),
                 token_url: "https://slack.com/api/oauth.access".to_string(),
                 scopes: None,
+                extra_params: None,
             },
             v.clone(),
             false,
@@ -178,19 +192,9 @@ pub fn build_basic_client(
     login: bool,
     base_url: &str,
 ) -> (String, OClient) {
-    let mut auth_url = Url::parse(&config.auth_url).expect("Invalid authorization endpoint URL");
+    let auth_url = Url::parse(&config.auth_url).expect("Invalid authorization endpoint URL");
     let token_url = Url::parse(&config.token_url).expect("Invalid token endpoint URL");
 
-    if ["gcal", "gdrive", "gsheets", "gcloud", "gmail"]
-        .into_iter()
-        .any(|x| x == name)
-    {
-        auth_url
-            .query_pairs_mut()
-            .append_pair("access_type", "offline")
-            .append_pair("prompt", "consent");
-    }
-
     let redirect_url = if login {
         format!("{base_url}/user/login_callback/{name}")
     } else {
@@ -234,23 +238,24 @@ pub struct SlackBotToken {
     bot_access_token: String,
 }
 
-#[derive(Deserialize)]
-struct ConnectScopes {
-    scopes: Option<String>,
-}
 async fn connect(
     Path(client_name): Path<String>,
-    Query(ConnectScopes { scopes }): Query<ConnectScopes>,
+    Query(query): Query<HashMap<String, String>>,
     Extension(clients): Extension<Arc<AllClients>>,
     cookies: Cookies,
 ) -> error::Result<Redirect> {
+    let mut query = query.clone();
     let connects = &clients.connects;
-    oauth_redirect(
-        connects,
-        client_name,
-        cookies,
-        scopes.map(|x| x.split('+').map(|x| x.to_owned()).collect()),
-    )
+    let scopes = query
+        .get("scopes")
+        .map(|x| x.split('+').map(|x| x.to_owned()).collect());
+    query.remove("scopes");
+    let extra_params = if query.is_empty() {
+        None
+    } else {
+        Some(query.clone())
+    };
+    oauth_redirect(connects, client_name, cookies, scopes, extra_params)
 }
 
 #[derive(Deserialize)]
@@ -279,7 +284,8 @@ async fn create_account(
         payload.refresh_token
     )
     .fetch_one(&mut tx)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("creating account in {w_id}: {e}")))?;
     tx.commit().await?;
     Ok(id.to_string())
 }
@@ -328,14 +334,27 @@ async fn list_logins(
     ))
 }
 
+#[derive(Serialize)]
+struct ScopesAndParams {
+    scopes: Vec<String>,
+    extra_params: Option<HashMap<String, String>>,
+}
 async fn list_connects(
     Extension(clients): Extension<Arc<AllClients>>,
-) -> error::JsonResult<HashMap<String, Vec<String>>> {
+) -> error::JsonResult<HashMap<String, ScopesAndParams>> {
     Ok(Json(
         (&clients.connects)
             .into_iter()
-            .map(|(k, v)| (k.to_owned(), v.scopes.clone()))
-            .collect::<HashMap<String, Vec<String>>>(),
+            .map(|(k, v)| {
+                (
+                    k.to_owned(),
+                    ScopesAndParams {
+                        scopes: v.scopes.clone(),
+                        extra_params: v.extra_params.clone(),
+                    },
+                )
+            })
+            .collect::<HashMap<String, ScopesAndParams>>(),
     ))
 }
 
@@ -402,7 +421,7 @@ async fn login(
     cookies: Cookies,
 ) -> error::Result<Redirect> {
     let clients = &clients.logins;
-    oauth_redirect(clients, client_name, cookies, None)
+    oauth_redirect(clients, client_name, cookies, None, None)
 }
 
 #[derive(Deserialize)]
@@ -894,6 +913,7 @@ fn oauth_redirect(
     client_name: String,
     cookies: Cookies,
     scopes: Option<Vec<String>>,
+    extra_params: Option<HashMap<String, String>>,
 ) -> error::Result<Redirect> {
     let client_w_scopes = clients
         .get(&client_name)
@@ -910,9 +930,25 @@ fn oauth_redirect(
         client.add_scope(scope);
     }
 
-    let url = client.authorize_url(&state);
+    let mut auth_url = client.authorize_url(&state);
+
+    if let Some(extra_params) = extra_params {
+        let mut query_string = auth_url.query_pairs_mut();
+        for (key, value) in extra_params {
+            query_string.append_pair(&key, &value);
+        }
+    }
+    // if ["gcal", "gdrive", "gsheets", "gcloud", "gmail"]
+    //     .into_iter()
+    //     .any(|x| x == name)
+    // {
+    //     auth_url
+    //         .query_pairs_mut()
+    //         .append_pair("access_type", "offline")
+    //         .append_pair("prompt", "consent");
+    // }
     set_cookie(&state, cookies);
-    Ok(Redirect::to(url.as_str()))
+    Ok(Redirect::to(auth_url.as_str()))
 }
 
 fn set_cookie(state: &State, cookies: Cookies) {

backend/src/parser.rs

@@ -6,19 +6,8 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
-use std::collections::HashMap;
-
-use itertools::Itertools;
-use regex::Regex;
 use serde::Serialize;
-use serde_json::json;
 
-use crate::error;
-
-use rustpython_parser::{
-    ast::{ExpressionType, Located, Number, StatementType, StringGroup, Varargs},
-    parser,
-};
 #[derive(Serialize)]
 pub struct MainArgSignature {
     pub star_args: bool,
@@ -28,27 +17,25 @@ pub struct MainArgSignature {
 
 #[derive(Serialize, Clone)]
 #[serde(rename_all(serialize = "lowercase"))]
-pub enum InnerTyp {
-    Str,
-    Int,
-    Float,
-    Bytes,
-    Email,
+pub struct ObjectProperty {
+    pub key: String,
+    pub typ: Box<Typ>,
 }
 
 #[derive(Serialize, Clone)]
 #[serde(rename_all(serialize = "lowercase"))]
 pub enum Typ {
-    Str,
+    Str(Option<Vec<String>>),
     Int,
     Float,
     Bool,
-    Dict,
-    List(InnerTyp),
+    List(Box<Typ>),
     Bytes,
     Datetime,
     Resource(String),
     Email,
+    Sql,
+    Object(Vec<ObjectProperty>),
     Unknown,
 }
 
@@ -59,713 +46,3 @@ pub struct Arg {
     pub default: Option<serde_json::Value>,
     pub has_default: bool,
 }
-
-pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
-    let ast = parser::parse_program(code)
-        .map_err(|e| error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())))?
-        .statements;
-    let param = ast.into_iter().find_map(|x| match x {
-        Located {
-            location: _,
-            node:
-                StatementType::FunctionDef {
-                    is_async: _,
-                    name,
-                    args,
-                    body: _,
-                    decorator_list: _,
-                    returns: _,
-                },
-        } if &name == "main" => Some(*args),
-        _ => None,
-    });
-    if let Some(params) = param {
-        //println!("{:?}", params);
-        let def_arg_start = params.args.len() - params.defaults.len();
-        Ok(MainArgSignature {
-            star_args: params.vararg != Varargs::None,
-            star_kwargs: params.vararg != Varargs::None,
-            args: params
-                .args
-                .into_iter()
-                .enumerate()
-                .map(|(i, x)| {
-                    let default = if i >= def_arg_start {
-                        to_value(&params.defaults[i - def_arg_start].node)
-                    } else {
-                        None
-                    };
-                    Arg {
-                        name: x.arg,
-                        typ: x.annotation.map_or(Typ::Unknown, |e| match *e {
-                            Located { location: _, node: ExpressionType::Identifier { name } } => {
-                                match name.as_ref() {
-                                    "str" => Typ::Str,
-                                    "float" => Typ::Float,
-                                    "int" => Typ::Int,
-                                    "bool" => Typ::Bool,
-                                    "dict" => Typ::Dict,
-                                    "list" => Typ::List(InnerTyp::Str),
-                                    "bytes" => Typ::Bytes,
-                                    "datetime" => Typ::Datetime,
-                                    "datetime.datetime" => Typ::Datetime,
-                                    _ => Typ::Unknown,
-                                }
-                            }
-                            _ => Typ::Unknown,
-                        }),
-                        has_default: default.is_some(),
-                        default,
-                    }
-                })
-                .collect(),
-        })
-    } else {
-        Err(error::Error::ExecutionErr(
-            "main function was not findable".to_string(),
-        ))
-    }
-}
-
-use swc_common::{sync::Lrc, FileName, SourceMap};
-use swc_ecma_ast::{
-    AssignPat, BindingIdent, Decl, ExportDecl, FnDecl, Ident, ModuleDecl, ModuleItem, Pat,
-    TsArrayType, TsEntityName, TsKeywordTypeKind, TsType, TsTypeRef,
-};
-use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsConfig};
-
-pub fn parse_deno_signature(code: &str) -> error::Result<MainArgSignature> {
-    let cm: Lrc<SourceMap> = Default::default();
-    let fm = cm.new_source_file(FileName::Custom("test.ts".into()), code.into());
-    let lexer = Lexer::new(
-        // We want to parse ecmascript
-        Syntax::Typescript(TsConfig::default()),
-        // EsVersion defaults to es5
-        Default::default(),
-        StringInput::from(&*fm),
-        None,
-    );
-
-    let mut parser = Parser::new_from(lexer);
-
-    let mut err_s = "".to_string();
-    for e in parser.take_errors() {
-        err_s += &e.into_kind().msg().to_string();
-    }
-
-    let ast = parser
-        .parse_module()
-        .map_err(|e| {
-            error::Error::ExecutionErr(format!("impossible to parse module: {err_s}\n{e:?}"))
-        })?
-        .body;
-
-    // println!("{ast:?}");
-    let params =
-        ast.into_iter().find_map(|x| match x {
-            ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl {
-                decl:
-                    Decl::Fn(FnDecl {
-                        ident: Ident { span: _, sym, optional: _ },
-                        declare: _,
-                        function,
-                    }),
-                span: _,
-            })) if &sym.to_string() == "main" => Some(function.params),
-            _ => None,
-        });
-    if let Some(params) = params {
-        Ok(MainArgSignature {
-            star_args: false,
-            star_kwargs: false,
-            args: params
-                .into_iter()
-                .map(|x| match x.pat {
-                    Pat::Ident(ident) => {
-                        let (name, typ) = binding_ident_to_arg(&ident)?;
-                        Ok(Arg { name, typ, default: None, has_default: ident.id.optional })
-                    }
-                    Pat::Assign(AssignPat { span: _, left, right, type_ann: _ }) => {
-                        let (name, typ) =
-                            left.as_ident().map(binding_ident_to_arg).ok_or_else(|| {
-                                error::Error::ExecutionErr(format!(
-                                    "Arg {left:?} has unexpected syntax"
-                                ))
-                            })??;
-                        Ok(Arg {
-                            name,
-                            typ,
-                            default: serde_json::to_value(right)
-                                .map_err(|e| error::Error::ExecutionErr(e.to_string()))?
-                                .as_object()
-                                .and_then(|x| x.get("value").to_owned())
-                                .cloned(),
-
-                            has_default: true,
-                        })
-                    }
-                    _ => Err(error::Error::ExecutionErr(format!(
-                        "Arg {x:?} has unexpected syntax"
-                    ))),
-                })
-                .collect::<Result<Vec<Arg>, error::Error>>()?,
-        })
-    } else {
-        Err(error::Error::ExecutionErr(
-            "main function was not findable (expected to find 'export main function(...)'"
-                .to_string(),
-        ))
-    }
-}
-
-fn binding_ident_to_arg(
-    BindingIdent { id, type_ann }: &BindingIdent,
-) -> anyhow::Result<(String, Typ)> {
-    Ok((
-        id.sym.to_string(),
-        type_ann
-            .as_ref()
-            .map(|x| {
-                match &*x.type_ann {
-                    TsType::TsKeywordType(t) => match t.kind {
-                        TsKeywordTypeKind::TsObjectKeyword => Typ::Dict,
-                        TsKeywordTypeKind::TsBooleanKeyword => Typ::Bool,
-                        TsKeywordTypeKind::TsBigIntKeyword => Typ::Int,
-                        TsKeywordTypeKind::TsNumberKeyword => Typ::Float,
-                        TsKeywordTypeKind::TsStringKeyword => Typ::Str,
-                        _ => Typ::Unknown,
-                    },
-                    // TODO: we can do better here and extract the inner type of array
-                    TsType::TsArrayType(TsArrayType { span: _, elem_type }) => {
-                        match &**elem_type {
-                            TsType::TsTypeRef(TsTypeRef {
-                                span: _,
-                                type_name: TsEntityName::Ident(Ident { span: _, sym, optional: _ }),
-                                type_params: _,
-                            }) => match sym.to_string().as_str() {
-                                "Base64" => Typ::List(InnerTyp::Bytes),
-                                "Email" => Typ::List(InnerTyp::Email),
-                                "bigint" => Typ::List(InnerTyp::Int),
-                                "number" => Typ::List(InnerTyp::Float),
-                                _ => Typ::List(InnerTyp::Str),
-                            },
-                            //TsType::TsKeywordType(())
-                            _ => Typ::List(InnerTyp::Str),
-                        }
-                    }
-                    TsType::TsTypeRef(TsTypeRef { span: _, type_name, type_params }) => {
-                        let sym = match type_name {
-                            TsEntityName::Ident(Ident { span: _, sym, optional: _ }) => sym,
-                            TsEntityName::TsQualifiedName(p) => &*p.right.sym,
-                        };
-                        match sym.to_string().as_str() {
-                            "Resource" => Typ::Resource(
-                                type_params
-                                    .as_ref()
-                                    .and_then(|x| {
-                                        x.params.get(0).and_then(|y| {
-                                            y.as_ts_lit_type().and_then(|z| {
-                                                z.lit
-                                                    .as_str()
-                                                    .map(|a| a.to_owned().value.to_string())
-                                            })
-                                        })
-                                    })
-                                    .unwrap_or_else(|| "unknown".to_string()),
-                            ),
-                            "Base64" => Typ::Bytes,
-                            "Email" => Typ::Email,
-                            _ => Typ::Unknown,
-                        }
-                    }
-                    _ => Typ::Unknown,
-                }
-            })
-            .unwrap_or(Typ::Unknown),
-    ))
-}
-
-const STDIMPORTS: [&str; 301] = [
-    "__future__",
-    "_abc",
-    "_aix_support",
-    "_ast",
-    "_asyncio",
-    "_bisect",
-    "_blake2",
-    "_bootsubprocess",
-    "_bz2",
-    "_codecs",
-    "_codecs_cn",
-    "_codecs_hk",
-    "_codecs_iso2022",
-    "_codecs_jp",
-    "_codecs_kr",
-    "_codecs_tw",
-    "_collections",
-    "_collections_abc",
-    "_compat_pickle",
-    "_compression",
-    "_contextvars",
-    "_crypt",
-    "_csv",
-    "_ctypes",
-    "_curses",
-    "_curses_panel",
-    "_datetime",
-    "_dbm",
-    "_decimal",
-    "_elementtree",
-    "_frozen_importlib",
-    "_frozen_importlib_external",
-    "_functools",
-    "_gdbm",
-    "_hashlib",
-    "_heapq",
-    "_imp",
-    "_io",
-    "_json",
-    "_locale",
-    "_lsprof",
-    "_lzma",
-    "_markupbase",
-    "_md5",
-    "_msi",
-    "_multibytecodec",
-    "_multiprocessing",
-    "_opcode",
-    "_operator",
-    "_osx_support",
-    "_overlapped",
-    "_pickle",
-    "_posixshmem",
-    "_posixsubprocess",
-    "_py_abc",
-    "_pydecimal",
-    "_pyio",
-    "_queue",
-    "_random",
-    "_sha1",
-    "_sha256",
-    "_sha3",
-    "_sha512",
-    "_signal",
-    "_sitebuiltins",
-    "_socket",
-    "_sqlite3",
-    "_sre",
-    "_ssl",
-    "_stat",
-    "_statistics",
-    "_string",
-    "_strptime",
-    "_struct",
-    "_symtable",
-    "_thread",
-    "_threading_local",
-    "_tkinter",
-    "_tracemalloc",
-    "_uuid",
-    "_warnings",
-    "_weakref",
-    "_weakrefset",
-    "_winapi",
-    "_zoneinfo",
-    "abc",
-    "aifc",
-    "antigravity",
-    "argparse",
-    "array",
-    "ast",
-    "asynchat",
-    "asyncio",
-    "asyncore",
-    "atexit",
-    "audioop",
-    "base64",
-    "bdb",
-    "binascii",
-    "binhex",
-    "bisect",
-    "builtins",
-    "bz2",
-    "cProfile",
-    "calendar",
-    "cgi",
-    "cgitb",
-    "chunk",
-    "cmath",
-    "cmd",
-    "code",
-    "codecs",
-    "codeop",
-    "collections",
-    "colorsys",
-    "compileall",
-    "concurrent",
-    "configparser",
-    "contextlib",
-    "contextvars",
-    "copy",
-    "copyreg",
-    "crypt",
-    "csv",
-    "ctypes",
-    "curses",
-    "dataclasses",
-    "datetime",
-    "dbm",
-    "decimal",
-    "difflib",
-    "dis",
-    "distutils",
-    "doctest",
-    "email",
-    "encodings",
-    "ensurepip",
-    "enum",
-    "errno",
-    "faulthandler",
-    "fcntl",
-    "filecmp",
-    "fileinput",
-    "fnmatch",
-    "fractions",
-    "ftplib",
-    "functools",
-    "gc",
-    "genericpath",
-    "getopt",
-    "getpass",
-    "gettext",
-    "glob",
-    "graphlib",
-    "grp",
-    "gzip",
-    "hashlib",
-    "heapq",
-    "hmac",
-    "html",
-    "http",
-    "idlelib",
-    "imaplib",
-    "imghdr",
-    "imp",
-    "importlib",
-    "inspect",
-    "io",
-    "ipaddress",
-    "itertools",
-    "json",
-    "keyword",
-    "lib2to3",
-    "linecache",
-    "locale",
-    "logging",
-    "lzma",
-    "mailbox",
-    "mailcap",
-    "marshal",
-    "math",
-    "mimetypes",
-    "mmap",
-    "modulefinder",
-    "msilib",
-    "msvcrt",
-    "multiprocessing",
-    "netrc",
-    "nis",
-    "nntplib",
-    "nt",
-    "ntpath",
-    "nturl2path",
-    "numbers",
-    "opcode",
-    "operator",
-    "optparse",
-    "os",
-    "ossaudiodev",
-    "pathlib",
-    "pdb",
-    "pickle",
-    "pickletools",
-    "pipes",
-    "pkgutil",
-    "platform",
-    "plistlib",
-    "poplib",
-    "posix",
-    "posixpath",
-    "pprint",
-    "profile",
-    "pstats",
-    "pty",
-    "pwd",
-    "py_compile",
-    "pyclbr",
-    "pydoc",
-    "pydoc_data",
-    "pyexpat",
-    "queue",
-    "quopri",
-    "random",
-    "re",
-    "readline",
-    "reprlib",
-    "resource",
-    "rlcompleter",
-    "runpy",
-    "sched",
-    "secrets",
-    "select",
-    "selectors",
-    "shelve",
-    "shlex",
-    "shutil",
-    "signal",
-    "site",
-    "smtpd",
-    "smtplib",
-    "sndhdr",
-    "socket",
-    "socketserver",
-    "spwd",
-    "sqlite3",
-    "sre_compile",
-    "sre_constants",
-    "sre_parse",
-    "ssl",
-    "stat",
-    "statistics",
-    "string",
-    "stringprep",
-    "struct",
-    "subprocess",
-    "sunau",
-    "symtable",
-    "sys",
-    "sysconfig",
-    "syslog",
-    "tabnanny",
-    "tarfile",
-    "telnetlib",
-    "tempfile",
-    "termios",
-    "textwrap",
-    "this",
-    "threading",
-    "time",
-    "timeit",
-    "tkinter",
-    "token",
-    "tokenize",
-    "trace",
-    "traceback",
-    "tracemalloc",
-    "tty",
-    "turtle",
-    "turtledemo",
-    "types",
-    "typing",
-    "unicodedata",
-    "unittest",
-    "urllib",
-    "uu",
-    "uuid",
-    "venv",
-    "warnings",
-    "wave",
-    "weakref",
-    "webbrowser",
-    "winreg",
-    "winsound",
-    "wsgiref",
-    "xdrlib",
-    "xml",
-    "xmlrpc",
-    "zipapp",
-    "zipfile",
-    "zipimport",
-    "",
-];
-
-fn to_value(et: &ExpressionType) -> Option<serde_json::Value> {
-    match et {
-        ExpressionType::String { value: StringGroup::Constant { value } } => Some(json!(value)),
-        ExpressionType::Number { value } => match value {
-            Number::Integer { value } => Some(json!(value.to_string().parse::<i64>().unwrap())),
-            Number::Float { value } => Some(json!(value)),
-            _ => None,
-        },
-        ExpressionType::True => Some(json!(true)),
-        ExpressionType::False => Some(json!(false)),
-
-        ExpressionType::Dict { elements } => {
-            let v = elements
-                .into_iter()
-                .map(|(k, v)| {
-                    let key = k
-                        .as_ref()
-                        .and_then(|x| to_value(&x.node))
-                        .and_then(|x| match x {
-                            serde_json::Value::String(s) => Some(s),
-                            _ => None,
-                        })
-                        .unwrap_or_else(|| "no_key".to_string());
-                    (key, to_value(&v.node))
-                })
-                .collect::<HashMap<String, _>>();
-            Some(json!(v))
-        }
-        ExpressionType::List { elements } => {
-            let v = elements
-                .into_iter()
-                .map(|x| to_value(&x.node))
-                .collect::<Vec<_>>();
-            Some(json!(v))
-        }
-        ExpressionType::None => Some(json!(null)),
-
-        ExpressionType::Call { function: _, args: _, keywords: _ } => {
-            Some(json!("<function call>"))
-        }
-
-        _ => None,
-    }
-}
-
-pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
-    let find_requirements = code
-        .lines()
-        .find_position(|x| x.starts_with("#requirements:"));
-    let re = Regex::new(r"^\#(\S+)$").unwrap();
-    if let Some((pos, _)) = find_requirements {
-        let lines = code
-            .lines()
-            .skip(pos + 1)
-            .map_while(|x| {
-                re.captures(x)
-                    .map(|x| x.get(1).unwrap().as_str().to_string())
-            })
-            .collect();
-        Ok(lines)
-    } else {
-        let ast = parser::parse_program(code)
-            .map_err(|e| {
-                error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
-            })?
-            .statements;
-        let imports = ast
-            .into_iter()
-            .filter_map(|x| match x {
-                Located { location: _, node } => match node {
-                    StatementType::Import { names } => Some(
-                        names
-                            .into_iter()
-                            .map(|x| x.symbol.split('.').next().unwrap_or("").to_string())
-                            .collect::<Vec<String>>(),
-                    ),
-                    StatementType::ImportFrom { level: _, module: Some(mod_), names: _ } => {
-                        Some(vec![mod_
-                            .split('.')
-                            .next()
-                            .unwrap_or("")
-                            .to_string()
-                            .replace("_", "-")])
-                    }
-                    _ => None,
-                },
-            })
-            .flatten()
-            .filter(|x| !STDIMPORTS.contains(&x.as_str()))
-            .unique()
-            .collect();
-        Ok(imports)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-
-    // Note this useful idiom: importing names from outer (for mod tests) scope.
-    use super::*;
-
-    #[test]
-    fn test_parse_python_sig() -> anyhow::Result<()> {
-        //let code = "print(2 + 3, fd=sys.stderr)";
-        let code = "
-
-import os
-
-def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = bytes(1)):
-
-	print(f\"Hello World and a warm welcome especially to {name}\")
-	print(\"The env variable at `all/pretty_secret`: \", os.environ.get(\"ALL_PRETTY_SECRET\"))
-	return {\"len\": len(name), \"splitted\": name.split() }
-
-";
-        println!("{}", serde_json::to_string(&parse_python_signature(code)?)?);
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_parse_python_imports() -> anyhow::Result<()> {
-        //let code = "print(2 + 3, fd=sys.stderr)";
-        let code = "
-
-import os
-import wmill
-from zanzibar.estonie import talin
-import matplotlib.pyplot as plt
-
-def main():
-    pass
-
-";
-        let r = parse_python_imports(code)?;
-        println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
-        Ok(())
-    }
-
-    #[test]
-    fn test_parse_python_imports2() -> anyhow::Result<()> {
-        //let code = "print(2 + 3, fd=sys.stderr)";
-        let code = "
-#requirements:
-#burkina=0.4
-#nigeria
-#
-#congo
-
-import os
-import wmill
-from zanzibar.estonie import talin
-
-def main():
-    pass
-
-";
-        let r = parse_python_imports(code)?;
-        println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
-
-        Ok(())
-    }
-
-    #[test]
-    fn test_parse_deno_sig() -> anyhow::Result<()> {
-        let code = "
-
-export function main(test1?: string, test2: string = \"burkina\",
-    test3: wmill.Resource<'postgres'>, b64: Base64, ls: Base64[], email: Email) {
-    console.log(42)
-}
-
-";
-        println!("{}", serde_json::to_string(&parse_deno_signature(code)?)?);
-
-        Ok(())
-    }
-}

backend/src/parser_py.rs

@@ -0,0 +1,571 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use std::collections::HashMap;
+
+use itertools::Itertools;
+use phf::phf_map;
+use regex::Regex;
+use serde_json::json;
+
+use crate::{
+    error,
+    parser::{Arg, MainArgSignature, Typ},
+};
+
+use rustpython_parser::{
+    ast::{ExpressionType, Located, Number, StatementType, StringGroup, Varargs},
+    parser,
+};
+
+pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
+    let ast = parser::parse_program(code)
+        .map_err(|e| error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())))?
+        .statements;
+    let param = ast.into_iter().find_map(|x| match x {
+        Located {
+            location: _,
+            node:
+                StatementType::FunctionDef {
+                    is_async: _,
+                    name,
+                    args,
+                    body: _,
+                    decorator_list: _,
+                    returns: _,
+                },
+        } if &name == "main" => Some(*args),
+        _ => None,
+    });
+    if let Some(params) = param {
+        //println!("{:?}", params);
+        let def_arg_start = params.args.len() - params.defaults.len();
+        Ok(MainArgSignature {
+            star_args: params.vararg != Varargs::None,
+            star_kwargs: params.vararg != Varargs::None,
+            args: params
+                .args
+                .into_iter()
+                .enumerate()
+                .map(|(i, x)| {
+                    let default = if i >= def_arg_start {
+                        to_value(&params.defaults[i - def_arg_start].node)
+                    } else {
+                        None
+                    };
+                    Arg {
+                        name: x.arg,
+                        typ: x.annotation.map_or(Typ::Unknown, |e| match *e {
+                            Located { location: _, node: ExpressionType::Identifier { name } } => {
+                                match name.as_ref() {
+                                    "str" => Typ::Str(None),
+                                    "float" => Typ::Float,
+                                    "int" => Typ::Int,
+                                    "bool" => Typ::Bool,
+                                    "dict" => Typ::Object(vec![]),
+                                    "list" => Typ::List(Box::new(Typ::Str(None))),
+                                    "bytes" => Typ::Bytes,
+                                    "datetime" => Typ::Datetime,
+                                    "datetime.datetime" => Typ::Datetime,
+                                    _ => Typ::Unknown,
+                                }
+                            }
+                            _ => Typ::Unknown,
+                        }),
+                        has_default: default.is_some(),
+                        default,
+                    }
+                })
+                .collect(),
+        })
+    } else {
+        Err(error::Error::ExecutionErr(
+            "main function was not findable".to_string(),
+        ))
+    }
+}
+
+fn to_value(et: &ExpressionType) -> Option<serde_json::Value> {
+    match et {
+        ExpressionType::String { value: StringGroup::Constant { value } } => Some(json!(value)),
+        ExpressionType::Number { value } => match value {
+            Number::Integer { value } => Some(json!(value.to_string().parse::<i64>().unwrap())),
+            Number::Float { value } => Some(json!(value)),
+            _ => None,
+        },
+        ExpressionType::True => Some(json!(true)),
+        ExpressionType::False => Some(json!(false)),
+
+        ExpressionType::Dict { elements } => {
+            let v = elements
+                .into_iter()
+                .map(|(k, v)| {
+                    let key = k
+                        .as_ref()
+                        .and_then(|x| to_value(&x.node))
+                        .and_then(|x| match x {
+                            serde_json::Value::String(s) => Some(s),
+                            _ => None,
+                        })
+                        .unwrap_or_else(|| "no_key".to_string());
+                    (key, to_value(&v.node))
+                })
+                .collect::<HashMap<String, _>>();
+            Some(json!(v))
+        }
+        ExpressionType::List { elements } => {
+            let v = elements
+                .into_iter()
+                .map(|x| to_value(&x.node))
+                .collect::<Vec<_>>();
+            Some(json!(v))
+        }
+        ExpressionType::None => Some(json!(null)),
+
+        ExpressionType::Call { function: _, args: _, keywords: _ } => {
+            Some(json!("<function call>"))
+        }
+
+        _ => None,
+    }
+}
+
+static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_map! {
+    "psycopg2" => "psycopg2-binary"
+};
+
+fn replace_import(x: String) -> String {
+    PYTHON_IMPORTS_REPLACEMENT
+        .get(&x)
+        .map(|x| x.to_owned())
+        .unwrap_or(&x)
+        .to_string()
+}
+
+pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
+    let find_requirements = code
+        .lines()
+        .find_position(|x| x.starts_with("#requirements:"));
+    let re = Regex::new(r"^\#(\S+)$").unwrap();
+    if let Some((pos, _)) = find_requirements {
+        let lines = code
+            .lines()
+            .skip(pos + 1)
+            .map_while(|x| {
+                re.captures(x)
+                    .map(|x| x.get(1).unwrap().as_str().to_string())
+            })
+            .collect();
+        Ok(lines)
+    } else {
+        let ast = parser::parse_program(code)
+            .map_err(|e| {
+                error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
+            })?
+            .statements;
+        let imports = ast
+            .into_iter()
+            .filter_map(|x| match x {
+                Located { location: _, node } => match node {
+                    StatementType::Import { names } => Some(
+                        names
+                            .into_iter()
+                            .map(|x| x.symbol.split('.').next().unwrap_or("").to_string())
+                            .map(replace_import)
+                            .collect::<Vec<String>>(),
+                    ),
+                    StatementType::ImportFrom { level: _, module: Some(mod_), names: _ } => {
+                        let imprt = mod_.split('.').next().unwrap_or("").replace("_", "-");
+
+                        Some(vec![replace_import(imprt)])
+                    }
+                    _ => None,
+                },
+            })
+            .flatten()
+            .filter(|x| !STDIMPORTS.contains(&x.as_str()))
+            .unique()
+            .collect();
+        Ok(imports)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    // Note this useful idiom: importing names from outer (for mod tests) scope.
+    use super::*;
+
+    #[test]
+    fn test_parse_python_sig() -> anyhow::Result<()> {
+        //let code = "print(2 + 3, fd=sys.stderr)";
+        let code = "
+
+import os
+
+def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = bytes(1)):
+
+	print(f\"Hello World and a warm welcome especially to {name}\")
+	print(\"The env variable at `all/pretty_secret`: \", os.environ.get(\"ALL_PRETTY_SECRET\"))
+	return {\"len\": len(name), \"splitted\": name.split() }
+
+";
+        println!("{}", serde_json::to_string(&parse_python_signature(code)?)?);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_python_imports() -> anyhow::Result<()> {
+        //let code = "print(2 + 3, fd=sys.stderr)";
+        let code = "
+
+import os
+import wmill
+from zanzibar.estonie import talin
+import matplotlib.pyplot as plt
+
+def main():
+    pass
+
+";
+        let r = parse_python_imports(code)?;
+        println!("{}", serde_json::to_string(&r)?);
+        assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
+        Ok(())
+    }
+
+    #[test]
+    fn test_parse_python_imports2() -> anyhow::Result<()> {
+        //let code = "print(2 + 3, fd=sys.stderr)";
+        let code = "
+#requirements:
+#burkina=0.4
+#nigeria
+#
+#congo
+
+import os
+import wmill
+from zanzibar.estonie import talin
+
+def main():
+    pass
+
+";
+        let r = parse_python_imports(code)?;
+        println!("{}", serde_json::to_string(&r)?);
+        assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
+
+        Ok(())
+    }
+}
+
+const STDIMPORTS: [&str; 301] = [
+    "__future__",
+    "_abc",
+    "_aix_support",
+    "_ast",
+    "_asyncio",
+    "_bisect",
+    "_blake2",
+    "_bootsubprocess",
+    "_bz2",
+    "_codecs",
+    "_codecs_cn",
+    "_codecs_hk",
+    "_codecs_iso2022",
+    "_codecs_jp",
+    "_codecs_kr",
+    "_codecs_tw",
+    "_collections",
+    "_collections_abc",
+    "_compat_pickle",
+    "_compression",
+    "_contextvars",
+    "_crypt",
+    "_csv",
+    "_ctypes",
+    "_curses",
+    "_curses_panel",
+    "_datetime",
+    "_dbm",
+    "_decimal",
+    "_elementtree",
+    "_frozen_importlib",
+    "_frozen_importlib_external",
+    "_functools",
+    "_gdbm",
+    "_hashlib",
+    "_heapq",
+    "_imp",
+    "_io",
+    "_json",
+    "_locale",
+    "_lsprof",
+    "_lzma",
+    "_markupbase",
+    "_md5",
+    "_msi",
+    "_multibytecodec",
+    "_multiprocessing",
+    "_opcode",
+    "_operator",
+    "_osx_support",
+    "_overlapped",
+    "_pickle",
+    "_posixshmem",
+    "_posixsubprocess",
+    "_py_abc",
+    "_pydecimal",
+    "_pyio",
+    "_queue",
+    "_random",
+    "_sha1",
+    "_sha256",
+    "_sha3",
+    "_sha512",
+    "_signal",
+    "_sitebuiltins",
+    "_socket",
+    "_sqlite3",
+    "_sre",
+    "_ssl",
+    "_stat",
+    "_statistics",
+    "_string",
+    "_strptime",
+    "_struct",
+    "_symtable",
+    "_thread",
+    "_threading_local",
+    "_tkinter",
+    "_tracemalloc",
+    "_uuid",
+    "_warnings",
+    "_weakref",
+    "_weakrefset",
+    "_winapi",
+    "_zoneinfo",
+    "abc",
+    "aifc",
+    "antigravity",
+    "argparse",
+    "array",
+    "ast",
+    "asynchat",
+    "asyncio",
+    "asyncore",
+    "atexit",
+    "audioop",
+    "base64",
+    "bdb",
+    "binascii",
+    "binhex",
+    "bisect",
+    "builtins",
+    "bz2",
+    "cProfile",
+    "calendar",
+    "cgi",
+    "cgitb",
+    "chunk",
+    "cmath",
+    "cmd",
+    "code",
+    "codecs",
+    "codeop",
+    "collections",
+    "colorsys",
+    "compileall",
+    "concurrent",
+    "configparser",
+    "contextlib",
+    "contextvars",
+    "copy",
+    "copyreg",
+    "crypt",
+    "csv",
+    "ctypes",
+    "curses",
+    "dataclasses",
+    "datetime",
+    "dbm",
+    "decimal",
+    "difflib",
+    "dis",
+    "distutils",
+    "doctest",
+    "email",
+    "encodings",
+    "ensurepip",
+    "enum",
+    "errno",
+    "faulthandler",
+    "fcntl",
+    "filecmp",
+    "fileinput",
+    "fnmatch",
+    "fractions",
+    "ftplib",
+    "functools",
+    "gc",
+    "genericpath",
+    "getopt",
+    "getpass",
+    "gettext",
+    "glob",
+    "graphlib",
+    "grp",
+    "gzip",
+    "hashlib",
+    "heapq",
+    "hmac",
+    "html",
+    "http",
+    "idlelib",
+    "imaplib",
+    "imghdr",
+    "imp",
+    "importlib",
+    "inspect",
+    "io",
+    "ipaddress",
+    "itertools",
+    "json",
+    "keyword",
+    "lib2to3",
+    "linecache",
+    "locale",
+    "logging",
+    "lzma",
+    "mailbox",
+    "mailcap",
+    "marshal",
+    "math",
+    "mimetypes",
+    "mmap",
+    "modulefinder",
+    "msilib",
+    "msvcrt",
+    "multiprocessing",
+    "netrc",
+    "nis",
+    "nntplib",
+    "nt",
+    "ntpath",
+    "nturl2path",
+    "numbers",
+    "opcode",
+    "operator",
+    "optparse",
+    "os",
+    "ossaudiodev",
+    "pathlib",
+    "pdb",
+    "pickle",
+    "pickletools",
+    "pipes",
+    "pkgutil",
+    "platform",
+    "plistlib",
+    "poplib",
+    "posix",
+    "posixpath",
+    "pprint",
+    "profile",
+    "pstats",
+    "pty",
+    "pwd",
+    "py_compile",
+    "pyclbr",
+    "pydoc",
+    "pydoc_data",
+    "pyexpat",
+    "queue",
+    "quopri",
+    "random",
+    "re",
+    "readline",
+    "reprlib",
+    "resource",
+    "rlcompleter",
+    "runpy",
+    "sched",
+    "secrets",
+    "select",
+    "selectors",
+    "shelve",
+    "shlex",
+    "shutil",
+    "signal",
+    "site",
+    "smtpd",
+    "smtplib",
+    "sndhdr",
+    "socket",
+    "socketserver",
+    "spwd",
+    "sqlite3",
+    "sre_compile",
+    "sre_constants",
+    "sre_parse",
+    "ssl",
+    "stat",
+    "statistics",
+    "string",
+    "stringprep",
+    "struct",
+    "subprocess",
+    "sunau",
+    "symtable",
+    "sys",
+    "sysconfig",
+    "syslog",
+    "tabnanny",
+    "tarfile",
+    "telnetlib",
+    "tempfile",
+    "termios",
+    "textwrap",
+    "this",
+    "threading",
+    "time",
+    "timeit",
+    "tkinter",
+    "token",
+    "tokenize",
+    "trace",
+    "traceback",
+    "tracemalloc",
+    "tty",
+    "turtle",
+    "turtledemo",
+    "types",
+    "typing",
+    "unicodedata",
+    "unittest",
+    "urllib",
+    "uu",
+    "uuid",
+    "venv",
+    "warnings",
+    "wave",
+    "weakref",
+    "webbrowser",
+    "winreg",
+    "winsound",
+    "wsgiref",
+    "xdrlib",
+    "xml",
+    "xmlrpc",
+    "zipapp",
+    "zipfile",
+    "zipimport",
+    "",
+];

backend/src/parser_ts.rs

@@ -0,0 +1,253 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use crate::{
+    error,
+    parser::{Arg, MainArgSignature, ObjectProperty, Typ},
+};
+
+use swc_common::{sync::Lrc, FileName, SourceMap};
+use swc_ecma_ast::{
+    AssignPat, BindingIdent, Decl, ExportDecl, Expr, FnDecl, Ident, ModuleDecl, ModuleItem, Pat,
+    Str, TsArrayType, TsEntityName, TsKeywordType, TsKeywordTypeKind, TsLit, TsLitType,
+    TsOptionalType, TsPropertySignature, TsType, TsTypeElement, TsTypeLit, TsTypeRef,
+    TsUnionOrIntersectionType, TsUnionType,
+};
+use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsConfig};
+
+pub fn parse_deno_signature(code: &str) -> error::Result<MainArgSignature> {
+    let cm: Lrc<SourceMap> = Default::default();
+    let fm = cm.new_source_file(FileName::Custom("test.ts".into()), code.into());
+    let lexer = Lexer::new(
+        // We want to parse ecmascript
+        Syntax::Typescript(TsConfig::default()),
+        // EsVersion defaults to es5
+        Default::default(),
+        StringInput::from(&*fm),
+        None,
+    );
+
+    let mut parser = Parser::new_from(lexer);
+
+    let mut err_s = "".to_string();
+    for e in parser.take_errors() {
+        err_s += &e.into_kind().msg().to_string();
+    }
+
+    let ast = parser
+        .parse_module()
+        .map_err(|_| {
+            error::Error::ExecutionErr(format!(
+                "Error while parsing code, it is invalid typescript"
+            ))
+        })?
+        .body;
+
+    // println!("{ast:?}");
+    let params = ast.into_iter().find_map(|x| match x {
+        ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl {
+            decl: Decl::Fn(FnDecl { ident: Ident { sym, .. }, function, .. }),
+            ..
+        })) if &sym.to_string() == "main" => Some(function.params),
+        _ => None,
+    });
+    if let Some(params) = params {
+        Ok(MainArgSignature {
+            star_args: false,
+            star_kwargs: false,
+            args: params
+                .into_iter()
+                .map(|x| match x.pat {
+                    Pat::Ident(ident) => {
+                        let (name, typ, nullable) = binding_ident_to_arg(&ident);
+                        Ok(Arg {
+                            name,
+                            typ,
+                            default: None,
+                            has_default: ident.id.optional || nullable,
+                        })
+                    }
+                    Pat::Assign(AssignPat { left, right, .. }) => {
+                        let (name, typ, _nullable) =
+                            left.as_ident().map(binding_ident_to_arg).ok_or_else(|| {
+                                error::Error::ExecutionErr(format!(
+                                    "Arg {left:?} has unexpected syntax"
+                                ))
+                            })?;
+                        Ok(Arg {
+                            name,
+                            typ,
+                            default: serde_json::to_value(right)
+                                .map_err(|e| error::Error::ExecutionErr(e.to_string()))?
+                                .as_object()
+                                .and_then(|x| x.get("value").to_owned())
+                                .cloned(),
+
+                            has_default: true,
+                        })
+                    }
+                    _ => Err(error::Error::ExecutionErr(format!(
+                        "Arg {x:?} has unexpected syntax"
+                    ))),
+                })
+                .collect::<Result<Vec<Arg>, error::Error>>()?,
+        })
+    } else {
+        Err(error::Error::ExecutionErr(
+            "main function was not findable (expected to find 'export main function(...)'"
+                .to_string(),
+        ))
+    }
+}
+
+fn binding_ident_to_arg(BindingIdent { id, type_ann }: &BindingIdent) -> (String, Typ, bool) {
+    let (typ, nullable) = type_ann
+        .as_ref()
+        .map(|x| tstype_to_typ(&*x.type_ann))
+        .unwrap_or((Typ::Unknown, false));
+    (id.sym.to_string(), typ, nullable)
+}
+
+fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
+    //println!("{:?}", ts_type);
+    match ts_type {
+        TsType::TsKeywordType(t) => (
+            match t.kind {
+                TsKeywordTypeKind::TsObjectKeyword => Typ::Object(vec![]),
+                TsKeywordTypeKind::TsBooleanKeyword => Typ::Bool,
+                TsKeywordTypeKind::TsBigIntKeyword => Typ::Int,
+                TsKeywordTypeKind::TsNumberKeyword => Typ::Float,
+                TsKeywordTypeKind::TsStringKeyword => Typ::Str(None),
+                _ => Typ::Unknown,
+            },
+            false,
+        ),
+        TsType::TsTypeLit(TsTypeLit { members, .. }) => {
+            let properties = members
+                .into_iter()
+                .filter_map(|x| match x {
+                    TsTypeElement::TsPropertySignature(TsPropertySignature {
+                        key,
+                        type_ann,
+                        ..
+                    }) => match (*key.to_owned(), type_ann) {
+                        (Expr::Ident(Ident { sym, .. }), type_ann) => Some(ObjectProperty {
+                            key: sym.to_string(),
+                            typ: type_ann
+                                .as_ref()
+                                .map(|typ| Box::new(tstype_to_typ(&*typ.type_ann).0))
+                                .unwrap_or(Box::new(Typ::Unknown)),
+                        }),
+                        _ => None,
+                    },
+                    _ => None,
+                })
+                .collect();
+            (Typ::Object(properties), false)
+        }
+        // TODO: we can do better here and extract the inner type of array
+        TsType::TsArrayType(TsArrayType { elem_type, .. }) => {
+            (Typ::List(Box::new(tstype_to_typ(&**elem_type).0)), false)
+        }
+        TsType::TsLitType(TsLitType { lit: TsLit::Str(Str { value, .. }), .. }) => {
+            (Typ::Str(Some(vec![value.to_string()])), false)
+        }
+        TsType::TsOptionalType(TsOptionalType { type_ann, .. }) => {
+            (tstype_to_typ(type_ann).0, true)
+        }
+        TsType::TsUnionOrIntersectionType(TsUnionOrIntersectionType::TsUnionType(
+            TsUnionType { types, .. },
+        )) => {
+            if let Some(p) = if types.len() != 2 {
+                None
+            } else {
+                types.into_iter().position(|x| match **x {
+                    TsType::TsKeywordType(TsKeywordType { kind, .. }) => {
+                        kind == TsKeywordTypeKind::TsUndefinedKeyword
+                            || kind == TsKeywordTypeKind::TsNullKeyword
+                    }
+                    _ => false,
+                })
+            } {
+                let other_p = if p == 0 { 1 } else { 0 };
+                (tstype_to_typ(&types[other_p]).0, true)
+            } else {
+                let literals = types
+                    .into_iter()
+                    .map(|x| match &**x {
+                        TsType::TsLitType(TsLitType {
+                            lit: TsLit::Str(Str { value, .. }), ..
+                        }) => Some(value.to_string()),
+                        _ => None,
+                    })
+                    .collect::<Vec<_>>();
+                if literals.iter().find(|x| x.is_none()).is_some() {
+                    (Typ::Unknown, false)
+                } else {
+                    (
+                        Typ::Str(Some(literals.into_iter().filter_map(|x| x).collect())),
+                        false,
+                    )
+                }
+            }
+        }
+        TsType::TsTypeRef(TsTypeRef { type_name, type_params, .. }) => {
+            let sym = match type_name {
+                TsEntityName::Ident(Ident { sym, .. }) => sym,
+                TsEntityName::TsQualifiedName(p) => &*p.right.sym,
+            };
+            match sym.to_string().as_str() {
+                "Resource" => (
+                    Typ::Resource(
+                        type_params
+                            .as_ref()
+                            .and_then(|x| {
+                                x.params.get(0).and_then(|y| {
+                                    y.as_ts_lit_type().and_then(|z| {
+                                        z.lit.as_str().map(|a| a.to_owned().value.to_string())
+                                    })
+                                })
+                            })
+                            .unwrap_or_else(|| "unknown".to_string()),
+                    ),
+                    false,
+                ),
+                "Base64" => (Typ::Bytes, false),
+                "Email" => (Typ::Email, false),
+                "Sql" => (Typ::Sql, false),
+                _ => (Typ::Unknown, false),
+            }
+        }
+        _ => (Typ::Unknown, false),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    // Note this useful idiom: importing names from outer (for mod tests) scope.
+    use super::*;
+
+    #[test]
+    fn test_parse_deno_sig() -> anyhow::Result<()> {
+        let code = "
+
+export function main(test1?: string, test2: string = \"burkina\",
+    test3: wmill.Resource<'postgres'>, b64: Base64, ls: Base64[], 
+    email: Email, literal: \"test\", literal_union: \"test\" | \"test2\",
+    opt_type?: string | null, opt_type_union: string | null, opt_type_union_union2: string | undefined,
+    min_object: {a: string, b: number}) {
+    console.log(42)
+}
+
+";
+        println!("{}", serde_json::to_string(&parse_deno_signature(code)?)?);
+
+        Ok(())
+    }
+}

backend/src/schedule.rs

@@ -11,7 +11,7 @@ use std::str::FromStr;
 use crate::{
     audit::{audit_log, ActionKind},
     db::{UserDB, DB},
-    error::{self, JsonResult, Result},
+    error::{self, Error, JsonResult, Result},
     jobs::{self, push, JobPayload},
     users::Authed,
     utils::{get_owner_from_path, Pagination, StripPath},
@@ -146,10 +146,11 @@ async fn create_schedule(
         ns.script_path,
         ns.is_flow,
         ns.args,
-        ns.enabled
+        ns.enabled.unwrap_or(false),
     )
     .fetch_one(&mut tx)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("inserting schedule in {w_id}: {e}")))?;
 
     audit_log(
         &mut tx,
@@ -249,7 +250,8 @@ async fn edit_schedule(
         w_id,
     )
     .fetch_one(&mut tx)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("updating schedule in {w_id}: {e}")))?;
 
     if schedule.enabled {
         tx = push_scheduled_job(tx, schedule).await?;

backend/src/scripts.rs

@@ -14,7 +14,7 @@ use crate::{
     audit::{audit_log, ActionKind},
     db::{UserDB, DB},
     error::{to_anyhow, Error, JsonResult, Result},
-    jobs, parser,
+    jobs, parser, parser_py, parser_ts,
     users::{owner_to_token_owner, truncate_token, Authed, Tokened},
     utils::{http_get_from_hub, list_elems_from_hub, require_admin, Pagination, StripPath},
 };
@@ -67,6 +67,16 @@ pub enum ScriptLang {
     Deno,
     Python3,
 }
+
+impl ScriptLang {
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            ScriptLang::Deno => "deno",
+            ScriptLang::Python3 => "python3",
+        }
+    }
+}
+
 #[derive(sqlx::Type, PartialEq, Debug, Hash, Clone, Copy)]
 #[sqlx(transparent)]
 pub struct ScriptHash(pub i64);
@@ -416,7 +426,7 @@ async fn create_script(
     .await?;
 
     let mut tx = if ns.lock.is_none() && ns.language == ScriptLang::Python3 {
-        let dependencies = parser::parse_python_imports(&ns.content)?;
+        let dependencies = parser_py::parse_python_imports(&ns.content)?;
         let (_, tx) = jobs::push(
             tx,
             &w_id,
@@ -619,7 +629,8 @@ async fn archive_script_by_path(
         &w_id
     )
     .fetch_one(&db)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("archiving script in {w_id}: {e}")))?;
     audit_log(
         &mut tx,
         &authed.username,
@@ -647,7 +658,9 @@ async fn archive_script_by_hash(
     )
     .bind(&hash.0)
     .fetch_one(&mut tx)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("archiving script in {w_id}: {e}")))?;
+
     audit_log(
         &mut tx,
         &authed.username,
@@ -679,7 +692,9 @@ async fn delete_script_by_hash(
     .bind(&hash.0)
     .bind(&w_id)
     .fetch_one(&db)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("deleting script by hash {w_id}: {e}")))?;
+
     audit_log(
         &mut tx,
         &authed.username,
@@ -698,13 +713,13 @@ async fn delete_script_by_hash(
 async fn parse_python_code_to_jsonschema(
     Json(code): Json<String>,
 ) -> JsonResult<parser::MainArgSignature> {
-    parser::parse_python_signature(&code).map(Json)
+    parser_py::parse_python_signature(&code).map(Json)
 }
 
 async fn parse_deno_code_to_jsonschema(
     Json(code): Json<String>,
 ) -> JsonResult<parser::MainArgSignature> {
-    parser::parse_deno_signature(&code).map(Json)
+    parser_ts::parse_deno_signature(&code).map(Json)
 }
 
 pub fn to_i64(s: &str) -> Result<i64> {

backend/src/static_assets.rs

@@ -12,6 +12,7 @@ use axum::{
     response::IntoResponse,
 };
 
+use mime_guess::mime;
 use rust_embed::RustEmbed;
 
 // static_handler is a handler that serves static files from the
@@ -46,12 +47,22 @@ fn serve_path(path: String) -> Response<BoxBody> {
         Some(content) => {
             let body = body::boxed(body::Full::from(content.data));
             let mime = mime_guess::from_path(path).first_or_octet_stream();
-            Response::builder()
-                .header(header::CONTENT_TYPE, mime.as_ref())
-                .header(header::CACHE_CONTROL, "max-age=3600".to_owned())
-                .body(body)
-                .unwrap()
+            let mut res = Response::builder().header(header::CONTENT_TYPE, mime.as_ref());
+            if mime.as_ref() == mime::APPLICATION_JAVASCRIPT {
+                res = res.header(header::CACHE_CONTROL, "max-age=31536000");
+            } else if (mime.type_(), mime.subtype()) == (mime::TEXT, mime::CSS) {
+                res = res.header(header::CACHE_CONTROL, "max-age=31536000");
+            } else if (mime.type_()) == (mime::IMAGE) {
+                res = res.header(header::CACHE_CONTROL, "max-age=31536000");
+            } else {
+                res = res.header(header::CACHE_CONTROL, "no-cache, no-store, must-revalidate");
+            }
+            res.body(body).unwrap()
         }
+        None if path.as_str().starts_with("_app/") => Response::builder()
+            .status(404)
+            .body(body::boxed(body::Empty::new()))
+            .unwrap(),
         None => serve_path("200.html".to_owned()),
     }
 }

backend/src/tracing_init.rs

@@ -58,7 +58,7 @@ fn filter_metadata(meta: &Metadata) -> bool {
     meta.target().starts_with("windmill")
 }
 
-pub async fn initialize_tracing() -> anyhow::Result<()> {
+pub fn initialize_tracing() {
     let tokio_console = std::env::var("TOKIO_CONSOLE")
         .map(|x| x == "true")
         .unwrap_or(false);
@@ -70,8 +70,8 @@ pub async fn initialize_tracing() -> anyhow::Result<()> {
 
     let nenv_filter = if tokio_console {
         env_filter
-            .add_directive("runtime=trace".parse()?)
-            .add_directive("tokio=trace".parse()?)
+            .add_directive("runtime=trace".parse().unwrap())
+            .add_directive("tokio=trace".parse().unwrap())
     } else {
         env_filter
     };
@@ -93,5 +93,4 @@ pub async fn initialize_tracing() -> anyhow::Result<()> {
             .with(compact_layer().with_filter(filter_fn(filter_metadata)))
             .init(),
     }
-    Ok(())
 }

backend/src/users.rs

@@ -653,7 +653,9 @@ async fn global_whoami(
         email
     )
     .fetch_one(&db)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("fetching global identity: {e}")))?;
+
     Ok(Json(user))
 }
 
@@ -1037,7 +1039,8 @@ async fn set_password(
         &email
     )
     .fetch_one(&mut tx)
-    .await?
+    .await
+    .map_err(|e| Error::InternalErr(format!("setting password: {e}")))?
     .unwrap_or("".to_string());
 
     if custom_type != "password".to_string() {
@@ -1422,8 +1425,10 @@ async fn delete_token(
         error::Error::BadRequest(format!("Only users with email can create tokens"))
     })?;
     let tokens_deleted: Vec<String> = sqlx::query_scalar(
-        "DELETE FROM token WHERE email = $1 AND
-     token LIKE concat($3, '%') RETURNING concat(substring(token for 10), '*****')",
+        "DELETE FROM token
+               WHERE email = $1
+                 AND token LIKE concat($2::text, '%')
+           RETURNING concat(substring(token for 10), '*****')",
     )
     .bind(&email)
     .bind(&token_prefix)

backend/src/utils.rs

@@ -43,7 +43,8 @@ pub async fn require_super_admin<'c>(
         email.as_ref()
     )
     .fetch_one(db)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("fetching super admin: {e}")))?;
     if !is_admin {
         Err(Error::NotAuthorized(
             "This endpoint require caller to be a super admin".to_owned(),

backend/src/variables.rs

@@ -442,7 +442,9 @@ pub async fn build_crypt<'c>(
         w_id
     )
     .fetch_one(db)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("fetching crypt key: {e}")))?;
+
     Ok(magic_crypt::new_magic_crypt!(key, 256))
 }
 

backend/src/worker.rs

@@ -19,12 +19,13 @@ use std::{
 
 use crate::{
     db::DB,
-    error::Error,
+    error::{self, Error},
     jobs::{
-        add_completed_job, add_completed_job_error, postprocess_queued_job, pull, JobKind,
-        QueuedJob,
+        add_completed_job, add_completed_job_error, get_queued_job, postprocess_queued_job, pull,
+        JobKind, QueuedJob,
     },
-    parser::{self, Typ},
+    parser::Typ,
+    parser_py,
     scripts::{ScriptHash, ScriptLang},
     users::{create_token_for_owner, get_email_from_username},
     variables,
@@ -39,12 +40,11 @@ use tokio::{
     fs::{DirBuilder, File},
     io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader},
     process::{Child, Command},
-    sync::Mutex,
+    sync::mpsc,
     time::Instant,
 };
 
 use async_recursion::async_recursion;
-use tokio::sync::mpsc;
 
 const TMP_DIR: &str = "/tmp/windmill";
 const PIP_CACHE_DIR: &str = "/tmp/windmill/cache/pip";
@@ -56,6 +56,10 @@ const NSJAIL_CONFIG_DOWNLOAD_CONTENT: &str = include_str!("../../nsjail/download
 const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str =
     include_str!("../../nsjail/run.python3.config.proto");
 const NSJAIL_CONFIG_RUN_DENO_CONTENT: &str = include_str!("../../nsjail/run.deno.config.proto");
+const MAX_LOG_SIZE: u32 = 200000;
+pub struct Metrics {
+    pub jobs_failed: prometheus::IntCounter,
+}
 
 pub async fn run_worker(
     db: &DB,
@@ -64,13 +68,12 @@ pub async fn run_worker(
     worker_name: String,
     i_worker: u64,
     num_workers: u64,
-    _mutex: Arc<Mutex<i32>>,
     ip: &str,
     sleep_queue: u64,
     base_url: &str,
     disable_nuser: bool,
     disable_nsjail: bool,
-    tx: tokio::sync::broadcast::Sender<()>,
+    mut rx: tokio::sync::broadcast::Receiver<()>,
 ) {
     let worker_dir = format!("{TMP_DIR}/{worker_name}");
     tracing::debug!(worker_dir = %worker_dir, worker_name = %worker_name, "Creating worker dir");
@@ -89,8 +92,39 @@ pub async fn run_worker(
 
     insert_initial_ping(worker_instance, &worker_name, ip, db).await;
 
+    prometheus::register_int_gauge!(prometheus::Opts::new(
+        "start_time_seconds",
+        "Start time of worker as seconds since unix epoch",
+    )
+    .const_label("name", &worker_name))
+    .expect("register prometheus metric")
+    .set(
+        std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .ok()
+            .map(|duration| duration.as_secs() as i64)
+            .unwrap_or(0),
+    );
+
+    let job_duration_seconds = prometheus::register_histogram_vec!(
+        prometheus::HistogramOpts::new(
+            "job_duration_seconds",
+            "Duration between receiving a job and completing it",
+        )
+        .const_label("name", &worker_name),
+        &["workspace_id", "language"],
+    )
+    .expect("register prometheus metric");
+
+    let jobs_failed = prometheus::register_int_counter_vec!(
+        prometheus::Opts::new("jobs_failed", "Number of failed jobs",)
+            .const_label("name", &worker_name),
+        &["workspace_id", "language"],
+    )
+    .expect("register prometheus metric");
+
     let mut jobs_executed = 0;
-    let mut rx = tx.subscribe();
+
     loop {
         if last_ping.elapsed().as_secs() > NUM_SECS_ENV_CHECK {
             sqlx::query!(
@@ -108,12 +142,24 @@ pub async fn run_worker(
 
         match pull(db).await {
             Ok(Some(job)) => {
+                let label_values = [
+                    &job.workspace_id,
+                    job.language.as_ref().map(|l| l.as_str()).unwrap_or(""),
+                ];
+
+                let _timer = job_duration_seconds
+                    .with_label_values(label_values.as_slice())
+                    .start_timer();
+
                 jobs_executed += 1;
 
+                let metrics =
+                    Metrics { jobs_failed: jobs_failed.with_label_values(label_values.as_slice()) };
+
                 tracing::info!(worker = %worker_name, id = %job.id, "Fetched job");
-                let job2 = job.clone();
+
                 if let Some(err) = handle_queued_job(
-                    job,
+                    job.clone(),
                     db,
                     timeout,
                     &worker_name,
@@ -121,36 +167,70 @@ pub async fn run_worker(
                     base_url,
                     disable_nuser,
                     disable_nsjail,
+                    &metrics,
                 )
                 .await
                 .err()
                 {
-                    let err_string = err.to_string().clone();
-                    let _ = add_completed_job_error(
+                    let m = add_completed_job_error(
                         db,
-                        &job2,
+                        &job,
                         "Unexpected error during job execution:\n".to_string(),
-                        err,
+                        &err,
+                        &metrics,
+                    )
+                    .await
+                    .map(|(_, m)| m)
+                    .unwrap_or_else(|_| Map::new());
+
+                    let _ = postprocess_queued_job(
+                        job.is_flow_step,
+                        job.schedule_path.clone(),
+                        job.script_path.clone(),
+                        &job.workspace_id,
+                        job.id,
+                        db,
                     )
                     .await;
 
-                    {
-                        let job = job2.clone();
-                        let _ = postprocess_queued_job(
-                            job.schedule_path,
-                            job.script_path,
-                            &job2.workspace_id,
-                            job2.id,
+                    if let Some(parent_job_id) = job.parent_job {
+                        let updated_flow = update_flow_status_after_job_completion(
                             db,
+                            &job,
+                            false,
+                            serde_json::Value::Object(m),
+                            &metrics,
                         )
                         .await;
-                    }
+                        if let Err(err) = updated_flow {
+                            if let Ok(mut tx) = db.begin().await {
+                                if let Ok(Some(parent_job)) =
+                                    get_queued_job(parent_job_id, &job.workspace_id, &mut tx).await
+                                {
+                                    let _ = add_completed_job_error(
+                                        db,
+                                        &parent_job,
+                                        format!("Unexpected error during flow job error handling:\n{err}")
+                                            ,
+                                        err,
+                                        &metrics,
+                                    )
+                                    .await;
 
-                    if job2.parent_job.is_some() {
-                        let _ =
-                            update_flow_status_after_job_completion(db, &job2, false, None).await;
+                                    let _ = postprocess_queued_job(
+                                        parent_job.is_flow_step,
+                                        parent_job.schedule_path.clone(),
+                                        parent_job.script_path.clone(),
+                                        &job.workspace_id,
+                                        parent_job.id,
+                                        db,
+                                    )
+                                    .await;
+                                }
+                            }
+                        }
                     }
-                    tracing::error!(job_id = %job2.id, "Error handling job: {err_string}");
+                    tracing::error!(job_id = %job.id, "Error handling job: {err}");
                 };
             }
             Ok(None) => (),
@@ -190,17 +270,19 @@ async fn handle_queued_job(
     base_url: &str,
     disable_nuser: bool,
     disable_nsjail: bool,
+    metrics: &Metrics,
 ) -> crate::error::Result<()> {
     let job_id = job.id;
     let w_id = &job.workspace_id.clone();
 
     match job.job_kind {
         JobKind::FlowPreview | JobKind::Flow => {
-            let args = match &job.args {
-                Some(serde_json::Value::Object(m)) => Some(m.to_owned()),
-                _ => None,
-            };
-            handle_flow(&job, db, args).await?;
+            handle_flow(
+                &job,
+                db,
+                job.args.clone().unwrap_or_else(|| serde_json::Value::Null),
+            )
+            .await?;
         }
         _ => {
             let mut logs = "".to_string();
@@ -233,31 +315,41 @@ async fn handle_queued_job(
 
             match execution {
                 Ok(r) => {
-                    add_completed_job(db, &job, true, false, r.result.clone(), logs).await?;
+                    add_completed_job(db, &job, true, false, r.clone(), logs).await?;
                     if job.is_flow_step {
-                        update_flow_status_after_job_completion(db, &job, true, r.result).await?;
+                        update_flow_status_after_job_completion(db, &job, true, r, metrics).await?;
                     }
                 }
                 Err(e) => {
-                    let (_, output_map) = add_completed_job_error(db, &job, logs, e).await?;
+                    let (_, output_map) =
+                        add_completed_job_error(db, &job, logs, e, &metrics).await?;
                     if job.is_flow_step {
-                        update_flow_status_after_job_completion(db, &job, false, Some(output_map))
-                            .await?;
+                        update_flow_status_after_job_completion(
+                            db,
+                            &job,
+                            false,
+                            serde_json::Value::Object(output_map),
+                            metrics,
+                        )
+                        .await?;
                     }
                 }
             };
 
-            let _ =
-                postprocess_queued_job(job.schedule_path, job.script_path, &w_id, job_id, db).await;
+            let _ = postprocess_queued_job(
+                job.is_flow_step,
+                job.schedule_path,
+                job.script_path,
+                &w_id,
+                job_id,
+                db,
+            )
+            .await;
         }
     }
     Ok(())
 }
 
-struct JobResult {
-    result: Option<Map<String, Value>>,
-}
-
 async fn write_file(dir: &str, path: &str, content: &str) -> Result<File, Error> {
     let path = format!("{}/{}", dir, path);
     let mut file = File::create(&path).await?;
@@ -266,34 +358,42 @@ async fn write_file(dir: &str, path: &str, content: &str) -> Result<File, Error>
 }
 
 #[async_recursion]
-async fn transform_json_value(token: &str, workspace: &str, base_url: &str, v: Value) -> Value {
+async fn transform_json_value(
+    token: &str,
+    workspace: &str,
+    base_url: &str,
+    v: Value,
+) -> error::Result<Value> {
     match v {
         Value::String(y) if y.starts_with("$var:") => {
             let path = y.strip_prefix("$var:").unwrap();
-            let v = crate::client::get_variable(workspace, path, token, base_url)
-                .await
-                .unwrap_or_else(|_| format!("error fetching variable {path}"));
-            Value::String(v)
+            let v = crate::client::get_variable(workspace, path, token, base_url).await?;
+            Ok(Value::String(v))
         }
         Value::String(y) if y.starts_with("$res:") => {
             let path = y.strip_prefix("$res:").unwrap();
             if path.split("/").count() < 2 {
-                return Value::String(format!("resource path: {path} is ill-defined"));
+                return Err(Error::InternalErr(
+                    format!("invalid resource path: {path}",),
+                ));
             }
             let v = crate::client::get_resource(workspace, path, token, base_url)
-                .await
-                .ok()
-                .flatten()
-                .unwrap_or_else(|| Value::String(format!("error fetching resource {path}")));
+                .await?
+                .ok_or_else(|| {
+                    error::Error::InternalErr(format!("resource path: {path} not found",))
+                })?;
             transform_json_value(token, workspace, base_url, v).await
         }
         Value::Object(mut m) => {
             for (a, b) in m.clone().into_iter() {
-                m.insert(a, transform_json_value(token, workspace, base_url, b).await);
+                m.insert(
+                    a,
+                    transform_json_value(token, workspace, base_url, b).await?,
+                );
             }
-            Value::Object(m)
+            Ok(Value::Object(m))
         }
-        a @ _ => a,
+        a @ _ => Ok(a),
     }
 }
 
@@ -309,10 +409,11 @@ async fn handle_job(
     base_url: &str,
     disable_nuser: bool,
     disable_nsjail: bool,
-) -> Result<JobResult, Error> {
+) -> Result<serde_json::Value, Error> {
     tracing::info!(
         worker = %worker_name,
         job_id = %job.id,
+        workspace_id = %job.workspace_id,
         "handling job"
     );
 
@@ -348,14 +449,14 @@ async fn handle_job(
     tokio::fs::remove_dir_all(job_dir).await?;
 
     if status.is_ok() && status.as_ref().unwrap().success() {
-        let result = serde_json::from_str::<Map<String, Value>>(last_line).map_err(|e| {
+        let result = serde_json::from_str::<serde_json::Value>(last_line).map_err(|e| {
             Error::ExecutionErr(format!(
                 "result {} is not parsable.\n err: {}",
                 last_line,
                 e.to_string()
             ))
         })?;
-        Ok(JobResult { result: Some(result) })
+        Ok(result)
     } else {
         let err = match status {
             Ok(_) => {
@@ -397,7 +498,7 @@ async fn handle_nondep_job(
             .map(|x| matches!(x, ScriptLang::Python3))
             .unwrap_or(false)
         {
-            Some(parser::parse_python_imports(&code)?.join("\n"))
+            Some(parser_py::parse_python_imports(&code)?.join("\n"))
         } else {
             None
         };
@@ -413,6 +514,7 @@ async fn handle_nondep_job(
         .await?
         .ok_or_else(|| Error::InternalErr(format!("expected content and lock")))?
     };
+    let worker_name = worker_dir.split("/").last().unwrap_or("unknown");
     match language {
         None => {
             return Err(Error::ExecutionErr(
@@ -437,6 +539,12 @@ async fn handle_nondep_job(
             }
             let _ = write_file(job_dir, "requirements.txt", &requirements).await?;
 
+            tracing::info!(
+                worker_name = %worker_name,
+                job_id = %job.id,
+                workspace_id = %job.workspace_id,
+                "started setup python dependencies"
+            );
             let child = if !disable_nsjail {
                 Command::new("nsjail")
                     .current_dir(job_dir)
@@ -467,7 +575,13 @@ async fn handle_nondep_job(
 
             logs.push_str("\n--- PIP DEPENDENCIES INSTALL ---\n");
             *status = handle_child(job, db, logs, last_line, timeout, child).await;
-
+            tracing::info!(
+                worker_name = %worker_name,
+                job_id = %job.id,
+                workspace_id = %job.workspace_id,
+                is_ok = status.is_ok(),
+                "finished setup python dependencies"
+            );
             if status.is_ok() {
                 logs.push_str("\n\n--- PYTHON CODE EXECUTION ---\n");
 
@@ -475,7 +589,7 @@ async fn handle_nondep_job(
 
                 let _ = write_file(job_dir, "inner.py", &inner_content).await?;
 
-                let sig = crate::parser::parse_python_signature(&inner_content)?;
+                let sig = crate::parser_py::parse_python_signature(&inner_content)?;
                 let transforms = sig
                     .args
                     .into_iter()
@@ -519,7 +633,7 @@ async fn handle_nondep_job(
                 let args = if let Some(args) = &job.args {
                     Some(
                         transform_json_value(&token, &job.workspace_id, base_url, args.clone())
-                            .await,
+                            .await?,
                     )
                 } else {
                     None
@@ -538,17 +652,11 @@ inner_script = __import__("inner")
 
 with open("args.json") as f:
     kwargs = json.load(f, strict=False)
-for k, v in kwargs.items():
+for k, v in list(kwargs.items()):
     if v == '<function call>':
-        kwargs[k] = None
+        del kwargs[k]
 {transforms}
 res = inner_script.main(**kwargs)
-if res is None:
-    res = {{}}
-if isinstance(res, tuple):
-    res = {{f"res{{i+1}}": v for i, v in enumerate(res)}}
-if not isinstance(res, dict):
-    res = {{ "res1": res }}
 res_json = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '')
 print()
 print("result:")
@@ -573,6 +681,12 @@ print(res_json)
                         .insert("PYTHONPATH".to_string(), format!("{job_dir}/dependencies"));
                 }
 
+                tracing::info!(
+                    worker_name = %worker_name,
+                    job_id = %job.id,
+                    workspace_id = %job.workspace_id,
+                    "started python code execution"
+                );
                 let child = if !disable_nuser {
                     Command::new("nsjail")
                         .current_dir(job_dir)
@@ -600,6 +714,13 @@ print(res_json)
                         .spawn()?
                 };
                 *status = handle_child(job, db, logs, last_line, timeout, child).await;
+                tracing::info!(
+                    worker_name = %worker_name,
+                    job_id = %job.id,
+                    workspace_id = %job.workspace_id,
+                    is_ok = status.is_ok(),
+                    "finished python code execution"
+                );
             }
         }
         Some(ScriptLang::Deno) => {
@@ -609,7 +730,7 @@ print(res_json)
 
             let _ = write_file(job_dir, "inner.ts", &inner_content).await?;
 
-            let sig = crate::parser::parse_deno_signature(&inner_content)?;
+            let sig = crate::parser_ts::parse_deno_signature(&inner_content)?;
             //             let transforms = sig.args.clone().into_iter().map(|x| match x.typ {
             //     Typ::Bytes => format!("if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n    kwargs[\"{}\"] = base64.b64decode(kwargs[\"{}\"])\n", x.name, x.name, x.name, x.name),
             //     Typ::Datetime => format!("if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n    kwargs[\"{}\"] = datetime.strptime(kwargs[\"{}\"], '%Y-%m-%dT%H:%M')\n", x.name, x.name, x.name, x.name),
@@ -633,7 +754,7 @@ print(res_json)
             .await?;
 
             let args = if let Some(args) = &job.args {
-                Some(transform_json_value(&token, &job.workspace_id, base_url, args.clone()).await)
+                Some(transform_json_value(&token, &job.workspace_id, base_url, args.clone()).await?)
             } else {
                 None
             };
@@ -652,14 +773,7 @@ const args = await Deno.readTextFile("args.json")
 
 async function run() {{
     let res: any = await main(...args);
-    if (res == undefined) {{
-        res = {{}}
-    }}
-    if (typeof res !== 'object' || Array.isArray(res)) {{
-        res = {{ res1: res }}
-    }}
-
-    const res_json = JSON.stringify(res);
+    const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value);
     console.log();
     console.log("result:");
     console.log(res_json);
@@ -687,6 +801,12 @@ run();
                 .await?;
             }
 
+            tracing::info!(
+                worker_name = %worker_name,
+                job_id = %job.id,
+                workspace_id = %job.workspace_id,
+                "started deno code execution"
+            );
             let child = if !disable_nsjail {
                 Command::new("nsjail")
                     .current_dir(job_dir)
@@ -723,6 +843,13 @@ run();
                     .spawn()?
             };
             *status = handle_child(job, db, logs, last_line, timeout, child).await;
+            tracing::info!(
+                worker_name = %worker_name,
+                job_id = %job.id,
+                workspace_id = %job.workspace_id,
+                is_ok = status.is_ok(),
+                "finished deno code execution"
+            );
         }
     }
     Ok(())
@@ -845,7 +972,7 @@ async fn handle_child(
 
     let done2 = done.clone();
     let done3 = done.clone();
-
+    let done4 = done.clone();
     // Ensure the child process is spawned in the runtime so it can
     // make progress on its own while we await for any output.
     let handle = tokio::spawn(async move {
@@ -856,15 +983,12 @@ async fn handle_child(
                 Ok(r?)
             }
             _ = async move {
-                loop {
-                    if done2.load(Ordering::Relaxed) {
-                        break;
-                    }
-                    tokio::time::sleep(Duration::from_secs(5)).await;
+                while !done2.load(Ordering::Relaxed) {
+                    tokio::time::sleep(Duration::from_secs(1)).await;
                 }
             } => {
                 child.kill().await?;
-                return Err(Error::ExecutionErr("execution interrupted (likely timeout or cancel)".to_string()).into())
+                return Err(Error::ExecutionErr("execution interrupted".to_string()).into())
             }
         };
         r
@@ -874,12 +998,28 @@ async fn handle_child(
     let id = job.id;
 
     tokio::spawn(async move {
-        loop {
+        while !done4.load(Ordering::Relaxed) {
             let send = tokio::select! {
                 Ok(Some(out)) = reader.next_line() => {
-                    tx.send(out).await
+                    if out.len() > MAX_LOG_SIZE as usize {
+                        tracing::info!("Line is too big");
+                        let _ = tx.send(format!("Line is too big")).await;
+                        done4.store(true, Ordering::Relaxed);
+                        break;
+                    } else {
+                        tx.send(out).await
+                    }
+                },
+                Ok(Some(err)) = stderr_reader.next_line() => {
+                    if err.len() > MAX_LOG_SIZE as usize {
+                        tracing::info!("Line is too big");
+                        let _ = tx.send(format!("Line is too big")).await;
+                        done4.store(true, Ordering::Relaxed);
+                        break;
+                    } else {
+                        tx.send(err).await
+                    }
                 },
-                Ok(Some(err)) = stderr_reader.next_line() => tx.send(err).await,
                 else => {
                     break
                 },
@@ -911,10 +1051,18 @@ async fn handle_child(
     });
 
     let mut start = logs.chars().count();
+    let mut last_update = chrono::Utc::now().timestamp_millis();
 
-    loop {
+    while !done.load(Ordering::Relaxed) {
+        let diff = 500 - (chrono::Utc::now().timestamp_millis() - last_update);
+        let sleeping_future = if diff > 0 as i64 {
+            tokio::time::sleep(Duration::from_millis(diff as u64))
+        } else {
+            //TODO make it just resolve immediately
+            tokio::time::sleep(Duration::from_millis(0))
+        };
         tokio::select! {
-            _ = tokio::time::sleep(Duration::from_millis(500)) => {
+            _ = sleeping_future => {
                 let end = logs.chars().count();
 
                 let to_send = logs.chars().skip(start).collect::<String>();
@@ -927,9 +1075,10 @@ async fn handle_child(
                 let canceled = sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", id)
                     .fetch_one(db)
                     .await
-                    .map_err(|_| tracing::error!("error getting canceled for id {}", id));
+                    .map_err(|e| tracing::error!("error getting canceled for id {}: {e}", id))
+                    .unwrap_or(false);
 
-                if canceled.unwrap_or(false) {
+                if canceled {
                     tracing::info!("killed after cancel: {}", job.id);
                     done.store(true, Ordering::Relaxed);
                 }
@@ -953,11 +1102,20 @@ async fn handle_child(
                         tracing::error!("error setting canceled for id {}", id);
                     }
                 }
+                last_update = chrono::Utc::now().timestamp_millis();
             },
             nl = rx.recv() => {
+
                 if let Some(nl) = nl {
+                    if logs.chars().count() > MAX_LOG_SIZE as usize{
+                        tracing::info!("Too many logs lines: {}", job.id);
+                        logs.push_str("Too many logs lines. Limit is 200000 chars. Killing job.");
+                        done.store(true, Ordering::Relaxed);
+                    }
+
                     logs.push('\n');
                     logs.push_str(&nl);
+
                     *last_line = nl;
                 } else {
                     let to_send = logs.chars().skip(start).collect::<String>();
@@ -1008,17 +1166,17 @@ pub async fn restart_zombie_jobs_periodically(
     mut rx: tokio::sync::broadcast::Receiver<()>,
 ) {
     loop {
-        let restarted = sqlx::query_scalar!(
-            "UPDATE queue SET running = false WHERE last_ping < $1 RETURNING id",
-            chrono::Utc::now() - chrono::Duration::seconds(timeout as i64 * 2)
+        let restarted = sqlx::query!(
+            "UPDATE queue SET running = false WHERE last_ping < $1 and running = true RETURNING id, workspace_id",
+            chrono::Utc::now() - chrono::Duration::seconds(timeout as i64 * 5)
         )
         .fetch_all(db)
         .await
         .ok()
         .unwrap_or_else(|| vec![]);
 
-        if restarted.len() > 0 {
-            tracing::info!("restarted zombie jobs {restarted:?}");
+        for r in restarted {
+            tracing::info!("restarted zombie job {} {}", r.id, r.workspace_id);
         }
 
         tokio::select! {
@@ -1030,3 +1188,260 @@ pub async fn restart_zombie_jobs_periodically(
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use sqlx::{postgres::PgListener, query_scalar};
+
+    use crate::{
+        db::DB,
+        flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
+        jobs::{push, JobPayload, RawCode},
+        scripts::ScriptLang,
+    };
+
+    use super::*;
+
+    async fn initialize_tracing() {
+        use std::sync::Once;
+
+        static ONCE: Once = Once::new();
+        ONCE.call_once(|| crate::tracing_init::initialize_tracing());
+    }
+
+    /// it's important this is unique between tests as there is one prometheus registry and
+    /// run_worker shouldn't register the same metric with the same worker name more than once.
+    fn next_worker_name() -> String {
+        use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
+
+        static ID: AtomicUsize = AtomicUsize::new(0);
+
+        // n.b.: when tests are run with RUST_TEST_THREADS or --test-threads set to 1, the name
+        // will be "main"... The id provides uniqueness & thread_name gives context.
+        let id = ID.fetch_add(1, SeqCst);
+        let thread = std::thread::current();
+        let thread_name = thread.name().unwrap_or("no thread name");
+        format!("{id}/{thread_name}")
+    }
+
+    #[sqlx::test(fixtures("base"))]
+    async fn test_deno_flow(db: DB) {
+        initialize_tracing().await;
+
+        let numbers = "export function main() { return [1, 2, 3]; }";
+        let doubles = "export function main(n) { return n * 2; }";
+
+        let flow = FlowValue {
+            modules: vec![
+                FlowModule {
+                    value: FlowModuleValue::RawScript(RawCode {
+                        language: ScriptLang::Deno,
+                        content: numbers.to_string(),
+                        path: None,
+                    }),
+                    input_transform: Default::default(),
+                    stop_after_if_expr: Default::default(),
+                    skip_if_stopped: Default::default(),
+                    summary: Default::default(),
+                },
+                FlowModule {
+                    value: FlowModuleValue::ForloopFlow {
+                        iterator: InputTransform::Javascript { expr: "result".to_string() },
+                        skip_failures: false,
+                        value: FlowValue {
+                            modules: vec![FlowModule {
+                                value: FlowModuleValue::RawScript(RawCode {
+                                    language: ScriptLang::Deno,
+                                    content: doubles.to_string(),
+                                    path: None,
+                                }),
+                                input_transform: [(
+                                    "n".to_string(),
+                                    InputTransform::Javascript {
+                                        expr: "previous_result.iter.value".to_string(),
+                                    },
+                                )]
+                                .into(),
+                                stop_after_if_expr: Default::default(),
+                                skip_if_stopped: Default::default(),
+                                summary: Default::default(),
+                            }],
+                            failure_module: Default::default(),
+                        }
+                        .into(),
+                    },
+                    input_transform: Default::default(),
+                    stop_after_if_expr: Default::default(),
+                    skip_if_stopped: Default::default(),
+                    summary: Default::default(),
+                },
+            ],
+            failure_module: Default::default(),
+        };
+
+        let job = JobPayload::RawFlow { value: flow, path: None };
+
+        let result = run_job_in_new_worker_until_complete(&db, job).await;
+
+        assert_eq!(result, serde_json::json!([2, 4, 6]));
+    }
+
+    #[sqlx::test(fixtures("base"))]
+    async fn test_python_flow(db: DB) {
+        initialize_tracing().await;
+
+        let numbers = "def main(): return [1, 2, 3]";
+        let doubles = "def main(n): return n * 2";
+
+        let flow: FlowValue = serde_json::from_value(serde_json::json!( {
+            "input_transform": {},
+            "modules": [
+                {
+                    "value": {
+                        "type": "rawscript",
+                        "language": "python3",
+                        "content": numbers,
+                    },
+                },
+                {
+                    "value": {
+                        "type": "forloopflow",
+                        "iterator": { "type": "javascript", "expr": "result" },
+                        "skip_failures": false,
+                        "value": {
+                            "modules": [{
+                                "value": {
+                                    "type": "rawscript",
+                                    "language": "python3",
+                                    "content": doubles,
+                                },
+                                "input_transform": {
+                                    "n": {
+                                        "type": "javascript",
+                                        "expr": "previous_result.iter.value",
+                                    },
+                                },
+                            }],
+                        }
+                    },
+                },
+            ],
+        }))
+        .unwrap();
+
+        let result = run_job_in_new_worker_until_complete(
+            &db,
+            JobPayload::RawFlow { value: flow, path: None },
+        )
+        .await;
+
+        assert_eq!(result, serde_json::json!([2, 4, 6]));
+    }
+
+    #[sqlx::test(fixtures("base"))]
+    async fn test_python_job(db: DB) {
+        initialize_tracing().await;
+
+        let content = r#"
+def main():
+    return "hello world"
+        "#
+        .to_owned();
+
+        let job = JobPayload::Code(RawCode { content, path: None, language: ScriptLang::Python3 });
+
+        let result = run_job_in_new_worker_until_complete(&db, job).await;
+
+        assert_eq!(result, serde_json::json!("hello world"));
+    }
+
+    async fn run_job_in_new_worker_until_complete(db: &DB, job: JobPayload) -> serde_json::Value {
+        let (uuid, tx) = push(
+            db.begin().await.unwrap(),
+            "test-workspace",
+            job,
+            /* TODO None doesn't work for args for python because the setup transformer doesn't
+             * handle it  */
+            /* args */
+            Some(Default::default()),
+            /* user */ "test-user",
+            /* permissioned_as */ Default::default(),
+            /* scheduled_for_o */ None,
+            /* schedule_path */ None,
+            /* parent_job */ None,
+            /* is_flow_step */ false,
+        )
+        .await
+        .unwrap();
+
+        tx.commit().await.unwrap();
+
+        run_worker_until_complete(db, &uuid).await;
+
+        query_scalar("SELECT result FROM completed_job WHERE id = $1")
+            .bind(&uuid)
+            .fetch_one(db)
+            .await
+            .unwrap()
+    }
+
+    async fn run_worker_until_complete(db: &DB, wait_for: &uuid::Uuid) {
+        let mut listener = PgListener::connect_with(db).await.unwrap();
+        listener.listen("insert on completed_job").await.unwrap();
+
+        let (tx, rx) = tokio::sync::broadcast::channel(1);
+        /* drop tx at the end of this block to close the channel and stop the worker */
+
+        let worker = {
+            let timeout = 4_000;
+            let worker_instance: &str = "test worker instance";
+            let worker_name: String = next_worker_name();
+            let i_worker: u64 = Default::default();
+            let num_workers: u64 = Default::default();
+            let ip: &str = Default::default();
+            let sleep_queue: u64 = Default::default();
+            let base_url: &str = Default::default();
+            let disable_nuser = false;
+            let disable_nsjail = false;
+
+            run_worker(
+                &db,
+                timeout,
+                worker_instance,
+                worker_name,
+                i_worker,
+                num_workers,
+                ip,
+                sleep_queue,
+                base_url,
+                disable_nuser,
+                disable_nsjail,
+                rx,
+            )
+        };
+
+        let worker = tokio::time::timeout(std::time::Duration::from_secs(19), worker);
+
+        tokio::pin!(worker);
+
+        while wait_for
+            != &tokio::select! {
+                biased;
+                notify = listener.recv() => notify,
+                res = &mut worker => if res.is_ok() {
+                    panic!("worker quit early")
+                } else {
+                    panic!("worker timed out")
+                },
+            }
+            .unwrap()
+            .payload()
+            .parse::<uuid::Uuid>()
+            .unwrap()
+        {}
+
+        /* ensure the worker quits before we return */
+        drop(tx);
+        worker.await.expect("worker timed out")
+    }
+}

backend/src/worker_flow.rs

@@ -10,6 +10,7 @@ use crate::{
     },
     js_eval::{eval_timeout, EvalCreds},
     users::create_token_for_owner,
+    worker,
 };
 use async_recursion::async_recursion;
 use serde::{Deserialize, Serialize};
@@ -48,7 +49,8 @@ pub async fn update_flow_status_after_job_completion(
     db: &DB,
     job: &QueuedJob,
     success: bool,
-    result: Option<Map<String, Value>>,
+    result: serde_json::Value,
+    metrics: &worker::Metrics,
 ) -> error::Result<()> {
     tracing::debug!("HANDLE FLOW: {job:?} {success} {result:?}");
 
@@ -66,7 +68,12 @@ pub async fn update_flow_status_after_job_completion(
         w_id
     )
     .fetch_one(&mut tx)
-    .await?
+    .await
+    .map_err(|e| {
+        Error::InternalErr(format!(
+            "fetching flow status {flow} while reporting {success} {result:?}: {e}"
+        ))
+    })?
     .ok_or_else(|| Error::InternalErr(format!("requiring a previous status")))?;
 
     let old_status = serde_json::from_value::<FlowStatus>(old_status_json)
@@ -125,18 +132,13 @@ pub async fn update_flow_status_after_job_completion(
         .bind(flow)
         .fetch_one(&mut tx)
         .await
-        .map_err(|e| {
-            Error::InternalErr(format!(
-                "error during retrieval of stop_early_expr from state: {e}"
-            ))
-        })?;
+        .map_err(|e| Error::InternalErr(format!("retrieval of stop_early_expr from state: {e}")))?;
 
     tracing::debug!("UPDATE: {:?}", new_status);
 
     let flow_job = get_queued_job(flow, w_id, &mut tx)
         .await?
         .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
-    tx.commit().await?;
 
     let stop_early = success
         && if let Some(expr) = stop_early_expr {
@@ -156,20 +158,20 @@ pub async fn update_flow_status_after_job_completion(
                    WHERE id = ANY($1)
                      AND workspace_id = $2
                      AND success = true
+                ORDER BY args->>'_index'
                     ",
                 )
                 .bind(jobs.as_slice())
                 .bind(w_id)
-                .fetch(db)
+                .fetch(&mut tx)
                 .map_ok(|(v,)| v)
                 .try_collect::<Vec<serde_json::Value>>()
                 .await?;
-                let mut results_map = serde_json::Map::new();
-                results_map.insert("res1".to_string(), serde_json::json!(results));
-                Some(results_map)
+                serde_json::json!(results)
             }
             _ => result.clone(),
         };
+        tx.commit().await?;
 
         let logs = if stop_early {
             "Flow job stopped early".to_string()
@@ -188,6 +190,8 @@ pub async fn update_flow_status_after_job_completion(
         .await?;
         true
     } else {
+        tx.commit().await?;
+
         match handle_flow(&flow_job, db, result.clone()).await {
             Err(err) => {
                 let _ = add_completed_job_error(
@@ -195,6 +199,7 @@ pub async fn update_flow_status_after_job_completion(
                     &flow_job,
                     "Unexpected error during flow chaining:\n".to_string(),
                     err,
+                    metrics,
                 )
                 .await;
                 true
@@ -205,6 +210,7 @@ pub async fn update_flow_status_after_job_completion(
 
     if done {
         postprocess_queued_job(
+            flow_job.is_flow_step,
             flow_job.schedule_path.clone(),
             flow_job.script_path.clone(),
             &w_id,
@@ -214,9 +220,10 @@ pub async fn update_flow_status_after_job_completion(
         .await?;
 
         if flow_job.parent_job.is_some() {
-            return Ok(
-                update_flow_status_after_job_completion(db, &flow_job, success, result).await?,
-            );
+            return Ok(update_flow_status_after_job_completion(
+                db, &flow_job, success, result, metrics,
+            )
+            .await?);
         }
     }
 
@@ -243,11 +250,7 @@ async fn skip_loop_failures<'c>(
     .map_err(|e| Error::InternalErr(format!("error during retrieval of skip_loop_failures: {e}")))
 }
 
-async fn compute_stop_early(
-    expr: String,
-    result: Option<Map<String, Value>>,
-) -> error::Result<bool> {
-    let result = serde_json::Value::Object(result.clone().unwrap_or_else(|| Map::new()));
+async fn compute_stop_early(expr: String, result: serde_json::Value) -> error::Result<bool> {
     match eval_timeout(expr, [("result".to_string(), result)].into(), None, vec![]).await? {
         serde_json::Value::Bool(true) => Ok(true),
         serde_json::Value::Bool(false) => Ok(false),
@@ -274,15 +277,11 @@ pub async fn update_flow_status_in_progress(
     let step = get_step_of_flow_status(db, flow).await?;
     sqlx::query(&format!(
         "UPDATE queue
-            SET flow_status = jsonb_set(flow_status, '{{modules, {}}}', $1)
-            WHERE id = $2 AND workspace_id = $3",
-        step
+            SET flow_status = jsonb_set(jsonb_set(flow_status, '{{modules, {step}, job}}', $1), '{{modules, {step}, type}}', $2)
+            WHERE id = $3 AND workspace_id = $4",
     ))
-    .bind(serde_json::json!(FlowStatusModule::InProgress {
-        job: job_in_progress,
-        iterator: None,
-        forloop_jobs: None,
-    }))
+    .bind(json!(job_in_progress.to_string()))
+    .bind(json!("InProgress"))
     .bind(flow)
     .bind(w_id)
     .execute(db)
@@ -297,7 +296,8 @@ pub async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result<i32> {
         id
     )
     .fetch_one(db)
-    .await?
+    .await
+    .map_err(|e| Error::InternalErr(format!("fetching step flow status: {e}")))?
     .ok_or_else(|| Error::InternalErr(format!("not found step")))?;
     Ok(r)
 }
@@ -305,12 +305,12 @@ pub async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result<i32> {
 #[instrument(level = "trace", skip_all)]
 async fn transform_input(
     flow_args: &Option<serde_json::Value>,
-    last_result: Option<Map<String, serde_json::Value>>,
+    last_result: serde_json::Value,
     input_transform: &HashMap<String, InputTransform>,
     workspace: &str,
     token: &str,
     steps: Vec<String>,
-) -> anyhow::Result<Option<Map<String, serde_json::Value>>> {
+) -> anyhow::Result<Map<String, serde_json::Value>> {
     let mut mapped = serde_json::Map::new();
 
     for (key, val) in input_transform.into_iter() {
@@ -327,8 +327,7 @@ async fn transform_input(
         match val {
             InputTransform::Static { value: _ } => (),
             InputTransform::Javascript { expr } => {
-                let previous_result =
-                    serde_json::Value::Object(last_result.clone().unwrap_or_else(|| Map::new()));
+                let previous_result = last_result.clone();
                 let flow_input = flow_args.clone().unwrap_or_else(|| json!({}));
                 let v = eval_timeout(
                     expr.to_string(),
@@ -352,14 +351,14 @@ async fn transform_input(
         }
     }
 
-    Ok(Some(mapped))
+    Ok(mapped)
 }
 
 #[instrument(level = "trace", skip_all)]
 pub async fn handle_flow(
     flow_job: &QueuedJob,
     db: &sqlx::Pool<sqlx::Postgres>,
-    last_result: Option<Map<String, serde_json::Value>>,
+    last_result: serde_json::Value,
 ) -> anyhow::Result<()> {
     let value = flow_job
         .raw_flow
@@ -367,6 +366,13 @@ pub async fn handle_flow(
         .ok_or_else(|| Error::InternalErr(format!("requiring a raw flow value")))?
         .to_owned();
     let flow = serde_json::from_value::<FlowValue>(value.to_owned())?;
+
+    if flow.modules.len() == 0 {
+        Err(Error::BadRequest(format!(
+            "A flow needs at least one module to run"
+        )))?;
+    }
+
     push_next_flow_job(
         flow_job,
         flow,
@@ -384,7 +390,7 @@ async fn push_next_flow_job(
     flow: FlowValue,
     schedule_path: Option<String>,
     db: &sqlx::Pool<sqlx::Postgres>,
-    last_result: Option<Map<String, serde_json::Value>>,
+    last_result: serde_json::Value,
 ) -> anyhow::Result<()> {
     let flow_status_json = flow_job.flow_status.as_ref().ok_or_else(|| {
         Error::InternalErr(format!("not found status for flow job {:?}", flow_job.id))
@@ -449,15 +455,13 @@ async fn push_next_flow_job(
             module.value,
             status.modules[i]
         );
-        let (forloop_args, forloop_iterator) = match &module.value {
+        let (compute_input_transform, forloop_args, forloop_iterator) = match &module.value {
             FlowModuleValue::ForloopFlow { iterator, .. } => match &status.modules[i] {
                 FlowStatusModule::WaitingForPriorSteps { .. } => {
                     let itered = match iterator {
                         InputTransform::Static { value } => value.clone(),
                         InputTransform::Javascript { expr } => {
-                            let result = serde_json::Value::Object(
-                                last_result.clone().unwrap_or_else(|| Map::new()),
-                            );
+                            let result = last_result.clone();
                             eval_timeout(
                                 expr.to_string(),
                                 [("result".to_string(), result)].into(),
@@ -467,16 +471,18 @@ async fn push_next_flow_job(
                             .await?
                         }
                     };
-
-                    let mut args = last_result.clone().unwrap_or_else(Map::new);
-
-                    args.insert(
-                        "_index".to_string(),
+                    let mut args = Map::new();
+                    let mut iteration_map = Map::new();
+                    iteration_map.insert(
+                        "index".to_string(),
                         serde_json::Value::Number(serde_json::Number::from(0)),
                     );
-                    args.insert("_value".to_string(), itered[0].clone());
+                    iteration_map.insert("value".to_string(), itered[0].clone());
+                    args.insert("iter".to_string(), serde_json::Value::Object(iteration_map));
                     match itered {
-                        serde_json::Value::Array(arr) => (Some(args), Some((0 as u8, arr, vec![]))),
+                        serde_json::Value::Array(arr) => {
+                            (true, Some(args), Some((0 as u8, arr, vec![])))
+                        }
                         a @ _ => Err(Error::BadRequest(format!(
                             "Expected an array value, found: {:?}",
                             a
@@ -490,15 +496,18 @@ async fn push_next_flow_job(
                 } if index.to_owned() + 1 < itered.len() as u8 => {
                     let mut args = args.clone();
                     let nindex = index.to_owned() + 1;
-                    args.insert(
-                        "_index".to_string(),
+                    let mut iteration_map = Map::new();
+                    iteration_map.insert(
+                        "index".to_string(),
                         serde_json::Value::Number(serde_json::Number::from(nindex.to_owned())),
                     );
-                    args.insert(
-                        "_value".to_string(),
+                    iteration_map.insert(
+                        "value".to_string(),
                         itered[nindex.to_owned() as usize].clone(),
                     );
+                    args.insert("iter".to_string(), serde_json::Value::Object(iteration_map));
                     (
+                        false,
                         Some(args),
                         Some((nindex, itered.clone(), forloop_jobs.clone())),
                     )
@@ -508,11 +517,33 @@ async fn push_next_flow_job(
                     a
                 )))?,
             },
-            _ => (None, None),
+            _ => (true, None, None),
         };
 
+        let mut args = if compute_input_transform {
+            let steps = status
+                .modules
+                .into_iter()
+                .map(|x| match x {
+                    FlowStatusModule::Success { job, forloop_jobs: _ } => job.to_string(),
+                    _ => "invalid step status".to_string(),
+                })
+                .collect();
+
+            transform_input(
+                &flow_job.args,
+                last_result.clone(),
+                &input_transform,
+                &flow_job.workspace_id,
+                &token,
+                steps,
+            )
+            .await?
+        } else {
+            Map::new()
+        };
         let args = if forloop_args.is_some() {
-            let mut args = forloop_args.unwrap();
+            args.extend(forloop_args.unwrap());
             if let Some(flow_args) = &flow_job.args {
                 match flow_args {
                     serde_json::Value::Object(obj) => {
@@ -528,35 +559,16 @@ async fn push_next_flow_job(
                     }
                 }
             }
-            Some(args)
+            args
         } else {
-            let steps = status
-                .modules
-                .into_iter()
-                .map(|x| match x {
-                    FlowStatusModule::Success { job, forloop_jobs: _ } => job.to_string(),
-                    _ => "invalid step status".to_string(),
-                })
-                .collect();
-
-            let transformed = transform_input(
-                &flow_job.args,
-                last_result.clone(),
-                &input_transform,
-                &flow_job.workspace_id,
-                &token,
-                steps,
-            )
-            .await?;
-
-            transformed
+            args
         };
 
         let (uuid, mut tx) = push(
             tx,
             &flow_job.workspace_id,
             job_payload,
-            args.clone(),
+            Some(args.clone()),
             &flow_job.created_by,
             flow_job.permissioned_as.to_owned(),
             None,
@@ -570,11 +582,7 @@ async fn push_next_flow_job(
             forloop_jobs.push(uuid.to_owned());
             serde_json::json!(FlowStatusModule::InProgress {
                 job: uuid,
-                iterator: Some(Iterator {
-                    index: index,
-                    itered: itered,
-                    args: args.unwrap_or_else(|| Map::new()),
-                }),
+                iterator: Some(Iterator { index: index, itered: itered, args: args }),
                 forloop_jobs: Some(forloop_jobs),
             })
         } else {

backend/src/workspaces.rs

@@ -191,7 +191,9 @@ async fn get_settings(
         &w_id
     )
     .fetch_one(&mut tx)
-    .await?;
+    .await
+    .map_err(|e| Error::InternalErr(format!("getting settings: {e}")))?;
+
     tx.commit().await?;
     Ok(Json(settings))
 }

community/resource_types/airtable.json

@@ -4,14 +4,13 @@
   "schema": {
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "properties": {
-      "token": {
+      "apiKey": {
         "type": "string",
-        "description": ""
+        "description": "",
+        "default": "keyXXXXXXXXXXXXXX"
       }
     },
-    "required": [
-      "token"
-    ],
+    "required": [],
     "type": "object"
   },
   "description": "The airtable OAuth token"

community/resource_types/basecamp.json

@@ -0,0 +1,18 @@
+{
+  "workspace_id": "starter",
+  "name": "basecamp",
+  "schema": {
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "properties": {
+      "token": {
+        "type": "string",
+        "description": "OAuth token"
+      }
+    },
+    "required": [
+      "token"
+    ],
+    "type": "object"
+  },
+  "description": "basecamp OAuth token"
+}

community/resource_types/discord_webhook.json

@@ -0,0 +1,16 @@
+{
+  "workspace_id": "starter",
+  "name": "discord_webhook",
+  "schema": {
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "properties": {
+      "webhook_url": {
+        "type": "string",
+        "description": "The webhook url"
+      }
+    },
+    "required": [],
+    "type": "object"
+  },
+  "description": "Discord webhook url to send messages. See: <https://support.discord.com/hc/en-us/articles/228383668-Intro-to-Webhooks>"
+}

community/resource_types/toggl.json

@@ -0,0 +1,18 @@
+{
+  "workspace_id": "starter",
+  "name": "toggl",
+  "schema": {
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "properties": {
+      "token": {
+        "type": "string",
+        "description": ""
+      }
+    },
+    "required": [
+      "token"
+    ],
+    "type": "object"
+  },
+  "description": "Toggle Track API Token"
+}

community/resources/g/all/demodb.json

@@ -2,12 +2,12 @@
   "workspace_id": "starter",
   "path": "g/all/demodb",
   "value": {
-    "host": "demodb.service.consul",
-    "port": "6543",
-    "user": "postgres",
+    "host": "postgres-demodb.node.consul",
+    "port": "5432",
+    "user": "myuser",
     "dbname": "demodb",
     "sslmode": "disable",
-    "password": "demodb"
+    "password": "password"
   },
   "description": "demodb",
   "resource_type": "postgresql",

deno-client/mod.ts

@@ -9,6 +9,7 @@ export {
 
 export type Email = string
 export type Base64 = string
+export type Sql = string
 export type Resource<S extends string> = any
 
 /**
@@ -36,7 +37,7 @@ export async function getResource(path: string, undefinedIfEmpty?: boolean): Pro
     const conf = createConf()
     try {
         const resource = await new ResourceApi(conf).getResource(conf.workspace_id, path)
-        return await transformLeaves(resource.value)
+        return await _transformLeaf(resource.value)
     } catch (e) {
         if (undefinedIfEmpty && e.code === 404) {
             return undefined

frontend/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "windmill",
-	"version": "1.26.2",
+	"version": "1.30.0",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
@@ -10,28 +10,30 @@
 		"lint": "prettier --ignore-path .gitignore --check --plugin-search-dir=. . && eslint --ignore-path .gitignore .",
 		"format": "prettier --ignore-path .gitignore --write --plugin-search-dir=. .",
 		"package": "svelte-kit package && cd package && rm README.md && rm README_DEV.md && sed -i -e 's/windmill/windmill-components/g' package.json",
-		"generate-backend-client": "openapi --input ../backend/openapi.yaml --output ./src/lib/gen --useOptions",
+		"generate-backend-client": "openapi --input ../backend/openapi.yaml --output ./src/lib/gen --useOptions && sed -i '213 i \\    request.referrerPolicy = \"no-referrer\"\n' src/lib/gen/core/request.ts",
 		"pretest": "tsc --incremental -p tests/tsconfig.json",
 		"test": "playwright test --config=tests-out/playwright.config.js"
 	},
 	"devDependencies": {
-		"@playwright/test": "^1.24.0",
-		"@sveltejs/adapter-static": "^1.0.0-next.37",
+		"@playwright/test": "^1.24.2",
+		"@sveltejs/adapter-static": "^1.0.0-next.39",
 		"@sveltejs/kit": "next",
 		"@tailwindcss/forms": "^0.5.1",
-		"@tailwindcss/typography": "^0.5.3",
-		"@types/vscode": "~1.67.0",
-		"@typescript-eslint/eslint-plugin": "^5.31.0",
-		"@typescript-eslint/parser": "^5.31.0",
+		"@tailwindcss/typography": "^0.5.4",
+		"@types/vscode": "~1.70.0",
+		"@typescript-eslint/eslint-plugin": "^5.33.0",
+		"@typescript-eslint/parser": "^5.33.0",
 		"@zerodevx/svelte-toast": "^0.7.2",
-		"autoprefixer": "^10.4.1",
+		"autoprefixer": "^10.4.8",
 		"cssnano": "^5.1.12",
-		"eslint": "^8.20.0",
+		"eslint": "^8.21.0",
 		"eslint-config-prettier": "^8.3.0",
 		"eslint-plugin-svelte3": "^4.0.0",
+		"flowbite": "^1.5.1",
+		"flowbite-svelte": "^0.24.2",
 		"openapi-typescript-codegen": "^0.23.0",
 		"path-browserify": "^1.0.1",
-		"postcss": "^8.4.5",
+		"postcss": "^8.4.16",
 		"postcss-load-config": "^4.0.1",
 		"prettier": "^2.7.1",
 		"prettier-plugin-svelte": "^2.4.0",
@@ -40,14 +42,15 @@
 		"svelte": "^3.49.0",
 		"svelte-awesome": "^3.0.0",
 		"svelte-check": "^2.8.0",
-		"svelte-highlight": "^6.2.0",
+		"svelte-heros": "^2.3.3",
+		"svelte-highlight": "^6.2.1",
 		"svelte-markdown": "^0.2.1",
 		"svelte-overlay": "^1.4.1",
 		"svelte-popperjs": "^1.2.11",
 		"svelte-preprocess": "^4.9.8",
 		"svelte-split-pane": "^0.1.2",
-		"svelte2tsx": "^0.5.12",
-		"tailwindcss": "^3.1.6",
+		"svelte2tsx": "^0.5.13",
+		"tailwindcss": "^3.1.8",
 		"tslib": "^2.3.1",
 		"typescript": "^4.7.4"
 	},

frontend/src/app.css

@@ -3,6 +3,12 @@
 @tailwind components;
 @tailwind utilities;
 
+html {
+
+	/* Avoid content shifting */
+	overflow-y: overlay;
+} 
+
 @layer base {
 	h1 {
 		@apply text-2xl;
@@ -71,6 +77,11 @@
 		@apply rounded-sm;
 	}
 
+	.blue-button {
+		@apply bg-blue-500 !important;
+		@apply hover:bg-blue-700 !important;
+	}
+
 	.default-button {
 		/*this styling does not impact the Dropdown component */
 		@apply bg-blue-500;
@@ -212,3 +223,8 @@
 		@apply text-sm;
 	}
 }
+
+.inline-highlight pre code.hljs {
+	padding: 0;
+	@apply text-xs;
+}

frontend/src/lib/common.ts

@@ -11,6 +11,7 @@ export interface SchemaProperty {
 	contentEncoding?: 'base64' | 'binary'
 	format?: string
 	items?: { type?: 'string' | 'number' | 'bytes', contentEncoding?: 'base64' },
+	properties?: { [name: string]: SchemaProperty }
 }
 
 export type Schema = {

frontend/src/lib/components/AppConnect.svelte

@@ -1,34 +1,49 @@
 <script lang="ts" context="module">
-	const apiTokenApps: Record<string, { img?: string; instructions: string }> = {
+	const apiTokenApps: Record<string, { img?: string; instructions: string; key?: string }> = {
 		airtable: {
 			img: 'airtable_connect.png',
 			instructions: 'Click on the top-right avatar -> Account -> Api'
+		},
+		discord_webhook: {
+			img: 'discord_webhook.png',
+			instructions: 'Server Settings -> Integration -> Webhooks',
+			key: 'webhook_url'
+		},
+		toggl: {
+			img: 'toggl_connect.png',
+			instructions: 'Go to https://track.toggl.com/profile -> API Token'
 		}
 	}
 </script>
 
 <script lang="ts">
+	import { oauthStore, userStore, workspaceStore } from '$lib/stores'
+	import { faMinus, faPlus } from '@fortawesome/free-solid-svg-icons'
 	import IconedResourceType from './IconedResourceType.svelte'
 	import PageHeader from './PageHeader.svelte'
-	import { workspaceStore, userStore, oauthStore } from '$lib/stores'
-	import { faMinus, faPlus } from '@fortawesome/free-solid-svg-icons'
 
 	import { OauthService, ResourceService, VariableService, type TokenResponse } from '$lib/gen'
 
-	import { createEventDispatcher, onMount } from 'svelte'
-	import Modal from './Modal.svelte'
-	import Icon from 'svelte-awesome'
-	import Path from './Path.svelte'
-	import Password from './Password.svelte'
+	import { page } from '$app/stores'
 	import { sendUserToast, truncateRev } from '$lib/utils'
+	import { createEventDispatcher } from 'svelte'
+	import Icon from 'svelte-awesome'
+	import Modal from './Modal.svelte'
+	import Password from './Password.svelte'
+	import Path from './Path.svelte'
 
 	let manual = false
 	let value: string = ''
 	let valueToken: TokenResponse
-	let connects: Record<string, string[]> = {}
-	let connectsManual: [string, { img?: string; instructions: string }][] = []
+	let connects: Record<string, { scopes: string[]; extra_params?: Record<string, string> }> = {}
+	let connectsManual: [string, { img?: string; instructions: string; key?: string }][] = []
+	let key: string = 'token'
+
+	$: key = apiTokenApps[resource_type]?.key ?? 'token'
 
 	let scopes: string[] = []
+	let extra_params: [string, string][] = []
+
 	let path: string
 
 	let modal: Modal
@@ -39,12 +54,19 @@
 
 	let pathError = ''
 
-	export function open(rt?: string) {
+	export async function open(rt?: string) {
 		step = 1
 		value = ''
-		resource_type = ''
 		no_back = false
 		resource_type = rt ?? ''
+
+		await loadConnects()
+
+		const connect = connects[resource_type]
+		if (connect) {
+			scopes = connect.scopes
+			extra_params = Object.entries(connect.extra_params ?? {})
+		}
 		modal.openModal()
 	}
 
@@ -74,7 +96,12 @@
 		if (step < 3 && manual) {
 			step += 1
 		} else if (step == 1 && !manual) {
-			window.location.href = `/api/oauth/connect/${resource_type}?scopes=${scopes.join('+')}`
+			const url = new URL(`/api/oauth/connect/${resource_type}`, $page.url.origin)
+			url.searchParams.append('scopes', scopes.join('+'))
+			if (extra_params.length > 0) {
+				extra_params.forEach(([key, value]) => url.searchParams.append(key, value))
+			}
+			window.location.href = url.toString()
 		} else {
 			let exists = await VariableService.existsVariable({
 				workspace: $workspaceStore!,
@@ -93,7 +120,7 @@
 			}
 
 			let account: number | undefined = undefined
-			if (valueToken.refresh_token != undefined && valueToken.expires_in != undefined) {
+			if (valueToken?.refresh_token != undefined && valueToken?.expires_in != undefined) {
 				account = Number(
 					await OauthService.createAccount({
 						workspace: $workspaceStore!,
@@ -106,25 +133,28 @@
 					})
 				)
 			}
+			const description = `${manual ? 'Token' : 'OAuth token'} for ${resource_type}`
 			await VariableService.createVariable({
 				workspace: $workspaceStore!,
 				requestBody: {
 					path,
 					value,
 					is_secret: true,
-					description: `OAuth token for ${resource_type}`,
-					is_oauth: true,
+					description,
+					is_oauth: !manual,
 					account: account
 				}
 			})
+			const resourceValue = {}
+			resourceValue[key] = `$var:${path}`
 			await ResourceService.createResource({
 				workspace: $workspaceStore!,
 				requestBody: {
 					resource_type,
 					path,
-					value: { token: `$var:${path}` },
-					description: `OAuth token for ${resource_type}`,
-					is_oauth: true
+					value: resourceValue,
+					description,
+					is_oauth: !manual
 				}
 			})
 			dispatch('refresh')
@@ -155,7 +185,7 @@
 	<div slot="title">Connect an App</div>
 	<div slot="content">
 		{#if step == 1}
-			{#if resource_type && !connects[resource_type] && !connectsManual[resource_type]}
+			{#if resource_type && !connects[resource_type] && !connectsManual.find((x) => x[0] == resource_type)}
 				<div class="bg-red-100 border-l-4 border-red-600 text-orange-700 p-4" role="alert">
 					<p class="font-bold">No app integration for {resource_type}</p>
 					<p>
@@ -173,7 +203,9 @@
 						on:click={() => {
 							manual = false
 							resource_type = key
-							scopes = values
+							scopes = values.scopes
+							extra_params = Object.entries(values.extra_params ?? {})
+
 							dispatch('click')
 						}}
 					>
@@ -204,7 +236,34 @@
 			{:else}
 				<p class="italic text-sm">Pick an OAuth app and customize the scopes here</p>
 			{/if}
-			<PageHeader title="API token apps" />
+			<PageHeader title="Extra Params" primary={false} />
+			{#if !manual && resource_type != ''}
+				{#each extra_params as [k, v], i}
+					<div class="flex flex-row max-w-md">
+						<input type="text" bind:value={k} />
+						<input type="text" bind:value={v} />
+
+						<button
+							class="default-button-secondary mx-6"
+							on:click={() => {
+								extra_params = extra_params.filter((el) => el[0] != k)
+							}}><Icon data={faMinus} class="mb-1" /></button
+						>
+					</div>
+				{/each}
+				<button
+					class="default-button-secondary mt-1"
+					on:click={() => {
+						extra_params.push(['', ''])
+						extra_params = extra_params
+					}}>Add item &nbsp;<Icon data={faPlus} class="mb-1" /></button
+				><span class="ml-2"
+					>{(extra_params ?? []).length} item{(extra_params ?? []).length > 1 ? 's' : ''}</span
+				>
+			{:else}
+				<p class="italic text-sm">Pick an OAuth app and customize the extra parameters here</p>
+			{/if}
+			<PageHeader title="Non OAuth apps" />
 			<div class="grid sm:grid-cols-2 md:grid-cols-3 gap-x-2 gap-y-1 items-center mb-2">
 				{#each connectsManual as [key, instructions]}
 					<button

frontend/src/lib/components/ArgInput.svelte

@@ -1,22 +1,32 @@
 <script lang="ts">
-	import Tooltip from './Tooltip.svelte'
-
 	import { slide } from 'svelte/transition'
 
-	import { faChevronDown, faChevronUp, faMinus, faPlus } from '@fortawesome/free-solid-svg-icons'
+	import {
+		faArrowRotateLeft,
+		faChevronDown,
+		faChevronUp,
+		faMinus,
+		faPlus
+	} from '@fortawesome/free-solid-svg-icons'
 
-	import StringTypeNarrowing from './StringTypeNarrowing.svelte'
-	import Icon from 'svelte-awesome'
-	import ResourcePicker from './ResourcePicker.svelte'
-	import ObjectTypeNarrowing from './ObjectTypeNarrowing.svelte'
-	import ObjectResourceInput from './ObjectResourceInput.svelte'
-	import FieldHeader from './FieldHeader.svelte'
-	import { createEventDispatcher } from 'svelte'
 	import { setInputCat as computeInputCat, type InputCat } from '$lib/utils'
+	import { Button, Tooltip } from 'flowbite-svelte'
+	import { createEventDispatcher } from 'svelte'
+	import Icon from 'svelte-awesome'
+	import Editor from './Editor.svelte'
+	import FieldHeader from './FieldHeader.svelte'
+	import ObjectResourceInput from './ObjectResourceInput.svelte'
+	import ObjectTypeNarrowing from './ObjectTypeNarrowing.svelte'
+	import ResourcePicker from './ResourcePicker.svelte'
+	import StringTypeNarrowing from './StringTypeNarrowing.svelte'
+	import SchemaForm from './SchemaForm.svelte'
+	import type { Schema, SchemaProperty } from '$lib/common'
 
 	export let label: string = ''
 	export let value: any
+
 	export let defaultValue: any = undefined
+
 	export let description: string = ''
 	export let format: string = ''
 	export let contentEncoding: 'base64' | 'binary' | undefined = undefined
@@ -33,7 +43,7 @@
 		| { type?: 'string' | 'number' | 'bytes'; contentEncoding?: 'base64' }
 		| undefined = undefined
 	export let displayHeader = true
-	export let numberAsString = false
+	export let properties: { [name: string]: SchemaProperty } | undefined = undefined
 
 	let seeEditable: boolean = enum_ != undefined || pattern != undefined
 	const dispatch = createEventDispatcher()
@@ -45,6 +55,8 @@
 
 	let error: string = ''
 
+	export let editor: Editor | undefined = undefined
+
 	let rawValue: string | undefined = undefined
 
 	$: {
@@ -62,14 +74,18 @@
 			evalValueToRaw()
 			validateInput(pattern, value)
 		}
-		if (defaultValue) {
-			let stringified = JSON.stringify(defaultValue, null, 4)
-			if (stringified.length > 50) {
-				minRows = 3
-			}
-			if (type != 'string') {
-				minRows = Math.max(minRows, Math.min(stringified.split(/\r\n|\r|\n/).length + 1, maxRows))
-			}
+	}
+
+	$: {
+		defaultValue && recomputeRowSize(JSON.stringify(defaultValue, null, 4))
+	}
+
+	function recomputeRowSize(str: string) {
+		if (str.length > 50) {
+			minRows = 3
+		}
+		if (type != 'string') {
+			minRows = Math.max(minRows, Math.min(str.split(/\r\n|\r|\n/).length + 1, maxRows))
 		}
 	}
 
@@ -126,7 +142,7 @@
 	$: inputCat = computeInputCat(type, format, itemsType?.type, enum_, contentEncoding)
 </script>
 
-<div class="flex flex-col w-full">
+<div class="flex flex-col w-full mb-2">
 	<div>
 		{#if displayHeader}
 			<FieldHeader {label} {required} {type} {contentEncoding} {format} {itemsType} />
@@ -138,17 +154,15 @@
 					on:click={() => {
 						seeEditable = !seeEditable
 					}}
-					>Customize argument<Icon
-						class="ml-2"
-						data={seeEditable ? faChevronUp : faChevronDown}
-						scale={0.7}
-					/></span
 				>
+					Customize argument
+					<Icon class="ml-2" data={seeEditable ? faChevronUp : faChevronDown} scale={0.7} />
+				</span>
 
 				{#if seeEditable}
 					<div transition:slide class="mt-2">
-						<label class="text-gray-700"
-							>Description
+						<label class="text-gray-700">
+							Description
 							<textarea rows="1" bind:value={description} placeholder="Edit description" />
 							{#if type == 'string' && !contentEncoding && format != 'date-time'}
 								<StringTypeNarrowing bind:format bind:pattern bind:enum_ bind:contentEncoding />
@@ -172,15 +186,20 @@
 		{/if}
 
 		<div class="grid grid-cols-2">
-			<div class="text-sm italic pb-1">
-				{description}
-			</div>
-			<div class="text-right text-xs {error === '' ? 'text-white' : 'font-bold text-red-600'}">
-				{error === '' ? '...' : error}
-			</div>
+			{#if description || error}
+				<div class="text-sm italic pb-1">
+					{description}
+				</div>
+				<div class="text-right text-xs {error === '' ? 'text-white' : 'font-bold text-red-600'}">
+					{error === '' ? '...' : error}
+				</div>
+			{:else}
+				<div class="mt-1" />
+			{/if}
 		</div>
-		<div class="flex flex-row gap-1">
-			{#if inputCat == 'number' && !numberAsString}
+
+		<div class="flex space-x-1">
+			{#if inputCat == 'number'}
 				<input
 					{disabled}
 					type="number"
@@ -227,8 +246,10 @@
 										if (value.length == 0) {
 											value = undefined
 										}
-									}}><Icon data={faMinus} class="mb-1" /></button
+									}}
 								>
+									<Icon data={faMinus} class="mb-1" />
+								</button>
 							</div>
 						{/each}
 					</div>
@@ -239,22 +260,37 @@
 								value = []
 							}
 							value = value.concat('')
-						}}>Add item &nbsp;<Icon data={faPlus} class="mb-1" /></button
-					><span class="ml-2">{(value ?? []).length} item{(value ?? []).length > 1 ? 's' : ''}</span
+						}}
 					>
+						<Icon data={faPlus} class="mr-2" />
+						Add item
+					</button>
+					<span class="ml-2">
+						{(value ?? []).length} item{(value ?? []).length > 1 ? 's' : ''}
+					</span>
 				</div>
 			{:else if inputCat == 'resource-object'}
 				<ObjectResourceInput {format} bind:value />
 			{:else if inputCat == 'object'}
-				<textarea
-					{disabled}
-					style="min-height: {minHeight}; max-height: {maxHeight}"
-					class="col-span-10 {valid
-						? ''
-						: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'}"
-					placeholder={defaultValue ? JSON.stringify(defaultValue, null, 4) : ''}
-					bind:value={rawValue}
-				/>
+				{#if properties}
+					<div class="p-4 pl-8 border rounded w-full">
+						<SchemaForm
+							schema={{ properties, $schema: '', required: [], type: 'object' }}
+							bind:args={value}
+						/>
+					</div>
+				{:else}
+					<textarea
+						{disabled}
+						style="min-height: {minHeight}; max-height: {maxHeight}"
+						on:input={async () => recomputeRowSize(rawValue ?? '')}
+						class="col-span-10 {valid
+							? ''
+							: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'}"
+						placeholder={defaultValue ? JSON.stringify(defaultValue, null, 4) : ''}
+						bind:value={rawValue}
+					/>
+				{/if}
 			{:else if inputCat == 'enum'}
 				<select {disabled} class="px-6" bind:value>
 					{#each enum_ ?? [] as e}
@@ -263,6 +299,20 @@
 				</select>
 			{:else if inputCat == 'date'}
 				<input class="inline-block" type="datetime-local" bind:value />
+			{:else if inputCat == 'sql'}
+				<div class="border rounded mb-4 w-full border-gray-700">
+					<Editor
+						on:focus={() => dispatch('focus')}
+						on:blur={() => dispatch('blur')}
+						bind:this={editor}
+						lang="sql"
+						bind:code={value}
+						class="two-lines-editor"
+						on:change={async () => {
+							dispatch('input', { rawValue: value, isRaw: false })
+						}}
+					/>
+				</div>
 			{:else if inputCat == 'base64'}
 				<input
 					type="file"
@@ -277,9 +327,10 @@
 						? format.substring('resource-'.length)
 						: undefined}
 				/>
-			{:else if inputCat == 'string' || (inputCat == 'number' && numberAsString)}
+			{:else if inputCat == 'string'}
 				<textarea
 					on:focus={() => dispatch('focus')}
+					on:blur={() => dispatch('blur')}
 					{disabled}
 					style="height: {minHeight}; max-height: {maxHeight}"
 					class="col-span-10 {valid
@@ -287,19 +338,26 @@
 						: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'}"
 					placeholder={defaultValue ?? ''}
 					bind:value
-					on:input={() => dispatch('input', { rawValue: value, isRaw: false })}
+					on:input={async () => {
+						recomputeRowSize(value)
+						dispatch('input', { rawValue: value, isRaw: false })
+					}}
 				/>
 			{/if}
 			{#if !required && inputCat != 'resource-object'}
-				<div class="flex flex-row-reverse">
-					<button
-						{disabled}
-						class="default-button-secondary items-center leading-4 py-0 my-px px-1 float-right"
+				<Tooltip placement="bottom" content="Reset to default value">
+					<Button
 						on:click={() => (value = undefined)}
-						>Reset&nbsp;<Tooltip>Reset to default value</Tooltip></button
+						{disabled}
+						color="alternative"
+						size="sm"
+						class="h-8"
 					>
-				</div>
+						<Icon data={faArrowRotateLeft} />
+					</Button>
+				</Tooltip>
 			{/if}
+			<slot name="actions" />
 		</div>
 	</div>
 </div>

frontend/src/lib/components/CenteredPage.svelte

@@ -1,5 +1,5 @@
 <div class="max-w-screen-lg flex grow w-full h-full xl:-ml-20">
-	<div class="grow w-full h-full">
+	<div class="grow w-full h-full mt-4">
 		<slot />
 	</div>
 </div>

frontend/src/lib/components/CollapseLink.svelte

@@ -1,16 +1,15 @@
 <script lang="ts">
-	import { faChevronUp, faChevronDown } from '@fortawesome/free-solid-svg-icons'
+	import { faChevronDown, faChevronUp } from '@fortawesome/free-solid-svg-icons'
 
 	import Icon from 'svelte-awesome'
-	import { slide } from 'svelte/transition'
 
 	export let open = false
 	export let text: string
 </script>
 
-<button class="text-gray-600 underline" on:click={() => (open = !open)}
-	>{text}&nbsp;<Icon data={open ? faChevronUp : faChevronDown} scale={0.5} /></button
->
+<button class="text-gray-600 underline" on:click={() => (open = !open)}>
+	{text}&nbsp;<Icon data={open ? faChevronUp : faChevronDown} scale={0.5} />
+</button>
 {#if open}
-	<div transition:slide><slot /></div>
+	<div><slot /></div>
 {/if}

frontend/src/lib/components/DisplayResult.svelte

@@ -6,20 +6,16 @@
 
 	export let result: any
 
-	let resultKind: 'json' | 'table-col' | 'table-row' | 'png' | 'file' | undefined =
+	let resultKind: 'json' | 'table-col' | 'table-row' | 'png' | 'file' | 'jpeg' | 'gif' | undefined =
 		inferResultKind(result)
 
-	function isArray(obj: any) {
-		return Object.prototype.toString.call(obj) === '[object Array]'
-	}
-
 	function isRectangularArray(obj: any) {
-		if (!isArray(obj) || obj.length == 0) {
+		if (!Array.isArray(obj) || obj.length == 0) {
 			return false
 		}
 		if (
 			!Object.values(obj)
-				.map(isArray)
+				.map(Array.isArray)
 				.reduce((a, b) => a && b)
 		) {
 			return false
@@ -39,12 +35,14 @@
 		if (result) {
 			try {
 				let keys = Object.keys(result)
-				if (keys.length == 1 && isRectangularArray(result[keys[0]])) {
+				if (isRectangularArray(result)) {
 					return 'table-row'
-				} else if (keys.map((k) => isArray(result[k])).reduce((a, b) => a && b)) {
+				} else if (keys.map((k) => Array.isArray(result[k])).reduce((a, b) => a && b)) {
 					return 'table-col'
 				} else if (keys.length == 1 && keys[0] == 'png') {
 					return 'png'
+				} else if (keys.length == 1 && keys[0] == 'jpeg') {
+					return 'jpeg'
 				} else if (keys.length == 1 && keys[0] == 'file') {
 					return 'file'
 				}
@@ -55,7 +53,7 @@
 </script>
 
 {#if result}
-	{#if Object.keys(result).length > 0}<div>
+	{#if typeof result == 'object' && Object.keys(result).length > 0}<div>
 			The result keys are: <b>{Object.keys(result).join(', ')}</b>
 		</div>
 	{/if}
@@ -81,10 +79,10 @@
 		>
 			<TableCustom>
 				<tbody slot="body">
-					{#each asListOfList(Object.values(result)[0]) as row}
+					{#each asListOfList(result) as row}
 						<tr>
 							{#each row as v}
-								<td>{truncate(v, 200) ?? ''}</td>
+								<td>{truncate(JSON.stringify(v), 200) ?? ''}</td>
 							{/each}
 						</tr>
 					{/each}
@@ -99,6 +97,22 @@
 				src="data:image/png;base64,{result.png}"
 			/>
 		</div>
+	{:else if resultKind == 'jpeg'}
+		<div class="h-full">
+			Result is an image: <img
+				alt="jpeg rendered"
+				class="w-auto h-full"
+				src="data:image/jpeg;base64,{result.jpeg}"
+			/>
+		</div>
+	{:else if resultKind == 'gif'}
+		<div class="h-full">
+			Result is an image: <img
+				alt="gif rendered"
+				class="w-auto h-full"
+				src="data:image/gif;base64,{result.gif}"
+			/>
+		</div>
 	{:else if resultKind == 'file'}
 		<div>
 			Result is a file: <a

frontend/src/lib/components/Editor.svelte

@@ -1,3 +1,39 @@
+<script lang="ts" context="module">
+	import * as monaco from 'monaco-editor'
+	import { language as sql } from 'monaco-editor/esm/vs/basic-languages/sql/sql'
+	monaco.languages.register({
+		id: 'python',
+		extensions: ['.py'],
+		aliases: ['python'],
+		mimetypes: ['application/text']
+	})
+
+	monaco.languages.register({
+		id: 'sql',
+		extensions: ['.sql'],
+		aliases: ['SQL'],
+		mimetypes: ['application/text']
+	})
+
+	monaco.languages.registerInlineCompletionsProvider('sql', {
+		provideInlineCompletions: (a, b, c, d) => {
+			const w = a.getWordUntilPosition(b).word.toLowerCase()
+			return {
+				items:
+					w.length > 0
+						? ['SELECT', 'FROM', 'WHILE', 'AND', 'OR', 'DELETE', 'UPDATE', 'SET', 'WHEN', 'LIMIT']
+								.concat(sql.keywords)
+								.filter((x) => x.toLowerCase().startsWith(w))
+								.map((x) => ({
+									text: x + ' '
+								}))
+						: []
+			}
+		},
+		freeInlineCompletions: () => {}
+	})
+</script>
+
 <script lang="ts">
 	import { browser, dev } from '$app/env'
 	import { page } from '$app/stores'
@@ -7,14 +43,17 @@
 	import jsonWorker from 'monaco-editor/esm/vs/language/json/json.worker?worker'
 	import tsWorker from 'monaco-editor/esm/vs/language/typescript/ts.worker?worker'
 
-	import * as monaco from 'monaco-editor'
-
-	import type { DocumentUri, MessageTransports, MonacoLanguageClient } from 'monaco-languageclient'
-	import { createEventDispatcher, onDestroy, onMount } from 'svelte'
 	import { buildWorkerDefinition } from 'monaco-editor-workers'
+	import type {
+		Disposable,
+		DocumentUri,
+		MessageTransports,
+		MonacoLanguageClient
+	} from 'monaco-languageclient'
+	import { createEventDispatcher, onDestroy, onMount } from 'svelte'
 
-	import { StandaloneServices } from 'vscode/services'
 	import getMessageServiceOverride from 'vscode/service-override/messages'
+	import { StandaloneServices } from 'vscode/services'
 
 	StandaloneServices.initialize({
 		...getMessageServiceOverride(document.body)
@@ -33,15 +72,17 @@
 	export let websocketAlive = { pyright: false, black: false, deno: false }
 	export let extraLib: string = ''
 	export let extraLibPath: string = ''
+	export let shouldBindKey: boolean = true
 
 	let websockets: [MonacoLanguageClient, WebSocket][] = []
 	let websocketInterval: NodeJS.Timer | undefined
 	let lastWsAttempt: Date = new Date()
 	let nbWsAttempt = 0
-	let uri: string = ''
 	let disposeMethod: () => void | undefined
 	const dispatch = createEventDispatcher()
 
+	const uri = `file:///${hash}.${langToExt(lang)}`
+
 	if (browser) {
 		if (dev) {
 			buildWorkerDefinition(
@@ -75,6 +116,23 @@
 		}
 	}
 
+	function langToExt(lang: string): string {
+		switch (lang) {
+			case 'typescript':
+				return 'ts'
+			case 'python':
+				return 'py'
+			case 'javascript':
+				return 'js'
+			case 'json':
+				return 'json'
+			case 'sql':
+				return 'sql'
+			default:
+				return 'unknown'
+		}
+	}
+
 	export function insertAtBeginning(code: string): void {
 		if (editor) {
 			const range = { startLineNumber: 1, startColumn: 1, endLineNumber: 1, endColumn: 1 }
@@ -100,6 +158,9 @@
 		}
 	}
 
+	let command: Disposable | undefined = undefined
+	let monacoServices: Disposable | undefined = undefined
+
 	export async function reloadWebsocket() {
 		await closeWebsockets()
 		if (lang == 'python' || deno) {
@@ -110,9 +171,10 @@
 			)
 			const vscode = await import('vscode')
 			const { RequestType } = await import('vscode-jsonrpc')
+			// install Monaco language client services
 			const { MonacoServices } = await import('monaco-languageclient')
 
-			MonacoServices.install()
+			monacoServices = MonacoServices.install()
 
 			function createLanguageClient(
 				transports: MessageTransports,
@@ -160,32 +222,18 @@
 				try {
 					const webSocket = new WebSocket(url)
 
-					webSocket.onopen = () => {
+					webSocket.onopen = async () => {
 						const socket = toSocket(webSocket)
 						const reader = new WebSocketMessageReader(socket)
 						const writer = new WebSocketMessageWriter(socket)
 						const languageClient = createLanguageClient({ reader, writer }, name, options)
 						websockets.push([languageClient, webSocket])
 
-						languageClient.start()
-						lastWsAttempt = new Date()
-						nbWsAttempt = 0
-						if (name == 'deno') {
-							vscode.commands.getCommands().then((v) => {
-								if (!v.includes('deno.cache')) {
-									vscode.commands.registerCommand('deno.cache', (uris: DocumentUri[] = []) => {
-										languageClient.sendRequest(new RequestType('deno/cache'), {
-											referrer: { uri },
-											uris: uris.map((uri) => ({ uri }))
-										})
-									})
-								}
-							})
-						}
-						reader.onClose(() => {
+						reader.onClose(async () => {
 							try {
 								console.log('CLOSE')
-								languageClient.stop()
+								websocketAlive[name] = false
+								await languageClient.stop()
 							} catch (err) {
 								console.error(err)
 							}
@@ -194,6 +242,31 @@
 							websocketAlive[name] = false
 						})
 
+						try {
+							console.log('started client')
+							await languageClient.start()
+						} catch (err) {
+							console.log('err at client')
+							console.error(err)
+							throw new Error(err)
+						}
+
+						lastWsAttempt = new Date()
+						nbWsAttempt = 0
+						if (name == 'deno') {
+							command && command.dispose()
+							command = undefined
+							command = vscode.commands.registerCommand(
+								'deno.cache',
+								(uris: DocumentUri[] = []) => {
+									languageClient.sendRequest(new RequestType('deno/cache'), {
+										referrer: { uri },
+										uris: uris.map((uri) => ({ uri }))
+									})
+								}
+							)
+						}
+
 						websocketAlive[name] = true
 					}
 				} catch (err) {
@@ -282,11 +355,18 @@
 	}
 
 	async function closeWebsockets() {
+		command && command.dispose()
+		command = undefined
+		monacoServices && monacoServices.dispose()
+		monacoServices = undefined
 		for (const x of websockets) {
 			try {
 				await x[0].stop()
 				x[1].close()
 			} catch (err) {
+				try {
+					x[1].close()
+				} catch (err) {}
 				console.log('error disposing websocket', err)
 			}
 		}
@@ -295,26 +375,6 @@
 	}
 
 	async function loadMonaco() {
-		if (lang == 'python') {
-			monaco.languages.register({
-				id: 'python',
-				extensions: ['.py'],
-				aliases: ['python'],
-				mimetypes: ['application/text']
-			})
-		}
-
-		let path: string = 'unknown'
-		if (lang == 'python') {
-			path = `${hash}.py`
-		} else if (lang == 'json') {
-			path = `${hash}.json`
-		} else if (lang == 'javascript') {
-			path = `${hash}.js`
-		} else if (lang == 'typescript') {
-			path = `${hash}.ts`
-		}
-		uri = `file:///${path}`
 		const model = monaco.editor.createModel(code, lang, monaco.Uri.parse(uri))
 
 		model.updateOptions({ tabSize: 2, insertSpaces: true })
@@ -324,6 +384,7 @@
 			language: lang,
 			automaticLayout,
 			readOnly: false,
+			fixedOverflowWidgets: true,
 			autoDetectHighContrast: true,
 			//lineNumbers: 'off',
 			//lineDecorationsWidth: 0,
@@ -334,23 +395,22 @@
 			minimap: {
 				enabled: false
 			},
-			scrollbar: {
-				alwaysConsumeMouseWheel: false
-			},
 			lightbulb: {
 				enabled: true
 			}
 		})
 
-		editor.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyS, function () {
-			format()
-		})
+		if (shouldBindKey) {
+			editor.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyS, function () {
+				format()
+			})
 
-		editor.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.Enter, function () {
-			if (cmdEnterAction) {
-				cmdEnterAction()
-			}
-		})
+			editor.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.Enter, function () {
+				if (cmdEnterAction) {
+					cmdEnterAction()
+				}
+			})
+		}
 
 		editor.onDidChangeModelContent((event) => {
 			code = getCode()
@@ -365,58 +425,53 @@
 			dispatch('blur')
 		})
 
-		if (lang == 'json') {
+		if (lang == 'sql') {
+			monaco.languages.register
+		} else if (lang == 'json') {
 			monaco.languages.json.jsonDefaults.setDiagnosticsOptions({
 				validate: true,
 				allowComments: false,
 				schemas: [],
 				enableSchemaRequest: true
 			})
-		}
-
-		if (lang == 'javascript' || lang == 'typescript') {
+		} else if (lang == 'javascript' || lang == 'typescript') {
 			monaco.languages.typescript.javascriptDefaults.setCompilerOptions({
 				target: monaco.languages.typescript.ScriptTarget.Latest,
 				allowNonTsExtensions: true,
 				noLib: true
 			})
-		}
-
-		if (lang == 'typescript') {
-			monaco.languages.typescript.typescriptDefaults.setCompilerOptions({
-				target: monaco.languages.typescript.ScriptTarget.Latest,
-				allowNonTsExtensions: true,
-				noLib: true
-			})
-			if (deno) {
-				monaco.languages.typescript.typescriptDefaults.setDiagnosticsOptions({
-					noSemanticValidation: true,
-					noSuggestionDiagnostics: true,
-					noSyntaxValidation: true
+			if (lang == 'typescript') {
+				monaco.languages.typescript.typescriptDefaults.setCompilerOptions({
+					target: monaco.languages.typescript.ScriptTarget.Latest,
+					allowNonTsExtensions: true,
+					noLib: true
 				})
+				if (deno) {
+					monaco.languages.typescript.typescriptDefaults.setDiagnosticsOptions({
+						noSemanticValidation: true,
+						noSuggestionDiagnostics: true,
+						noSyntaxValidation: true
+					})
+				}
 			}
-		}
 
-		if (lang == 'javascript') {
-			if (extraLib != '' && extraLibPath != '') {
-				monaco.languages.typescript.javascriptDefaults.addExtraLib(extraLib, extraLibPath)
+			if (lang == 'javascript') {
+				if (extraLib != '' && extraLibPath != '') {
+					monaco.languages.typescript.javascriptDefaults.addExtraLib(extraLib, extraLibPath)
+				}
 			}
 		}
 
 		if (lang == 'python' || deno) {
-			// install Monaco language client services
-
 			reloadWebsocket()
 		}
 
 		return () => {
-			if (editor) {
-				try {
-					closeWebsockets()
-					editor.dispose()
-				} catch (err) {
-					console.log('error disposing editor', err)
-				}
+			try {
+				closeWebsockets()
+				editor && editor.dispose()
+			} catch (err) {
+				console.log('error disposing editor', err)
 			}
 		}
 	}
@@ -455,4 +510,9 @@
 		/* stylelint-disable-next-line unit-allowed-list */
 		height: 80px;
 	}
+
+	.two-lines-editor {
+		/* stylelint-disable-next-line unit-allowed-list */
+		height: 40px;
+	}
 </style>

frontend/src/lib/components/EditorBar.svelte

@@ -2,17 +2,18 @@
 	import { ResourceService, ScriptService, VariableService } from '$lib/gen'
 	import { getScriptByPath, loadHubScripts, sendUserToast } from '$lib/utils'
 
-	import Icon from 'svelte-awesome'
 	import { faSearch } from '@fortawesome/free-solid-svg-icons'
+	import Icon from 'svelte-awesome'
 
-	import { workspaceStore, hubScripts } from '$lib/stores'
-	import ItemPicker from './ItemPicker.svelte'
-	import VariableEditor from './VariableEditor.svelte'
-	import ResourceEditor from './ResourceEditor.svelte'
+	import { hubScripts, workspaceStore } from '$lib/stores'
+	import { Button } from 'flowbite-svelte'
 	import { Highlight } from 'svelte-highlight'
 	import { python, typescript } from 'svelte-highlight/languages'
-	import Modal from './Modal.svelte'
 	import type Editor from './Editor.svelte'
+	import ItemPicker from './ItemPicker.svelte'
+	import Modal from './Modal.svelte'
+	import ResourceEditor from './ResourceEditor.svelte'
+	import VariableEditor from './VariableEditor.svelte'
 
 	export let lang: 'python3' | 'deno'
 	export let editor: Editor
@@ -172,41 +173,56 @@
 
 <VariableEditor bind:this={variableEditor} on:create={variablePicker.openModal} />
 
-<div class="flex flex-row justify-around w-full">
-	<button
-		class="default-button-secondary font-semibold py-px mr-2 text-xs align-middle max-h-8"
-		on:click|stopPropagation={() => {
-			variablePicker.openModal()
-		}}
-		>Variable picker <Icon data={faSearch} scale={0.7} />
-	</button>
+<div class="flex flex-row w-full">
+	<div class="flex flex-row w-full space-x-2 whitespace-nowrap">
+		<Button
+			size="xs"
+			color="alternative"
+			on:click={() => {
+				variablePicker.openModal()
+			}}
+		>
+			<Icon data={faSearch} scale={0.7} class="mr-2" />
+			Variable picker
+		</Button>
 
-	<button
-		class="default-button-secondary font-semibold py-px text-xs mr-2 align-middle max-h-8"
-		on:click|stopPropagation={() => {
-			resourcePicker.openModal()
-		}}
-		>Resource picker <Icon data={faSearch} scale={0.7} />
-	</button>
-	<button
-		class="default-button-secondary font-semibold py-px text-xs mr-2 align-middle max-h-8"
-		on:click|stopPropagation={() => {
-			scriptPicker.openModal()
-		}}
-		>Script explorer <Icon data={faSearch} scale={0.7} />
-	</button>
+		<Button
+			size="xs"
+			color="alternative"
+			on:click={() => {
+				resourcePicker.openModal()
+			}}
+		>
+			<Icon data={faSearch} scale={0.7} class="mr-2" />
+			Resource picker
+		</Button>
 
-	<button
-		class="default-button-secondary py-px max-h-8 text-xs"
-		on:click|stopPropagation={() => {
-			editor.reloadWebsocket()
-		}}
-	>
-		Reload assistants (status: {#if lang == 'deno'}<span
-				class={websocketAlive.deno ? 'text-green-600' : 'text-red-600'}>deno</span
-			>{:else if lang == 'python3'}<span
-				class={websocketAlive.pyright ? 'text-green-600' : 'text-red-600'}>pyright</span
-			>
-			<span class={websocketAlive.black ? 'text-green-600' : 'text-red-600'}> black</span>{/if})
-	</button>
+		<Button
+			size="xs"
+			color="alternative"
+			on:click={() => {
+				scriptPicker.openModal()
+			}}
+		>
+			<Icon data={faSearch} scale={0.7} class="mr-2" />
+			Script explorer
+		</Button>
+	</div>
+	<div class="flex flex-row-reverse w-full">
+		<Button
+			size="xs"
+			color="alternative"
+			on:click={() => {
+				editor.reloadWebsocket()
+			}}
+		>
+			Reload assistants (status:&nbsp;
+			{#if lang == 'deno'}
+				<span class={websocketAlive.deno ? 'text-green-600' : 'text-red-600'}>Deno</span>
+			{:else if lang == 'python3'}
+				<span class={websocketAlive.pyright ? 'text-green-600' : 'text-red-600'}>Pyright</span>
+				<span class={websocketAlive.black ? 'text-green-600' : 'text-red-600'}>Black</span>
+			{/if})
+		</Button>
+	</div>
 </div>

frontend/src/lib/components/FlowBuilder.svelte

@@ -2,19 +2,24 @@
 	import { goto } from '$app/navigation'
 	import { page } from '$app/stores'
 	import { FlowService, ScheduleService, type Flow } from '$lib/gen'
-	import { clearPreviewResults, workspaceStore } from '$lib/stores'
+	import { clearPreviewResults, previewResults, workspaceStore } from '$lib/stores'
 	import {
 		encodeState,
 		formatCron,
 		loadHubScripts,
+		pathIsEmpty,
 		sendUserToast,
 		setQueryWithoutLoad
 	} from '$lib/utils'
-	import { onMount } from 'svelte'
+	import { faPlay } from '@fortawesome/free-solid-svg-icons'
+	import { Breadcrumb, BreadcrumbItem, Button } from 'flowbite-svelte'
+	import { onDestroy, onMount } from 'svelte'
+	import Icon from 'svelte-awesome'
 	import { OFFSET } from './CronInput.svelte'
 	import FlowEditor from './FlowEditor.svelte'
+	import FlowPreviewContent from './FlowPreviewContent.svelte'
 	import { flowStore, mode } from './flows/flowStore'
-	import { flowToMode } from './flows/utils'
+	import { flowToMode, jobsToResults } from './flows/utils'
 
 	import ScriptSchema from './ScriptSchema.svelte'
 
@@ -22,9 +27,12 @@
 	let pathError = ''
 
 	let scheduleArgs: Record<string, any>
+	let previewArgs: Record<string, any>
 	let scheduleEnabled
 	let scheduleCron: string
 
+	let previewOpen = false
+
 	$: step = Number($page.url.searchParams.get('step')) || 1
 
 	async function createSchedule(path: string) {
@@ -111,51 +119,53 @@
 	}
 
 	async function changeStep(step: number) {
+		if (step === 2 && previewOpen) {
+			previewOpen = false
+		}
 		goto(`?step=${step}`)
 	}
 
 	flowStore.subscribe((flow: Flow) => {
-		setQueryWithoutLoad($page.url, 'state', encodeState(flowToMode(flow, $mode)))
+		if (flow) {
+			setQueryWithoutLoad($page.url, 'state', encodeState(flowToMode(flow, $mode)))
+		}
 	})
 
 	onMount(() => {
 		loadHubScripts()
 		clearPreviewResults()
 	})
+
+	onDestroy(() => {
+		//@ts-ignore
+		$flowStore = undefined
+	})
 </script>
 
-<div class="flex flex-col h-screen max-w-screen-lg xl:-ml-20 xl:pl-4 w-full -mt-4 pt-4 md:mx-10 ">
-	<!-- Nav between steps-->
-	<div class="flex flex-col w-full">
-		<div class="justify-between flex flex-row drop-shadow-sm w-full">
-			<div class="wizard-nav flex flex-row w-full">
-				<button
-					disabled={pathError != ''}
-					class="{step === 1
-						? 'default-button-disabled text-gray-700'
-						: 'default-button-secondary'} min-w-max ml-2"
-					on:click={() => {
-						changeStep(1)
-					}}>Step 1: Flow</button
-				>
-				<button
-					disabled={pathError != ''}
-					class="{step === 2
-						? 'default-button-disabled text-gray-700'
-						: 'default-button-secondary'} min-w-max ml-2"
-					on:click={() => {
-						changeStep(2)
-					}}>Step 2: UI customisation</button
-				>
-			</div>
+<div class="flex flex-row w-full h-full justify-between">
+	<div
+		class={`flex flex-col mb-96 m-auto w-full sm:w-3/4 lg:w-2/3	${previewOpen ? 'xl:w-1/2' : ''}`}
+	>
+		<!-- Nav between steps-->
+		<div class="justify-between flex flex-row w-full my-4">
+			<Breadcrumb>
+				<BreadcrumbItem>
+					<button on:click={() => changeStep(1)} class={step === 1 ? 'font-bold' : null}>
+						Flow Editor
+					</button>
+				</BreadcrumbItem>
+				<BreadcrumbItem>
+					<button on:click={() => changeStep(2)} class={step === 2 ? 'font-bold' : null}>
+						UI customisation
+					</button>
+				</BreadcrumbItem>
+			</Breadcrumb>
 			<div class="flex flex-row-reverse ml-2">
 				{#if step == 1}
 					<button
 						disabled={pathError != ''}
 						class="default-button px-6 max-h-8"
-						on:click={() => {
-							changeStep(step + 1)
-						}}
+						on:click={() => changeStep(2)}
 					>
 						Next
 					</button>
@@ -177,34 +187,57 @@
 				{$flowStore?.path}
 			</span>
 		</div>
+
+		<!-- metadata -->
+
+		{#if $flowStore}
+			{#if step === 1}
+				<FlowEditor
+					bind:pathError
+					bind:initialPath
+					bind:scheduleEnabled
+					bind:scheduleCron
+					bind:scheduleArgs
+					bind:previewArgs
+				/>
+				<Button
+					disabled={pathIsEmpty($flowStore.path)}
+					size="lg"
+					pill
+					on:click={() => (previewOpen = !previewOpen)}
+					class={`blue-button fixed bottom-10 right-10 ${previewOpen ? 'hidden' : ''}`}
+				>
+					Preview flow
+					{pathIsEmpty($flowStore.path) ? '(pick a name first!)' : ''}
+
+					<Icon data={faPlay} class="ml-2" />
+				</Button>
+			{:else if step === 2}
+				<ScriptSchema
+					synchronizedHeader={false}
+					bind:summary={$flowStore.summary}
+					bind:description={$flowStore.description}
+					bind:schema={$flowStore.schema}
+				/>
+			{/if}
+		{:else}
+			<p>Loading</p>
+		{/if}
 	</div>
 
-	<!-- metadata -->
-
-	{#if step === 1}
-		<FlowEditor
-			bind:pathError
-			bind:initialPath
-			bind:scheduleEnabled
-			bind:scheduleCron
-			bind:scheduleArgs
-		/>
-	{:else if step === 2}
-		<ScriptSchema
-			synchronizedHeader={false}
-			bind:summary={$flowStore.summary}
-			bind:description={$flowStore.description}
-			bind:schema={$flowStore.schema}
-		/>
-	{/if}
+	<div class={`relative h-screen w-1/3 ${previewOpen ? '' : 'hidden'}`}>
+		<div class="absolute top-0 h-full">
+			{#if $flowStore && step === 1}
+				<div class="fixed border-l-2 right-0 h-screen w-1/2 sm:w-1/3">
+					<FlowPreviewContent
+						bind:args={previewArgs}
+						on:close={() => (previewOpen = !previewOpen)}
+						on:change={(e) => {
+							previewResults.set(jobsToResults(e.detail))
+						}}
+					/>
+				</div>
+			{/if}
+		</div>
+	</div>
 </div>
-
-<style>
-	/* .wizard-nav {
-		@apply w-1/2 sm:w-1/4;
-	} */
-
-	.wizard-nav button {
-		max-height: 30px;
-	}
-</style>

frontend/src/lib/components/FlowEditor.svelte

@@ -1,28 +1,16 @@
 <script lang="ts">
 	import { ScheduleService } from '$lib/gen'
 
-	import { faFileExport, faFileImport, faGlobe, faPlus } from '@fortawesome/free-solid-svg-icons'
-	import Icon from 'svelte-awesome'
-	import FlowPreview from './FlowPreview.svelte'
-	import CopyFirstStepSchema from './flows/CopyFirstStepSchema.svelte'
-	import { addModule, flowStore, initFlow, mode } from './flows/flowStore'
-	import ModuleStep from './ModuleStep.svelte'
-	import Path from './Path.svelte'
-	import RadioButtonV2 from './RadioButtonV2.svelte'
-	import SchemaEditor from './SchemaEditor.svelte'
-	import Required from './Required.svelte'
-	import { flowToHubUrl, pathIsEmpty, sendUserToast } from '$lib/utils'
-	import Dropdown from './Dropdown.svelte'
-	import Editor from './Editor.svelte'
-	import Modal from './Modal.svelte'
-	import FlowViewer from './FlowViewer.svelte'
-	import { flowToMode } from './flows/utils'
-	import CronInput from './CronInput.svelte'
-	import CollapseLink from './CollapseLink.svelte'
-	import Toggle from './Toggle.svelte'
-	import SchemaForm from './SchemaForm.svelte'
-	import Tooltip from './Tooltip.svelte'
 	import { workspaceStore } from '$lib/stores'
+	import { pathIsEmpty } from '$lib/utils'
+	import { faPlus } from '@fortawesome/free-solid-svg-icons'
+	import { Button } from 'flowbite-svelte'
+	import Icon from 'svelte-awesome'
+	import FlowInput from './flows/FlowInput.svelte'
+	import FlowSettings from './flows/FlowSettings.svelte'
+	import { addModule, flowStore, mode } from './flows/flowStore'
+	import ModuleStep from './ModuleStep.svelte'
+	import Tooltip from './Tooltip.svelte'
 
 	export let pathError = ''
 	export let initialPath: string = ''
@@ -30,251 +18,90 @@
 	export let scheduleArgs: Record<string, any> = {}
 	export let scheduleEnabled = false
 	export let scheduleCron: string = '0 */5 * * *'
+	export let previewArgs: Record<string, any> = {}
 
-	let jsonSetter: Modal
-	let jsonViewer: Modal
-
-	let jsonValue: string = ''
+	let scheduleLoaded = false
 
 	async function loadSchedule() {
-		const existsSchedule = await ScheduleService.existsSchedule({
-			workspace: $workspaceStore ?? '',
-			path: initialPath
-		})
-		if (existsSchedule) {
-			const schedule = await ScheduleService.getSchedule({
+		if (!scheduleLoaded) {
+			scheduleLoaded = true
+
+			const existsSchedule = await ScheduleService.existsSchedule({
 				workspace: $workspaceStore ?? '',
 				path: initialPath
 			})
-			scheduleEnabled = schedule.enabled
-			scheduleCron = schedule.schedule
-			scheduleArgs = schedule.args ?? {}
+			if (existsSchedule) {
+				const schedule = await ScheduleService.getSchedule({
+					workspace: $workspaceStore ?? '',
+					path: initialPath
+				})
+
+				scheduleEnabled = schedule.enabled!
+				scheduleCron = schedule.schedule
+				scheduleArgs = schedule.args ?? {}
+				previewArgs = JSON.parse(JSON.stringify(scheduleArgs))
+			}
 		}
 	}
 
-	$: if ($workspaceStore && initialPath != '') {
+	$: if ($flowStore && $workspaceStore && initialPath != '') {
 		loadSchedule()
 	}
 
 	let open = 0
-	let args: Record<string, any> = {}
 </script>
 
-<Modal bind:this={jsonSetter}>
-	<div slot="title">Import JSON</div>
-	<div slot="content" class="h-full">
-		<Editor bind:code={jsonValue} lang={'json'} class="h-full" />
-	</div>
-	<div slot="submission">
-		<button
-			class="default-button px-4 py-2 font-semibold"
+{#if $flowStore}
+	<div class="flex space-y-8 flex-col items-center line">
+		<FlowSettings
+			bind:pathError
+			bind:initialPath
+			bind:scheduleArgs
+			{previewArgs}
+			bind:scheduleCron
+			bind:scheduleEnabled
+			bind:open
+		/>
+		<FlowInput />
+		{#each $flowStore?.value.modules as mod, i}
+			<ModuleStep bind:open bind:mod bind:args={previewArgs} {i} />
+			{#if i == 0 && $mode == 'pull'}
+				<div class="flex justify-center bg-white shadow p-2">
+					<p>
+						Starting from here, the flow for loop over items from the 1st step's result right above.
+						&nbsp; <br />For-loops insertable at other points is not supported yet but coming soon
+						(See
+						<a href="https://github.com/windmill-labs/windmill/issues/350">#350</a>.)
+					</p>
+					<Tooltip>
+						This flow being in 'Pull' mode, the rest of the flow will for loop over the list of
+						items returned by the trigger script right above. Retrieve the item value using
+						`flow_input._value`
+					</Tooltip>
+				</div>
+			{/if}
+		{/each}
+		<Button
+			disabled={pathIsEmpty($flowStore.path)}
+			class="blue-button"
 			on:click={() => {
-				Object.assign($flowStore, JSON.parse(jsonValue))
-				initFlow($flowStore)
-				open = -1
-				sendUserToast('OpenFlow imported from JSON')
-				jsonSetter.closeModal()
+				addModule()
+				open = $flowStore?.value.modules.length - 1
 			}}
 		>
-			Import
-		</button>
-	</div></Modal
->
-
-<Modal bind:this={jsonViewer}>
-	<div slot="title">See JSON</div>
-	<div slot="content" class="h-full">
-		<FlowViewer flow={flowToMode($flowStore, $mode)} tab="json" />
+			<Icon class="text-white mr-2" data={faPlus} />
+			Add step {pathIsEmpty($flowStore?.path) ? '(pick a name first!)' : ''}
+		</Button>
 	</div>
-</Modal>
-
-{#if $flowStore}
-	<div class="flow-root bg-gray-50 rounded-xl border  border-gray-200">
-		<ul class="relative -mt-10">
-			<span class="absolute top-0 left-1/2  h-full w-1 bg-gray-400" aria-hidden="true" />
-			<div class="relative">
-				<li class="flex flex-row flex-shrink max-w-full mx-auto mt-20">
-					<div
-						class="bg-white border border-gray xl-rounded shadow-lg w-full max-w-4xl mx-4 md:mx-auto p-4"
-					>
-						<div class="flex flex-row-reverse mr-4">
-							<Dropdown
-								dropdownItems={[
-									{
-										displayName: 'Import from a JSON OpenFlow',
-										icon: faFileImport,
-										action: () => {
-											jsonSetter.openModal()
-										}
-									},
-									{
-										displayName: 'Export to a JSON OpenFlow',
-										icon: faFileExport,
-										action: () => {
-											jsonViewer.openModal()
-										}
-									},
-									{
-										displayName: 'Publish to Hub',
-										icon: faGlobe,
-										action: () => {
-											const url = flowToHubUrl(flowToMode($flowStore, $mode))
-											window.open(url, '_blank')?.focus()
-										}
-									}
-								]}
-								relative={false}
-							/>
-						</div>
-						<div class="mb-8 p-4">
-							<Path
-								bind:error={pathError}
-								bind:path={$flowStore.path}
-								{initialPath}
-								namePlaceholder="my_flow"
-								kind="flow"
-							>
-								<div slot="ownerToolkit">
-									Flow permissions depend on their path. Select the group <span class="font-mono"
-										>all</span
-									>
-									to share your flow, and <span class="font-mono">user</span> to keep it private.
-									<a href="https://docs.windmill.dev/docs/reference/namespaces">docs</a>
-								</div>
-							</Path>
-
-							<label class="block mt-4">
-								<span class="text-gray-700">Summary <Required required={false} /></span>
-								<textarea
-									bind:value={$flowStore.summary}
-									class="
-					mt-1
-					block
-					w-full
-					rounded-md
-					border-gray-300
-					shadow-sm
-					focus:border-indigo-300 focus:ring focus:ring-indigo-200 focus:ring-opacity-50
-					"
-									placeholder="A very short summary of the flow displayed when the flow is listed"
-									rows="1"
-								/>
-							</label>
-						</div>
-						<RadioButtonV2
-							options={[
-								[
-									{
-										title: 'UI or webhook triggered',
-										desc: 'Trigger this flow through the generated UI, a manual schedule or by calling the associated webhook'
-									},
-									'push'
-								],
-								[
-									{
-										title: 'Watching changes regularly',
-										desc: 'The first module of this flow is a trigger script whose purpose is to pull data from an external source and return all new items since last run. This flow is meant to be scheduled very regularly to reduce latency to react to new events. It will trigger the rest of the flow once per item. If no new items, the flow will be skipped.'
-									},
-									'pull'
-								]
-							]}
-							bind:value={$mode}
-						/>
-						{#if $mode == 'pull'}
-							<div class="p-4">
-								<CollapseLink text="set primary schedule" open={true}>
-									<Tooltip
-										>The primary schedule of a flow is simply a schedule that has the same name as a
-										flow. It can be set and enabled directly within the flow editor. "Watching for
-										new changes" flows are meant to be watching regularly for new items in an
-										external systems. The primary schedule purpose is there to set the periodicity
-										at which you want this watcher to operate.
-									</Tooltip> &nbsp;
-									<Toggle
-										bind:checked={scheduleEnabled}
-										options={{
-											left: { label: 'disabled', value: false },
-											right: { label: 'enabled', value: true }
-										}}
-									/>
-									<div class="p-2 mt-2 rounded" class:bg-gray-300={!scheduleEnabled}>
-										{#if !scheduleEnabled}
-											<span class="font-black">No next scheduled run when disabled</span>
-										{/if}
-										<CronInput bind:schedule={scheduleCron} />
-									</div>
-									<SchemaForm schema={$flowStore.schema} bind:args={scheduleArgs} />
-								</CollapseLink>
-							</div>
-						{/if}
-					</div>
-				</li>
-				<li class="flex flex-row flex-shrink max-w-full mx-auto mt-20">
-					<div class="bg-white border border-gray xl-rounded shadow-lg w-full mx-4 xl:mx-20">
-						<div
-							class="flex items-center justify-between flex-wra px-4 py-5 border-b border-gray-200 sm:px-6"
-						>
-							<h3 class="text-lg leading-6 font-medium text-gray-900">Flow Input</h3>
-							<CopyFirstStepSchema />
-						</div>
-						<div class="p-4">
-							<SchemaEditor
-								on:change={() => {
-									$flowStore = $flowStore
-								}}
-								schema={$flowStore.schema}
-							/>
-							<div class="my-4" />
-							<FlowPreview
-								mode={$mode}
-								flow={$flowStore}
-								i={$flowStore?.value.modules.length}
-								bind:args
-							/>
-						</div>
-					</div>
-				</li>
-				{#each $flowStore?.value.modules as mod, i}
-					<li class="relative mt-16">
-						<div class="relative flex justify-center">
-							<button
-								class="default-button h-10 w-10 shadow-blue-600/40  border-blue-600 shadow"
-								on:click={() => {
-									addModule(i)
-									open = i
-								}}
-							>
-								<Icon class="text-white mb-1" data={faPlus} />
-							</button>
-						</div>
-					</li>
-					<ModuleStep bind:open bind:mod bind:args {i} mode={$mode} />
-				{/each}
-				<li class="relative m-20 ">
-					<div class="relative flex justify-center">
-						<button
-							disabled={pathIsEmpty($flowStore.path)}
-							class="default-button h-10 w-10 shadow"
-							on:click={() => {
-								addModule()
-								open = $flowStore?.value.modules.length - 1
-							}}
-						>
-							<Icon class="text-white mb-1" data={faPlus} />
-							Add step {pathIsEmpty($flowStore?.path) ? '(pick a name first!)' : ''}
-						</button>
-					</div>
-				</li>
-			</div>
-		</ul>
-	</div>
-	<div class="py-10 bg-white" />
 {:else}
 	<h3>Loading flow</h3>
 {/if}
 
 <style>
-	.shadow:not([disabled]) {
-		@apply border-blue-600 shadow-blue-600/40;
+	.line {
+		background-image: linear-gradient(#e5e7eb, #e5e7eb);
+		background-size: 2px 100%;
+		background-repeat: no-repeat;
+		background-position: center center;
 	}
 </style>

frontend/src/lib/components/FlowPreview.svelte

@@ -1,14 +1,13 @@
 <script lang="ts">
 	import type { Schema } from '$lib/common'
-	import { type InputTransform, Job, JobService, type Flow } from '$lib/gen'
+	import { Job, JobService, type Flow } from '$lib/gen'
 	import { workspaceStore } from '$lib/stores'
 	import { sendUserToast, truncateRev } from '$lib/utils'
 	import { faChevronDown, faChevronUp } from '@fortawesome/free-solid-svg-icons'
 	import { createEventDispatcher, onDestroy } from 'svelte'
 	import Icon from 'svelte-awesome'
 	import FlowJobResult from './FlowJobResult.svelte'
-	import type { FlowMode } from './flows/flowStore'
-	import { flowToMode } from './flows/utils'
+	import { runFlowPreview } from './flows/utils'
 	import FlowStatusViewer from './FlowStatusViewer.svelte'
 	import RunForm from './RunForm.svelte'
 	import Tabs from './Tabs.svelte'
@@ -17,7 +16,6 @@
 	export let i: number
 	export let flow: Flow
 	export let schemas: Schema[] = []
-	export let mode: FlowMode
 
 	export let args: Record<string, any> = {}
 
@@ -35,19 +33,14 @@
 
 	$: dispatch('change', jobs)
 
-	export async function runPreview(args) {
+	export async function runPreview(args: any) {
 		viewPreview = true
 		intervalId && clearInterval(intervalId)
-		let newFlow = flowToMode(flow, mode)
-		newFlow = tab == 'upto' ? truncateFlow(newFlow) : extractStep(newFlow)
-		jobId = await JobService.runFlowPreview({
-			workspace: $workspaceStore ?? '',
-			requestBody: {
-				args,
-				value: newFlow.value,
-				path: newFlow.path
-			}
-		})
+
+		let newFlow: Flow =
+			tab == 'upto' ? truncateFlow(flow) : setInputTransformFromArgs(extractStep(flow), args)
+		jobId = await runFlowPreview(args, newFlow)
+
 		jobs = []
 		intervalId = setInterval(loadJob, 1000)
 		sendUserToast(`started preview ${truncateRev(jobId, 10)}`)
@@ -63,13 +56,19 @@
 		const localFlow = JSON.parse(JSON.stringify(flow))
 		localFlow.value.modules = flow.value.modules.slice(i, i + 1)
 		localFlow.schema = schemas[i]
-		stepArgs = {}
-		Object.entries(flow.value.modules[i].input_transform).forEach((x) => {
-			if (x[1].type == 'static') {
-				stepArgs[x[0]] = x[1].value
+		return localFlow
+	}
+
+	function setInputTransformFromArgs(flow: Flow, args: any) {
+		let input_transform = {}
+		Object.entries(args).forEach(([key, value]) => {
+			input_transform[key] = {
+				type: 'static',
+				value: value
 			}
 		})
-		return localFlow
+		flow.value.modules[0].input_transform = input_transform
+		return flow
 	}
 
 	async function loadJob() {
@@ -97,7 +96,8 @@
 >
 	<h2 class="flex justify-center text-gray-600">
 		<div>
-			Preview<Icon class="ml-1" data={viewPreview ? faChevronUp : faChevronDown} scale={1} />
+			Preview
+			<Icon class="ml-1" data={viewPreview ? faChevronUp : faChevronDown} scale={1} />
 		</div>
 	</h2>
 </button>

frontend/src/lib/components/FlowPreviewContent.svelte

@@ -0,0 +1,75 @@
+<script lang="ts">
+	import { Job, JobService } from '$lib/gen'
+	import { workspaceStore } from '$lib/stores'
+	import { sendUserToast, truncateRev } from '$lib/utils'
+	import { faClose } from '@fortawesome/free-solid-svg-icons'
+	import { Button } from 'flowbite-svelte'
+	import { createEventDispatcher, onDestroy } from 'svelte'
+	import Icon from 'svelte-awesome'
+	import FlowJobResult from './FlowJobResult.svelte'
+	import { flowStore } from './flows/flowStore'
+	import { runFlowPreview } from './flows/utils'
+	import FlowStatusViewer from './FlowStatusViewer.svelte'
+	import SchemaForm from './SchemaForm.svelte'
+
+	const dispatch = createEventDispatcher()
+
+	export let args: Record<string, any> = {}
+
+	let intervalId: NodeJS.Timer
+	let job: Job | undefined
+	let jobs = []
+	let jobId: string
+	let isValid: boolean = false
+
+	$: dispatch('change', jobs)
+
+	export async function runPreview(args: Record<string, any>) {
+		intervalId && clearInterval(intervalId)
+		jobId = await runFlowPreview(args, $flowStore)
+		jobs = []
+		intervalId = setInterval(loadJob, 1000)
+		sendUserToast(`started preview ${truncateRev(jobId, 10)}`)
+	}
+
+	async function loadJob() {
+		try {
+			job = await JobService.getJob({ workspace: $workspaceStore!, id: jobId })
+			if (job?.type == 'CompletedJob') {
+				clearInterval(intervalId)
+			}
+		} catch (err) {
+			sendUserToast(err, true)
+		}
+	}
+
+	onDestroy(() => {
+		intervalId && clearInterval(intervalId)
+	})
+</script>
+
+<div class="flex flex-col space-y-4 h-screen bg-white">
+	<div class="flex flex-col space-y-4 p-6 border-b-2 overflow-y-auto grow">
+		<div class="flex justify-between">
+			<h3 class="text-lg leading-6 font-bold text-gray-900">Flow Preview</h3>
+
+			<Button color="alternative" on:click={() => dispatch('close')}>
+				<Icon data={faClose} />
+			</Button>
+		</div>
+		<SchemaForm schema={$flowStore.schema} bind:isValid bind:args />
+	</div>
+	<Button disabled={!isValid} class="blue-button  mx-4" on:click={() => runPreview(args)} size="md">
+		Preview
+	</Button>
+	<div class="h-full overflow-y-auto mb-16 grow">
+		{#if job}
+			<div class="w-full">
+				<FlowStatusViewer {job} bind:jobs />
+			</div>
+			{#if `result` in job}
+				<FlowJobResult {job} />
+			{/if}
+		{/if}
+	</div>
+</div>

frontend/src/lib/components/FlowStatusViewer.svelte

@@ -1,21 +1,23 @@
 <script lang="ts">
-	import { faHourglassHalf, faSpinner, faTimes } from '@fortawesome/free-solid-svg-icons'
 	import { scriptPathToHref, truncateRev } from '$lib/utils'
+	import { faHourglassHalf, faSpinner, faTimes } from '@fortawesome/free-solid-svg-icons'
 
 	import Icon from 'svelte-awesome'
 	import { check } from 'svelte-awesome/icons'
 
-	import { CompletedJob, FlowStatusModule, JobService, QueuedJob } from '$lib/gen'
+	import { CompletedJob, FlowStatusModule, Job, JobService, QueuedJob } from '$lib/gen'
 	import { workspaceStore } from '$lib/stores'
-	import JobStatus from './JobStatus.svelte'
 	import FlowJobResult from './FlowJobResult.svelte'
+	import JobStatus from './JobStatus.svelte'
 
 	export let job: QueuedJob | CompletedJob
-	export let jobs: (CompletedJob | CompletedJob[] | undefined)[] = []
+	export let jobs: (Job | Job[] | undefined)[] = []
+	export let fullyRetrieved = -1
+
+	let lastJobid: string | undefined
 
 	let forloop_selected = ''
 
-	let logs: { [key: number]: string } = {}
 	let pres: { [key: number]: HTMLElement } = {}
 
 	async function loadResults() {
@@ -25,20 +27,23 @@
 				let i = mods?.findIndex((x) => x.type == FlowStatusModule.type.IN_PROGRESS)
 				if (i != -1) {
 					let last = mods[i]
-					if (last?.type == FlowStatusModule.type.IN_PROGRESS) {
-						logs[i] =
-							(await JobService.getJob({ workspace: $workspaceStore ?? '', id: last.job ?? '' }))
-								.logs ?? ''
-						console.log(logs[i])
-						logs = logs
-						pres[i].scroll({ top: pres[i]?.scrollHeight, behavior: 'smooth' })
-					}
+					jobs[i] = await JobService.getJob({
+						workspace: $workspaceStore ?? '',
+						id: last.job ?? ''
+					})
+					jobs = jobs
+					pres[i]?.scroll({ top: pres[i]?.scrollHeight, behavior: 'smooth' })
 				}
 			}
 		}
+		if (job.id != lastJobid) {
+			lastJobid = job.id
+			jobs = []
+			fullyRetrieved = -1
+		}
 		job?.flow_status?.modules?.forEach(async (x, i) => {
 			if (
-				(i >= jobs.length && x.type == FlowStatusModule.type.SUCCESS) ||
+				(i > fullyRetrieved && x.type == FlowStatusModule.type.SUCCESS) ||
 				x.type == FlowStatusModule.type.FAILURE
 			) {
 				if (x.forloop_jobs) {
@@ -49,15 +54,20 @@
 							await JobService.getCompletedJob({ workspace: $workspaceStore!, id: j })
 						)
 					}
-					jobs.push(forloop_jobs)
+					jobs[i] = forloop_jobs
 				} else {
-					jobs.push(await JobService.getCompletedJob({ workspace: $workspaceStore!, id: x.job! }))
+					jobs[i] = await JobService.getCompletedJob({ workspace: $workspaceStore!, id: x.job! })
 				}
 				jobs = jobs
+				fullyRetrieved = i
 			}
 		})
 	}
 
+	function toJob(x: any): Job {
+		return x as Job
+	}
+
 	function toCompletedJob(x: any): CompletedJob {
 		return x as CompletedJob
 	}
@@ -69,32 +79,31 @@
 	$: $workspaceStore && job && loadResults()
 </script>
 
-<div class="flow-root w-full p-4">
-	<div class="flex flex-row-reverse">
+<div class="flow-root w-full p-6">
+	<div class="flex ">
 		{#if job}
 			<div class="flex-col">
-				<a href="/run/{job?.id}" class="font-medium text-blue-600"
-					>{truncateRev(job?.id ?? '', 10)}</a
-				>
-				<div>
-					<JobStatus {job} />
-				</div>
+				<a href="/run/{job?.id}" class="font-medium text-blue-600">
+					{truncateRev(job?.id ?? '', 10)}
+				</a>
 			</div>
 		{/if}
 	</div>
+	<JobStatus {job} />
 
 	<p class="text-gray-500 mb-6 w-full text-center">
 		Step
-		<span class="font-medium text-gray-900"
-			>{Math.min((job?.flow_status?.step ?? 0) + 1, job?.raw_flow?.modules.length ?? 0)}</span
-		>
+		<span class="font-medium text-gray-900">
+			{Math.min((job?.flow_status?.step ?? 0) + 1, job?.raw_flow?.modules.length ?? 0)}
+		</span>
 		out of <span class="font-medium text-gray-900">{job?.raw_flow?.modules.length}</span>
 		<span class="mt-4" />
 	</p>
-	<ul class="-mb-8 w-full">
+
+	<ul class="w-full">
 		{#each job?.raw_flow?.modules ?? [] as mod, i}
 			<li class="w-full">
-				<div class="relative pb-8 w-full">
+				<div class="relative w-full">
 					{#if i < (job?.raw_flow?.modules ?? []).length - 1}
 						<span
 							class="absolute top-4 left-4 -ml-px h-full w-0.5 bg-gray-200"
@@ -157,17 +166,18 @@
 									{#if job.flow_status?.modules[i].forloop_jobs}
 										{#each job.flow_status?.modules[i].forloop_jobs ?? [] as job}
 											<div class="flex flex-col">
-												<a href="/run/{job}" class="font-medium text-blue-600"
-													>{truncateRev(job ?? '', 10)}</a
-												>
+												<a href="/run/{job}" class="font-medium text-blue-600">
+													{truncateRev(job ?? '', 10)}
+												</a>
 											</div>
 										{/each}
 									{:else if job.flow_status?.modules[i].job}
 										<a
 											href="/run/{job.flow_status?.modules[i].job}"
 											class="font-medium text-blue-600"
-											>{truncateRev(job.flow_status?.modules[i].job ?? '', 10)}</a
 										>
+											{truncateRev(job.flow_status?.modules[i].job ?? '', 10)}
+										</a>
 									{/if}
 								</div>
 							</div>
@@ -191,21 +201,31 @@
 									>
 									{#if forloop_selected == job.id}
 										<svelte:self {job} />
+										{#if `result` in job}
+											<FlowJobResult {job} />
+										{/if}
 									{/if}
 								{/each}
 							</div>
-						{:else}
+						{:else if toJob(jobs[i]).type == 'CompletedJob'}
 							<FlowJobResult job={toCompletedJob(jobs[i])} />
-						{/if}
-					{:else}
-						<div class="mx-20">
-							<pre
-								bind:this={pres[i]}
-								class="break-all p-4 relative h-full mx-2 bg-gray-50 text-xs max-h-40 overflow-y-auto border">{logs[
-									i
-								] ?? ''}
+						{:else if jobs[i]}
+							{#if toJob(jobs[i])?.raw_flow}
+								<div class="border-2">
+									<h2>Forloop current iteration</h2>
+									<svelte:self job={jobs[i]} />
+								</div>
+							{:else}
+								<div class="max-w-2xl mt-2 h-full">
+									<pre
+										bind:this={pres[i]}
+										class="break-all p-4 relative h-full mx-2 bg-gray-50 text-xs max-h-40 overflow-y-auto border">{toJob(
+											jobs[i]
+										).logs ?? ''}
 							</pre>
-						</div>
+								</div>
+							{/if}
+						{/if}
 					{/if}
 				</div>
 			</li>

frontend/src/lib/components/FlowViewer.svelte

@@ -11,6 +11,8 @@
 	import Tabs from './Tabs.svelte'
 	import SchemaViewer from './SchemaViewer.svelte'
 	import FieldHeader from './FieldHeader.svelte'
+	import InputTransformsViewer from './InputTransformsViewer.svelte'
+	import SvelteMarkdown from 'svelte-markdown'
 
 	export let flow: {
 		summary: string
@@ -18,6 +20,7 @@
 		value: FlowValue
 		schema?: any
 	}
+	export let initialOpen: number | undefined = undefined
 
 	let flowFiltered = {
 		summary: flow.summary,
@@ -30,6 +33,9 @@
 
 	export let tab: 'ui' | 'json' | 'schema' = 'ui'
 	let open: { [id: number]: boolean } = {}
+	if (initialOpen) {
+		open[initialOpen] = true
+	}
 
 	function toAny(x: unknown): any {
 		return x as any
@@ -47,36 +53,40 @@
 	/>
 {/if}
 {#if tab == 'ui'}
-	<div class="flow-root w-full p-4">
-		<p class="font-black text-lg w-full ml-2">
-			<span>Inputs</span>
-		</p>
-		{#if flow.schema && flow.schema.properties && Object.keys(flow.schema.properties).length > 0 && flow.schema}
-			<ul class="my-4 ml-6">
-				{#each Object.entries(flow.schema.properties) as [inp, v]}
-					<li class="list-disc flex flex-row">
-						<FieldHeader
-							label={inp}
-							required={flow.schema.required?.includes(inp)}
-							type={toAny(v)?.type}
-							contentEncoding={toAny(v)?.contentEncoding}
-							format={toAny(v)?.format}
-							itemsType={toAny(v)?.itemsType}
-						/><span class="ml-4 mt-2 text-xs"
-							>{toAny(v)?.default != undefined
-								? 'default: ' + JSON.stringify(toAny(v)?.default)
-								: ''}</span
-						>
-					</li>
-				{/each}
-			</ul>
-		{:else}
-			<div class="text-gray-700 text-xs italic mb-4">
-				This script has no argument or is ill-defined
-			</div>
+	<div class="flow-root w-full pb-4">
+		{#if !embedded}
+			<h2 class="my-4">{flow.summary}</h2>
+			<SvelteMarkdown source={flow.description ?? ''} />
+
+			<p class="font-black text-lg w-full my-4">
+				<span>Inputs</span>
+			</p>
+			{#if flow.schema && flow.schema.properties && Object.keys(flow.schema.properties).length > 0 && flow.schema}
+				<ul class="my-2">
+					{#each Object.entries(flow.schema.properties) as [inp, v]}
+						<li class="list-disc flex flex-row">
+							<FieldHeader
+								label={inp}
+								required={flow.schema.required?.includes(inp)}
+								type={toAny(v)?.type}
+								contentEncoding={toAny(v)?.contentEncoding}
+								format={toAny(v)?.format}
+								itemsType={toAny(v)?.itemsType}
+							/><span class="ml-4 mt-2 text-xs"
+								>{toAny(v)?.default != undefined
+									? 'default: ' + JSON.stringify(toAny(v)?.default)
+									: ''}</span
+							>
+						</li>
+					{/each}
+				</ul>
+			{:else}
+				<div class="text-gray-700 text-xs italic mb-4">No inputs</div>
+			{/if}
 		{/if}
-		<p class="font-black text-lg my-6 w-full ml-2">
-			<span>{flow?.value?.modules?.length} Steps </span>
+
+		<p class="font-black text-lg my-6 w-full">
+			<span>{flow?.value?.modules?.length} Step{flow?.value?.modules?.length > 1 ? 's' : ''} </span>
 			<span class="mt-4" />
 		</p>
 		<ul class="-mb-8 w-full">
@@ -98,6 +108,7 @@
 							</div>
 							<div class="min-w-0 flex-1 pt-1.5 flex justify-between space-x-4 w-full">
 								<div class="w-full">
+									<span class="text-black">{mod?.summary ?? ''}</span>
 									<p class="text-sm text-gray-500">
 										{#if mod?.value?.type == 'script'}
 											Script at path <a
@@ -105,6 +116,34 @@
 												href={scriptPathToHref(mod?.value?.path ?? '')}
 												class="font-medium text-gray-900">{mod?.value?.path}</a
 											>
+											{#if mod?.value?.path?.startsWith('hub/')}
+												<div>
+													<button
+														on:click={async () => {
+															open[i] = !open[i]
+														}}
+														class="mb-2 underline text-black"
+													>
+														View code and inputs {open[i] ? '(-)' : '(+)'}</button
+													>
+													{#if open[i]}
+														<div class="border border-black p-2 bg-gray-50  divide-y">
+															<InputTransformsViewer inputTransforms={mod?.input_transform} />
+															<div class="w-full h-full mt-6">
+																<iframe
+																	style="height: 400px;"
+																	class="w-full h-full  text-sm"
+																	title="embedded script from hub"
+																	frameborder="0"
+																	src="https://hub.windmill.dev/embed/script/{mod?.value?.path?.substring(
+																		4
+																	)}"
+																/>
+															</div>
+														</div>
+													{/if}
+												</div>
+											{/if}
 										{:else if mod?.value?.type == 'rawscript'}
 											<button
 												on:click={() => (open[i] = !open[i])}
@@ -115,6 +154,8 @@
 
 											{#if open[i]}
 												<div transition:slide class="border border-black p-2 bg-gray-50 w-full">
+													<InputTransformsViewer inputTransforms={mod?.input_transform} />
+
 													<Highlight
 														language={mod?.value?.language == 'deno' ? typescript : python}
 														code={mod?.value?.content}
@@ -124,7 +165,7 @@
 										{:else if mod?.value?.type == 'flow'}
 											Flow at path {mod?.value?.path}
 										{:else if mod?.value?.type == 'forloopflow'}
-											For loop over step {i}'s result':
+											For loop over all the elements of the list returned as a result of step {i}:
 											<svelte:self flow={mod?.value} embedded={true} />
 										{/if}
 									</p>
@@ -148,5 +189,6 @@
 		<Highlight language={json} code={JSON.stringify(flowFiltered, null, 4)} />
 	</div>
 {:else if tab == 'schema'}
+	<div class="my-4" />
 	<SchemaViewer schema={flow.schema} />
 {/if}

frontend/src/lib/components/IconedResourceType.svelte

@@ -1,18 +1,19 @@
 <script lang="ts">
-	import Mysql from './icons/Mysql.svelte'
-	import Mail from './icons/Mail.svelte'
-	import DbIcon from './icons/DbIcon.svelte'
-	import PostgresIcon from './icons/PostgresIcon.svelte'
 	import { faGithub } from '@fortawesome/free-brands-svg-icons'
-	import Slack from './icons/Slack.svelte'
 	import Icon from 'svelte-awesome'
-	import GmailIcon from './icons/GmailIcon.svelte'
-	import GSheetsIcon from './icons/GSheetsIcon.svelte'
-	import GitlabIcon from './icons/GItlabIcon.svelte'
+	import AirtableIcon from './icons/AirtableIcon.svelte'
+	import DbIcon from './icons/DbIcon.svelte'
+	import GcalIcon from './icons/GcalIcon.svelte'
 	import GCloudIcon from './icons/GCloudIcon.svelte'
 	import GdriveIcon from './icons/GdriveIcon.svelte'
-	import GcalIcon from './icons/GcalIcon.svelte'
-	import AirtableIcon from './icons/AirtableIcon.svelte'
+	import GitlabIcon from './icons/GItlabIcon.svelte'
+	import GmailIcon from './icons/GmailIcon.svelte'
+	import GSheetsIcon from './icons/GSheetsIcon.svelte'
+	import Mail from './icons/Mail.svelte'
+	import Mysql from './icons/Mysql.svelte'
+	import PostgresIcon from './icons/PostgresIcon.svelte'
+	import Slack from './icons/Slack.svelte'
+	import TogglIcon from './icons/TogglIcon.svelte'
 
 	export let name: string
 	export let after: boolean = false
@@ -51,6 +52,8 @@
 		<GdriveIcon {height} {width} />
 	{:else if name === 'airtable'}
 		<AirtableIcon {height} {width} />
+	{:else if name === 'toggl'}
+		<TogglIcon {height} {width} />
 	{/if}
 	{#if after}
 		{name}

frontend/src/lib/components/InputTransformForm.svelte

@@ -0,0 +1,223 @@
+<script lang="ts">
+	import type { Schema } from '$lib/common'
+	import type { InputTransform } from '$lib/gen'
+	import type { InputCat } from '$lib/utils'
+	import { faChain } from '@fortawesome/free-solid-svg-icons'
+	import { Button, Tooltip } from 'flowbite-svelte'
+	import Icon from 'svelte-awesome'
+	import ArgInput from './ArgInput.svelte'
+	import Editor from './Editor.svelte'
+	import FieldHeader from './FieldHeader.svelte'
+	import DynamicInputHelpBox from './flows/DynamicInputHelpBox.svelte'
+	import { codeToStaticTemplate } from './flows/flowStore'
+	import { getDefaultExpr, isCodeInjection } from './flows/utils'
+	import OverlayPropertyPicker from './propertyPicker/OverlayPropertyPicker.svelte'
+	import Toggle from './Toggle.svelte'
+
+	export let schema: Schema
+	export let arg: InputTransform | any
+	export let argName: string
+	export let extraLib: string = 'missing extraLib'
+	export let inputCheck: { [id: string]: boolean }
+	export let i: number | undefined = undefined
+	export let pickableProperties: Object | undefined = undefined
+
+	let overlays: { [id: string]: OverlayPropertyPicker } = {}
+	let monacos: { [id: string]: Editor } = {}
+
+	let inputCats: { [id: string]: InputCat } = {}
+	let propertyType = getPropertyType(arg)
+
+	function getPropertyType(arg: InputTransform | any): 'static' | 'javascript' {
+		let type: 'static' | 'javascript' = arg.type
+		if (type == 'javascript') {
+			if (codeToStaticTemplate(arg.expr)) {
+				type = 'static'
+			}
+		}
+		return type
+	}
+
+	function setPropertyType(rawValue: string) {
+		if (!arg) {
+			return
+		}
+
+		if (isCodeInjection(rawValue)) {
+			arg.expr = getDefaultExpr(i!, argName, `\`${rawValue}\``)
+			arg.type = 'javascript'
+			propertyType = 'static'
+		} else {
+			if (arg.type === 'javascript' && propertyType === 'static') {
+				arg.type = 'static'
+			}
+			if (arg.type) {
+				propertyType = arg.type
+			}
+		}
+	}
+
+	function isStaticTemplate(inputCat: InputCat) {
+		return inputCat === 'string' || inputCat === 'sql'
+	}
+
+	function focusProp(argName: string, monacoEditor: boolean = false) {
+		Object.keys(overlays).forEach((k) => {
+			if (k == argName && (isStaticTemplate(inputCats[argName]) || monacoEditor)) {
+				overlays[k].focus()
+			} else {
+				overlays[k].unfocus()
+			}
+		})
+	}
+
+	function connectProperty(argName: string, rawValue: string) {
+		if (isStaticTemplate(inputCats[argName])) {
+			arg.value = `\$\{${rawValue}}`
+			setPropertyType(arg.value)
+		} else {
+			arg.expr = getDefaultExpr(i ?? -1, undefined, rawValue)
+			arg.type = 'javascript'
+			propertyType = 'javascript'
+		}
+
+		if (monacos[argName]) {
+			monacos[argName].setCode(arg.value)
+		}
+	}
+
+	$: checked = propertyType == 'javascript'
+</script>
+
+{#if arg != undefined}
+	<div class="flex justify-between items-center mb-2">
+		<div class="flex items-center">
+			<FieldHeader
+				label={argName}
+				format={schema.properties[argName].format}
+				contentEncoding={schema.properties[argName].contentEncoding}
+				required={schema.required.includes(argName)}
+				type={schema.properties[argName].type}
+				itemsType={schema.properties[argName].items}
+			/>
+
+			{#if !checked && arg.type === 'javascript'}
+				<span class="bg-blue-100 text-blue-800 text-sm font-medium mr-2 px-2.5 py-0.5 rounded ml-2">
+					{'${...}'}
+				</span>
+			{/if}
+		</div>
+		<Toggle
+			bind:checked
+			options={{
+				right: 'Raw Javascript Editor'
+			}}
+			on:change={(e) => {
+				const type = e.detail ? 'javascript' : 'static'
+				const staticTemplate = isStaticTemplate(inputCats[argName])
+				if (type === 'javascript') {
+					arg.expr = getDefaultExpr(
+						i ?? -1,
+						argName,
+						staticTemplate ? `\`${arg.value ?? ''}\`` : arg.value
+					)
+
+					arg.value = undefined
+					propertyType = 'javascript'
+				} else {
+					arg.value = staticTemplate ? codeToStaticTemplate(arg.expr) : undefined
+
+					arg.expr = undefined
+					propertyType = 'static'
+				}
+
+				arg.type = type
+			}}
+		/>
+	</div>
+	<div class="max-w-xs" />
+
+	{#if propertyType === undefined || !checked}
+		<OverlayPropertyPicker
+			{pickableProperties}
+			bind:this={overlays[argName]}
+			on:select={({ detail }) => {
+				if (detail.pickerVariation === 'connect') {
+					connectProperty(argName, detail.propPath)
+				} else {
+					const toAppend = `\$\{${detail.propPath}}`
+					arg.value = `${arg.value ?? ''}${toAppend}`
+					if (monacos[argName]) {
+						monacos[argName].setCode(arg.value)
+					}
+					if (isStaticTemplate(inputCats[argName])) {
+						setPropertyType(arg.value)
+					}
+				}
+			}}
+		>
+			<ArgInput
+				on:focus={() => focusProp(argName)}
+				label={argName}
+				bind:editor={monacos[argName]}
+				bind:description={schema.properties[argName].description}
+				bind:value={arg.value}
+				type={schema.properties[argName].type}
+				required={schema.required.includes(argName)}
+				bind:pattern={schema.properties[argName].pattern}
+				bind:valid={inputCheck[argName]}
+				defaultValue={schema.properties[argName].default}
+				bind:enum_={schema.properties[argName].enum}
+				bind:format={schema.properties[argName].format}
+				contentEncoding={schema.properties[argName].contentEncoding}
+				bind:itemsType={schema.properties[argName].items}
+				properties={schema.properties[argName].properties}
+				displayHeader={false}
+				bind:inputCat={inputCats[argName]}
+				on:input={(e) => {
+					if (isStaticTemplate(inputCats[argName])) {
+						setPropertyType(e.detail.rawValue)
+					}
+				}}
+			>
+				<div slot="actions">
+					<div on:click={() => overlays[argName]?.focus('connect')}>
+						<Tooltip placement="bottom" content="Input connect">
+							<Button size="sm" class="blue-button h-8">
+								<Icon data={faChain} />
+							</Button>
+						</Tooltip>
+					</div>
+				</div>
+			</ArgInput>
+		</OverlayPropertyPicker>
+	{:else if checked}
+		{#if arg.expr != undefined}
+			<OverlayPropertyPicker
+				bind:this={overlays[argName]}
+				{pickableProperties}
+				on:select={(event) => {
+					monacos[argName].insertAtCursor(event.detail.propPath)
+				}}
+			>
+				<div class="border rounded p-2 mt-2 border-gray-300">
+					<Editor
+						bind:this={monacos[argName]}
+						on:focus={() => focusProp(argName, true)}
+						bind:code={arg.expr}
+						lang="javascript"
+						class="few-lines-editor"
+						{extraLib}
+						extraLibPath="file:///node_modules/@types/windmill@{i}/index.d.ts"
+						shouldBindKey={false}
+					/>
+				</div>
+			</OverlayPropertyPicker>
+			<DynamicInputHelpBox {i} />
+		{/if}
+	{:else}
+		<p>Not recognized arg type {arg.type}</p>
+	{/if}
+{:else}
+	<p>Arg at {argName} is undefined</p>
+{/if}

frontend/src/lib/components/InputTransformsViewer.svelte

@@ -0,0 +1,22 @@
+<script lang="ts">
+	import type { InputTransform } from '$lib/gen'
+	import { Highlight } from 'svelte-highlight'
+	import ObjectViewer from './propertyPicker/ObjectViewer.svelte'
+	import typescript from 'svelte-highlight/languages/typescript'
+
+	export let inputTransforms: Record<string, InputTransform>
+</script>
+
+<ul class="mb-1">
+	{#each Object.entries(inputTransforms) as [key, val]}
+		<li>
+			<span class="font-black text-gray-700">{key}</span>: {#if val.type == 'static'}<ObjectViewer
+					json={val.value}
+				/>{:else}
+				<span class="inline-highlight">
+					<Highlight offsetTop={0} language={typescript} code={val.expr} />
+				</span>
+			{/if}
+		</li>
+	{/each}
+</ul>

frontend/src/lib/components/JobStatus.svelte

@@ -1,17 +1,18 @@
 <script lang="ts">
+	import { displayDate, forLater, msToSec } from '$lib/utils'
 	import {
 		faCalendar,
+		faCheck,
 		faCircle,
 		faClock,
 		faHourglassHalf,
 		faTimes
 	} from '@fortawesome/free-solid-svg-icons'
-	import { displayDate, forLater, msToSec } from '$lib/utils'
 
 	import Icon from 'svelte-awesome'
-	import { check } from 'svelte-awesome/icons'
 
 	import type { CompletedJob, QueuedJob } from '$lib/gen'
+	import { Badge } from 'flowbite-svelte'
 
 	const SMALL_ICON_SCALE = 0.7
 
@@ -19,32 +20,39 @@
 </script>
 
 {#if job && 'success' in job && job.success}
-	<Icon class="text-green-600" data={check} scale={SMALL_ICON_SCALE} />
-	<span class="mx-2">Succeeded {job.is_skipped ? '(Skipped)' : ''}</span>
-	<div>
-		<Icon class="text-gray-700" data={faHourglassHalf} scale={SMALL_ICON_SCALE} /><span class="mx-2"
-			>Job ran in {msToSec(job.duration_ms)}
-			s</span
-		>
-	</div>
+	<Badge large color="green">
+		<Icon data={faCheck} scale={SMALL_ICON_SCALE} class="mr-2" />
+		Succeeded {job.is_skipped ? '(Skipped)' : ''}
+	</Badge>
+
+	<Badge large>
+		<Icon data={faHourglassHalf} scale={SMALL_ICON_SCALE} class="mr-2" />
+		Job ran in {msToSec(job.duration_ms)} s
+	</Badge>
 {:else if job && 'success' in job}
-	<Icon class="text-red-700" data={faTimes} scale={SMALL_ICON_SCALE} />
-	<span class="mx-2">Failed</span>
-	<div>
-		<Icon class="text-gray-700" data={faHourglassHalf} scale={SMALL_ICON_SCALE} /><span class="mx-2"
-			>Job ran in {msToSec(job.duration_ms)}
-			s</span
-		>
-	</div>
+	<Badge large color="red">
+		<Icon data={faTimes} scale={SMALL_ICON_SCALE} class="mr-2" />
+		Failed
+	</Badge>
+	<Badge large>
+		<Icon data={faHourglassHalf} scale={SMALL_ICON_SCALE} class="mr-2" />
+		Job ran in {msToSec(job.duration_ms)}s
+	</Badge>
 {:else if job && 'running' in job && job.running}
-	<Icon class="text-yellow-500" data={faCircle} scale={SMALL_ICON_SCALE} />
-	<span class="mx-2">Running</span>
+	<Badge large color="yellow">
+		<Icon data={faCircle} scale={SMALL_ICON_SCALE} class="mr-2" />
+		Running
+	</Badge>
 {:else if job && 'running' in job && 'scheduled_for' in job && job.scheduled_for && forLater(job.scheduled_for)}
-	<Icon class="text-gray-700" data={faCalendar} scale={SMALL_ICON_SCALE} />
-	<span class="mx-2">Scheduled for {displayDate(job.scheduled_for)}</span>
+	<Badge>
+		<Icon data={faCalendar} scale={SMALL_ICON_SCALE} class="mr-2" />
+		Scheduled for {displayDate(job.scheduled_for)}
+	</Badge>
 {:else if job && 'running' in job}
-	<Icon class="text-gray-700" data={faClock} scale={SMALL_ICON_SCALE} />
-	<span class="mx-2">Queued</span>
+	<Badge>
+		<Icon data={faClock} scale={SMALL_ICON_SCALE} class="mr-2" />
+		Queued
+	</Badge>
 {:else}
 	<Icon class="text-gray-200" data={faCircle} scale={SMALL_ICON_SCALE} />
 {/if}

frontend/src/lib/components/Modal.svelte

@@ -37,10 +37,10 @@
 
 	<div class="fixed top-0 w-screen h-screen {z}">
 		<div
-			class="fixed right-0 flex flex-col w-3/4 sm:w-2/3 lg:w-1/2 h-screen border border-gray-300 shadow-xl"
+			class="fixed right-0 top-0 flex flex-col w-3/4 sm:w-2/3 lg:w-1/2 h-screen border border-gray-300 shadow-xl"
 		>
 			{#if open}
-				<div class="flex flex-row justify-between p-2  bg-white border-b border-gray-200">
+				<div class="flex flex-row justify-between p-2 bg-white border-b border-gray-200">
 					<button
 						on:click={() => {
 							open = false

frontend/src/lib/components/ModuleStep.svelte

@@ -1,28 +1,29 @@
 <script lang="ts">
-	import { type FlowModule, RawScript } from '$lib/gen'
+	import { RawScript, type FlowModule } from '$lib/gen'
 	import { previewResults } from '$lib/stores'
 	import { buildExtraLib, objectToTsType, schemaToTsType } from '$lib/utils'
-	import { faChevronDown, faChevronUp } from '@fortawesome/free-solid-svg-icons'
+	import { faChevronDown, faChevronUp, faPlus } from '@fortawesome/free-solid-svg-icons'
 	import Icon from 'svelte-awesome'
 	import Editor from './Editor.svelte'
 	import EditorBar from './EditorBar.svelte'
 	import FlowPreview from './FlowPreview.svelte'
+	import FlowBox from './flows/FlowBox.svelte'
 	import FlowInputs from './flows/FlowInputs.svelte'
 	import FlowModuleHeader from './flows/FlowModuleHeader.svelte'
 	import {
+		addModule,
 		createInlineScriptModule,
 		flowStore,
 		loadSchema,
+		mode,
 		pickScript,
-		schemasStore,
-		type FlowMode
+		schemasStore
 	} from './flows/flowStore'
-	import { getPickableProperties } from './flows/utils'
+	import { getPickableProperties, jobsToResults } from './flows/utils'
 	import SchemaForm from './SchemaForm.svelte'
 	import Tooltip from './Tooltip.svelte'
 
 	export let open: number
-	export let mode: FlowMode
 	export let i: number
 	export let mod: FlowModule
 	export let args: Record<string, any> = {}
@@ -35,72 +36,69 @@
 
 	$: schema = $schemasStore[i]
 	$: shouldPick = 'path' in mod.value && mod.value.path === '' && !('language' in mod.value)
-	$: pickableProperties = getPickableProperties($flowStore?.schema, args, $previewResults, mode, i)
+	$: pickableProperties = getPickableProperties($flowStore?.schema, args, $previewResults, $mode, i)
 	$: extraLib = buildExtraLib(
 		schemaToTsType($flowStore?.schema),
 		i === 0 ? schemaToTsType($flowStore?.schema) : objectToTsType($previewResults[i])
 	)
 
-	function toResults(jobs: any) {
-		return jobs.map((x) => {
-			if (x.result) {
-				return x.result
-			} else if (x.length) {
-				return toResults(x)
-			}
-		})
-	}
+	const isTrigger = $mode === 'pull' && i === 0
 </script>
 
-<li class="flex flex-row flex-shrink max-w-full mx-auto mt-16">
-	<div
-		class="bg-white border border-gray xl-rounded shadow-lg w-full max-w-4xl mx-4 md:mx-auto"
-		id="module-{i}"
-	>
-		<div class="flex items-center justify-between flex-wra p-4 sm:px-6 z-10">
-			<FlowModuleHeader bind:open {mod} {i} {shouldPick}>
-				<div>
-					<h3 class="text-lg font-bold text-gray-900">Step {i + 1}</h3>
-					{#if i == 0 && mode == 'pull'}
-						<h3 class="font-bold">
-							Trigger Script <Tooltip
-								>When a flow is 'Pull', the first step is a trigger script. Trigger scripts are
-								scripts that must return a list which are the new items to be treated one by one by
-								the rest of the flow, usually the list of new items since last time the flow was
-								run. One can retrieve the item in the next step using `previous_result._value`. To
-								easily compute the diff, windmill provides some helpers under the form of
-								`getInternalState` and `setInternalState`.</Tooltip
-							>
-						</h3>{/if}
-					<p>
-						{#if 'path' in mod.value && mod.value.path}
-							{mod.value.path}
-						{/if}
-						{#if 'language' in mod.value && mod.value.language}
-							Inline {mod.value.language}
-						{/if}
-						{#if !('path' in mod.value) && !('language' in mod.value)}
-							Select a script
-						{/if}
-					</p>
-				</div>
-			</FlowModuleHeader>
-		</div>
-		<div class="border-b border-gray-200" />
+<button
+	class="rounded-full h-10 w-10 bg-white border-2 border-gray-400"
+	on:click={() => {
+		addModule(i)
+		open = i
+	}}
+>
+	<Icon class="text-gray-400 mb-1" data={faPlus} />
+</button>
+<FlowBox>
+	<div id="module-{i}">
+		<FlowModuleHeader bind:open {mod} {i} {shouldPick}>
+			<div>
+				<h3 class="text-lg font-bold text-gray-900">Step {i + 1}</h3>
+				{#if isTrigger}
+					<h3 class="font-bold">
+						Trigger Script
+						<Tooltip>
+							When a flow is 'Pull', the first step is a trigger script. Trigger scripts are scripts
+							that must return a list which are the new items to be treated one by one by the rest
+							of the flow, usually the list of new items since last time the flow was run. One can
+							retrieve the item in the next step using `previous_result._value`. To easily compute
+							the diff, windmill provides some helpers under the form of `getInternalState` and
+							`setInternalState`.
+						</Tooltip>
+					</h3>
+				{/if}
+				<p>
+					{#if 'path' in mod.value && mod.value.path}
+						{mod.value.path}
+					{/if}
+					{#if 'language' in mod.value && mod.value.language}
+						Inline {mod.value.language}
+					{/if}
+					{#if !('path' in mod.value) && !('language' in mod.value)}
+						Select a script
+					{/if}
+				</p>
+			</div>
+		</FlowModuleHeader>
 		{#if open == i}
 			<div class="p-6">
 				{#if shouldPick}
 					<FlowInputs
-						isTrigger={mode == 'pull' && i == 0}
+						{isTrigger}
 						on:pick={(e) => pickScript(e.detail.path, i)}
-						on:new={(e) => createInlineScriptModule(e.detail.language, i, mode)}
+						on:new={(e) => createInlineScriptModule(e.detail.language, i, $mode)}
 					/>
 				{/if}
 				{#if mod.value.type === 'rawscript'}
-					<div class="p-1 overflow-hidden">
+					<div class="mb-2 overflow-hidden">
 						<EditorBar {editor} {websocketAlive} lang={mod.value.language ?? 'deno'} />
 					</div>
-					<div>
+					<div on:mouseleave={() => loadSchema(i)}>
 						<Editor
 							bind:websocketAlive
 							bind:this={editor}
@@ -145,10 +143,9 @@
 						bind:args
 						flow={$flowStore}
 						{i}
-						{mode}
 						schemas={$schemasStore}
 						on:change={(e) => {
-							previewResults.set(toResults(e.detail))
+							previewResults.set(jobsToResults(e.detail))
 						}}
 					/>
 				</div>
@@ -156,27 +153,16 @@
 		{/if}
 		{#if open == i}
 			<div>
-				<button class="w-full h-full" on:click={() => (open = -1)}
-					><Icon data={faChevronUp} scale={1.0} /></button
-				>
+				<button class="w-full h-full" on:click={() => (open = -1)}>
+					<Icon data={faChevronUp} scale={1.0} />
+				</button>
 			</div>
 		{:else}
 			<div>
-				<button class="w-full h-full" on:click={() => (open = i)}
-					><Icon data={faChevronDown} scale={1.0} /></button
-				>
+				<button class="w-full h-full" on:click={() => (open = i)}>
+					<Icon data={faChevronDown} scale={1.0} />
+				</button>
 			</div>
 		{/if}
 	</div>
-</li>
-{#if i == 0 && mode == 'pull'}
-	<li class="relative m-20 ">
-		<div class="relative flex justify-center bg-white shadow p-2">
-			Starting from here, the flow for loop over items from step 1's result above &nbsp;<Tooltip
-				>This flow being in 'Pull' mode, the rest of the flow will for loop over the list of items
-				returned by the trigger script right above. Retrieve the item value using
-				`flow_input._value`</Tooltip
-			>
-		</div>
-	</li>
-{/if}
+</FlowBox>

frontend/src/lib/components/ObjectResourceInput.svelte

@@ -1,15 +1,14 @@
 <script lang="ts">
 	import { ResourceService } from '$lib/gen'
 
-	import ResourcePicker from './ResourcePicker.svelte'
 	import { workspaceStore } from '$lib/stores'
+	import RadioButton from './RadioButton.svelte'
+	import ResourcePicker from './ResourcePicker.svelte'
 	import SchemaForm from './SchemaForm.svelte'
-	import RadioButtonV3 from './RadioButtonV3.svelte'
 
 	export let format: string
 	export let value: any
 
-	console.log(value)
 	function isString(value: any) {
 		return typeof value === 'string' || value instanceof String
 	}
@@ -17,12 +16,6 @@
 	let path: string = ''
 	let args: Record<string, any> = {}
 
-	if (!isString(value) && value) {
-		args = value
-	}
-
-	valueToPath()
-
 	let schema: any | undefined = undefined
 	let isValid = true
 	let resourceTypeName: string = ''
@@ -39,23 +32,35 @@
 	}
 
 	function argToValue() {
-		if (option == 'resource') {
-			value = `$res:${path}`
-		} else {
-			value = args
-		}
+		value = args
 	}
+
+	function resourceToValue() {
+		value = `$res:${path}`
+	}
+
+	function isResource() {
+		return isString(value) && value.length >= '$res:'.length
+	}
+
 	function valueToPath() {
-		path =
-			isString(value) && value.length >= '$res:'.length ? value.substr('$res:'.length) : undefined
+		if (!isString(value) && value) {
+			args = value
+		}
+
+		if (isResource()) {
+			path = value.substr('$res:'.length)
+		} else if (value !== undefined && value !== '') {
+			option = 'raw'
+		}
 	}
 
 	$: value && valueToPath()
 </script>
 
-<div class="flex flex-row w-full gap-2">
-	<div class="shrink w-40">
-		<RadioButtonV3
+<div class="flex flex-row w-full flex-wrap gap-x-2">
+	<div class="shrink">
+		<RadioButton
 			options={[
 				[`Resource (${resourceTypeName})`, 'resource'],
 				[`Raw object value`, 'raw']
@@ -70,7 +75,7 @@
 				on:refresh={() => loadSchema(format)}
 				on:change={(e) => {
 					path = e.detail
-					argToValue()
+					resourceToValue()
 				}}
 				bind:value={path}
 				resourceType={format.split('-').length > 1

frontend/src/lib/components/RadioButton.svelte

@@ -1,9 +1,10 @@
 <script lang="ts">
 	export let label = ''
-	export let options: ([string, any] | string)[]
+	export let options: [string | { title: string; desc: string }, any][]
 	export let value: any
 
 	import { createEventDispatcher } from 'svelte'
+	import Tooltip from './Tooltip.svelte'
 
 	const dispatch = createEventDispatcher()
 </script>
@@ -14,7 +15,7 @@
 		{#each options as [label, val]}
 			<label
 				class:item-button-selected={val == value}
-				class="item-button text-center text-sm h-full p-2 grow"
+				class="item-button text-center text-sm h-full p-2 grow whitespace-nowrap"
 			>
 				<input
 					type="radio"
@@ -24,7 +25,12 @@
 					aria-labelledby="memory-option-0-label"
 					on:click={() => dispatch('change', val)}
 				/>
-				<p>{label}</p>
+				<p>
+					{#if typeof label !== 'string'}
+						{label.title}
+						<Tooltip>{label.desc}</Tooltip>
+					{:else}{label}{/if}
+				</p>
 			</label>
 		{/each}
 	</div>

frontend/src/lib/components/RadioButtonV2.svelte

@@ -1,35 +0,0 @@
-<script lang="ts">
-	export let options: [{ title: string; desc?: string }, string][]
-	export let value: any
-
-	import { createEventDispatcher } from 'svelte'
-	import Tooltip from './Tooltip.svelte'
-
-	$: dispatch('change', value)
-
-	const dispatch = createEventDispatcher()
-</script>
-
-<div class="flex flex-row gap-x-4 m-4 reveal flex-wrap ">
-	{#each options as [label, val]}
-		<button
-			type="button"
-			on:click={() => {
-				value = val
-			}}
-			class="flex flex-col default-secondary-button-v2 mb-2 grow"
-			class:selected={value == val}
-		>
-			<h2 class="mb-2">{label.title} <Tooltip>{label.desc}</Tooltip></h2>
-		</button>
-	{/each}
-</div>
-
-<style>
-	.selected {
-		@apply bg-blue-500/70 text-white;
-	}
-	.selected > h2 {
-		@apply text-white;
-	}
-</style>

frontend/src/lib/components/ResourceEditor.svelte

@@ -180,6 +180,7 @@
 								enum_={resourceSchema.properties[fieldName]?.enum}
 								contentEncoding={resourceSchema.properties[fieldName]?.contentEncoding}
 								itemsType={resourceSchema.properties[fieldName]?.items}
+								properties={resourceSchema.properties[fieldName]?.properties}
 								format={resourceSchema.properties[fieldName]?.format}
 							/>
 							<div class="pb-3 ml-2 relative">

frontend/src/lib/components/ResourcePicker.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import { type Resource, ResourceService } from '$lib/gen'
+	import { ResourceService, type Resource } from '$lib/gen'
 	import { workspaceStore } from '$lib/stores'
 	import { createEventDispatcher } from 'svelte'
 	import ResourceEditor from './ResourceEditor.svelte'
@@ -27,7 +27,7 @@
 </script>
 
 <ResourceEditor bind:this={resourceEditor} on:refresh={() => loadResources(resourceType)} />
-<select class="mt-1" bind:value placeholder="Pick a resource {resourceType}">
+<select bind:value placeholder="Pick a resource {resourceType}">
 	<option value={undefined} />
 	{#each resources as r}
 		<option value={r.path}>{r.path}{r.description ? ' | ' + r.description : ''}</option>

frontend/src/lib/components/RunForm.svelte

@@ -1,14 +1,14 @@
 <script lang="ts">
 	import { page } from '$app/stores'
-	import type { Script, Flow } from '$lib/gen'
+	import type { Flow, Script } from '$lib/gen'
 	import { decodeState, getToday } from '$lib/utils'
 	import { slide } from 'svelte/transition'
 
 	import { faChevronDown, faChevronUp } from '@fortawesome/free-solid-svg-icons'
 	import Icon from 'svelte-awesome'
-	import Tooltip from './Tooltip.svelte'
 	import SvelteMarkdown from 'svelte-markdown'
 	import SchemaForm from './SchemaForm.svelte'
+	import Tooltip from './Tooltip.svelte'
 
 	export let runnable: Script | Flow | undefined
 	export let runAction: (scheduledForStr: string | undefined, args: Record<string, any>) => void
@@ -60,17 +60,16 @@
 	{#if runnable?.schema}
 		{#if detailed}
 			<h2>
-				Arguments <Tooltip
-					>The optional fields, if left blank, will use the placeholder value as default.</Tooltip
-				>
+				Arguments
+				<Tooltip>
+					The optional fields, if left blank, will use the placeholder value as default.
+				</Tooltip>
 			</h2>
 		{/if}
 		{#if !runnable.schema.properties || Object.keys(runnable.schema.properties).length === 0}
 			<div class="text-sm p-4">No arguments</div>
 		{:else}
-			<div
-				class="bg-gray-50 border border shadow-md shadow-blue-100 shadow-inner rounded border-gray-300 p-6"
-			>
+			<div class="bg-gray-50 border  shadow-blue-100 shadow-inner rounded border-gray-300 p-6">
 				<SchemaForm schema={runnable.schema} bind:isValid bind:args />
 			</div>
 		{/if}
@@ -97,8 +96,10 @@
 						class="default-button-secondary mx-2 mb-1"
 						on:click={() => {
 							scheduledForStr = undefined
-						}}>clear</button
+						}}
 					>
+						Clear
+					</button>
 				</div>
 			</div>
 		</div>
@@ -113,10 +114,8 @@
 		>
 			{#if schedulable}
 				<div>
-					schedule to run later <Icon
-						data={viewOptions ? faChevronUp : faChevronDown}
-						scale={0.5}
-					/>
+					Schedule to run later
+					<Icon data={viewOptions ? faChevronUp : faChevronDown} scale={0.5} />
 				</div>
 			{/if}
 		</button>

frontend/src/lib/components/SchemaEditor.svelte

@@ -1,12 +1,17 @@
 <script lang="ts">
-	import SchemaModal, { DEFAULT_PROPERTY, schemaToModal } from './SchemaModal.svelte'
-	import type { ModalSchemaProperty } from './SchemaModal.svelte'
 	import type { Schema, SchemaProperty } from '$lib/common'
-	import Editor from './Editor.svelte'
-	import { emptySchema, sendUserToast, truncate } from '$lib/utils'
-	import Tooltip from './Tooltip.svelte'
-	import TableCustom from './TableCustom.svelte'
+	import { emptySchema, sendUserToast } from '$lib/utils'
+	import { faPen, faPlus, faTrash } from '@fortawesome/free-solid-svg-icons'
+	import { Button } from 'flowbite-svelte'
 	import { createEventDispatcher } from 'svelte'
+	import Icon from 'svelte-awesome'
+	import Editor from './Editor.svelte'
+	import SchemaEditorProperty from './SchemaEditorProperty.svelte'
+	import type { ModalSchemaProperty } from './SchemaModal.svelte'
+	import SchemaModal, { DEFAULT_PROPERTY, schemaToModal } from './SchemaModal.svelte'
+	import TableCustom from './TableCustom.svelte'
+	import Toggle from './Toggle.svelte'
+	import Tooltip from './Tooltip.svelte'
 
 	const dispatch = createEventDispatcher()
 
@@ -58,7 +63,6 @@
 		} else if (Object.keys(schema.properties).includes(modalProperty.name) && !editing) {
 			argError = 'There is already an argument with this name'
 		} else {
-			console.log('XXXXXX')
 			schema.properties[modalProperty.name] = modalToSchema(modalProperty)
 			if (modalProperty.required) {
 				schema.required = [...schema.required, modalProperty.name]
@@ -102,6 +106,13 @@
 		try {
 			if (Object.keys(schema.properties).includes(argName)) {
 				delete schema.properties[argName]
+
+				const requiredIndex = schema.required.findIndex((arg) => arg === argName)
+
+				if (requiredIndex > -1) {
+					schema.required = schema.required.slice(requiredIndex, 1)
+				}
+
 				schema = schema
 				schemaString = JSON.stringify(schema, null, '\t')
 				dispatch('change', schema)
@@ -127,95 +138,88 @@
 </script>
 
 <div class="flex flex-col">
-	<div class="w-full">
-		<div class="flex flex-row text-base">
-			<button
-				class="text-xs sm:text-base py-1 px-6 block hover:text-blue-500 focus:outline-noneborder-gray-200  {viewJsonSchema
-					? 'text-gray-500 '
-					: 'text-gray-700 font-semibold  '}"
-				on:click={() => (viewJsonSchema ? switchTab() : null)}
-			>
-				arguments
-			</button><button
-				class="py-1 px-6 block hover:text-blue-500 focus:outline-none border-gray-200  {viewJsonSchema
-					? 'text-gray-700 font-semibold '
-					: 'text-gray-500'}"
-				on:click={() => (viewJsonSchema ? null : switchTab())}
-			>
-				advanced <Tooltip
-					>Arguments can be edited either using the wizard, or by editing their json-schema <a
-						href="https://docs.windmill.dev/docs/reference/script_arguments_reference">docs</a
-					></Tooltip
-				>
-			</button>
-			<button
-				class="default-button-secondary grow mb-1"
-				on:click={() => {
-					modalProperty = Object.assign({}, DEFAULT_PROPERTY)
-					schemaModal.openModal()
-				}}>Add argument</button
-			>
+	<div class="flex justify-between gap-x-2">
+		<Button
+			on:click={() => {
+				modalProperty = Object.assign({}, DEFAULT_PROPERTY)
+				schemaModal.openModal()
+			}}
+			class="blue-button"
+		>
+			<Icon data={faPlus} class="mr-1" />
+			Add argument
+		</Button>
+
+		<div class="flex items-center">
+			<Toggle
+				on:change={() => switchTab()}
+				options={{
+					right: 'Json Schema Editor'
+				}}
+			/>
+			<div class="ml-2">
+				<Tooltip>
+					Arguments can be edited either using the wizard, or by editing their json-schema
+					<a href="https://docs.windmill.dev/docs/reference/script_arguments_reference">docs</a>
+				</Tooltip>
+			</div>
 		</div>
 	</div>
+
 	<!--json schema or table view-->
-	<div class="border-t py-1  h-full overflow-y-auto">
+	<div class="h-full overflow-y-auto">
 		{#if !viewJsonSchema}
 			<div class="h-full">
 				{#if schema.properties && Object.keys(schema.properties).length > 0 && schema.required}
-					<TableCustom class="w-full min-h-full">
-						<tr slot="header-row" class="underline">
-							<th>name</th>
-							<th>type</th>
-							<th>description</th>
-							<th>default</th>
-							<th>required</th>
+					<TableCustom>
+						<tr slot="header-row">
+							<th>Name</th>
+							<th>Type</th>
+							<th>Description</th>
+							<th>Default</th>
+							<th>Required</th>
+							<th />
 						</tr>
 						<tbody slot="body">
 							{#each Object.entries(schema.properties) as [name, property] (name)}
 								<tr>
-									<td>{name}</td>
-									<td
-										>{property.type ?? 'any'}
-										{property.format ? `(format: ${property.format})` : ''}
-										{property.contentEncoding ? `(encoding: ${property.contentEncoding})` : ''}
+									<td class="font-bold">{name}</td>
+									<td>
+										<SchemaEditorProperty {property} />
 									</td>
-									<td>{truncate(property.description, 20)}</td>
-									<td>{truncate(JSON.stringify(property.default) ?? '', 20)}</td>
-									<td>{schema.required.includes(name) ? 'required' : 'optional'}</td>
-									<td class="">
-										<button class="mr-2" on:click={() => handleDeleteArgument(name)}
-											><svg
-												class="w-4 h-4"
-												fill="none"
-												stroke="currentColor"
-												viewBox="0 0 24 14"
-												xmlns="http://www.w3.org/2000/svg"
-											>
-												<path
-													stroke-linecap="round"
-													stroke-linejoin="round"
-													stroke-width="2"
-													d="M6 18L18 6M6 6l12 12"
-												/>
-											</svg></button
-										>
-										<button
-											class="default-button-secondary text-xs inline-flex"
-											on:click={() => {
-												startEditArgument(name)
-											}}>edit</button
-										></td
+									<td>{property.description}</td>
+									<td>{property.default ? JSON.stringify(property.default) : ''}</td>
+									<td
+										>{#if schema.required.includes(name)}
+											<span class="text-red-600 font-bold text-lg">*</span>
+										{/if}</td
 									>
+									<td class="justify-end flex">
+										<Button
+											color="red"
+											outline
+											class="mr-2"
+											size="xs"
+											on:click={() => handleDeleteArgument(name)}
+										>
+											<Icon data={faTrash} class="mr-2" scale={0.8} />
+											Delete
+										</Button>
+										<Button color="alternative" size="xs" on:click={() => startEditArgument(name)}>
+											<Icon data={faPen} class="mr-2" scale={0.8} />
+											Edit
+										</Button>
+									</td>
 								</tr>
 							{/each}
 						</tbody>
 					</TableCustom>
 				{:else}
-					<div class="text-gray-700 text-xs italic">This script has no argument</div>
+					<div class="text-gray-700 text-xs italic mt-2">This schema has no arguments.</div>
 				{/if}
 			</div>
 		{:else}
-			<div>
+			<div class="border rounded mt-4 p-2">
 				<Editor
 					on:change={() => {
 						try {
@@ -225,7 +229,7 @@
 						}
 					}}
 					bind:code={schemaString}
-					lang={'json'}
+					lang="json"
 					class="small-editor"
 				/>
 			</div>

frontend/src/lib/components/SchemaEditorProperty.svelte

@@ -0,0 +1,16 @@
+<script lang="ts">
+	import type { SchemaProperty } from '$lib/common'
+	import { Badge } from 'flowbite-svelte'
+
+	export let property: SchemaProperty
+</script>
+
+<div class="flex flex-row flex-wrap gap-1">
+	<Badge color="blue">{property.type?.toUpperCase() ?? 'ANY'}</Badge>
+	{#if property.format}
+		<Badge color="green">{property.format?.toUpperCase()}</Badge>
+	{/if}
+	{#if property.contentEncoding}
+		<Badge color="indigo">{property.contentEncoding?.toUpperCase()}</Badge>
+	{/if}
+</div>

frontend/src/lib/components/SchemaForm.svelte

@@ -1,201 +1,50 @@
 <script lang="ts">
 	import type { Schema } from '$lib/common'
 	import type { InputTransform } from '$lib/gen'
-	import { allTrue, type InputCat } from '$lib/utils'
+	import { allTrue } from '$lib/utils'
 	import ArgInput from './ArgInput.svelte'
-	import Editor from './Editor.svelte'
-	import FieldHeader from './FieldHeader.svelte'
-	import DynamicInputHelpBox from './flows/DynamicInputHelpBox.svelte'
-	import { codeToStaticTemplate } from './flows/flowStore'
-	import { getCodeInjectionExpr, getDefaultExpr, isCodeInjection } from './flows/utils'
-	import OverlayPropertyPicker from './propertyPicker/OverlayPropertyPicker.svelte'
-	import Toggle from './Toggle.svelte'
+	import InputTransformForm from './InputTransformForm.svelte'
 
 	export let inputTransform = false
 	export let schema: Schema
 	export let args: Record<string, InputTransform | any> = {}
 	export let editableSchema = false
-	export let extraLib: string = 'missing extraLib'
 	export let isValid: boolean = true
-
-	export let i: number | undefined = undefined
 	export let pickableProperties: Object | undefined = undefined
+	export let extraLib: string = 'missing extraLib'
+	export let i: number | undefined = undefined
 
 	let inputCheck: { [id: string]: boolean } = {}
-	let overlays: { [id: string]: OverlayPropertyPicker } = {}
-	let monacos: { [id: string]: Editor } = {}
-
 	$: isValid = allTrue(inputCheck) ?? false
 
-	let propertiesTypes: { [id: string]: 'javascript' | 'static' } = inputTransform
-		? Object.keys(args).reduce((acc, key) => {
-				let type = args[key].type
-				if (type == 'javascript') {
-					if (codeToStaticTemplate(args[key].expr)) {
-						type = 'static'
-					}
-				}
-				acc[key] = type
-				return acc
-		  }, {})
-		: {}
-
-	let inputCats: { [id: string]: InputCat } = {}
-
-	function setPropertyType(id: string, rawValue: string, isRaw: boolean) {
-		const arg = args[id]
-		if (!arg) {
-			return
-		}
-
-		if (isCodeInjection(rawValue)) {
-			args[id].expr = getCodeInjectionExpr(rawValue, isRaw)
-			args[id].type = 'javascript'
-			propertiesTypes[id] = 'static'
-		} else {
-			if (args[id].type === 'javascript' && propertiesTypes[id] === 'static') {
-				args[id].type = 'static'
-				if (inputCats[id] == 'number') {
-					args[id].value = Number(args[id].value)
-				}
-			}
-			if (arg.type) {
-				propertiesTypes[id] = arg.type
-			}
-		}
+	$: if (!args) {
+		args = {}
 	}
 
-	function isStaticTemplate(inputCat: InputCat) {
-		return inputCat === 'string' || inputCat === 'number'
-	}
-
-	function focusProp(argName: string) {
-		Object.keys(overlays).forEach((k) => {
-			if (k == argName) {
-				overlays[k].focus()
-			} else {
-				overlays[k].unfocus()
+	function removeExtraKey() {
+		Object.keys(args ?? {}).forEach((key) => {
+			if (!Object.keys(schema?.properties ?? {}).includes(key)) {
+				delete args[key]
 			}
 		})
 	}
+
+	$: Object.keys(schema?.properties ?? {}).length > 0 && removeExtraKey()
 </script>
 
 <div class="w-full">
 	{#if Object.keys(schema?.properties ?? {}).length > 0}
-		{#each Object.keys(schema?.properties ?? {}) as argName, index}
+		{#each Object.keys(schema?.properties ?? {}) as argName}
 			{#if inputTransform}
-				{#if args[argName] != undefined}
-					<div class={index > 0 ? 'mt-8' : ''} />
-					<div class="flex justify-between items-center">
-						<div class="flex items-center">
-							<FieldHeader
-								label={argName}
-								format={schema.properties[argName].format}
-								contentEncoding={schema.properties[argName].contentEncoding}
-								required={schema.required.includes(argName)}
-								type={schema.properties[argName].type}
-								itemsType={schema.properties[argName].items}
-							/>
-							{#if propertiesTypes[argName] === 'static' && args[argName].type === 'javascript'}
-								<span
-									class="bg-blue-100 text-blue-800 text-sm font-medium mr-2 px-2.5 py-0.5 rounded ml-2"
-								>
-									{'${...}'}
-								</span>
-							{/if}
-						</div>
-						<Toggle
-							options={{
-								left: { label: '', value: 'static' },
-								right: { label: 'Raw Javascript Editor', value: 'javascript' }
-							}}
-							bind:value={propertiesTypes[argName]}
-							on:change={(e) => {
-								const staticTemplate = isStaticTemplate(inputCats[argName])
-								if (e.detail === 'javascript') {
-									args[argName].expr = getDefaultExpr(
-										i ?? -1,
-										argName,
-										staticTemplate ? args[argName].value : undefined
-									)
-									args[argName].value = undefined
-								} else {
-									args[argName].value = staticTemplate
-										? codeToStaticTemplate(args[argName].expr)
-										: undefined
-									args[argName].expr = undefined
-								}
-
-								args[argName].type = e.detail
-							}}
-						/>
-					</div>
-					<div class="max-w-xs" />
-
-					{#if propertiesTypes[argName] === undefined || propertiesTypes[argName] === 'static'}
-						<OverlayPropertyPicker
-							bind:this={overlays[argName]}
-							{pickableProperties}
-							disabled={!isStaticTemplate(inputCats[argName])}
-							on:select={(event) => {
-								const toAppend = `\$\{${event.detail}}`
-								args[argName].value = `${args[argName].value ?? ''}${toAppend}`
-								setPropertyType(argName, args[argName].value, false)
-							}}
-						>
-							<ArgInput
-								on:focus={() => focusProp(argName)}
-								label={argName}
-								bind:description={schema.properties[argName].description}
-								bind:value={args[argName].value}
-								type={schema.properties[argName].type}
-								required={schema.required.includes(argName)}
-								bind:pattern={schema.properties[argName].pattern}
-								bind:valid={inputCheck[argName]}
-								defaultValue={schema.properties[argName].default}
-								bind:enum_={schema.properties[argName].enum}
-								bind:format={schema.properties[argName].format}
-								contentEncoding={schema.properties[argName].contentEncoding}
-								bind:itemsType={schema.properties[argName].items}
-								displayHeader={false}
-								bind:inputCat={inputCats[argName]}
-								numberAsString={true}
-								on:input={(e) => {
-									if (isStaticTemplate(inputCats[argName])) {
-										setPropertyType(argName, e.detail.rawValue, e.detail.isRaw)
-									}
-								}}
-							/>
-						</OverlayPropertyPicker>
-					{:else if propertiesTypes[argName] === 'javascript'}
-						{#if args[argName].expr != undefined}
-							<OverlayPropertyPicker
-								bind:this={overlays[argName]}
-								{pickableProperties}
-								on:select={(event) => {
-									monacos[argName].insertAtCursor(event.detail)
-								}}
-							>
-								<div class="border rounded p-2 mt-2 border-gray-300">
-									<Editor
-										bind:this={monacos[argName]}
-										on:focus={() => focusProp(argName)}
-										bind:code={args[argName].expr}
-										lang="javascript"
-										class="few-lines-editor"
-										{extraLib}
-										extraLibPath="file:///node_modules/@types/windmill@{i}/index.d.ts"
-									/>
-								</div>
-							</OverlayPropertyPicker>
-							<DynamicInputHelpBox />
-						{/if}
-					{:else}
-						<p>Not recognized arg type {args[argName].type}</p>
-					{/if}
-				{:else}
-					<p>Arg at {argName} is undefined</p>
-				{/if}
+				<InputTransformForm
+					bind:arg={args[argName]}
+					bind:schema
+					bind:argName
+					bind:inputCheck
+					bind:pickableProperties
+					bind:extraLib
+					bind:i
+				/>
 			{:else}
 				<ArgInput
 					label={argName}
@@ -209,6 +58,7 @@
 					bind:enum_={schema.properties[argName].enum}
 					bind:format={schema.properties[argName].format}
 					contentEncoding={schema.properties[argName].contentEncoding}
+					properties={schema.properties[argName].properties}
 					bind:itemsType={schema.properties[argName].items}
 					{editableSchema}
 				/>

frontend/src/lib/components/SchemaViewer.svelte

@@ -63,7 +63,11 @@
 									>{property.format ?? ''}
 									{property.contentEncoding ? `(encoding: ${property.contentEncoding})` : ''}</td
 								>
-								<td>{schema.required.includes(name) ? 'required' : 'optional'}</td>
+								<td
+									>{#if schema.required.includes(name)}
+										<span class="text-red-600 font-bold text-lg">*</span>
+									{/if}</td
+								>
 							</tr>
 						{/each}
 					</tbody>

frontend/src/lib/components/ScriptBuilder.svelte

@@ -1,19 +1,20 @@
 <script lang="ts">
 	import { ScriptService, type Script } from '$lib/gen'
 
-	import { emptySchema, encodeState, sendUserToast, setQueryWithoutLoad } from '$lib/utils'
-	import { onDestroy } from 'svelte'
-	import ScriptEditor from './ScriptEditor.svelte'
-	import { page } from '$app/stores'
 	import { goto } from '$app/navigation'
-	import Path from './Path.svelte'
-	import SvelteMarkdown from 'svelte-markdown'
-	import { workspaceStore } from '$lib/stores'
-	import ScriptSchema from './ScriptSchema.svelte'
+	import { page } from '$app/stores'
 	import { inferArgs } from '$lib/infer'
-	import Required from './Required.svelte'
 	import { DENO_INIT_CODE, DENO_INIT_CODE_TRIGGER, initialCode } from '$lib/script_helpers'
-	import RadioButtonV3 from './RadioButtonV3.svelte'
+	import { workspaceStore } from '$lib/stores'
+	import { emptySchema, encodeState, sendUserToast, setQueryWithoutLoad } from '$lib/utils'
+	import { Breadcrumb, BreadcrumbItem } from 'flowbite-svelte'
+	import { onDestroy } from 'svelte'
+	import SvelteMarkdown from 'svelte-markdown'
+	import Path from './Path.svelte'
+	import RadioButton from './RadioButton.svelte'
+	import Required from './Required.svelte'
+	import ScriptEditor from './ScriptEditor.svelte'
+	import ScriptSchema from './ScriptSchema.svelte'
 
 	let editor: ScriptEditor
 	let scriptSchema: ScriptSchema
@@ -96,34 +97,33 @@
 <div class="flex flex-col h-screen max-w-screen-lg xl:-ml-20 xl:pl-4 w-full -mt-4 pt-4 md:mx-10 ">
 	<!-- Nav between steps-->
 	<div class="flex flex-col w-full">
-		<div class="justify-between flex flex-row drop-shadow-sm w-full">
+		<div class="justify-between flex flex-row drop-shadow-sm w-full mt-4">
 			<div class="wizard-nav flex flex-row w-full">
-				<button
-					class="{step === 1
-						? 'default-button-disabled text-gray-700'
-						: 'default-button-secondary'} min-w-max ml-2"
-					on:click={() => {
-						changeStep(1)
-					}}>Step 1: Metadata</button
-				>
-				<button
-					disabled={pathError != ''}
-					class="{step === 2
-						? 'default-button-disabled text-gray-700'
-						: 'default-button-secondary'} min-w-max ml-2"
-					on:click={() => {
-						changeStep(2)
-					}}>Step 2: Code</button
-				>
-				<button
-					disabled={pathError != ''}
-					class="{step === 3
-						? 'default-button-disabled text-gray-700'
-						: 'default-button-secondary'} min-w-max ml-2"
-					on:click={() => {
-						changeStep(3)
-					}}>Step 3: UI customisation</button
-				>
+				<Breadcrumb>
+					<BreadcrumbItem>
+						<button on:click={() => changeStep(1)} class={step === 1 ? 'font-bold' : null}>
+							Metadata
+						</button>
+					</BreadcrumbItem>
+					<BreadcrumbItem>
+						<button
+							on:click={() => changeStep(2)}
+							class={step === 2 ? 'font-bold' : null}
+							disabled={pathError != ''}
+						>
+							Code
+						</button>
+					</BreadcrumbItem>
+					<BreadcrumbItem>
+						<button
+							on:click={() => changeStep(3)}
+							class={step === 3 ? 'font-bold' : null}
+							disabled={pathError != ''}
+						>
+							UI customisation
+						</button>
+					</BreadcrumbItem>
+				</Breadcrumb>
 			</div>
 			<div class="flex flex-row-reverse ml-2">
 				{#if step != 3}
@@ -177,7 +177,7 @@
 			</Path>
 			<h3 class="text-gray-700 pb-1 border-b">Language</h3>
 			<div class="max-w-md">
-				<RadioButtonV3
+				<RadioButton
 					label="Language"
 					options={[
 						['Typescript (Deno)', 'deno'],
@@ -279,7 +279,6 @@
 	/* .wizard-nav {
 		@apply w-1/2 sm:w-1/4;
 	} */
-
 	.wizard-nav button {
 		max-height: 30px;
 	}

frontend/src/lib/components/ScriptEditor.svelte

@@ -1,9 +1,8 @@
 <script lang="ts">
-	import { JobService, Job, CompletedJob } from '$lib/gen'
-	import { emptySchema, displayDate } from '$lib/utils'
 	import type { Schema } from '$lib/common'
-	import { fade } from 'svelte/transition'
-	import Icon from 'svelte-awesome'
+	import { CompletedJob, Job, JobService } from '$lib/gen'
+	import { userStore, workspaceStore } from '$lib/stores'
+	import { displayDate, emptySchema } from '$lib/utils'
 	import {
 		faCheck,
 		faChevronDown,
@@ -12,26 +11,25 @@
 		faSpinner,
 		faTimes
 	} from '@fortawesome/free-solid-svg-icons'
-	import Editor from './Editor.svelte'
-	import Tooltip from './Tooltip.svelte'
 	import { onDestroy, onMount } from 'svelte'
-	import { userStore, workspaceStore } from '$lib/stores'
-	import TableCustom from './TableCustom.svelte'
+	import Icon from 'svelte-awesome'
 	import { check } from 'svelte-awesome/icons'
+	import { fade } from 'svelte/transition'
+	import Editor from './Editor.svelte'
 	import Modal from './Modal.svelte'
+	import TableCustom from './TableCustom.svelte'
+	import Tooltip from './Tooltip.svelte'
 
 	import { inferArgs } from '$lib/infer'
 
 	// @ts-ignore
-	import { VSplitPane } from 'svelte-split-pane'
-	import SchemaForm from './SchemaForm.svelte'
-	import DisplayResult from './DisplayResult.svelte'
 	import type { Preview } from '$lib/gen/models/Preview'
-	import EditorBar from './EditorBar.svelte'
 	import { Highlight } from 'svelte-highlight'
-	import typescript from 'svelte-highlight/languages/typescript'
-	import python from 'svelte-highlight/languages/python'
-	import json from 'svelte-highlight/languages/json'
+	import { json, python, typescript } from 'svelte-highlight/languages'
+	import { VSplitPane } from 'svelte-split-pane'
+	import DisplayResult from './DisplayResult.svelte'
+	import EditorBar from './EditorBar.svelte'
+	import SchemaForm from './SchemaForm.svelte'
 
 	// Exported
 	export let schema: Schema = emptySchema()
@@ -164,7 +162,9 @@
 		await inferArgs(lang, editor.getCode(), schema)
 		schema = schema
 
-		isDefault.forEach((key) => (args[key] = schema.properties[key].default))
+		isDefault
+			.filter((key) => schema.properties[key] != undefined)
+			.forEach((key) => (args[key] = schema.properties[key].default))
 		for (const key of Object.keys(args)) {
 			if (schema.properties[key] == undefined) {
 				delete args[key]
@@ -240,7 +240,13 @@
 			<div class="header">
 				<EditorBar {editor} {lang} {websocketAlive} />
 			</div>
-			<div class="flex-1 overflow-hidden">
+			<div
+				class="flex-1 overflow-hidden border p-2 rounded"
+				on:mouseleave={() => {
+					code = getEditor().getCode()
+					inferSchema()
+				}}
+			>
 				<Editor
 					{code}
 					bind:websocketAlive
@@ -371,7 +377,7 @@
 				</div>
 			</div>
 		</div>
-		<div bind:this={div} class="preview flex-1 overflow-hidden p-3">
+		<div bind:this={div} class="preview flex-1 p-3">
 			{#if previewTab === 'logs'}
 				<pre
 					class="break-all relative h-full mx-2">{#if previewJob && previewJob.logs}{previewJob.logs}
@@ -423,7 +429,7 @@
 					{:else}No local save{/if}
 				</div>
 			{:else if previewTab === 'history'}
-				<TableCustom paginated={false}>
+				<TableCustom>
 					<tr slot="header-row">
 						<th class="text-xs">id</th>
 						<th class="text-xs">created at</th>
@@ -435,9 +441,9 @@
 					<tbody slot="body">
 						{#each pastPreviews as { id, created_at, success, result }}
 							<tr class="">
-								<td class="text-xs"
-									><a class="pr-3" href="/run/{id}" target="_blank">{id.substring(30)}</a></td
-								>
+								<td class="text-xs">
+									<a class="pr-3" href="/run/{id}" target="_blank">{id.substring(30)}</a>
+								</td>
 								<td class="text-xs">{displayDate(created_at)}</td>
 								<td class="text-xs">
 									{#if success}
@@ -454,9 +460,10 @@
 											modalViewerContent = result
 											modalViewerMode = 'result'
 											modalViewer.openModal()
-										}}>{JSON.stringify(result).substring(0, 30)}...</a
-									></td
-								>
+										}}
+										>{JSON.stringify(result).substring(0, 30)}...
+									</a>
+								</td>
 								<td class="text-xs"
 									><a
 										href="#code"
@@ -502,12 +509,7 @@
 
 <style>
 	.header {
-		max-height: 28px;
-		@apply border;
-		@apply px-2 py-1;
-		@apply border-l border-r;
-		@apply bg-gray-100;
-		@apply text-xs text-gray-700;
+		@apply py-2;
 	}
 
 	.preview {

frontend/src/lib/components/ScriptPicker.svelte

@@ -12,7 +12,7 @@
 	import python from 'svelte-highlight/languages/python'
 
 	import { getScriptByPath } from '$lib/utils'
-	import RadioButtonV3 from './RadioButtonV3.svelte'
+	import RadioButton from './RadioButton.svelte'
 
 	export let scriptPath: string | undefined = undefined
 	export let allowFlow = false
@@ -65,7 +65,7 @@
 <div class="flex flex-row items-center space-x-5">
 	<div class="w-80">
 		{#if options.length > 1}
-			<RadioButtonV3 bind:value={itemKind} {options} />
+			<RadioButton bind:value={itemKind} {options} />
 		{/if}
 	</div>
 

frontend/src/lib/components/SharedBadge.svelte

@@ -23,12 +23,12 @@
 			reason = 'This item was shared to you personally'
 		} else {
 			let writeGroup = pgroups.find((x) => extraPermsKeys.includes(x) && extraPerms[x])
-			if (pgroups.find((x) => x in extraPermsKeys && extraPerms[x])) {
+			if (writeGroup) {
 				kind = 'write'
 				reason = `This item was write shared to the group ${writeGroup} which you are a member of`
 			} else {
-				let readGroup = pgroups.find((x) => extraPermsKeys.includes(x) && extraPerms[x])
-				if (pgroups.find((x) => extraPermsKeys.includes(x))) {
+				let readGroup = pgroups.find((x) => extraPermsKeys.includes(x))
+				if (readGroup) {
 					kind = 'read'
 					reason = `This item was read-only shared to the group ${readGroup} which you are a member of`
 				} else {

frontend/src/lib/components/StringTypeNarrowing.svelte

@@ -1,5 +1,5 @@
 <script lang="ts">
-	import RadioButtonV3 from './RadioButtonV3.svelte'
+	import RadioButton from './RadioButton.svelte'
 	import ResourceTypePicker from './ResourceTypePicker.svelte'
 
 	export let pattern: string | undefined
@@ -31,7 +31,7 @@
 	$: contentEncoding = kind == 'base64' ? 'base64' : undefined
 </script>
 
-<RadioButtonV3
+<RadioButton
 	label="Kind"
 	options={[
 		['None', 'none'],

frontend/src/lib/components/TableCustom.svelte

@@ -13,7 +13,7 @@
 - the second slot should be a <tbody>, containing th elements
 -->
 <div class="flex flex-col {$$props.class} min-w-full">
-	<div class="inline-block min-w-full py-2 align-middle md:px-6 lg:px-8">
+	<div class="inline-block min-w-full py-2 align-middle">
 		<table id="table-custom" class="min-w-full divide-y divide-gray-300 table-auto">
 			<thead>
 				<slot name="header-row" />
@@ -25,19 +25,16 @@
 		<div class="flex flex-row-reverse text-gray-500 mb-6">
 			<button
 				class="ml-2 drop-shadow-md {showNext ? 'visible' : 'invisible'}"
-				on:click={() => {
-					dispatch('next')
-				}}>Next</button
+				on:click={() => dispatch('next')}
 			>
+				Next
+			</button>
 			<button
 				class="mx-2 drop-shadow-md {currentPage === 1 ? 'hidden' : ''}"
-				on:click={() => {
-					dispatch('previous')
-				}}>Previous</button
+				on:click={() => dispatch('previous')}
 			>
+				Previous
+			</button>
 		</div>
 	{/if}
 </div>
-
-<style>
-</style>

frontend/src/lib/components/TableSimple.svelte

@@ -12,7 +12,7 @@
 </script>
 
 <div class="mt-8 flex flex-col {$$props.class}">
-	<div class="inline-block min-w-full py-2 align-middle md:px-6 lg:px-8">
+	<div class="inline-block min-w-full align-middle">
 		<table class="min-w-full divide-y divide-gray-300 table-auto">
 			<thead>
 				<tr class={twTextSize}>

frontend/src/lib/components/Toggle.svelte

@@ -1,23 +1,20 @@
 <script lang="ts">
 	import { createEventDispatcher } from 'svelte'
-	export let value: any = undefined
-	export let options:
-		| {
-				left: { label?: string; value: any }
-				right: { label?: string; value: any }
-		  }
-		| undefined = undefined
-	export let checked = options?.right.value === value
+
+	export let options: {
+		left?: string
+		right?: string
+	} = {}
+	export let checked: boolean = false
 
 	const id = (Math.random() + 1).toString(36).substring(10)
-
 	const dispatch = createEventDispatcher()
 </script>
 
 <span>
 	<label for={id} class="inline-flex items-center cursor-pointer mt-2">
-		{#if Boolean(options?.left.label)}
-			<span class="mr-2 text-sm font-medium text-gray-900">{options?.left.label}</span>
+		{#if Boolean(options?.left)}
+			<span class="mr-2 text-sm font-medium text-gray-900">{options?.left}</span>
 		{/if}
 		<div class="relative">
 			<input
@@ -27,16 +24,15 @@
 				class="sr-only peer"
 				bind:checked
 				on:change={() => {
-					value = options ? (checked ? options?.right.value : options?.left.value) : checked
-					dispatch('change', value ?? checked)
+					dispatch('change', checked)
 				}}
 			/>
 			<div
 				class="w-11 h-6 bg-gray-200 rounded-full peer peer-focus:ring-4 peer-focus:ring-blue-300 peer-checked:after:translate-x-full peer-checked:after:border-white after:content-[''] after:absolute after:top-0.5 after:left-[2px] after:bg-white after:border-gray-300 after:border after:rounded-full after:h-5 after:w-5 after:transition-all peer-checked:bg-blue-600"
 			/>
 		</div>
-		{#if Boolean(options?.right.label)}
-			<span class="ml-2 text-sm font-medium text-gray-900">{options?.right.label}</span>
+		{#if Boolean(options?.right)}
+			<span class="ml-2 text-sm font-medium text-gray-900">{options?.right}</span>
 		{/if}
 	</label>
 </span>

frontend/src/lib/components/Tooltip.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
-	import Icon from 'svelte-awesome'
 	import { faInfoCircle } from '@fortawesome/free-solid-svg-icons'
+	import Icon from 'svelte-awesome'
 	import { createPopperActions } from 'svelte-popperjs'
 	import { fade } from 'svelte/transition'
 	const [popperRef, popperContent] = createPopperActions({
@@ -50,6 +50,7 @@
 		use:popperContent={extraOpts}
 		on:mouseenter={open}
 		on:mouseleave={close}
+		class="w-96 text-left"
 	>
 		<slot />
 		<div id="arrow" data-popper-arrow />

frontend/src/lib/components/flows/CopyFirstStepSchema.svelte

@@ -1,11 +1,13 @@
 <script lang="ts">
+	import { Button } from 'flowbite-svelte'
 	import { copyFirstStepSchema, isCopyFirstStepSchemaDisabled } from './flowStore'
 </script>
 
-<button
-	class="default-secondary-button-v2"
+<Button
+	color="alternative"
+	size="sm"
 	disabled={$isCopyFirstStepSchemaDisabled}
 	on:click={copyFirstStepSchema}
 >
-	Copy from step 1's schema
-</button>
+	Copy from first step schema
+</Button>

frontend/src/lib/components/flows/DynamicInputHelpBox.svelte

@@ -3,6 +3,8 @@
 	import Icon from 'svelte-awesome'
 	import { slide } from 'svelte/transition'
 
+	export let i: number = 1
+
 	$: opened = false
 </script>
 
@@ -27,29 +29,29 @@
 	>
 		<p class="font-bold">Dynamic arg help</p>
 		<p>
-			When a field is "dynamic", its value is computed dynamically as the evaluation of its
-			corresponding typescript snippet.
+			When a field is using the "Raw Javascript Editor", its value is computed dynamically as the
+			evaluation of its corresponding javascript snippet.
 		</p>
-		That snippet can be single line:
+		That snippet can be a single line:
 		<pre><code>last_result.myarg</code></pre>
-		or multiline:
+		or a multiline:
 		<pre><code
 				>let x = 5;
 x + 2</code
 			></pre>
 		<p>
-			If it is multiline, the last statement before the final expression<b
+			If it is multiline, the statement before the final expression <b
 				>MUST END WITH ; and a newline</b
 			>
 		</p>
 		The snippet can also be a string template:
-		<pre><code
-				>`Hello $&#123;params.name&#125;, all your base $&#123;previous_result.base_name&#125;
-belong to us`</code
-			></pre>
+		<code
+			>`Hello $&#123;params.name&#125;, all your base $&#123;previous_result.base_name&#125; belong
+			to us`</code
+		>
 		However, the last line must always be the final expression.
 		<p>
-			The snippet can use any typescript primitives, and the following flow specific objects and
+			The snippet can use any javascript primitives, and the following flow specific objects and
 			functions:
 		</p>
 		<ul class="ml-4">
@@ -70,5 +72,9 @@ belong to us`</code
 				<b>resource(path)</b>: the function returning the resource at a given path as an object
 			</li>
 		</ul>
+		<p>To re-enable editor assistance, import the helper functions types using:</p>
+		<code
+			>{`import { previous_result, flow_input, step, variable, resource, params } from 'windmill@${i}'`}</code
+		>
 	</div>
 {/if}

frontend/src/lib/components/flows/FlowBox.svelte

@@ -0,0 +1,3 @@
+<div class="bg-white border border-gray-300 rounded-lg w-full shadow-md">
+	<slot />
+</div>

frontend/src/lib/components/flows/FlowBoxHeader.svelte

@@ -0,0 +1,10 @@
+<script lang="ts">
+	export let title: string | undefined = undefined
+</script>
+
+<div class="flex items-center justify-between flex-wra px-6 py-4 border-b border-gray-300">
+	{#if title}
+		<h3 class="text-lg leading-6 font-bold text-gray-900">{title}</h3>
+	{/if}
+	<slot />
+</div>

frontend/src/lib/components/flows/FlowInput.svelte

@@ -0,0 +1,21 @@
+<script lang="ts">
+	import SchemaEditor from '../SchemaEditor.svelte'
+	import { flowStore } from './../flows/flowStore'
+	import CopyFirstStepSchema from './CopyFirstStepSchema.svelte'
+	import FlowBox from './FlowBox.svelte'
+	import FlowBoxHeader from './FlowBoxHeader.svelte'
+</script>
+
+<FlowBox>
+	<FlowBoxHeader title="Flow Inputs">
+		<CopyFirstStepSchema />
+	</FlowBoxHeader>
+	<div class="p-6 ">
+		<SchemaEditor
+			on:change={() => {
+				$flowStore = $flowStore
+			}}
+			schema={$flowStore.schema}
+		/>
+	</div>
+</FlowBox>

frontend/src/lib/components/flows/FlowInputs.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
 	import { RawScript } from '$lib/gen'
 
-	import { faCode } from '@fortawesome/free-solid-svg-icons'
+	import { faCode, faRepeat } from '@fortawesome/free-solid-svg-icons'
 	import { createEventDispatcher } from 'svelte'
 	import FlowScriptPicker from './pickers/FlowScriptPicker.svelte'
 	import PickHubScript from './pickers/PickHubScript.svelte'
@@ -15,17 +15,35 @@
 <div class="columns-2">
 	<PickScript {isTrigger} on:pick />
 	<PickHubScript {isTrigger} on:pick />
+	<FlowScriptPicker
+		label={`Create a for-loop here (coming soon)`}
+		disabled={true}
+		icon={faRepeat}
+		iconColor="text-blue-500"
+	/>
+
+	<FlowScriptPicker
+		label={`New Postgres SQL query`}
+		icon={faCode}
+		iconColor="text-blue-800"
+		on:click={() => dispatch('pick', { path: 'hub/130/execute_custom_query_postgresql' })}
+	/>
+
+	<FlowScriptPicker
+		disabled={isTrigger}
+		label="New Python {isTrigger ? 'trigger ' : ''}script (3.10)"
+		icon={faCode}
+		iconColor="text-green-500"
+		on:click={() => dispatch('new', { language: RawScript.language.PYTHON3 })}
+		tooltip={isTrigger
+			? 'Python is not supported for trigger scripts yet but is supported for every other steps'
+			: undefined}
+	/>
+
 	<FlowScriptPicker
 		label="New Typescript {isTrigger ? 'trigger ' : ''}script (Deno)"
 		icon={faCode}
 		iconColor="text-blue-800"
 		on:click={() => dispatch('new', { language: RawScript.language.DENO })}
 	/>
-	<FlowScriptPicker
-		disabled={isTrigger}
-		label="New Python {isTrigger ? 'trigger ' : ''}script (3.10)"
-		icon={faCode}
-		iconColor="text-yellow-500"
-		on:click={() => dispatch('new', { language: RawScript.language.PYTHON3 })}
-	/>
 </div>

frontend/src/lib/components/flows/FlowModuleHeader.svelte

@@ -2,11 +2,13 @@
 	import type { FlowModule } from '$lib/gen'
 	import { getScriptByPath } from '$lib/utils'
 	import { faCode, faCodeBranch, faSave, faTrashAlt } from '@fortawesome/free-solid-svg-icons'
+	import { Button } from 'flowbite-svelte'
 	import Icon from 'svelte-awesome'
 	import { Highlight } from 'svelte-highlight'
-	import typescript from 'svelte-highlight/languages/typescript'
 	import python from 'svelte-highlight/languages/python'
+	import typescript from 'svelte-highlight/languages/typescript'
 	import Modal from '../Modal.svelte'
+	import FlowBoxHeader from './FlowBoxHeader.svelte'
 	import { createScriptFromInlineScript, fork, removeModule } from './flowStore'
 	import { scrollIntoView } from './utils'
 
@@ -36,54 +38,54 @@
 	}
 </script>
 
-<div class="flex flex-row w-full space-x-2">
-	<slot />
-	<a href="#module-{i}" class="grow" on:click={() => (open = i)} on:click|preventDefault={scrollTo}
-		><div class="w-full" /></a
+<FlowBoxHeader>
+	<a
+		href="#module-{i}"
+		class="grow text-inherit"
+		on:click={() => (open = i)}
+		on:click|preventDefault={scrollTo}
 	>
+		<slot />
+	</a>
 
 	<div class="flex flex-row space-x-2">
 		{#if mod.value.type === 'script' && !shouldPick}
-			<button
-				type="button"
+			<Button
 				on:click={() => {
 					open = i
 					fork(i)
 				}}
-				class="default-secondary-button-v2 text-xs"
+				size="sm"
+				color="alternative"
 			>
-				<Icon data={faCodeBranch} class={`w-4 mr-2 h-4`} />
+				<Icon data={faCodeBranch} class="mr-2" />
 				Fork
-			</button>
-			<button type="button" on:click={viewCode} class="default-secondary-button-v2 text-xs">
-				<Icon data={faCode} class="w-4 mr-2 h-4" />
+			</Button>
+			<Button size="sm" color="alternative" on:click={viewCode}>
+				<Icon data={faCode} class="mr-2" />
 				View code
-			</button>
+			</Button>
 		{/if}
 
 		{#if mod.value.type === 'rawscript' && !shouldPick}
-			<button
-				type="button"
-				on:click={() => createScriptFromInlineScript(i)}
-				class="default-secondary-button-v2 text-sm"
-			>
-				<Icon data={faSave} class="w-4 mr-4" />
+			<Button size="sm" color="alternative" on:click={() => createScriptFromInlineScript(i)}>
+				<Icon data={faSave} class="mr-2" />
 				Save to workspace
-			</button>
+			</Button>
 		{/if}
-		<button
-			type="button"
+		<Button
+			size="sm"
+			color="alternative"
 			on:click={() => {
 				open = -1
 				removeModule(i)
 			}}
-			class="default-secondary-button-v2 text-xs"
 		>
-			<Icon data={faTrashAlt} class="w-4 mr-2 h-4" />
+			<Icon data={faTrashAlt} class="mr-2" />
 			Remove step
-		</button>
+		</Button>
 	</div>
-</div>
+</FlowBoxHeader>
 
 <Modal bind:this={modalViewer}>
 	<div slot="title">Script {'path' in mod?.value ? mod?.value.path : ''}</div>

frontend/src/lib/components/flows/FlowSettings.svelte

@@ -0,0 +1,189 @@
+<script lang="ts">
+	import { sendUserToast } from '$lib/utils'
+	import { faFileExport, faFileImport, faGlobe } from '@fortawesome/free-solid-svg-icons'
+	import { Button, Dropdown, DropdownItem } from 'flowbite-svelte'
+	import Icon from 'svelte-awesome'
+	import Editor from '../Editor.svelte'
+	import FlowViewer from '../FlowViewer.svelte'
+	import Modal from '../Modal.svelte'
+	import RadioButton from '../RadioButton.svelte'
+	import CollapseLink from './../CollapseLink.svelte'
+	import CronInput from './../CronInput.svelte'
+	import FlowBox from './../flows/FlowBox.svelte'
+	import { flowStore, initFlow, mode } from './../flows/flowStore'
+	import { flowToMode } from './../flows/utils'
+	import Path from './../Path.svelte'
+	import Required from './../Required.svelte'
+	import SchemaForm from './../SchemaForm.svelte'
+	import Toggle from './../Toggle.svelte'
+	import Tooltip from './../Tooltip.svelte'
+	import FlowBoxHeader from './FlowBoxHeader.svelte'
+
+	export let pathError = ''
+	export let initialPath: string = ''
+
+	export let previewArgs: Record<string, any> = {}
+	export let scheduleArgs: Record<string, any> = {}
+	export let scheduleEnabled = false
+	export let scheduleCron: string = '0 */5 * * *'
+	export let open: number
+
+	let jsonSetter: Modal
+	let jsonViewer: Modal
+
+	let jsonValue: string = ''
+</script>
+
+<Modal bind:this={jsonSetter}>
+	<div slot="title">Import JSON</div>
+	<div slot="content" class="h-full">
+		<Editor bind:code={jsonValue} lang={'json'} class="h-full" />
+	</div>
+	<div slot="submission">
+		<button
+			class="default-button px-4 py-2 font-semibold"
+			on:click={() => {
+				Object.assign($flowStore, JSON.parse(jsonValue))
+				initFlow($flowStore)
+				open = -1
+				sendUserToast('OpenFlow imported from JSON')
+				jsonSetter.closeModal()
+			}}
+		>
+			Import
+		</button>
+	</div>
+</Modal>
+
+<Modal bind:this={jsonViewer}>
+	<div slot="title">See JSON</div>
+	<div slot="content" class="h-full">
+		<FlowViewer flow={flowToMode($flowStore, $mode)} tab="json" />
+	</div>
+</Modal>
+
+<FlowBox>
+	<FlowBoxHeader title="Flow Settings">
+		<div class="flex flex-row-reverse">
+			<Dropdown class="w-fit" placement="bottom-end">
+				<button slot="trigger" class="text-gray-900 bg-white dark:text-white dark:bg-gray-800">
+					...
+				</button>
+				<DropdownItem
+					on:click={() => {
+						jsonSetter.openModal()
+					}}
+				>
+					<Icon data={faFileImport} scale={0.6} class="inline mr-2" />
+					Import from a JSON OpenFlow
+				</DropdownItem>
+				<DropdownItem
+					on:click={() => {
+						jsonViewer.openModal()
+					}}
+				>
+					<Icon data={faFileExport} scale={0.6} class="inline mr-2" />
+					Export to a JSON OpenFlow
+				</DropdownItem>
+				<DropdownItem
+					on:click={() => {
+						const url = new URL('https://hub.windmill.dev/flows/add')
+						const openFlow = {
+							value: $flowStore.value,
+							summary: $flowStore.summary,
+							description: $flowStore.description,
+							schema: $flowStore.schema
+						}
+						url.searchParams.append('flow', btoa(JSON.stringify(flowToMode(openFlow, $mode))))
+						window.open(url, '_blank')?.focus()
+					}}
+				>
+					<Icon data={faGlobe} scale={0.6} class="inline mr-2" />
+					Publish to Hub
+				</DropdownItem>
+			</Dropdown>
+		</div>
+	</FlowBoxHeader>
+
+	<div class="p-6">
+		<Path
+			bind:error={pathError}
+			bind:path={$flowStore.path}
+			{initialPath}
+			namePlaceholder="my_flow"
+			kind="flow"
+		>
+			<div slot="ownerToolkit">
+				Flow permissions depend on their path. Select the group <span class="font-mono">all</span>
+				to share your flow, and <span class="font-mono">user</span> to keep it private.
+				<a href="https://docs.windmill.dev/docs/reference/namespaces">docs</a>
+			</div>
+		</Path>
+
+		<label class="block mt-4">
+			<span class="text-gray-700">Summary <Required required={false} /></span>
+			<textarea
+				bind:value={$flowStore.summary}
+				class="mt-1 block w-full rounded-md border-gray-300 shadow-sm focus:border-indigo-300 focus:ring focus:ring-indigo-200 focus:ring-opacity-50"
+				placeholder="A very short summary of the flow displayed when the flow is listed"
+				rows="1"
+			/>
+		</label>
+
+		<div class="mt-4">
+			<RadioButton
+				options={[
+					[
+						{
+							title: 'UI or webhook triggered',
+							desc: 'Trigger this flow through the generated UI, a manual schedule or by calling the associated webhook'
+						},
+						'push'
+					],
+					[
+						{
+							title: 'Watching changes regularly',
+							desc: 'The first module of this flow is a trigger script whose purpose is to pull data from an external source and return all new items since last run. This flow is meant to be scheduled very regularly to reduce latency to react to new events. It will trigger the rest of the flow once per item. If no new items, the flow will be skipped.'
+						},
+						'pull'
+					]
+				]}
+				bind:value={$mode}
+			/>
+		</div>
+		{#if $mode == 'pull'}
+			<CollapseLink text="set primary schedule" open={true}>
+				<Tooltip>
+					The primary schedule of a flow is simply a schedule that has the same name as a flow. It
+					can be set and enabled directly within the flow editor. "Watching for new changes" flows
+					are meant to be watching regularly for new items in an external systems. The primary
+					schedule purpose is there to set the periodicity at which you want this watcher to
+					operate.
+				</Tooltip>
+				<Toggle
+					bind:checked={scheduleEnabled}
+					options={{
+						left: 'disabled',
+						right: 'enabled'
+					}}
+				/>
+				<div class="p-2 my-2 rounded" class:bg-gray-300={!scheduleEnabled}>
+					{#if !scheduleEnabled}
+						<span class="font-black">No next scheduled run when disabled</span>
+					{/if}
+					<CronInput bind:schedule={scheduleCron} />
+				</div>
+				<div class="flex flex-row-reverse">
+					<Button
+						color="alternative"
+						size="sm"
+						on:click={() => (scheduleArgs = JSON.parse(JSON.stringify(previewArgs)))}
+					>
+						Copy from preview arguments
+					</Button>
+				</div>
+				<SchemaForm schema={$flowStore.schema} bind:args={scheduleArgs} />
+			</CollapseLink>
+		{/if}
+	</div>
+</FlowBox>

frontend/src/lib/components/flows/flowStore.ts

@@ -1,9 +1,15 @@
 import type { Schema } from '$lib/common'
-import { type InputTransform, RawScript, ScriptService, type Flow, type FlowModule } from '$lib/gen'
+import { RawScript, ScriptService, type Flow, type FlowModule } from '$lib/gen'
 import { initialCode } from '$lib/script_helpers'
 import { userStore, workspaceStore } from '$lib/stores'
+import { emptySchema } from '$lib/utils'
 import { derived, get, writable } from 'svelte/store'
-import { createInlineScriptModuleFromPath, getFirstStepSchema, loadSchemaFromModule, scrollIntoView } from './utils'
+import {
+	createInlineScriptModuleFromPath,
+	getFirstStepSchema,
+	loadSchemaFromModule,
+	scrollIntoView
+} from './utils'
 
 export type FlowMode = 'push' | 'pull'
 
@@ -17,7 +23,8 @@ export function initFlow(flow: Flow) {
 	flow = flattenForloopFlows(flow)
 	flow.value.modules.forEach((mod) => {
 		Object.values(mod.input_transform).forEach((inp) => {
-			if (inp.type == 'javascript') {				// for now we use the value for dynamic expression when done in the static editor so we have to resort to this
+			if (inp.type == 'javascript') {
+				// for now we use the value for dynamic expression when done in the static editor so we have to resort to this
 
 				//@ts-ignore
 				inp.value = codeToStaticTemplate(inp.expr)
@@ -48,6 +55,7 @@ export function codeToStaticTemplate(code?: string): string | undefined {
 	}
 	return undefined
 }
+
 export function flattenForloopFlows(flow: Flow): Flow {
 	let newFlow: Flow = JSON.parse(JSON.stringify(flow))
 	const mod = newFlow.value.modules[1]?.value
@@ -63,6 +71,7 @@ export const isCopyFirstStepSchemaDisabled = derived(flowStore, (flow: Flow | un
 	if (flow) {
 		const modules = flow.value.modules
 		const [firstModule] = modules
+
 		return (
 			modules.length === 0 || (firstModule.value.type === 'script' && firstModule.value.path === '')
 		)
@@ -77,7 +86,6 @@ export function addModule(i?: number) {
 		input_transform: {}
 	}
 
-
 	flowStore.update((flow: Flow) => {
 		const insertAt = i ?? flow.value.modules.length
 
@@ -87,6 +95,19 @@ export function addModule(i?: number) {
 
 		return flow
 	})
+
+	schemasStore.update((schemas: Schema[]) => {
+		if (typeof i !== 'undefined') {
+			const previousSchema = schemas[i]
+			const hadSchema = Boolean(previousSchema)
+			if (hadSchema) {
+				schemas[i] = emptySchema()
+				schemas[i + 1] = previousSchema
+			}
+		}
+
+		return schemas
+	})
 }
 
 export async function pickScript(path: string, step: number) {
@@ -101,13 +122,17 @@ export async function pickScript(path: string, step: number) {
 	await loadSchema(step)
 }
 
-export async function createInlineScriptModule(language: RawScript.language, step: number, mode: FlowMode) {
-	const code = initialCode(language, (mode === 'pull' && step == 0))
+export async function createInlineScriptModule(
+	language: RawScript.language,
+	step: number,
+	mode: FlowMode
+) {
+	const code = initialCode(language, mode === 'pull' && step == 0)
 	flowStore.update((flow: Flow) => {
 		flow.value.modules[step].value = {
 			type: 'rawscript',
 			content: code,
-			language,
+			language
 		}
 
 		return flow
@@ -152,7 +177,6 @@ export async function createScriptFromInlineScript(step: number) {
 	const schemas = get(schemasStore)
 	const user = get(userStore)
 
-
 	const flowModuleValue = flow.value.modules[step].value
 
 	if (flowModuleValue.type != 'rawscript') {

frontend/src/lib/components/flows/pickers/FlowScriptPicker.svelte

@@ -1,14 +1,19 @@
 <script lang="ts">
 	import type { IconDefinition } from '@fortawesome/free-brands-svg-icons'
+	import { Tooltip } from 'flowbite-svelte'
 	import Icon from 'svelte-awesome'
 
 	export let disabled: boolean = false
 	export let icon: IconDefinition
 	export let label: string
 	export let iconColor: string
+	export let tooltip: string | undefined = undefined
 </script>
 
 <button {disabled} type="button" on:click class="default-secondary-button-v2 mb-2 w-full">
 	<Icon data={icon} class={`w-4 h-4 mr-2 -ml-2 ${iconColor}`} />
 	{label}
+	{#if tooltip}
+		<Tooltip placement="bottom" content={tooltip}>(?)</Tooltip>
+	{/if}
 </button>