chore(main): release 1.96.3 (#1551 )

* chore(main): release 1.96.3 * Apply automatic changes --------- Co-authored-by: rubenfiszel <rubenfiszel@users.noreply.github.com>
fix(cli): variable updating
2026-09-20 16:02:28 +00:00 · 2023-05-09 01:20:19 +02:00 · 2023-05-09 01:17:09 +02:00 · 2023-05-09 00:43:19 +02:00 · 2023-05-09 00:17:01 +02:00 · 2023-05-09 00:16:50 +02:00
374 changed files with 16206 additions and 8401 deletions
@@ -3,5 +3,5 @@ DB_PASSWORD=changeme
 # this is the url that your instance is publicly exposed to
 WM_BASE_URL=http://localhost

-# this is the url that caddy will reverse proxy from. It might be different than WM_BASE_URL if you re using a second proxy/load balancer in front
-CADDY_REVERSE_PROXY=http://localhost
+# this is the addres caddy will listen on
+CADDY_REVERSE_PROXY=":80"
@@ -40,4 +40,4 @@ jobs:
            backend -> target
      - name: cargo test
        timeout-minutes: 10
-        run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill DISABLE_NSJAIL=false cargo test --all -- --nocapture
+        run: mkdir frontend/build && cd backend && touch windmill-api/openapi-deref.yaml && DATABASE_URL=postgres://postgres:changeme@postgres:5432/windmill cargo test --all -- --nocapture
@@ -4,6 +4,8 @@ on:
    types: [opened,synchronize,reopened,closed]
    paths:
      - "frontend/**"
+  merge_group:
+
 jobs:
  npm_check:
    runs-on: ubuntu-latest
@@ -1,6 +1,236 @@
 # Changelog


+## [1.96.3](https://github.com/windmill-labs/windmill/compare/v1.96.2...v1.96.3) (2023-05-08)
+
+
+### Bug Fixes
+
+* **cli:** add folder listing ([c598083](https://github.com/windmill-labs/windmill/commit/c5980839251cdc759c5afc688e7084f5d58ad57f))
+* **cli:** show diffs only with --show-diffs ([d254088](https://github.com/windmill-labs/windmill/commit/d254088fce00091352ad95888deeaf88bc6c9d6f))
+* **cli:** show diffs only with --show-diffs ([37f08e9](https://github.com/windmill-labs/windmill/commit/37f08e9357c7a74a1c6cdc18bad2d6dc4de5d33d))
+* **cli:** variable updating ([2639250](https://github.com/windmill-labs/windmill/commit/2639250b43c3c55f5b9a43f4020fc2f0747e792b))
+
+## [1.96.2](https://github.com/windmill-labs/windmill/compare/v1.96.1...v1.96.2) (2023-05-08)
+
+
+### Bug Fixes
+
+* **cli:** add debug mode to CLI + improve error output ([8f1cdf1](https://github.com/windmill-labs/windmill/commit/8f1cdf1d61adf80bf0d7c4a5160fd3085d3814ac))
+
+## [1.96.1](https://github.com/windmill-labs/windmill/compare/v1.96.0...v1.96.1) (2023-05-08)
+
+
+### Bug Fixes
+
+* **cli:** fix cli folder sync ([239f401](https://github.com/windmill-labs/windmill/commit/239f40199955d47e4943be4c72c3d150a58f2dd9))
+* **cli:** fix cli folder sync ([a90514b](https://github.com/windmill-labs/windmill/commit/a90514b8e99a419c26512eb370895322088b6aa9))
+
+## [1.96.0](https://github.com/windmill-labs/windmill/compare/v1.95.1...v1.96.0) (2023-05-08)
+
+
+### Features
+
+* add support for full fleged apps (react, svelte, vue) ([#1536](https://github.com/windmill-labs/windmill/issues/1536)) ([13242ab](https://github.com/windmill-labs/windmill/commit/13242abff153b021cac1ecaa3cbf65ae9d87fb69))
+* **frontend:** Add a custom deepEqualWithOrderedArray to handle orde… ([#1537](https://github.com/windmill-labs/windmill/issues/1537)) ([3a291f7](https://github.com/windmill-labs/windmill/commit/3a291f7108623b5c7194f0a7f6a3774499669313))
+* **frontend:** Add label, description, input style + add displayType… ([#1540](https://github.com/windmill-labs/windmill/issues/1540)) ([bef829d](https://github.com/windmill-labs/windmill/commit/bef829d4805bb6c5330b13dc17c9a89a84ad48ca))
+* **frontend:** app modal ([#1518](https://github.com/windmill-labs/windmill/issues/1518)) ([686f5bb](https://github.com/windmill-labs/windmill/commit/686f5bbe1847cb2a92678e7cfdbb51ecf6bbe2b6))
+
+## [1.95.1](https://github.com/windmill-labs/windmill/compare/v1.95.0...v1.95.1) (2023-05-06)
+
+
+### Bug Fixes
+
+* **cli:** cli flow sync improvements ([e585e3a](https://github.com/windmill-labs/windmill/commit/e585e3aea2b18b6dc0c9fa7ffa1e6c1dfb2a3ce2))
+
+## [1.95.0](https://github.com/windmill-labs/windmill/compare/v1.94.0...v1.95.0) (2023-05-05)
+
+
+### Features
+
+* **backend:** default parameters are used in python if missing from args ([8791a86](https://github.com/windmill-labs/windmill/commit/8791a86a936301d44ae05ea09d26c9815abf8929))
+* **frontend:** App Schema Form component ([#1533](https://github.com/windmill-labs/windmill/issues/1533)) ([85c0d93](https://github.com/windmill-labs/windmill/commit/85c0d939f59411d023cd4b173ce11224d3cbc9db))
+* **frontend:** App stepper ([#1529](https://github.com/windmill-labs/windmill/issues/1529)) ([15f1c94](https://github.com/windmill-labs/windmill/commit/15f1c947bb233147f7da261fd32054a51a9c6efa))
+* **frontend:** Merge run configuration + triggers ([#1530](https://github.com/windmill-labs/windmill/issues/1530)) ([1be4658](https://github.com/windmill-labs/windmill/commit/1be4658150ef20a9f1f0fe57b5f30ba3c2d4d94e))
+
+## [1.94.0](https://github.com/windmill-labs/windmill/compare/v1.93.1...v1.94.0) (2023-05-04)
+
+
+### Features
+
+* **frontend:** add eval badge + alert ([#1522](https://github.com/windmill-labs/windmill/issues/1522)) ([32f04c7](https://github.com/windmill-labs/windmill/commit/32f04c796856fa48ddc1548752ba1e7a8802083a))
+
+
+### Bug Fixes
+
+* **backend:** fix python transformers ([a07e3e8](https://github.com/windmill-labs/windmill/commit/a07e3e84386c0895a7209fc87a4b07218271feca))
+* **frontend:** fix ArrayStaticInputEditor width ([#1528](https://github.com/windmill-labs/windmill/issues/1528)) ([b423eec](https://github.com/windmill-labs/windmill/commit/b423eec019785a62c279db01fc93eb3fe08f7f1f))
+* **frontend:** fix select width ([#1526](https://github.com/windmill-labs/windmill/issues/1526)) ([f248c09](https://github.com/windmill-labs/windmill/commit/f248c09655889ddace24f451597a56e81443be3c))
+* **frontend:** preserve customise arguments ([b4867f1](https://github.com/windmill-labs/windmill/commit/b4867f12bb4f595b5b0e8142ab5d720307ecadd3))
+
+## [1.93.1](https://github.com/windmill-labs/windmill/compare/v1.93.0...v1.93.1) (2023-05-03)
+
+
+### Bug Fixes
+
+* **cli:** add yaml support for cli ([03e6017](https://github.com/windmill-labs/windmill/commit/03e6017860526784f1a8696eceed5750b25f1c5c))
+
+## [1.93.0](https://github.com/windmill-labs/windmill/compare/v1.92.2...v1.93.0) (2023-05-03)
+
+
+### Features
+
+* **frontend:** add recompute others to background scripts ([392d0f8](https://github.com/windmill-labs/windmill/commit/392d0f8b876c9b587fe85421098f3eceb8a74dec))
+
+
+### Bug Fixes
+
+* **frontend:** deploy path for apps ([7ac9677](https://github.com/windmill-labs/windmill/commit/7ac96771a5c3d44234c790e8cea3d621d8c1d00e))
+
+## [1.93.0](https://github.com/windmill-labs/windmill/compare/v1.92.2...v1.93.0) (2023-05-03)
+
+
+### Features
+
+* **frontend:** add recompute others to background scripts ([392d0f8](https://github.com/windmill-labs/windmill/commit/392d0f8b876c9b587fe85421098f3eceb8a74dec))
+
+
+### Bug Fixes
+
+* **frontend:** deploy path for apps ([7ac9677](https://github.com/windmill-labs/windmill/commit/7ac96771a5c3d44234c790e8cea3d621d8c1d00e))
+
+## [1.92.2](https://github.com/windmill-labs/windmill/compare/v1.92.1...v1.92.2) (2023-05-02)
+
+
+### Bug Fixes
+
+* **go-client:** fix go-client gen ([82c4d66](https://github.com/windmill-labs/windmill/commit/82c4d6629e134f00389c87a948c52878e5a3f4f5))
+
+## [1.92.1](https://github.com/windmill-labs/windmill/compare/v1.92.0...v1.92.1) (2023-05-02)
+
+
+### Bug Fixes
+
+* **go-client:** fix go-client gen ([df333d9](https://github.com/windmill-labs/windmill/commit/df333d9739f601714f7a0124f47422dfb113d320))
+
+## [1.92.0](https://github.com/windmill-labs/windmill/compare/v1.91.0...v1.92.0) (2023-05-02)
+
+
+### Features
+
+* **frontend:** add labels as table action ([64065c1](https://github.com/windmill-labs/windmill/commit/64065c17f305fb4c7c078c7fa4935d5423da8f66))
+* **frontend:** add labels as table action ([2ab1714](https://github.com/windmill-labs/windmill/commit/2ab1714dfa7cffc46b4b6aa40dabdd92c5a6270f))
+* **frontend:** allow running eval in every field ([62acbb5](https://github.com/windmill-labs/windmill/commit/62acbb5ab3a0727b306e25a80b74ac8216619501))
+* **frontend:** background script can run script and flows ([#1515](https://github.com/windmill-labs/windmill/issues/1515)) ([607c803](https://github.com/windmill-labs/windmill/commit/607c803be91921b53f329a2c2c3c129ce53d6c0c))
+
+
+### Bug Fixes
+
+* **frontend:** fix small ui issues ([#1513](https://github.com/windmill-labs/windmill/issues/1513)) ([f6ff8ca](https://github.com/windmill-labs/windmill/commit/f6ff8ca232f5725f86a36379956da2731def2580))
+
+## [1.91.0](https://github.com/windmill-labs/windmill/compare/v1.90.0...v1.91.0) (2023-05-01)
+
+
+### Features
+
+* add drafts for apps ([f7374c8](https://github.com/windmill-labs/windmill/commit/f7374c8204f85b4371e61f34dcd4b66857c0f8ab))
+* introduce backend persisted draft systems for scripts ([88e37fe](https://github.com/windmill-labs/windmill/commit/88e37fe0bed58f396690622e925d5e078c60140c))
+* introduce draft for flows ([a196642](https://github.com/windmill-labs/windmill/commit/a1966427e893dc8a58c8f2862ded752884843813))
+
+## [1.90.0](https://github.com/windmill-labs/windmill/compare/v1.89.0...v1.90.0) (2023-04-28)
+
+
+### Features
+
+* **backend:** add EXIT_AFTER_NO_JOB_FOR_SECS for ephemeral workers ([de9abd1](https://github.com/windmill-labs/windmill/commit/de9abd129db13dcdf0e69e2c1e2d3aa558fb783a))
+* **backend:** add JOB_RETENTION_SECS to delete completed jobs completed after expiration period ([0b7bad3](https://github.com/windmill-labs/windmill/commit/0b7bad3816e61841ef4765db7881227274c20b23))
+* **backend:** expose tag in the job  ([#1486](https://github.com/windmill-labs/windmill/issues/1486)) ([324d4f5](https://github.com/windmill-labs/windmill/commit/324d4f5e9e89e6de600e882f23bf545c0b1dc539))
+* **frontend:** adapt style ([#1488](https://github.com/windmill-labs/windmill/issues/1488)) ([41a24ec](https://github.com/windmill-labs/windmill/commit/41a24ecd36d9cc537cbd1dd0cd1de6f689be1b8c))
+* **frontend:** add an eval input component for flow ([#1494](https://github.com/windmill-labs/windmill/issues/1494)) ([2815f1e](https://github.com/windmill-labs/windmill/commit/2815f1ec71177bb6e89d0d62a8df89030d37b1fc))
+* **frontend:** Add new integration icons ([#1479](https://github.com/windmill-labs/windmill/issues/1479)) ([7adacd4](https://github.com/windmill-labs/windmill/commit/7adacd4c9f03d17b47abc515bc391e348a7e6ec1))
+* **frontend:** refactor inline script ([#1480](https://github.com/windmill-labs/windmill/issues/1480)) ([05c837e](https://github.com/windmill-labs/windmill/commit/05c837e64f61bfb22ae1f80263deb1c879030985))
+* **frontend:** Schedules run now ([#1475](https://github.com/windmill-labs/windmill/issues/1475)) ([47f0f35](https://github.com/windmill-labs/windmill/commit/47f0f35236e02958f6bc00b5652e06e25eabeaf5))
+* **frontend:** Small style fix ([#1473](https://github.com/windmill-labs/windmill/issues/1473)) ([7ad496a](https://github.com/windmill-labs/windmill/commit/7ad496ad3f746ffd782856a35c1456999792fa94))
+* **frontend:** Support TS union type with a select field ([#1457](https://github.com/windmill-labs/windmill/issues/1457)) ([8b76324](https://github.com/windmill-labs/windmill/commit/8b763249cb1360c122cc81d50a1a95d1ad3ddd5b))
+
+
+### Bug Fixes
+
+* **frontend:** Allow 0 as select default value ([#1474](https://github.com/windmill-labs/windmill/issues/1474)) ([d8529ff](https://github.com/windmill-labs/windmill/commit/d8529ff3ed6168de60bb24626a3e36ab4beae15c))
+* **frontend:** close the modal before deleting a form modal ([#1484](https://github.com/windmill-labs/windmill/issues/1484)) ([430c733](https://github.com/windmill-labs/windmill/commit/430c73399b7e1524ee81d0ce8d7d2eaf16117f9a))
+* **frontend:** fix apply connection ([#1487](https://github.com/windmill-labs/windmill/issues/1487)) ([cf59cc0](https://github.com/windmill-labs/windmill/commit/cf59cc04efb853fe06a23042115128689b5d26ee))
+* **frontend:** Fix frontend script ([#1476](https://github.com/windmill-labs/windmill/issues/1476)) ([b60a7f6](https://github.com/windmill-labs/windmill/commit/b60a7f63d04b1fc851479060e77613c77d20198a))
+* **frontend:** fix recomputa all ([#1491](https://github.com/windmill-labs/windmill/issues/1491)) ([fb05a09](https://github.com/windmill-labs/windmill/commit/fb05a09955f937000fbba0826c03c52e65aa146e))
+* **frontend:** Flow editor design updates ([#1477](https://github.com/windmill-labs/windmill/issues/1477)) ([50d814c](https://github.com/windmill-labs/windmill/commit/50d814c3dc55841b16cd1df8ae86d021de4e880c))
+* **frontend:** Minor app editor updates ([#1458](https://github.com/windmill-labs/windmill/issues/1458)) ([8fd10b1](https://github.com/windmill-labs/windmill/commit/8fd10b1f5813b3f4200a980de4a85e7d701660a7))
+* **frontend:** register applyConnection as a callback to remove unnecessary reactivit ([#1485](https://github.com/windmill-labs/windmill/issues/1485)) ([d915f6b](https://github.com/windmill-labs/windmill/commit/d915f6b004ea7cc27a3c55b3504df902f5db1aef))
+* **frontend:** reset ui job loading state when submitting preview job triggers error ([#1483](https://github.com/windmill-labs/windmill/issues/1483)) ([6f8616f](https://github.com/windmill-labs/windmill/commit/6f8616fb273b1f4a1489878a617d239f30ecb1c0))
+* **frontend:** Update CLI login request styling ([#1454](https://github.com/windmill-labs/windmill/issues/1454)) ([c77393c](https://github.com/windmill-labs/windmill/commit/c77393c15444868308b44a72bee89e49fc23d80f))
+* **frontend:** Update direct exports ([#1456](https://github.com/windmill-labs/windmill/issues/1456)) ([4a2af13](https://github.com/windmill-labs/windmill/commit/4a2af1359ee29001236591f685cacfa9df6715df))
+
+## [1.89.0](https://github.com/windmill-labs/windmill/compare/v1.88.1...v1.89.0) (2023-04-23)
+
+
+### Features
+
+* **backend:** global cache refactor for pip using tar for each dependency ([#1443](https://github.com/windmill-labs/windmill/issues/1443)) ([369dd0d](https://github.com/windmill-labs/windmill/commit/369dd0dac61e5430856ed9abf7129bbad3b75860))
+* **backend:** only run fully deployed scripts ([3d031c7](https://github.com/windmill-labs/windmill/commit/3d031c701705459f418b11d2ca83e71943e4079b))
+* **backend:** worker groups ([#1452](https://github.com/windmill-labs/windmill/issues/1452)) ([722783f](https://github.com/windmill-labs/windmill/commit/722783f7f630e3123ffd2605deb21a915188bd20))
+* **backend:** workers are instantly ready and sync with global cache in background ([670ba51](https://github.com/windmill-labs/windmill/commit/670ba51d9bd9a9a0b07a2ed064c316234bb5819d))
+* **ee:** sync cache in background ([c919827](https://github.com/windmill-labs/windmill/commit/c919827cf8eb9437b6d9bd57b3d3ad883a66de3b))
+* **ee:** sync cache in background ([0e77e37](https://github.com/windmill-labs/windmill/commit/0e77e37fbddbb517d5e1b1f07a27a40b63371439))
+* **frontend:** Add documentation links ([#1399](https://github.com/windmill-labs/windmill/issues/1399)) ([36acbf7](https://github.com/windmill-labs/windmill/commit/36acbf793b6714dffe4dbb0e2501b9438f034858))
+* **frontend:** Add seconds input ([#1445](https://github.com/windmill-labs/windmill/issues/1445)) ([30bf7ad](https://github.com/windmill-labs/windmill/commit/30bf7ad3e9785420b4dcd814ce3c7a444d23cc9f))
+* **frontend:** add toast actions ([#1411](https://github.com/windmill-labs/windmill/issues/1411)) ([d173232](https://github.com/windmill-labs/windmill/commit/d17323286a05aa0b1680ef94d7058d2c8902782f))
+* **frontend:** reorder array items in app editor ([#1426](https://github.com/windmill-labs/windmill/issues/1426)) ([3615fb2](https://github.com/windmill-labs/windmill/commit/3615fb26fb91d64626d82ace6e6275e424ece832))
+* **frontend:** support showing metadata on script add via query param ([#1438](https://github.com/windmill-labs/windmill/issues/1438)) ([3c98452](https://github.com/windmill-labs/windmill/commit/3c98452f50913ef639eba96996f3a6c80508bd63))
+
+
+### Bug Fixes
+
+* **backend:** avoid potential conflict between pull from tar and background sync ([d76e907](https://github.com/windmill-labs/windmill/commit/d76e90757e209263da7f79fa85052969e7efd63d))
+* **backend:** global cache synco only start if all piptars have been downloaded ([5f8a730](https://github.com/windmill-labs/windmill/commit/5f8a730fdfbb9e3d518555f7272a3bb297725f28))
+* **frontend:** App color picker overflow issue ([#1449](https://github.com/windmill-labs/windmill/issues/1449)) ([32903d2](https://github.com/windmill-labs/windmill/commit/32903d2839a082d53168bb9177fc88f6ab0ec482))
+* **frontend:** fix copy content button width ([#1428](https://github.com/windmill-labs/windmill/issues/1428)) ([d96d4a5](https://github.com/windmill-labs/windmill/commit/d96d4a524edebea65bc602194c8f11f5d69e920a))
+* **frontend:** Minor update of app default codes ([#1440](https://github.com/windmill-labs/windmill/issues/1440)) ([fe75aa1](https://github.com/windmill-labs/windmill/commit/fe75aa18f2f27745db35329aa60938694640a8c6))
+* **frontend:** Update app default codes ([#1432](https://github.com/windmill-labs/windmill/issues/1432)) ([c8acfbc](https://github.com/windmill-labs/windmill/commit/c8acfbc1ff0f6c23e5a2229ca83a3b09eec826c3))
+* **frontend:** Update app mobile preview width ([#1431](https://github.com/windmill-labs/windmill/issues/1431)) ([1764613](https://github.com/windmill-labs/windmill/commit/17646130bcf8cf646a4ccdfa39f9a8791876a137))
+* **frontend:** Update flow tooltip z-indexes ([#1433](https://github.com/windmill-labs/windmill/issues/1433)) ([17cb8fc](https://github.com/windmill-labs/windmill/commit/17cb8fc3fa0b39a9750e61ca2731a15bbda690ec))
+* **frontend:** Update flow viewer styling ([#1441](https://github.com/windmill-labs/windmill/issues/1441)) ([46a29b5](https://github.com/windmill-labs/windmill/commit/46a29b5d27b8d9f7ea38c1063fc081ed5933db5d))
+
+## [1.88.1](https://github.com/windmill-labs/windmill/compare/v1.88.0...v1.88.1) (2023-04-18)
+
+
+### Bug Fixes
+
+* **frontend:** fix hub list ([1144329](https://github.com/windmill-labs/windmill/commit/1144329972fb61e2df62873ca1e485c88fabc478))
+
+## [1.88.0](https://github.com/windmill-labs/windmill/compare/v1.87.0...v1.88.0) (2023-04-17)
+
+
+### Features
+
+* **backend:** install python scripts on save ([cb7e686](https://github.com/windmill-labs/windmill/commit/cb7e686dd95397d5b37edd5aac50b6d1429c4a71))
+* **frontend:** Add runs preview popup ([#1405](https://github.com/windmill-labs/windmill/issues/1405)) ([4ab023f](https://github.com/windmill-labs/windmill/commit/4ab023f95085958ab1ad01dc249d308c7ebf423e))
+* **frontend:** cancellable inline script editor run ([e828d26](https://github.com/windmill-labs/windmill/commit/e828d2673e62e95e5e1235eeca8107ac7cfb7e45))
+* **frontend:** Remove gap when button label is empty ([#1402](https://github.com/windmill-labs/windmill/issues/1402)) ([568f59e](https://github.com/windmill-labs/windmill/commit/568f59eefb104047b8ef063f273fe238075d6407))
+* **frontend:** Unify main lists ([#1406](https://github.com/windmill-labs/windmill/issues/1406)) ([48bbbd0](https://github.com/windmill-labs/windmill/commit/48bbbd0e872a12ed1c562a6d14967a2a0f7c4735))
+* **frontend:** Update airtable instructions ([#1403](https://github.com/windmill-labs/windmill/issues/1403)) ([7dc7ece](https://github.com/windmill-labs/windmill/commit/7dc7ecef55b465fc096f71fc9de5c8b543136ff7))
+* inputs library on run page ([92a2934](https://github.com/windmill-labs/windmill/commit/92a293488e8e58350229931ab69f7924d58474be))
+
+
+### Bug Fixes
+
+* **backend:** deno uses --no-check ([a5499c2](https://github.com/windmill-labs/windmill/commit/a5499c26f3ebd8b07541a7e0cbf33a7008a8f476))
+* **backend:** do not fail on schedule not existing anymore ([a5f6d73](https://github.com/windmill-labs/windmill/commit/a5f6d73f7d53d7af9d285a85460509763263c508))
+* **frontend:** Fix app file uploads ([#1408](https://github.com/windmill-labs/windmill/issues/1408)) ([ac489ac](https://github.com/windmill-labs/windmill/commit/ac489ac2da0fbf01f5e2877612c14cfaf1ef79c2))
+* **frontend:** fix buttons width ([#1407](https://github.com/windmill-labs/windmill/issues/1407)) ([75a0482](https://github.com/windmill-labs/windmill/commit/75a0482ef046dd7e30f6d6039dbc66880182dc5e))
+* **frontend:** fix enum sync ([#1410](https://github.com/windmill-labs/windmill/issues/1410)) ([98060ce](https://github.com/windmill-labs/windmill/commit/98060ce55d5efa59a8989cf9357935976d57650b))
+* **frontend:** Handle scheduled runs in preview ([#1413](https://github.com/windmill-labs/windmill/issues/1413)) ([accdc1a](https://github.com/windmill-labs/windmill/commit/accdc1ac59ce9611f66567222a73995d3c0a3f9d))
+* **frontend:** Keep selected tab during renaming ([#1409](https://github.com/windmill-labs/windmill/issues/1409)) ([82cd048](https://github.com/windmill-labs/windmill/commit/82cd048ef4d08f31660f6f31a96940676a28996c))
+* **frontend:** Queued-running jobs preview ([#1414](https://github.com/windmill-labs/windmill/issues/1414)) ([b2a40a0](https://github.com/windmill-labs/windmill/commit/b2a40a05805344c1c34f2ba917b4cdd52dfffc3f))
+* **frontend:** Remove output when deleting a component ([#1397](https://github.com/windmill-labs/windmill/issues/1397)) ([6aa1008](https://github.com/windmill-labs/windmill/commit/6aa100893352870d5a99fdd56d7f1425a221a273))
+
 ## [1.87.0](https://github.com/windmill-labs/windmill/compare/v1.86.0...v2.0.0) (2023-04-11)


@@ -39,7 +39,7 @@ WORKDIR /windmill
 ENV SQLX_OFFLINE=true
 ENV CARGO_INCREMENTAL=1

-FROM node:19-alpine as frontend
+FROM node:20-alpine as frontend

 # install dependencies
 WORKDIR /frontend
@@ -5,7 +5,7 @@
    <em>.</em>
 </p>
 <p align=center>
-Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
+Open-source developer infrastructure for internal tools. Self-hostable alternative to Airplane, Pipedream, Superblocks and a simplified Temporal with autogenerated UIsm and custom UIs to trigger workflows and scripts as internal apps. Scripts are turned into UIs and no-code modules, no-code modules can be composed into very rich flows, and script and flows can be triggered from internal UIs made with a low-code builder. The script languages supported are: Python, Typescript, Go, Bash, SQL.
 </p>

 <p align="center">
@@ -187,9 +187,9 @@ compiling from source or using without a postgres super user, see
 ### Docker compose

 ```
-curl https://github.com/windmill-labs/windmill/blob/main/docker-compose.yml -o docker-compose.yml
-curl https://github.com/windmill-labs/windmill/blob/main/CaddyFile -o Caddyfile
-curl https://github.com/windmill-labs/windmill/blob/main/.env -o .env
+curl https://raw.githubusercontent.com/windmill-labs/windmill/main/docker-compose.yml -o docker-compose.yml
+curl https://raw.githubusercontent.com/windmill-labs/windmill/main/Caddyfile -o Caddyfile
+curl https://raw.githubusercontent.com/windmill-labs/windmill/main/.env -o .env

 docker compose up -d --pull always
 ```
@@ -288,53 +288,61 @@ it being synced automatically everyday.

 ## Environment Variables

-| Environment Variable name | Default                | Description                                                                                                                                                                                        | Api Server/Worker/All |
-| ------------------------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
-| DATABASE_URL              |                        | The Postgres database url.                                                                                                                                                                         | All                   |
-| DISABLE_NSJAIL            | true                   | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
-| SERVER_BIND_ADDR          | 0.0.0.0                | IP Address on which to bind listening socket                                                                                                                                                       | Server                |
-| PORT                      | 8000                   | Exposed port                                                                                                                                                                                       | Server                |
-| NUM_WORKERS               | 3                      | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance)                                                                             | Worker                |
-| DISABLE_SERVER            | false                  | Binary would operate as a worker only instance                                                                                                                                                     | Worker                |
-| METRICS_ADDR              | None                   | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
-| JSON_FMT                  | false                  | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
-| BASE_URL                  | http://localhost:8000  | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
-| BASE_INTERNAL_URL         | http://localhost:8000  | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests.                                      | Worker                |
-| TIMEOUT                   | 300                    | The maximum time of execution of a script. When reached, the job is failed as having timedout.                                                                                                     | Worker                |
-| ZOMBIE_JOB_TIMEOUT        | 30                     | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s)                                  | Server                |
-| RESTART_ZOMBIE_JOBS       | true                   | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed                                                                              | Server                |
-| SLEEP_QUEUE               | 50                     | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker                |
-| MAX_LOG_SIZE              | 500000                 | The maximum number of characters a job can emit (log + result)                                                                                                                                     | Worker                |
-| DISABLE_NUSER             | false                  | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting                                                                                                                                 | Worker                |
-| KEEP_JOB_DIR              | false                  | Keep the job directory after the job is done. Useful for debugging.                                                                                                                                | Worker                |
-| LICENSE_KEY (EE only)     | None                   | License key checked at startup for the Enterprise Edition of Windmill                                                                                                                              | Worker                |
-| S3_CACHE_BUCKET (EE only) | None                   | The S3 bucket to sync the cache of the workers to                                                                                                                                                  | Worker                |
-| TAR_CACHE_RATE (EE only)  | 100                    | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed).                                                                          | Worker                |
-| SLACK_SIGNING_SECRET      | None                   | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack)                                                                | Server                |
-| COOKIE_DOMAIN             | None                   | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |
-| DENO_PATH                 | /usr/bin/deno          | The path to the deno binary.                                                                                                                                                                       | Worker                |
-| PYTHON_PATH               | /usr/local/bin/python3 | The path to the python binary.                                                                                                                                                                     | Worker                |
-| GO_PATH                   | /usr/bin/go            | The path to the go binary.                                                                                                                                                                         | Worker                |
-| GOPRIVATE                 |                        | The GOPRIVATE env variable to use private go modules                                                                                                                                               | Worker                |
-| NETRC                     |                        | The netrc content to use a private go registry                                                                                                                                                     | Worker                |
-| PIP_INDEX_URL             | None                   | The index url to pass for pip.                                                                                                                                                                     | Worker                |
-| PIP_EXTRA_INDEX_URL       | None                   | The extra index url to pass to pip.                                                                                                                                                                | Worker                |
-| PIP_TRUSTED_HOST          | None                   | The trusted host to pass to pip.                                                                                                                                                                   | Worker                |
-| PATH                      | None                   | The path environment variable, usually inherited                                                                                                                                                   | Worker                |
-| HOME                      | None                   | The home directory to use for Go and Bash , usually inherited                                                                                                                                      | Worker                |
-| DATABASE_CONNECTIONS      | 50 (Server)/3 (Worker) | The max number of connections in the database connection pool                                                                                                                                      | All                   |
-| SUPERADMIN_SECRET         | None                   | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev                                                                                                              | Server                |
-| TIMEOUT_WAIT_RESULT       | 20                     | The number of seconds to wait before timeout on the 'run_wait_result' endpoint                                                                                                                     | Worker                |
-| QUEUE_LIMIT_WAIT_RESULT   | None                   | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit.           | Worker                |
-| DENO_AUTH_TOKENS          | None                   | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
-| DENO_FLAGS                | None                   | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json"                                  | Worker                |
-| NPM_CONFIG_REGISTRY       | None                   | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry                                                              | Worker                |
-| PIP_LOCAL_DEPENDENCIES    | None                   | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |                       |
-| ADDITIONAL_PYTHON_PATHS   | None                   | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
-| INCLUDE_HEADERS           | None                   | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
-| WHITELIST_WORKSPACES      | None                   | Whitelist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
-| BLACKLIST_WORKSPACES      | None                   | Blacklist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
-| INSTANCE_EVENTS_WEBHOOK   | None                   | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails                                                                                        | Server                |
+| Environment Variable name           | Default                                    | Description                                                                                                                                                                                        | Api Server/Worker/All |
+| ----------------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------- |
+| DATABASE_URL                        |                                            | The Postgres database url.                                                                                                                                                                         | All                   |
+| DISABLE_NSJAIL                      | true                                       | Disable Nsjail Sandboxing                                                                                                                                                                          | Worker                |
+| SERVER_BIND_ADDR                    | 0.0.0.0                                    | IP Address on which to bind listening socket                                                                                                                                                       | Server                |
+| PORT                                | 8000                                       | Exposed port                                                                                                                                                                                       | Server                |
+| NUM_WORKERS                         | 3                                          | The number of worker per Worker instance (set to 1 on Eks to have 1 pod = 1 worker, set to 0 for an API only instance)                                                                             | Worker                |
+| DISABLE_SERVER                      | false                                      | Binary would operate as a worker only instance                                                                                                                                                     | Worker                |
+| METRICS_ADDR                        | None                                       | The socket addr at which to expose Prometheus metrics at the /metrics path. Set to "true" to expose it on port 8001                                                                                | All                   |
+| JSON_FMT                            | false                                      | Output the logs in json format instead of logfmt                                                                                                                                                   | All                   |
+| BASE_URL                            | http://localhost:8000                      | The base url that is exposed publicly to access your instance                                                                                                                                      | Server                |
+| BASE_INTERNAL_URL                   | http://localhost:8000                      | The base url that is reachable by your workers to talk to the Servers. This help avoiding going through the external load balancer for VPC-internal requests.                                      | Worker                |
+| TIMEOUT                             | 300                                        | The maximum time of execution of a script. When reached, the job is failed as having timedout.                                                                                                     | Worker                |
+| ZOMBIE_JOB_TIMEOUT                  | 30                                         | The timeout after which a job is considered to be zombie if the worker did not send pings about processing the job (every server check for zombie jobs every 30s)                                  | Server                |
+| RESTART_ZOMBIE_JOBS                 | true                                       | If true then a zombie job is restarted (in-place with the same uuid and some logs), if false the zombie job is failed                                                                              | Server                |
+| SLEEP_QUEUE                         | 50                                         | The number of ms to sleep in between the last check for new jobs in the DB. It is multiplied by NUM_WORKERS such that in average, for one worker instance, there is one pull every SLEEP_QUEUE ms. | Worker                |
+| MAX_LOG_SIZE                        | 500000                                     | The maximum number of characters a job can emit (log + result)                                                                                                                                     | Worker                |
+| DISABLE_NUSER                       | false                                      | If Nsjail is enabled, disable the nsjail's `clone_newuser` setting                                                                                                                                 | Worker                |
+| KEEP_JOB_DIR                        | false                                      | Keep the job directory after the job is done. Useful for debugging.                                                                                                                                | Worker                |
+| LICENSE_KEY (EE only)               | None                                       | License key checked at startup for the Enterprise Edition of Windmill                                                                                                                              | Worker                |
+| S3_CACHE_BUCKET (EE only)           | None                                       | The S3 bucket to sync the cache of the workers to                                                                                                                                                  | Worker                |
+| TAR_CACHE_RATE (EE only)            | 100                                        | The rate at which to tar the cache of the workers. 100 means every 100th job in average (uniformly randomly distributed).                                                                          | Worker                |
+| SLACK_SIGNING_SECRET                | None                                       | The signing secret of your Slack app. See [Slack documentation](https://api.slack.com/authentication/verifying-requests-from-slack)                                                                | Server                |
+| COOKIE_DOMAIN                       | None                                       | The domain of the cookie. If not set, the cookie will be set by the browser based on the full origin                                                                                               | Server                |
+| DENO_PATH                           | /usr/bin/deno                              | The path to the deno binary.                                                                                                                                                                       | Worker                |
+| PYTHON_PATH                         | /usr/local/bin/python3                     | The path to the python binary.                                                                                                                                                                     | Worker                |
+| GO_PATH                             | /usr/bin/go                                | The path to the go binary.                                                                                                                                                                         | Worker                |
+| GOPRIVATE                           |                                            | The GOPRIVATE env variable to use private go modules                                                                                                                                               | Worker                |
+| NETRC                               |                                            | The netrc content to use a private go registry                                                                                                                                                     | Worker                |
+| PIP_INDEX_URL                       | None                                       | The index url to pass for pip.                                                                                                                                                                     | Worker                |
+| PIP_EXTRA_INDEX_URL                 | None                                       | The extra index url to pass to pip.                                                                                                                                                                | Worker                |
+| PIP_TRUSTED_HOST                    | None                                       | The trusted host to pass to pip.                                                                                                                                                                   | Worker                |
+| PATH                                | None                                       | The path environment variable, usually inherited                                                                                                                                                   | Worker                |
+| HOME                                | None                                       | The home directory to use for Go and Bash , usually inherited                                                                                                                                      | Worker                |
+| DATABASE_CONNECTIONS                | 50 (Server)/3 (Worker)                     | The max number of connections in the database connection pool                                                                                                                                      | All                   |
+| SUPERADMIN_SECRET                   | None                                       | A token that would let the caller act as a virtual superadmin superadmin@windmill.dev                                                                                                              | Server                |
+| TIMEOUT_WAIT_RESULT                 | 20                                         | The number of seconds to wait before timeout on the 'run_wait_result' endpoint                                                                                                                     | Worker                |
+| QUEUE_LIMIT_WAIT_RESULT             | None                                       | The number of max jobs in the queue before rejecting immediately the request in 'run_wait_result' endpoint. Takes precedence on the query arg. If none is specified, there are no limit.           | Worker                |
+| DENO_AUTH_TOKENS                    | None                                       | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules                                                                                                                      | Worker                |
+| DENO_FLAGS                          | None                                       | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json"                                  | Worker                |
+| NPM_CONFIG_REGISTRY                 | None                                       | Registry to use for NPM dependencies, set if you have a private repository you need to use instead of the default public NPM registry                                                              | Worker                |
+| PIP_LOCAL_DEPENDENCIES              | None                                       | Specify dependencies that are installed locally and do not need to be solved nor installed again                                                                                                   |                       |
+| ADDITIONAL_PYTHON_PATHS             | None                                       | Specify python paths (separated by a :) to be appended to the PYTHONPATH of the python jobs. To be used with PIP_LOCAL_DEPENDENCIES to use python codebases within Windmill                        | Worker                |
+| INCLUDE_HEADERS                     | None                                       | Whitelist of headers that are passed to jobs as args (separated by a comma)                                                                                                                        | Server                |
+| WHITELIST_WORKSPACES                | None                                       | Whitelist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
+| BLACKLIST_WORKSPACES                | None                                       | Blacklist of workspaces this worker takes job from                                                                                                                                                 | Worker                |
+| INSTANCE_EVENTS_WEBHOOK             | None                                       | Webhook to notify of events such as new user added, signup/invite. Can hook back to windmill to send emails                                                                                        |
+| GLOBAL_CACHE_INTERVAL               | 10\*60                                     | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together.          | Worker                |
+| WORKER_TAGS                         | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers                                                                                                                                                         | Worker                |
+| CUSTOM_TAGS                         | None                                       | The custom tags assignable to scripts.                                                                                                                                                             | Server                |
+| JOB_RETENTION_SECS                  | 60*60*24\*60 //60 days                     | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete                                                                                                                   |
+| WAIT_RESULT_FAST_POLL_INTERVAL_MS   | 50                                         | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
+| WAIT_RESULT_SLOW_POLL_INTERVAL_MS   | 200                                        | The time in between polling for the run_wait_result endpoints in fast poll mode                                                                                                                    | Server                |
+| WAIT_RESULT_FAST_POLL_DURATION_SECS | 2                                          | The duration of fast poll mode before switching to slow poll                                                                                                                                       | Server                |
+| EXIT_AFTER_NO_JOB_FOR_SECS          | None                                       | Exit worker if no job is received after duration in secs if defined                                                                                                                                | Worker                |

 ## Run a local dev setup

@@ -1,3 +1,4 @@
 {
-  "python.analysis.typeCheckingMode": "basic"
+  "python.analysis.typeCheckingMode": "basic",
+  "rust-analyzer.linkedProjects": ["./windmill-common/Cargo.toml"]
 }
@@ -1,6 +1,6 @@
 [package]
 name = "windmill"
-version = "1.87.0"
+version = "1.96.3"
 authors.workspace = true
 edition.workspace = true

@@ -19,7 +19,7 @@ members = [
 ]

 [workspace.package]
-version = "1.87.0"
+version = "1.96.3"
 authors = ["Ruben Fiszel <ruben@windmill.dev>"]
 edition = "2021"

@@ -50,6 +50,10 @@ base64.workspace = true
 sha2.workspace = true
 rsmq_async.workspace = true
 url.workspace = true
+lazy_static.workspace = true
+once_cell.workspace = true
+prometheus.workspace = true
+uuid.workspace = true


 [dev-dependencies]
@@ -100,7 +104,7 @@ rand = "0.8.5"
 rand_core = { version = "^0", features = ["std"] }
 magic-crypt = "^3"
 git-version = "^0"
-rustpython-parser = { git = "https://github.com/RustPython/RustPython" }
+rustpython-parser = "0.2.0"
 cron = "^0"
 lettre = { version = "^0", features = [
    "rustls-tls",
@@ -158,5 +162,5 @@ async_zip = { version = "0.0.11", features = ["full"] }
 once_cell = "1.17.1"
 rsmq_async = { version = "5.1.5" }
 gosyn = "0.2.2"
+bytes = "1.4.0"

-[patch.crates-io]
@@ -1,6 +1,7 @@
 # Windmill Backend

-This folder holds all backend components, the [src/](./src/) folder only contains files used to build the "root" binary.
+This folder holds all backend components, the [src/](./src/) folder only
+contains files used to build the "root" binary.

 ## Components

@@ -13,3 +14,9 @@ This folder holds all backend components, the [src/](./src/) folder only contain
 | [windmill-queue](./windmill-queue/)           | Contains job & flow queuing functionality, commonly written to by the API server and read from by workers |
 | [windmill-worker](./windmill-worker/)         | The worker. Used to process and execute flows & jobs.                                                     |
 | [parsers](./parsers/)                         | Contains code to parse signatures in different langauges.                                                 |
+
+### Compile sqlx for offline ci
+
+```
+cargo sqlx prepare --merged -- --bin windmill --features enterprise
+```
@@ -0,0 +1 @@
+-- Add down migration script here
@@ -0,0 +1,3 @@
+-- Add up migration script here
+GRANT ALL ON input TO windmill_user;
+GRANT ALL ON input TO windmill_admin;
@@ -0,0 +1,4 @@
+-- Add down migration script here
+ALTER TABLE script DROP COLUMN tag;
+ALTER TABLE completed_job DROP COLUMN tag;
+ALTER TABLE queue DROP COLUMN tag;
@@ -0,0 +1,4 @@
+-- Add up migration script here
+ALTER TABLE script ADD COLUMN tag VARCHAR(50);
+ALTER TABLE completed_job  ADD COLUMN tag VARCHAR(50) NOT NULL DEFAULT 'other';
+ALTER TABLE queue ADD COLUMN tag VARCHAR(50) NOT NULL DEFAULT 'other';
@@ -0,0 +1,3 @@
+-- Add down migration script here
+DROP TABLE draft;
+DROP TYPE DRAFT_TYPE;
@@ -0,0 +1,18 @@
+-- Add up migration script here
+CREATE TYPE DRAFT_TYPE AS ENUM ('script', 'flow', 'app');
+
+CREATE TABLE draft (
+    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+    path VARCHAR(255) NOT NULL,
+    typ DRAFT_TYPE NOT NULL,
+    value JSONB NOT NULL,
+    created_at TIMESTAMP NOT NULL DEFAULT now(),
+    PRIMARY KEY (workspace_id, path, typ)
+);
+
+GRANT ALL ON draft TO windmill_admin;
+GRANT ALL ON draft TO windmill_user;
+
+ALTER TABLE script ADD COLUMN draft_only BOOLEAN;
+ALTER TABLE flow ADD COLUMN draft_only BOOLEAN;
+ALTER TABLE app ADD COLUMN draft_only BOOLEAN;
@@ -0,0 +1 @@
+-- Add down migration script here
@@ -0,0 +1,33 @@
+-- Add up migration script here
+CREATE TABLE raw_app (
+    path varchar(255) PRIMARY KEY,
+    version INTEGER NOT NULL DEFAULT 0,
+    workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
+    summary VARCHAR(1000) NOT NULL DEFAULT '',
+    edited_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
+    data TEXT NOT NULL,
+    extra_perms JSONB NOT NULL DEFAULT '{}'
+);
+
+CREATE POLICY see_own ON raw_app FOR ALL
+USING (SPLIT_PART(raw_app.path, '/', 1) = 'u' AND SPLIT_PART(raw_app.path, '/', 2) = current_setting('session.user'));
+
+CREATE POLICY see_member ON raw_app FOR ALL
+USING (SPLIT_PART(raw_app.path, '/', 1) = 'g' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
+
+CREATE POLICY see_extra_perms_user ON raw_app FOR ALL
+USING (extra_perms ? CONCAT('u/', current_setting('session.user')))
+WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
+
+CREATE POLICY see_extra_perms_groups ON raw_app FOR ALL
+USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
+WITH CHECK (exists(
+    SELECT key, value FROM jsonb_each_text(extra_perms) 
+    WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
+    AND value::boolean));
+
+CREATE POLICY see_folder_extra_perms_user ON raw_app FOR ALL
+USING (SPLIT_PART(raw_app.path, '/', 1) = 'f' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]))
+WITH CHECK (SPLIT_PART(raw_app.path, '/', 1) = 'f' AND SPLIT_PART(raw_app.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
+
+ALTER TYPE FAVORITE_KIND ADD VALUE 'raw_app';
@@ -96,7 +96,7 @@ fn parse_go_typ(typ: &Expression) -> (Option<String>, Typ) {
                        .as_ref()
                        .and_then(|x| x.value.strip_prefix("`json:\""))
                        .and_then(|x| x.strip_suffix("\"`"))
-                        .and_then(|x| x.split(',').last().map(|x| x.to_string()));
+                        .and_then(|x| x.split(',').next().map(|x| x.to_string()));
                    let (otyp, typ) = parse_go_typ(&field.typ);
                    let name = get_name(field);
                    let key = json_tag.unwrap_or_else(|| name.to_string());
@@ -17,8 +17,8 @@ use serde_json::json;
 use windmill_common::error;
 use windmill_parser::{json_to_typ, Arg, MainArgSignature, Typ};

-use rustpython_parser as parser;
 use rustpython_parser::ast::{Constant, ExprKind, Located, StmtKind};
+use rustpython_parser::parser::parse_program;

 const DEF_MAIN: &str = "def main(";
 const FUNCTION_CALL: &str = "<function call>";
@@ -64,7 +64,7 @@ pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
            "No main function found".to_string(),
        ));
    }
-    let ast = parser::parse_program(&filtered_code, "main.py").map_err(|e| {
+    let ast = parse_program(&filtered_code, "main.py").map_err(|e| {
        error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
    })?;
    let param = ast.into_iter().find_map(|x| match x {
@@ -134,10 +134,7 @@ fn to_value(et: &ExprKind) -> Option<serde_json::Value> {
                .into_iter()
                .zip(values)
                .map(|(k, v)| {
-                    let key = k
-                        .as_ref()
-                        .map(|x| x.node.clone())
-                        .and_then(|n| to_value(&n))
+                    let key = to_value(&k.node)
                        .and_then(|x| match x {
                            serde_json::Value::String(s) => Some(s),
                            _ => None,
@@ -215,7 +212,7 @@ pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
        Ok(lines)
    } else {
        let code = code.split(DEF_MAIN).next().unwrap_or("");
-        let ast = parser::parse_program(code, "main.py").map_err(|e| {
+        let ast = parse_program(code, "main.py").map_err(|e| {
            error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
        })?;
        let mut imports: Vec<String> = ast
@@ -268,7 +268,7 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
 pub fn eval_sync(code: &str) -> Result<serde_json::Value, String> {
    let mut context = JsRuntime::new(RuntimeOptions::default());
    let code = format!("let x = {}; x", code);
-    let res = context.execute_script("<anon>", code);
+    let res = context.execute_script("<anon>", code.into());
    match res {
        Ok(global) => {
            let scope = &mut context.handle_scope();
@@ -6,11 +6,24 @@
 * LICENSE-AGPL for a copy of the license.
 */

-use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::{
+    net::{IpAddr, Ipv4Addr, SocketAddr},
+    sync::Arc,
+};

 use git_version::git_version;
+use monitor::handle_zombie_jobs_periodically;
 use sqlx::{Pool, Postgres};
+use tokio::{
+    fs::{metadata, DirBuilder},
+    join,
+    sync::RwLock,
+};
 use windmill_common::{utils::rd_string, METRICS_ADDR};
+use windmill_worker::{
+    DENO_CACHE_DIR, DENO_TMP_CACHE_DIR, GO_CACHE_DIR, GO_TMP_CACHE_DIR, PIP_CACHE_DIR,
+    ROOT_TMP_CACHE_DIR, TAR_PIP_TMP_CACHE_DIR,
+};

 const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
 const DEFAULT_NUM_WORKERS: usize = 3;
@@ -18,6 +31,7 @@ const DEFAULT_PORT: u16 = 8000;
 const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0);

 mod ee;
+mod monitor;

 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
@@ -88,7 +102,7 @@ async fn main() -> anyhow::Result<()> {
    }

    let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
-    let shutdown_signal = windmill_common::shutdown_signal(tx);
+    let shutdown_signal = windmill_common::shutdown_signal(tx.clone(), rx.resubscribe());

    #[cfg(feature = "enterprise")]
    tracing::info!(
@@ -148,6 +162,14 @@ Windmill Community Edition {GIT_VERSION}
        "BLACKLIST_WORKSPACES",
        "INSTANCE_EVENTS_WEBHOOK",
        "CLOUD_HOSTED",
+        "GLOBAL_CACHE_INTERVAL",
+        "WORKER_TAGS",
+        "CUSTOM_TAGS",
+        "JOB_RETENTION_SECS",
+        "WAIT_RESULT_FAST_POLL_DURATION_SECS",
+        "WAIT_RESULT_SLOW_POLL_INTERVAL_MS",
+        "WAIT_RESULT_FAST_POLL_INTERVAL_MS",
+        "EXIT_AFTER_NO_JOB_FOR_SECS",
    ]);

    if server_mode || num_workers > 0 {
@@ -171,6 +193,8 @@ Windmill Community Edition {GIT_VERSION}
                    rsmq.clone(),
                )
                .await?;
+                tracing::info!("All workers exited.");
+                tx.send(())?; // signal server to shutdown
            }
            Ok(()) as anyhow::Result<()>
        };
@@ -230,9 +254,11 @@ pub fn monitor_db<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>
    let rx2 = rx.resubscribe();
    let base_internal_url = base_internal_url.to_string();
    tokio::spawn(async move {
-        windmill_worker::handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq).await
+        join!(
+            handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq),
+            windmill_api::delete_expired_items_perdiodically(&db2, rx2)
+        );
    });
-    tokio::spawn(async move { windmill_api::delete_expired_items_perdiodically(&db2, rx2).await });
 }

 pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static>(
@@ -263,6 +289,28 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '

    let mut handles = Vec::with_capacity(num_workers as usize);

+    if metadata(&ROOT_TMP_CACHE_DIR).await.is_ok() {
+        if let Err(e) = tokio::fs::remove_dir_all(&ROOT_TMP_CACHE_DIR).await {
+            tracing::info!(error = %e, "Could not remove root tmp cache dir");
+        }
+    }
+
+    for x in [
+        PIP_CACHE_DIR,
+        DENO_CACHE_DIR,
+        GO_CACHE_DIR,
+        TAR_PIP_TMP_CACHE_DIR,
+        DENO_TMP_CACHE_DIR,
+        GO_TMP_CACHE_DIR,
+    ] {
+        DirBuilder::new()
+            .recursive(true)
+            .create(x)
+            .await
+            .expect("could not create initial worker dir");
+    }
+
+    let sync_barrier = Arc::new(RwLock::new(None));
    for i in 1..(num_workers + 1) {
        let db1 = db.clone();
        let instance_name = instance_name.clone();
@@ -271,6 +319,7 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
        let rx = rx.resubscribe();
        let base_internal_url = base_internal_url.clone();
        let rsmq2 = rsmq.clone();
+        let sync_barrier = sync_barrier.clone();
        handles.push(tokio::spawn(monitor.instrument(async move {
            tracing::info!(worker = %worker_name, "starting worker");
            windmill_worker::run_worker(
@@ -278,10 +327,12 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
                &instance_name,
                worker_name,
                i as u64,
+                num_workers as u32,
                &ip,
                rx,
                &base_internal_url,
                rsmq2,
+                sync_barrier,
            )
            .await
        })));
@@ -0,0 +1,148 @@
+use std::time::Duration;
+
+use once_cell::sync::OnceCell;
+use sqlx::{Pool, Postgres};
+use tokio::sync::mpsc;
+use uuid::Uuid;
+use windmill_common::{error, jobs::{JobKind, QueuedJob}, METRICS_ENABLED};
+use windmill_worker::{
+    create_token_for_owner, handle_job_error, AuthedClient, SESSION_TOKEN_EXPIRY,
+};
+
+lazy_static::lazy_static! {
+    static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
+    .ok()
+    .and_then(|x| x.parse::<String>().ok())
+    .unwrap_or_else(|| "30".to_string());
+
+
+    pub static ref RESTART_ZOMBIE_JOBS: bool = std::env::var("RESTART_ZOMBIE_JOBS")
+    .ok()
+    .and_then(|x| x.parse::<bool>().ok())
+    .unwrap_or(true);
+
+    static ref QUEUE_ZOMBIE_RESTART_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
+        "queue_zombie_restart_count",
+        "Total number of jobs restarted due to ping timeout."
+    )
+    .unwrap();
+    static ref QUEUE_ZOMBIE_DELETE_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
+        "queue_zombie_delete_count",
+        "Total number of jobs deleted due to their ping timing out in an unrecoverable state."
+    )
+    .unwrap();
+}
+
+pub async fn handle_zombie_jobs_periodically<
+    R: rsmq_async::RsmqConnection + Send + Sync + Clone,
+>(
+    db: &Pool<Postgres>,
+    mut rx: tokio::sync::broadcast::Receiver<()>,
+    base_internal_url: &str,
+    rsmq: Option<R>,
+) {
+    loop {
+        handle_zombie_jobs(db, base_internal_url, rsmq.clone()).await;
+
+        tokio::select! {
+            _ = tokio::time::sleep(Duration::from_secs(30))    => (),
+            _ = rx.recv() => {
+                    println!("received killpill for monitor job");
+                    break;
+            }
+        }
+    }
+}
+
+async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
+    db: &Pool<Postgres>,
+    base_internal_url: &str,
+    rsmq: Option<R>,
+) {
+    if *RESTART_ZOMBIE_JOBS {
+        let restarted = sqlx::query!(
+                "UPDATE queue SET running = false, started_at = null, logs = logs || '\nRestarted job after not receiving job''s ping for too long the ' || now() || '\n\n' WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2 AND job_kind != $3 AND same_worker = false RETURNING id, workspace_id, last_ping",
+                *ZOMBIE_JOB_TIMEOUT,
+                JobKind::Flow: JobKind,
+                JobKind::FlowPreview: JobKind,
+            )
+            .fetch_all(db)
+            .await
+            .ok()
+            .unwrap_or_else(|| vec![]);
+
+        if *METRICS_ENABLED {
+            QUEUE_ZOMBIE_RESTART_COUNT.inc_by(restarted.len() as _);
+        }
+        for r in restarted {
+            tracing::info!(
+                "restarted zombie job {} {} {}",
+                r.id,
+                r.workspace_id,
+                r.last_ping
+            );
+        }
+    }
+
+    let mut timeout_query = "SELECT * FROM queue WHERE last_ping < now() - ($1 || ' seconds')::interval AND running = true AND job_kind != $2".to_string();
+    if *RESTART_ZOMBIE_JOBS {
+        timeout_query.push_str(" AND same_worker = true");
+    };
+    let timeouts = sqlx::query_as::<_, QueuedJob>(&timeout_query)
+        .bind(ZOMBIE_JOB_TIMEOUT.as_str())
+        .bind(JobKind::Flow)
+        .fetch_all(db)
+        .await
+        .ok()
+        .unwrap_or_else(|| vec![]);
+
+    if *METRICS_ENABLED {
+        QUEUE_ZOMBIE_DELETE_COUNT.inc_by(timeouts.len() as _);
+    }
+
+    for job in timeouts {
+        tracing::info!("timedout zombie job {} {}", job.id, job.workspace_id,);
+
+        // since the job is unrecoverable, the same worker queue should never be sent anything
+        let (same_worker_tx_never_used, _same_worker_rx_never_used) = mpsc::channel::<Uuid>(1);
+
+        let token = create_token_for_owner(
+            &db,
+            &job.workspace_id,
+            &job.permissioned_as,
+            "ephemeral-zombie-jobs",
+            *SESSION_TOKEN_EXPIRY,
+            &job.email,
+        )
+        .await
+        .expect("could not create job token");
+
+        let client = AuthedClient {
+            base_internal_url: base_internal_url.to_string(),
+            token,
+            workspace: job.workspace_id.to_string(),
+            client: OnceCell::new(),
+        };
+
+        let last_ping = job.last_ping.clone();
+        let _ = handle_job_error(
+            db,
+            &client,
+            job,
+            error::Error::ExecutionErr(format!(
+                "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {})",
+                last_ping
+                    .map(|x| x.to_string())
+                    .unwrap_or_else(|| "no ping".to_string()),
+                *ZOMBIE_JOB_TIMEOUT
+            )),
+            None,
+            true,
+            same_worker_tx_never_used,
+            "",
+            base_internal_url,
+            rsmq.clone(),
+        )
+        .await;
+    }
+}
@@ -1,14 +1,19 @@
+use std::sync::Arc;
+
 use futures::{stream, Stream};
 use serde::Deserialize;
 use serde_json::json;
 use sqlx::{postgres::PgListener, types::Uuid, Pool, Postgres, Transaction};
+use tokio::sync::RwLock;
 use windmill_api::jobs::{CompletedJob, Job};
+use windmill_api_client::types::{CreateFlowBody, RawScript};
 use windmill_common::{
    flow_status::{FlowStatus, FlowStatusModule},
    flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform},
+    jobs::{JobPayload, RawCode},
    scripts::ScriptLang,
 };
-use windmill_queue::{get_queued_job, JobPayload, RawCode};
+use windmill_queue::get_queued_job;

 async fn initialize_tracing() {
    use std::sync::Once;
@@ -128,8 +133,7 @@ mod suspend_resume {
    use futures::{Stream, StreamExt};
    use serde_json::json;
    use sqlx::{query_scalar, types::Uuid};
-    use windmill_common::flows::FlowValue;
-    use windmill_queue::JobPayload;
+    use windmill_common::{flows::FlowValue, jobs::JobPayload};

    use super::*;

@@ -393,7 +397,6 @@ mod retry {
    use serde_json::json;
    use tokio::io::{AsyncReadExt, AsyncWriteExt};
    use windmill_common::flows::FlowValue;
-    use windmill_queue::JobPayload;

    use super::*;

@@ -843,6 +846,7 @@ impl RunJob {
            /* running */ false,
            None,
            true,
+            None,
        )
        .await
        .expect("push has to succeed");
@@ -877,7 +881,7 @@ async fn in_test_worker<Fut: std::future::Future>(
    port: u16,
 ) -> <Fut as std::future::Future>::Output {
    let (quit, worker) = spawn_test_worker(db, port);
-    let worker = tokio::time::timeout(std::time::Duration::from_secs(19), worker);
+    let worker = tokio::time::timeout(std::time::Duration::from_secs(45), worker);
    tokio::pin!(worker);

    let res = tokio::select! {
@@ -912,7 +916,6 @@ fn spawn_test_worker(
    let db = db.to_owned();
    let worker_instance: &str = "test worker instance";
    let worker_name: String = next_worker_name();
-    let i_worker: u64 = Default::default();
    let ip: &str = Default::default();
    let future = async move {
        let base_internal_url = format!("http://localhost:{}", port);
@@ -920,11 +923,13 @@ fn spawn_test_worker(
            &db,
            worker_instance,
            worker_name,
-            i_worker,
+            1,
+            1,
            ip,
            rx,
            &base_internal_url,
            None,
+            Arc::new(RwLock::new(None)),
        )
        .await
    };
@@ -1002,6 +1007,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                        content: numbers.to_string(),
                        path: None,
                        lock: None,
+                        tag: None,
                    },
                    stop_after_if: Default::default(),
                    summary: Default::default(),
@@ -1029,6 +1035,7 @@ async fn test_deno_flow(db: Pool<Postgres>) {
                                content: doubles.to_string(),
                                path: None,
                                lock: None,
+                                tag: None,
                            },
                            stop_after_if: Default::default(),
                            summary: Default::default(),
@@ -1129,6 +1136,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                        content: write_file.clone(),
                        path: None,
                        lock: None,
+                        tag: None,
                    },
                    stop_after_if: Default::default(),
                    summary: Default::default(),
@@ -1167,6 +1175,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                    content: write_file,
                                    path: None,
                                    lock: None,
+                                    tag: None,
                                },
                                stop_after_if: Default::default(),
                                summary: Default::default(),
@@ -1192,6 +1201,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                                    .to_string(),
                                    path: None,
                                    lock: None,
+                                    tag: None,
                                },
                                stop_after_if: Default::default(),
                                summary: Default::default(),
@@ -1234,6 +1244,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) {
                        .to_string(),
                        path: None,
                        lock: None,
+                        tag: None,
                    },
                    stop_after_if: Default::default(),
                    summary: Default::default(),
@@ -2432,9 +2443,12 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
    client
        .create_flow(
            "test-workspace",
-            &windmill_api_client::types::OpenFlowWPath {
-                open_flow: flow,
-                path: "g/all/flow_lock_all".to_owned(),
+            &CreateFlowBody {
+                open_flow_w_path: windmill_api_client::types::OpenFlowWPath {
+                    open_flow: flow,
+                    path: "g/all/flow_lock_all".to_owned(),
+                },
+                draft_only: None,
            },
        )
        .await
@@ -2453,20 +2467,21 @@ async fn test_flow_lock_all(db: Pool<Postgres>) {
        .modules
        .into_iter()
        .for_each(|m| {
+            tracing::error!("Module: {:?}", m.value);
            assert!(matches!(
                m.value,
-                windmill_api_client::types::FlowModuleValue::Rawscript {
+                windmill_api_client::types::FlowModuleValue::RawScript(RawScript {
                    language: windmill_api_client::types::RawScriptLanguage::Deno | windmill_api_client::types::RawScriptLanguage::Bash,
                    lock: Some(ref lock),
                    ..
-                } if lock == "")
+                }) if lock == "")
                || matches!(
                m.value,
-                windmill_api_client::types::FlowModuleValue::Rawscript {
+                windmill_api_client::types::FlowModuleValue::RawScript(RawScript{
                    language: windmill_api_client::types::RawScriptLanguage::Go | windmill_api_client::types::RawScriptLanguage::Python3,
                    lock: Some(ref lock),
                    ..
-                } if lock.len() > 0)
+                }) if lock.len() > 0)
            );
        });
 }
@@ -21,5 +21,7 @@ rand.workspace = true
 base64.workspace = true

 [build-dependencies]
+prettyplease = "0.1.25"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor" }
 serde_json = "1.0"
+syn = "1.0"
@@ -13,7 +13,9 @@ fn main() {
    let spec = serde_json::from_reader(file).unwrap();
    let mut generator = progenitor::Generator::default();

-    let content = generator.generate_text(&spec).unwrap();
+    let tokens = generator.generate_tokens(&spec).unwrap();
+    let ast = syn::parse2(tokens).unwrap();
+    let content = prettyplease::unparse(&ast);

    let mut out_file = Path::new(&env::var("OUT_DIR").unwrap()).to_path_buf();
    out_file.push("codegen.rs");
@@ -72,3 +72,4 @@ prometheus.workspace = true
 async_zip.workspace = true
 rsmq_async.workspace = true
 regex.workspace = true
+bytes.workspace = true
@@ -1,7 +1,7 @@
 openapi: "3.0.3"

 info:
-  version: 1.87.0
+  version: 1.96.3
  title: Windmill API

  contact:
@@ -2125,6 +2125,41 @@ paths:
                items:
                  type: string

+
+  /w/{workspace}/drafts/create:
+    post:
+      summary: create draft
+      operationId: createDraft
+      tags:
+        - draft
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                path:
+                  type: string
+                typ:
+                  type: string
+                  enum: ["flow", "script", "app"]
+                value: {}
+              required:
+                - path
+                - typ
+                - enum
+      responses:
+        "201":
+          description: draft created
+          content:
+            text/plain:
+              schema:
+                type: string
+
+
  /w/{workspace}/scripts/create:
    post:
      summary: create script
@@ -2139,38 +2174,8 @@ paths:
        content:
          application/json:
            schema:
-              type: object
-              properties:
-                path:
-                  type: string
-                parent_hash:
-                  type: string
-                summary:
-                  type: string
-                description:
-                  type: string
-                content:
-                  type: string
-                schema:
-                  type: object
-                is_template:
-                  type: boolean
-                lock:
-                  type: array
-                  items:
-                    type: string
-                language:
-                  type: string
-                  enum: [python3, deno, go, bash]
-                kind:
-                  type: string
-                  enum: [script, failure, trigger, command, approval]
-              required:
-                - path
-                - summary
-                - description
-                - content
-                - language
+              $ref: "#/components/schemas/NewScript"
+
      responses:
        "201":
          description: script created
@@ -2179,6 +2184,22 @@ paths:
              schema:
                type: string

+  /workers/custom_tags:
+    get:
+      summary: get all instance custom tags (tags are used to dispatch jobs to different worker groups)
+      operationId: getCustomTags
+      tags:
+        - worker
+      responses:
+        "200":
+          description: list of custom tags
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+
  /scripts/python/tojsonschema:
    post:
      summary: inspect python code to infer jsonschema of arguments
@@ -2348,6 +2369,25 @@ paths:
              schema:
                $ref: "#/components/schemas/Script"

+  /w/{workspace}/scripts/get/draft/{path}:
+    get:
+      summary: get script by path with draft
+      operationId: getScriptByPathWithDraft
+      tags:
+        - script
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: script details
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/NewScriptWithDraft"
+
+
+
  /w/{workspace}/scripts/raw/p/{path}:
    get:
      summary: raw script by path
@@ -2663,7 +2703,14 @@ paths:
              schema:
                type: array
                items:
-                  $ref: "#/components/schemas/Flow"
+                  allOf:
+                    - $ref: "#/components/schemas/Flow"
+                    - type: object
+                      properties:
+                        has_draft:
+                          type: boolean
+                        draft_only:
+                          type: boolean

  /w/{workspace}/flows/get/{path}:
    get:
@@ -2682,6 +2729,29 @@ paths:
              schema:
                $ref: "#/components/schemas/Flow"

+  /w/{workspace}/flows/get/draft/{path}:
+    get:
+      summary: get flow by path with draft
+      operationId: getFlowByPathWithDraft
+      tags:
+        - flow
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: flow details with draft
+          content:
+            application/json:
+              schema:
+                allOf:
+                  - $ref: "#/components/schemas/Flow"
+                  - type: object
+                    properties:
+                      draft:
+                        $ref: "#/components/schemas/Flow"
+
+
  /w/{workspace}/flows/exists/{path}:
    get:
      summary: exists flow by path
@@ -2713,7 +2783,12 @@ paths:
        content:
          application/json:
            schema:
-              $ref: "#/components/schemas/OpenFlowWPath"
+              allOf:
+                - $ref: "#/components/schemas/OpenFlowWPath"
+                - type: object
+                  properties:
+                    draft_only:
+                      type: boolean

      responses:
        "201":
@@ -2757,6 +2832,16 @@ paths:
      parameters:
        - $ref: "#/components/parameters/WorkspaceId"
        - $ref: "#/components/parameters/ScriptPath"
+      requestBody:
+        description: archiveFlow
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                archived:
+                  type: boolean
      responses:
        "200":
          description: flow archived
@@ -2803,6 +2888,80 @@ paths:
                items:
                  $ref: "#/components/schemas/Input"

+  /w/{workspace}/raw_apps/list:
+    get:
+      summary: list all available raw apps
+      operationId: listRawApps
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Page"
+        - $ref: "#/components/parameters/PerPage"
+        - $ref: "#/components/parameters/OrderDesc"
+        - $ref: "#/components/parameters/CreatedBy"
+        - name: path_start
+          description: mask to filter matching starting path
+          in: query
+          schema:
+            type: string
+        - name: path_exact
+          description: mask to filter exact matching path
+          in: query
+          schema:
+            type: string
+        - name: starred_only
+          description: |
+            (default false)
+            show only the starred items
+          in: query
+          schema:
+            type: boolean
+      responses:
+        "200":
+          description: All available raw apps
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/ListableRawApp"
+
+  /w/{workspace}/raw_apps/exists/{path}:
+    get:
+      summary: does an app exisst at path
+      operationId: existsRawApp
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: app exists
+          content:
+            application/json:
+              schema:
+                type: boolean
+
+  /w/{workspace}/apps/get_data/{version}/{path}:
+    get:
+      summary: get app by path
+      operationId: getRawAppData
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/VersionId"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: app details
+          content:
+            text/javascript:
+              schema:
+                type: string
+
  /w/{workspace}/apps/list:
    get:
      summary: list all available apps
@@ -2842,6 +3001,44 @@ paths:
                items:
                  $ref: "#/components/schemas/ListableApp"

+  /w/{workspace}/apps/create:
+    post:
+      summary: create app
+      operationId: createApp
+      tags:
+        - app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: new app
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                path:
+                  type: string
+                value: {}
+                summary:
+                  type: string
+                policy:
+                  $ref: "#/components/schemas/Policy"
+                draft_only:
+                  type: boolean
+              required:
+                - path
+                - value
+                - summary
+                - policy
+      responses:
+        "201":
+          description: app created
+          content:
+            text/plain:
+              schema:
+                type: string
+
  /w/{workspace}/apps/exists/{path}:
    get:
      summary: does an app exisst at path
@@ -2876,6 +3073,24 @@ paths:
              schema:
                $ref: "#/components/schemas/AppWithLastVersion"

+  /w/{workspace}/apps/get/draft/{path}:
+    get:
+      summary: get app by path with draft
+      operationId: getAppByPathWithDraft
+      tags:
+        - app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+      responses:
+        "200":
+          description: app details with draft
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/AppWithLastVersionWDraft"
+
+
  /w/{workspace}/apps_u/public_app/{path}:
    get:
      summary: get public app by secret
@@ -2927,16 +3142,16 @@ paths:
              schema:
                $ref: "#/components/schemas/AppWithLastVersion"

-  /w/{workspace}/apps/create:
+  /w/{workspace}/raw_apps/create:
    post:
-      summary: create app
-      operationId: createApp
+      summary: create raw app
+      operationId: createRawApp
      tags:
-        - app
+        - raw_app
      parameters:
        - $ref: "#/components/parameters/WorkspaceId"
      requestBody:
-        description: new app
+        description: new raw app
        required: true
        content:
          application/json:
@@ -2945,19 +3160,65 @@ paths:
              properties:
                path:
                  type: string
-                value: {}
+                value:
+                  type: string
                summary:
                  type: string
-                policy:
-                  $ref: "#/components/schemas/Policy"
              required:
                - path
                - value
                - summary
-                - policy
      responses:
        "201":
-          description: app created
+          description: raw app created
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /w/{workspace}/raw_apps/update/{path}:
+    post:
+      summary: update app
+      operationId: updateRawApp
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/ScriptPath"
+      requestBody:
+        description: updateraw  app
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                path:
+                  type: string
+                summary:
+                  type: string
+                value:
+                  type: string
+      responses:
+        "200":
+          description: app updated
+          content:
+            text/plain:
+              schema:
+                type: string
+
+  /w/{workspace}/raw_apps/delete/{path}:
+    delete:
+      summary: delete raw app
+      operationId: deleteRawApp
+      tags:
+        - raw_app
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/Path"
+      responses:
+        "200":
+          description: app deleted
          content:
            text/plain:
              schema:
@@ -3233,6 +3494,7 @@ paths:
        - $ref: "#/components/parameters/Running"
        - $ref: "#/components/parameters/ArgsFilter"
        - $ref: "#/components/parameters/ResultFilter"
+        - $ref: "#/components/parameters/Tag"
      responses:
        "200":
          description: All available queued jobs
@@ -3263,6 +3525,7 @@ paths:
        - $ref: "#/components/parameters/JobKinds"
        - $ref: "#/components/parameters/ArgsFilter"
        - $ref: "#/components/parameters/ResultFilter"
+        - $ref: "#/components/parameters/Tag"
        - name: is_skipped
          description: is the job skipped
          in: query
@@ -3300,6 +3563,7 @@ paths:
        - $ref: "#/components/parameters/StartedAfter"
        - $ref: "#/components/parameters/JobKinds"
        - $ref: "#/components/parameters/ArgsFilter"
+        - $ref: "#/components/parameters/Tag"
        - $ref: "#/components/parameters/ResultFilter"
        - name: is_skipped
          description: is the job skipped
@@ -4391,7 +4655,7 @@ paths:
          schema:
            type: string
            enum:
-              [script, group_, resource, schedule, variable, flow, folder, app]
+              [script, group_, resource, schedule, variable, flow, folder, app, raw_app]
      responses:
        "200":
          description: acls
@@ -4417,7 +4681,7 @@ paths:
          schema:
            type: string
            enum:
-              [script, group_, resource, schedule, variable, flow, folder, app]
+              [script, group_, resource, schedule, variable, flow, folder, app, raw_app]
      requestBody:
        description: acl to add
        required: true
@@ -4454,7 +4718,7 @@ paths:
          schema:
            type: string
            enum:
-              [script, group_, resource, schedule, variable, flow, folder, app]
+              [script, group_, resource, schedule, variable, flow, folder, app, raw_app]
      requestBody:
        description: acl to add
        required: true
@@ -4534,7 +4798,7 @@ paths:
                  type: string
                favorite_kind:
                  type: string
-                  enum: [flow, app, script]
+                  enum: [flow, app, script, raw_app]
      responses:
        "200":
          description: star item
@@ -4557,7 +4821,7 @@ paths:
                  type: string
                favorite_kind:
                  type: string
-                  enum: [flow, app, script]
+                  enum: [flow, app, script, raw_app]
      responses:
        "200":
          description: unstar item
@@ -4691,6 +4955,12 @@ components:
      required: true
      schema:
        type: string
+    VersionId:
+      name: version
+      in: path
+      required: true
+      schema:
+        type: number
    Token:
      name: token
      in: path
@@ -4858,6 +5128,12 @@ components:
      in: query
      schema:
        type: string
+    Tag:
+      name: tag
+      description: filter on jobs with a given tag/worker group
+      in: query
+      schema:
+        type: string
    ResultFilter:
      name: result
      description: filter on jobs containing those result as a json subset (@> in postgres)
@@ -4987,6 +5263,12 @@ components:
          enum: [script, failure, trigger, command, approval]
        starred:
          type: boolean
+        tag:
+          type: string
+        has_draft:
+          type: boolean
+        draft_only:
+          type: boolean
      required:
        - hash
        - path
@@ -5003,6 +5285,57 @@ components:
        - kind
        - starred

+    NewScript:
+      type: object
+      properties:
+        path:
+          type: string
+        parent_hash:
+          type: string
+        summary:
+          type: string
+        description:
+          type: string
+        content:
+          type: string
+        schema:
+          type: object
+        is_template:
+          type: boolean
+        lock:
+          type: array
+          items:
+            type: string
+        language:
+          type: string
+          enum: [python3, deno, go, bash]
+        kind:
+          type: string
+          enum: [script, failure, trigger, command, approval]
+        tag:
+          type: string
+        draft_only:
+          type: boolean
+      required:
+        - path
+        - summary
+        - description
+        - content
+        - language
+
+    NewScriptWithDraft:
+      allOf:
+        - $ref: "#/components/schemas/NewScript"
+        - type: object
+          properties:
+            draft:
+              $ref: "#/components/schemas/NewScript"
+            hash:
+              type: string
+          required:
+            - hash
+
+
    ScriptArgs:
      type: object
      additionalProperties: {}
@@ -5138,6 +5471,8 @@ components:
          type: boolean
        mem_peak:
          type: integer
+        tag:
+          type: string
      required:
        - id
        - running
@@ -5147,6 +5482,7 @@ components:
        - is_flow_step
        - email
        - visible_to_owner
+        - tag

    CompletedJob:
      type: object
@@ -5226,6 +5562,8 @@ components:
          type: boolean
        mem_peak:
          type: integer
+        tag:
+          type: string
      required:
        - id
        - created_by
@@ -5240,6 +5578,7 @@ components:
        - is_skipped
        - email
        - visible_to_owner
+        - tag

    Job:
      allOf:
@@ -5280,6 +5619,10 @@ components:
          type: array
          items:
            type: string
+        folders_owners:
+          type: array
+          items:
+            type: string
        usage:
          $ref: "#/components/schemas/Usage"
      required:
@@ -5291,6 +5634,7 @@ components:
        - operator
        - disabled
        - folders
+        - folders_owners

    Usage:
      type: object
@@ -5630,6 +5974,8 @@ components:
        language:
          type: string
          enum: [python3, deno, go, bash]
+        tag:
+          type: string

      required:
        - content
@@ -6001,6 +6347,8 @@ components:
          type: boolean
        starred:
          type: boolean
+        draft_only:
+          type: boolean
      required:
        - path
        - edited_by
@@ -6083,6 +6431,34 @@ components:
        - edited_at
        - execution_mode

+    ListableRawApp:
+      type: object
+      properties:
+        workspace_id:
+          type: string
+        path:
+          type: string
+        summary:
+          type: string
+        extra_perms:
+          type: object
+          additionalProperties:
+            type: boolean
+        starred:
+          type: boolean
+        version:
+          type: number
+        edited_at:
+          type: string
+          format: date-time
+      required:
+        - workspace_id
+        - path
+        - summary
+        - extra_perms
+        - version
+        - edited_at
+
    AppWithLastVersion:
      type: object
      properties:
@@ -6126,6 +6502,15 @@ components:
        - execution_mode
        - extra_perms

+    AppWithLastVersionWDraft:
+      allOf:
+        - $ref: "#/components/schemas/AppWithLastVersion"
+        - type: object
+          properties:
+            draft_only:
+              type: boolean
+            draft: {}
+
    SlackToken:
      type: object
      properties:
@@ -9,7 +9,6 @@ use std::collections::HashMap;
 */
 use crate::{
    db::{UserDB, DB},
-    jobs::script_path_to_payload,
    users::{require_owner_of_path, Authed, OptAuthed},
    variables::build_crypt,
    webhook_util::{WebhookMessage, WebhookShared},
@@ -32,17 +31,19 @@ use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
    apps::ListAppQuery,
    error::{to_anyhow, Error, JsonResult, Result},
+    jobs::{script_path_to_payload, JobPayload, RawCode},
    users::username_to_permissioned_as,
    utils::{
        http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
    },
 };
-use windmill_queue::{push, JobPayload, QueueTransaction, RawCode};
+use windmill_queue::{push, QueueTransaction};

 pub fn workspaced_service() -> Router {
    Router::new()
        .route("/list", get(list_apps))
        .route("/get/p/*path", get(get_app))
+        .route("/get/draft/*path", get(get_app_w_draft))
        .route("/secret_of/*path", get(get_secret_id))
        .route("/get/v/*id", get(get_app_by_id))
        .route("/exists/*path", get(exists_app))
@@ -74,6 +75,9 @@ pub struct ListableApp {
    pub execution_mode: String,
    pub starred: bool,
    pub edited_at: chrono::DateTime<chrono::Utc>,
+    pub has_draft: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
 }

 #[derive(FromRow, Serialize, Deserialize)]
@@ -98,6 +102,23 @@ pub struct AppWithLastVersion {
    pub extra_perms: serde_json::Value,
 }

+#[derive(Serialize, Deserialize)]
+pub struct AppWithLastVersionAndDraft {
+    pub id: i64,
+    pub path: String,
+    pub summary: String,
+    pub policy: serde_json::Value,
+    pub versions: Vec<i64>,
+    pub value: serde_json::Value,
+    pub created_by: String,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    pub extra_perms: serde_json::Value,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
+}
+
 pub type StaticFields = Map<String, Value>;

 #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
@@ -126,6 +147,7 @@ pub struct CreateApp {
    pub summary: String,
    pub value: serde_json::Value,
    pub policy: Policy,
+    pub draft_only: Option<bool>,
 }

 #[derive(Deserialize)]
@@ -156,6 +178,8 @@ async fn list_apps(
            "app_version.created_at as edited_at",
            "app.extra_perms",
            "favorite.path IS NOT NULL as starred",
+            "draft.path IS NOT NULL as has_draft",
+            "draft_only"
        ])
        .left()
        .join("favorite")
@@ -168,6 +192,11 @@ async fn list_apps(
        .on(
            "app_version.id = versions[array_upper(versions, 1)]"
        )
+        .left()
+        .join("draft")
+        .on(
+            "draft.path = app.path AND draft.workspace_id = app.workspace_id AND draft.typ = 'app'"
+        )
        .order_desc("favorite.path IS NOT NULL")
        .order_by("app_version.created_at", true)
        .and_where("app.workspace_id = ?".bind(&w_id))
@@ -215,6 +244,37 @@ async fn get_app(
    Ok(Json(app))
 }

+async fn get_app_w_draft(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<AppWithLastVersionAndDraft> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    let app_o = sqlx::query_as!(
+        AppWithLastVersionAndDraft,
+        r#"SELECT app.id, app.path, app.summary, app.versions, app.policy,
+        app.extra_perms, app_version.value, 
+        app_version.created_at, app_version.created_by,
+        app.draft_only, draft.value as "draft?"
+        from app
+        INNER JOIN app_version ON
+        app_version.id = app.versions[array_upper(app.versions, 1)]
+        LEFT JOIN draft ON 
+        app.path = draft.path AND app.workspace_id = draft.workspace_id AND draft.typ = 'app' 
+        WHERE app.path = $1 AND app.workspace_id = $2"#,
+        path.to_owned(),
+        &w_id
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    let app = not_found_if_none(app_o, "App", path)?;
+    Ok(Json(app))
+}
+
 async fn get_app_by_id(
    authed: Authed,
    Extension(user_db): Extension<UserDB>,
@@ -324,14 +384,39 @@ async fn create_app(
        return Err(Error::BadRequest("App path cannot be empty".to_string()));
    }

+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)",
+        &app.path,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(false);
+
+    if exists {
+        return Err(Error::BadRequest(format!(
+            "App with path {} already exists",
+            &app.path
+        )));
+    }
+
+    sqlx::query!(
+        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
+        &app.path,
+        &w_id
+    )
+    .execute(&mut tx)
+    .await?;
+
    let id = sqlx::query_scalar!(
        "INSERT INTO app
-            (workspace_id, path, summary, policy, versions)
-            VALUES ($1, $2, $3, $4, '{}') RETURNING id",
+            (workspace_id, path, summary, policy, versions, draft_only)
+            VALUES ($1, $2, $3, $4, '{}', $5) RETURNING id",
        w_id,
        app.path,
        app.summary,
        json!(app.policy),
+        app.draft_only,
    )
    .fetch_one(&mut tx)
    .await?;
@@ -411,6 +496,14 @@ async fn delete_app(
    let path = path.to_path();
    let mut tx = user_db.begin(&authed).await?;

+    sqlx::query!(
+        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
+        path,
+        &w_id
+    )
+    .execute(&mut tx)
+    .await?;
+
    sqlx::query!(
        "DELETE FROM app WHERE path = $1 AND workspace_id = $2",
        path,
@@ -441,7 +534,6 @@ async fn update_app(
    authed: Authed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
-    Extension(db): Extension<DB>,
    Path((w_id, path)): Path<(String, StripPath)>,
    Json(ns): Json<EditApp>,
 ) -> Result<String> {
@@ -456,11 +548,25 @@ async fn update_app(
        sqlb.and_where_eq("path", "?".bind(&path));
        sqlb.and_where_eq("workspace_id", "?".bind(&w_id));

+        sqlb.set("draft_only", "NULL");
        if let Some(npath) = &ns.path {
            if npath != path {
-                if !authed.is_admin {
-                    require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db)
-                        .await?;
+                require_owner_of_path(&authed, path)?;
+
+                let exists = sqlx::query_scalar!(
+                    "SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)",
+                    npath,
+                    w_id
+                )
+                .fetch_one(&mut tx)
+                .await?
+                .unwrap_or(false);
+
+                if exists {
+                    return Err(Error::BadRequest(format!(
+                        "App with path {} already exists",
+                        npath
+                    )));
                }
            }
            sqlb.set_str("path", npath);
@@ -521,6 +627,15 @@ async fn update_app(
        .execute(&mut tx)
        .await?;
    }
+
+    sqlx::query!(
+        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
+        path,
+        &w_id
+    )
+    .execute(&mut tx)
+    .await?;
+
    audit_log(
        &mut tx,
        &authed.username,
@@ -643,16 +758,16 @@ async fn execute_component(
        }
    };

-    let (job_payload, args) = match &payload {
+    let (job_payload, args, tag) = match &payload {
        ExecuteApp { args, raw_code: Some(raw_code), path: None, .. } => {
            let content = &raw_code.content;
            let payload = JobPayload::Code(raw_code.clone());
            let path = digest(content);
            let args = build_args(policy, path, args)?;
-            (payload, args)
+            (payload, args, None)
        }
        ExecuteApp { args, raw_code: None, path: Some(path), .. } => {
-            let payload = if path.starts_with("script/") {
+            let (payload, tag) = if path.starts_with("script/") {
                script_path_to_payload(
                    path.strip_prefix("script/").unwrap(),
                    tx.transaction_mut(),
@@ -660,7 +775,10 @@ async fn execute_component(
                )
                .await?
            } else if path.starts_with("flow/") {
-                JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string())
+                (
+                    JobPayload::Flow(path.strip_prefix("flow/").unwrap().to_string()),
+                    None,
+                )
            } else {
                return Err(Error::BadRequest(format!(
                    "path must start with script/ or flow/ (got {})",
@@ -668,7 +786,7 @@ async fn execute_component(
                )));
            };
            let args = build_args(policy, path.to_string(), args)?;
-            (payload, args)
+            (payload, args, tag)
        }
        _ => unreachable!(),
    };
@@ -689,6 +807,7 @@ async fn execute_component(
        false,
        None,
        true,
+        tag,
    )
    .await?;
    tx.commit().await?;
@@ -0,0 +1,94 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2042
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+
+use crate::{
+    db::{UserDB, DB},
+    users::{maybe_refresh_folders, require_owner_of_path, Authed},
+};
+
+use axum::{
+    extract::{Extension, Path},
+    routing::post,
+    Json, Router,
+};
+use hyper::StatusCode;
+use serde::{Deserialize, Serialize};
+use windmill_common::error::Result;
+
+pub fn workspaced_service() -> Router {
+    Router::new().route("/create", post(create_draft))
+}
+
+#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
+#[sqlx(type_name = "DRAFT_TYPE", rename_all = "lowercase")]
+#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
+pub enum DraftType {
+    Script,
+    Flow,
+    App,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct Draft {
+    pub path: String,
+    pub value: serde_json::Value,
+    pub typ: DraftType,
+}
+
+async fn create_draft(
+    authed: Authed,
+    Extension(db): Extension<DB>,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Json(draft): Json<Draft>,
+) -> Result<(StatusCode, String)> {
+    let authed = maybe_refresh_folders(&draft.path, &w_id, authed, &db).await;
+
+    let mut tx = user_db.begin(&authed).await?;
+
+    require_owner_of_path(&authed, &draft.path)?;
+
+    sqlx::query!(
+        "INSERT INTO draft
+            (workspace_id, path, value, typ)
+            VALUES ($1, $2, $3, $4)
+            ON CONFLICT (workspace_id, path, typ) DO UPDATE SET value = $3",
+        &w_id,
+        draft.path,
+        draft.value,
+        draft.typ: DraftType,
+    )
+    .execute(&mut tx)
+    .await?;
+
+    tx.commit().await?;
+
+    Ok((StatusCode::CREATED, format!("draft {} created", draft.path)))
+}
+
+// async fn get_draft(
+//     authed: Authed,
+//     Extension(user_db): Extension<UserDB>,
+//     Path((w_id, path)): Path<(String, StripPath)>,
+// ) -> JsonResult<Draft> {
+//     let path = path.to_path();
+//     let mut tx = user_db.begin(&authed).await?;
+
+//     let script_o = sqlx::query_as!(
+//         Draft,
+//         r#"SELECT path, value, typ as "typ: DraftType" FROM draft WHERE path = $1 AND workspace_id = $2"#,
+//         path,
+//         w_id
+//     )
+//     .fetch_optional(&mut tx)
+//     .await?;
+//     tx.commit().await?;
+
+//     let draft = not_found_if_none(script_o, "draft", path)?;
+//     Ok(Json(draft))
+// }
@@ -29,6 +29,7 @@ pub enum FavoriteKind {
    Script,
    Flow,
    App,
+    Raw_App,
 }
 #[derive(Deserialize)]
 pub struct Favorite {
@@ -19,6 +19,7 @@ use axum::{
    Json, Router,
 };
 use hyper::StatusCode;
+use serde::{Deserialize, Serialize};
 use sql_builder::prelude::*;
 use sql_builder::SqlBuilder;
 use sqlx::{Postgres, Transaction};
@@ -26,12 +27,14 @@ use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
    error::{self, to_anyhow, Error, JsonResult, Result},
    flows::{Flow, ListFlowQuery, ListableFlow, NewFlow},
+    jobs::JobPayload,
    schedule::Schedule,
+    scripts::Schema,
    utils::{
        http_get_from_hub, list_elems_from_hub, not_found_if_none, paginate, Pagination, StripPath,
    },
 };
-use windmill_queue::{push, schedule::push_scheduled_job, JobPayload, QueueTransaction};
+use windmill_queue::{push, schedule::push_scheduled_job, QueueTransaction};

 pub fn workspaced_service() -> Router {
    Router::new()
@@ -41,6 +44,7 @@ pub fn workspaced_service() -> Router {
        .route("/archive/*path", post(archive_flow_by_path))
        .route("/delete/*path", delete(delete_flow_by_path))
        .route("/get/*path", get(get_flow_by_path))
+        .route("/get/draft/*path", get(get_flow_by_path_w_draft))
        .route("/exists/*path", get(exists_flow_by_path))
        .route("/list_paths", get(list_paths))
 }
@@ -71,6 +75,8 @@ async fn list_flows(
            "archived",
            "extra_perms",
            "favorite.path IS NOT NULL as starred",
+            "draft.path IS NOT NULL as has_draft",
+            "draft_only"
        ])
        .left()
        .join("favorite")
@@ -78,6 +84,11 @@ async fn list_flows(
            "favorite.favorite_kind = 'flow' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?"
                .bind(&authed.username),
        )
+        .left()
+        .join("draft")
+        .on(
+            "draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'flow'"
+        )
        .order_desc("favorite.path IS NOT NULL")
        .order_by("edited_at", lq.order_desc.unwrap_or(true))
        .and_where("o.workspace_id = ?".bind(&w_id))
@@ -85,9 +96,8 @@ async fn list_flows(
        .limit(per_page)
        .clone();

-    if !lq.show_archived.unwrap_or(false) {
-        sqlb.and_where_eq("archived", false);
-    }
+    sqlb.and_where_eq("archived", lq.show_archived.unwrap_or(false));
+
    if let Some(ps) = &lq.path_start {
        sqlb.and_where_like_left("path", "?".bind(ps));
    }
@@ -193,7 +203,7 @@ async fn create_flow(

    sqlx::query!(
        "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, \
-         schema, dependency_job) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL)",
+         schema, dependency_job, draft_only) VALUES ($1, $2, $3, $4, $5, $6, now(), $7::text::json, NULL, $8)",
        w_id,
        nf.path,
        nf.summary,
@@ -201,6 +211,15 @@ async fn create_flow(
        nf.value,
        &authed.username,
        nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
+        nf.draft_only
+    )
+    .execute(&mut tx)
+    .await?;
+
+    sqlx::query!(
+        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
+        nf.path,
+        &w_id
    )
    .execute(&mut tx)
    .await?;
@@ -242,6 +261,7 @@ async fn create_flow(
        false,
        None,
        true,
+        None,
    )
    .await?;

@@ -308,7 +328,7 @@ async fn update_flow(
    let old_dep_job = not_found_if_none(old_dep_job, "Flow", flow_path)?;
    sqlx::query!(
        "UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, \
-         edited_at = now(), schema = $6::text::json, dependency_job = NULL WHERE path = $7 AND workspace_id = $8",
+         edited_at = now(), schema = $6::text::json, dependency_job = NULL, draft_only = NULL WHERE path = $7 AND workspace_id = $8",
        nf.path,
        nf.summary,
        nf.description,
@@ -325,58 +345,65 @@ async fn update_flow(
        check_schedule_conflict(tx.transaction_mut(), &w_id, &nf.path).await?;

        if !authed.is_admin {
-            require_owner_of_path(&w_id, &authed.username, &authed.groups, &flow_path, &db).await?;
+            require_owner_of_path(&authed, flow_path)?;
        }
+    }

-        let mut schedulables: Vec<Schedule> = sqlx::query_as!(
-            Schedule,
-                "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
-                nf.path,
-                flow_path,
-                w_id,
-            )
-            .fetch_all(&mut tx)
-            .await?;
-
-        let schedule = sqlx::query_as!(Schedule,
-            "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
+    let mut schedulables: Vec<Schedule> = sqlx::query_as!(
+        Schedule,
+            "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND path != $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
            nf.path,
            flow_path,
            w_id,
        )
-        .fetch_optional(&mut tx)
+        .fetch_all(&mut tx)
        .await?;

-        if let Some(schedule) = schedule {
-            schedulables.push(schedule);
-        }
+    let schedule = sqlx::query_as!(Schedule,
+        "UPDATE schedule SET path = $1, script_path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow IS true RETURNING *",
+        nf.path,
+        flow_path,
+        w_id,
+    )
+    .fetch_optional(&mut tx)
+    .await?;

-        for schedule in schedulables.into_iter() {
-            // TODO: Why is this in the loop in the first place? Seems like it's just doing nothing after the first iteration? Should this use schedule.path?
-            clear_schedule(tx.transaction_mut(), flow_path, true).await?;
-
-            if schedule.enabled {
-                tx = push_scheduled_job(tx, schedule).await?;
-            }
-        }
-
-        audit_log(
-            &mut tx,
-            &authed.username,
-            "flows.update",
-            ActionKind::Create,
-            &w_id,
-            Some(&nf.path.to_string()),
-            Some(
-                [Some(("flow", nf.path.as_str()))]
-                    .into_iter()
-                    .flatten()
-                    .collect(),
-            ),
-        )
-        .await?;
+    if let Some(schedule) = schedule {
+        schedulables.push(schedule);
    }

+    for schedule in schedulables.into_iter() {
+        clear_schedule(tx.transaction_mut(), &schedule.path, true).await?;
+
+        if schedule.enabled {
+            tx = push_scheduled_job(tx, schedule).await?;
+        }
+    }
+
+    sqlx::query!(
+        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
+        flow_path,
+        &w_id
+    )
+    .execute(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "flows.update",
+        ActionKind::Create,
+        &w_id,
+        Some(&nf.path.to_string()),
+        Some(
+            [Some(("flow", nf.path.as_str()))]
+                .into_iter()
+                .flatten()
+                .collect(),
+        ),
+    )
+    .await?;
+
    webhook.send_message(
        w_id.clone(),
        WebhookMessage::UpdateFlow {
@@ -402,6 +429,7 @@ async fn update_flow(
        false,
        None,
        true,
+        None,
    )
    .await?;
    sqlx::query!(
@@ -444,6 +472,44 @@ async fn get_flow_by_path(
    Ok(Json(flow))
 }

+#[derive(Serialize, sqlx::FromRow)]
+pub struct FlowWDraft {
+    pub path: String,
+    pub summary: String,
+    pub description: String,
+    pub schema: Option<Schema>,
+    pub value: serde_json::Value,
+    pub extra_perms: serde_json::Value,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
+}
+
+async fn get_flow_by_path_w_draft(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<FlowWDraft> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    let flow_o = sqlx::query_as::<_, FlowWDraft>(
+        "SELECT flow.path, flow.summary, flow,description, flow.schema, flow.value, flow.extra_perms, flow.draft_only, draft.value as draft FROM flow
+        LEFT JOIN draft ON 
+        flow.path = draft.path AND flow.workspace_id = draft.workspace_id AND draft.typ = 'flow' 
+        WHERE flow.path = $1 AND flow.workspace_id = $2",
+    )
+    .bind(path)
+    .bind(w_id)
+    .fetch_optional(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    let flow = not_found_if_none(flow_o, "Flow", path)?;
+    Ok(Json(flow))
+}
+
 async fn exists_flow_by_path(
    Extension(db): Extension<DB>,
    Path((w_id, path)): Path<(String, StripPath)>,
@@ -462,17 +528,24 @@ async fn exists_flow_by_path(
    Ok(Json(exists))
 }

+#[derive(Deserialize)]
+struct Archived {
+    archived: Option<bool>,
+}
+
 async fn archive_flow_by_path(
    authed: Authed,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, path)): Path<(String, StripPath)>,
+    Json(archived): Json<Archived>,
 ) -> Result<String> {
    let path = path.to_path();
    let mut tx = user_db.begin(&authed).await?;

    sqlx::query!(
-        "UPDATE flow SET archived = true WHERE path = $1 AND workspace_id = $2",
+        "UPDATE flow SET archived = $1 WHERE path = $2 AND workspace_id = $3",
+        archived.archived.unwrap_or(true),
        path,
        &w_id
    )
@@ -507,6 +580,14 @@ async fn delete_flow_by_path(
    let path = path.to_path();
    let mut tx = user_db.begin(&authed).await?;

+    sqlx::query!(
+        "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
+        path,
+        &w_id
+    )
+    .execute(&mut tx)
+    .await?;
+
    sqlx::query!(
        "DELETE FROM flow WHERE path = $1 AND workspace_id = $2",
        path,
@@ -578,6 +659,7 @@ mod tests {
                        language: scripts::ScriptLang::Deno,
                        path: None,
                        lock: None,
+                        tag: None,
                    },
                    stop_after_if: Some(StopAfterIf {
                        expr: "foo = 'bar'".to_string(),
@@ -22,13 +22,13 @@ use lazy_static::lazy_static;
 use regex::Regex;
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
-    error::{self, to_anyhow, Error, JsonResult, Result},
+    error::{self, to_anyhow, JsonResult, Result},
    users::username_to_permissioned_as,
    utils::{not_found_if_none, paginate, Pagination},
 };

 use serde::{Deserialize, Serialize};
-use sqlx::{query_scalar, FromRow, Postgres, Transaction};
+use sqlx::{FromRow, Postgres, Transaction};

 pub fn workspaced_service() -> Router {
    Router::new()
@@ -41,7 +41,7 @@ pub fn workspaced_service() -> Router {
        .route("/delete/:name", delete(delete_folder))
        .route("/addowner/:name", post(add_owner))
        .route("/removeowner/:name", post(remove_owner))
-        .route("/is_owner/*path", get(is_owner))
+        .route("/is_owner/*path", get(is_owner_api))
 }

 #[derive(FromRow, Serialize, Deserialize, Clone)]
@@ -219,47 +219,29 @@ async fn create_folder(
    Ok(format!("Created folder {}", ng.name))
 }

-pub async fn is_owner(
-    Authed { username, is_admin, groups, .. }: Authed,
-    Extension(db): Extension<DB>,
-    Path((w_id, name)): Path<(String, String)>,
+pub async fn is_owner_api(
+    authed: Authed,
+    Path((_w_id, name)): Path<(String, String)>,
 ) -> JsonResult<bool> {
-    if is_admin {
-        Ok(Json(true))
+    Ok(Json(is_owner(&authed, &name)))
+}
+
+pub fn is_owner(Authed { is_admin, folders, .. }: &Authed, name: &str) -> bool {
+    if *is_admin {
+        true
    } else {
-        Ok(Json(
-            require_is_owner(&name, &username, &groups, &w_id, &db)
-                .await
-                .is_ok(),
-        ))
+        folders.into_iter().any(|x| x.0 == name && x.2)
    }
 }

-pub async fn require_is_owner(
-    folder_name: &str,
-    username: &str,
-    groups: &Vec<String>,
-    w_id: &str,
-    db: &DB,
-) -> Result<()> {
-    let is_owner = query_scalar!(
-        "SELECT EXISTS(SELECT 1 FROM folder WHERE CONCAT('u/', $1::text) = ANY(owners) AND name = $2 AND workspace_id = $4) OR exists(
-            SELECT 1 FROM folder, unnest(folder.owners) as o
-            WHERE o = ANY($3::text[]) AND folder.name = $2 AND folder.workspace_id = $4)",
-        username,
-        folder_name,
-        groups,
-        w_id,
-    ).fetch_one(db)
-    .await?
-    .unwrap_or(false);
-    if !is_owner {
-        Err(Error::BadRequest(format!(
-            "{} is not an owner of {} and hence is not authorized to perform this operation",
-            username, folder_name
-        )))
-    } else {
+pub fn require_is_owner(authed: &Authed, name: &str) -> Result<()> {
+    if is_owner(authed, name) {
        Ok(())
+    } else {
+        Err(windmill_common::error::Error::NotAuthorized(format!(
+            "You are not owner of the folder {}",
+            name
+        )))
    }
 }

@@ -491,7 +473,6 @@ async fn delete_folder(

 async fn add_owner(
    authed: Authed,
-    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, name)): Path<(String, String)>,
@@ -500,9 +481,7 @@ async fn add_owner(
    let mut tx = user_db.begin(&authed).await?;

    not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
-    if !authed.is_admin {
-        require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
-    }
+    require_is_owner(&authed, &name)?;

    sqlx::query!(
        "UPDATE folder SET owners = array_append(owners, $1) WHERE name = $2 AND workspace_id = $3 AND NOT $1 = ANY(owners) RETURNING name",
@@ -538,14 +517,14 @@ pub async fn get_folders_for_user(
    username: &str,
    groups: &[String],
    db: &DB,
-) -> Result<Vec<(String, bool)>> {
+) -> Result<Vec<(String, bool, bool)>> {
    let mut perms = groups
        .into_iter()
        .map(|x| format!("g/{}", x))
        .collect::<Vec<_>>();
    perms.insert(0, format!("u/{}", username));
    let folders = sqlx::query!(
-        "SELECT name, (EXISTS (SELECT 1 FROM (SELECT key, value FROM jsonb_each_text(extra_perms) WHERE key = ANY($1)) t  WHERE value::boolean IS true)) as write  FROM folder
+        "SELECT name, (EXISTS (SELECT 1 FROM (SELECT key, value FROM jsonb_each_text(extra_perms) WHERE key = ANY($1)) t  WHERE value::boolean IS true)) as write, $1 && owners::text[] as owner  FROM folder
        WHERE extra_perms ?| $1  AND workspace_id = $2",
        &perms[..],
        w_id,
@@ -553,7 +532,7 @@ pub async fn get_folders_for_user(
    .fetch_all(db)
    .await?
    .into_iter()
-    .map(|x| (x.name, x.write.unwrap_or(false)))
+    .map(|x| (x.name, x.write.unwrap_or(false), x.owner.unwrap_or(false)))
    .collect();

    Ok(folders)
@@ -561,7 +540,6 @@ pub async fn get_folders_for_user(

 async fn remove_owner(
    authed: Authed,
-    Extension(db): Extension<DB>,
    Extension(user_db): Extension<UserDB>,
    Extension(webhook): Extension<WebhookShared>,
    Path((w_id, name)): Path<(String, String)>,
@@ -570,9 +548,7 @@ async fn remove_owner(
    let mut tx = user_db.begin(&authed).await?;

    not_found_if_none(get_folderopt(&mut tx, &w_id, &name).await?, "Folder", &name)?;
-    if !authed.is_admin {
-        require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
-    }
+    require_is_owner(&authed, &name)?;

    sqlx::query!(
        "UPDATE folder SET owners = array_remove(owners, $1) WHERE name = $2 AND workspace_id = $3 RETURNING name",
@@ -43,25 +43,29 @@ async fn add_granular_acl(
    Json(GranularAcl { owner, write }): Json<GranularAcl>,
 ) -> Result<String> {
    let path = path.to_path();
+
    let (kind, path) = path
        .split_once('/')
        .ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
    let mut tx = user_db.begin(&authed).await?;

-    if !authed.is_admin {
-        if kind == "folder" {
-            crate::folders::require_is_owner(&path, &authed.username, &authed.groups, &w_id, &db)
-                .await?;
-        } else if kind == "group_" {
-        } else {
-            require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
-        }
-    }
    let identifier = if kind == "group_" || kind == "folder" {
        "name"
    } else {
        "path"
    };
+
+    if !authed.is_admin {
+        if kind == "folder" {
+            crate::folders::require_is_owner(&authed, path)?;
+        } else if kind == "group_" {
+            crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db)
+                .await?;
+        } else {
+            require_owner_of_path(&authed, path)?;
+        }
+    }
+
    let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
        "UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), \
         true) WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
@@ -86,12 +90,22 @@ async fn remove_granular_acl(
    Json(GranularAcl { owner, write: _ }): Json<GranularAcl>,
 ) -> Result<String> {
    let path = path.to_path();
-    if !authed.is_admin {
-        require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
-    }
+
    let (kind, path) = path
        .split_once('/')
        .ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
+
+    if !authed.is_admin {
+        if kind == "folder" {
+            crate::folders::require_is_owner(&authed, path)?;
+        } else if kind == "group_" {
+            crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db)
+                .await?;
+        } else {
+            require_owner_of_path(&authed, path)?;
+        }
+    }
+
    let mut tx = user_db.begin(&authed).await?;

    let identifier = if kind == "group_" || kind == "folder" {
@@ -99,6 +113,11 @@ async fn remove_granular_acl(
    } else {
        "path"
    };
+
+    if identifier == "path" {
+        require_owner_of_path(&authed, path)?;
+    }
+
    let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
        "UPDATE {kind} SET extra_perms = extra_perms - $1 WHERE {identifier} = $2 AND \
         workspace_id = $3 RETURNING extra_perms"
@@ -292,6 +292,12 @@ async fn delete_group(
 ) -> Result<String> {
    let mut tx = user_db.begin(&authed).await?;

+    if name == "all" {
+        return Err(Error::BadRequest(
+            "The group 'all' is a special group that contains all users and cannot be deleted".to_string(),
+        ));
+    }
+
    if !authed.is_admin {
        require_is_owner(&name, &authed.username, &authed.groups, &w_id, &db).await?;
    }
@@ -22,11 +22,10 @@ use std::{
 };
 use windmill_common::{
    error::JsonResult,
+    jobs::JobKind,
    scripts::to_i64,
    utils::{paginate, Pagination},
 };
-use windmill_queue::JobKind;
-
 pub fn workspaced_service() -> Router {
    Router::new()
        .route("/history", get(get_input_history))
@@ -113,9 +112,9 @@ async fn get_input_history(
    let mut tx = user_db.begin(&authed).await?;

    let sql = &format!(
-        "select distinct on (args) * from completed_job \
+        "select * from (select distinct on (args) * from completed_job \
        where {} = $1 and job_kind = $2 and workspace_id = $3 \
-        order by args, started_at desc limit $4 offset $5",
+        order by args, started_at desc) t ORDER BY started_at desc limit $4 offset $5",
        r.runnable_type.column_name()
    );

@@ -31,14 +31,13 @@ use windmill_common::{
    error::{self, to_anyhow, Error},
    flow_status::{Approval, FlowStatus, FlowStatusModule},
    flows::FlowValue,
+    jobs::{script_path_to_payload, JobKind, JobPayload, QueuedJob, RawCode},
    oauth2::HmacSha256,
    scripts::{ScriptHash, ScriptLang},
    users::username_to_permissioned_as,
    utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
 };
-use windmill_queue::{
-    get_queued_job, push, JobKind, JobPayload, QueueTransaction, QueuedJob, RawCode,
-};
+use windmill_queue::{get_queued_job, push, QueueTransaction};

 pub fn workspaced_service() -> Router {
    Router::new()
@@ -221,6 +220,26 @@ pub async fn get_path_for_hash<'c>(
    Ok(path)
 }

+pub async fn get_path_and_tag_for_hash<'c>(
+    db: &mut Transaction<'c, Postgres>,
+    w_id: &str,
+    hash: i64,
+) -> error::Result<(String, Option<String>)> {
+    let script = sqlx::query!(
+        "select path, tag from script where hash = $1 AND workspace_id = $2",
+        hash,
+        w_id
+    )
+    .fetch_one(db)
+    .await
+    .map_err(|e| {
+        Error::InternalErr(format!(
+            "querying getting path for hash {hash} in {w_id}: {e}"
+        ))
+    })?;
+    Ok((script.path, script.tag))
+}
+
 async fn get_job(
    Extension(db): Extension<DB>,
    Path((w_id, id)): Path<(String, Uuid)>,
@@ -307,6 +326,7 @@ pub struct CompletedJob {
    pub visible_to_owner: bool,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub mem_peak: Option<i32>,
+    pub tag: String,
 }

 #[derive(Deserialize, Clone)]
@@ -391,6 +411,7 @@ pub struct ListQueueQuery {
    pub suspended: Option<bool>,
    // filter by matching a subset of the args using base64 encoded json subset
    pub args: Option<String>,
+    pub tag: Option<String>,
 }

 fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> SqlBuilder {
@@ -413,6 +434,9 @@ fn list_queue_jobs_query(w_id: &str, lq: &ListQueueQuery, fields: &[&str]) -> Sq
    if let Some(cb) = &lq.created_by {
        sqlb.and_where_eq("created_by", "?".bind(cb));
    }
+    if let Some(t) = &lq.tag {
+        sqlb.and_where_eq("tag", "?".bind(t));
+    }
    if let Some(r) = &lq.running {
        sqlb.and_where_eq("running", &r);
    }
@@ -545,6 +569,7 @@ async fn list_jobs(
            job_kinds: lq.job_kinds,
            suspended: lq.suspended,
            args: lq.args,
+            tag: lq.tag,
        },
        &[
            "'QueuedJob' as typ",
@@ -574,6 +599,7 @@ async fn list_jobs(
            "visible_to_owner",
            "suspend",
            "mem_peak",
+            "tag",
        ],
    );
    let sqlc = list_completed_jobs_query(
@@ -609,6 +635,7 @@ async fn list_jobs(
            "visible_to_owner",
            "null as suspend",
            "mem_peak",
+            "tag",
        ],
    );
    let sql = format!(
@@ -627,7 +654,7 @@ async fn list_jobs(
 pub async fn resume_suspended_flow_as_owner(
    authed: Authed,
    Extension(db): Extension<DB>,
-    Path((w_id, flow_id)): Path<(String, Uuid)>,
+    Path((_w_id, flow_id)): Path<(String, Uuid)>,
    QueryOrBody(value): QueryOrBody<serde_json::Value>,
 ) -> error::Result<StatusCode> {
    let value = value.unwrap_or(serde_json::Value::Null);
@@ -635,16 +662,11 @@ pub async fn resume_suspended_flow_as_owner(

    let (flow, job_id) = get_suspended_flow_info(flow_id, &mut tx).await?;

-    if !authed.is_admin {
-        require_owner_of_path(
-            &w_id,
-            &authed.username,
-            &authed.groups,
-            &flow.script_path.clone().unwrap_or_else(|| String::new()),
-            &db,
-        )
-        .await?;
-    }
+    require_owner_of_path(
+        &authed,
+        &flow.script_path.clone().unwrap_or_else(|| String::new()),
+    )?;
+
    insert_resume_job(0, job_id, &flow, value, Some(authed.username), &mut tx).await?;

    resume_immediately_if_relevant(flow, job_id, &mut tx).await?;
@@ -1090,6 +1112,7 @@ struct UnifiedJob {
    visible_to_owner: bool,
    suspend: Option<i32>,
    mem_peak: Option<i32>,
+    tag: String,
 }

 impl From<UnifiedJob> for Job {
@@ -1125,6 +1148,7 @@ impl From<UnifiedJob> for Job {
                email: uj.email,
                visible_to_owner: uj.visible_to_owner,
                mem_peak: uj.mem_peak,
+                tag: uj.tag,
            }),
            "QueuedJob" => Job::QueuedJob(QueuedJob {
                workspace_id: uj.workspace_id,
@@ -1160,6 +1184,7 @@ impl From<UnifiedJob> for Job {
                mem_peak: uj.mem_peak,
                root_job: None,
                leaf_jobs: None,
+                tag: uj.tag,
            }),
            t => panic!("job type {} not valid", t),
        }
@@ -1176,6 +1201,7 @@ struct Preview {
    path: Option<String>,
    args: Option<serde_json::Map<String, serde_json::Value>>,
    language: ScriptLang,
+    tag: Option<String>,
 }

 #[derive(Deserialize)]
@@ -1244,7 +1270,6 @@ pub async fn run_flow_by_path(
    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
    let args = run_query.add_include_headers(headers, args.unwrap_or_default());
-
    let (uuid, tx) = push(
        tx,
        &w_id,
@@ -1261,6 +1286,7 @@ pub async fn run_flow_by_path(
        false,
        None,
        !run_query.invisible_to_owner.unwrap_or(false),
+        None,
    )
    .await?;
    tx.commit().await?;
@@ -1278,7 +1304,8 @@ pub async fn run_job_by_path(
 ) -> error::Result<(StatusCode, String)> {
    let script_path = script_path.to_path();
    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
-    let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
+    let (job_payload, tag) =
+        script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
    let args = run_query.add_include_headers(headers, args.unwrap_or_default());

@@ -1298,6 +1325,7 @@ pub async fn run_job_by_path(
        false,
        None,
        !run_query.invisible_to_owner.unwrap_or(false),
+        tag,
    )
    .await?;
    tx.commit().await?;
@@ -1347,14 +1375,13 @@ async fn run_wait_result<T>(
    uuid: Uuid,
    Path((w_id, _)): Path<(String, T)>,
 ) -> error::JsonResult<serde_json::Value> {
-    let mut result = None;
-    let iters = if timeout <= 0 {
-        20
-    } else if timeout <= 1 {
-        timeout * 10
+    let mut result;
+    let timeout_ms = if timeout <= 0 {
+        2000
    } else {
-        10 + ((timeout - 1) * 2)
+        (timeout * 1000) as u64
    };
+
    let mut g = Guard {
        done: false,
        id: uuid,
@@ -1362,7 +1389,10 @@ async fn run_wait_result<T>(
        db: user_db.clone(),
        authed: authed.clone(),
    };
-    for i in 0..iters {
+
+    let fast_poll_duration = *WAIT_RESULT_FAST_POLL_DURATION_SECS as u64 * 1000;
+    let mut accumulated_delay = 0 as u64;
+    loop {
        let mut tx = user_db.clone().begin(&authed).await?;
        result = sqlx::query_scalar!(
            "SELECT result FROM completed_job WHERE id = $1 AND workspace_id = $2",
@@ -1377,7 +1407,16 @@ async fn run_wait_result<T>(
        if result.is_some() {
            break;
        }
-        let delay = if i < 10 { 100 } else { 500 };
+
+        let delay = if accumulated_delay <= fast_poll_duration {
+            *WAIT_RESULT_FAST_POLL_INTERVAL_MS
+        } else {
+            *WAIT_RESULT_SLOW_POLL_INTERVAL_MS
+        };
+        accumulated_delay += delay;
+        if accumulated_delay > timeout_ms {
+            break;
+        };
        tokio::time::sleep(core::time::Duration::from_millis(delay)).await;
    }
    if let Some(result) = result {
@@ -1417,6 +1456,18 @@ lazy_static::lazy_static! {
        .ok()
        .and_then(|x| x.parse().ok())
        .unwrap_or(20);
+    pub static ref WAIT_RESULT_FAST_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_FAST_POLL_INTERVAL_MS")
+        .ok()
+        .and_then(|x| x.parse().ok())
+        .unwrap_or(50);
+    pub static ref WAIT_RESULT_FAST_POLL_DURATION_SECS: u16 = std::env::var("WAIT_RESULT_FAST_POLL_DURATION_SECS")
+        .ok()
+        .and_then(|x| x.parse().ok())
+        .unwrap_or(2);
+    pub static ref WAIT_RESULT_SLOW_POLL_INTERVAL_MS: u64 = std::env::var("WAIT_RESULT_SLOW_POLL_INTERVAL_MS")
+        .ok()
+        .and_then(|x| x.parse().ok())
+        .unwrap_or(200);
 }

 pub async fn run_wait_result_job_by_path_get(
@@ -1441,7 +1492,8 @@ pub async fn run_wait_result_job_by_path_get(
    check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
    let script_path = script_path.to_path();
    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
-    let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
+    let (job_payload, tag) =
+        script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;

    let (uuid, tx) = push(
        tx,
@@ -1459,6 +1511,7 @@ pub async fn run_wait_result_job_by_path_get(
        false,
        None,
        !run_query.invisible_to_owner.unwrap_or(false),
+        tag,
    )
    .await?;
    tx.commit().await?;
@@ -1486,7 +1539,8 @@ pub async fn run_wait_result_job_by_path(
    check_queue_too_long(db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
    let script_path = script_path.to_path();
    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
-    let job_payload = script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;
+    let (job_payload, tag) =
+        script_path_to_payload(script_path, tx.transaction_mut(), &w_id).await?;

    let args = run_query.add_include_headers(headers, args.unwrap_or_default());

@@ -1506,6 +1560,7 @@ pub async fn run_wait_result_job_by_path(
        false,
        None,
        !run_query.invisible_to_owner.unwrap_or(false),
+        tag,
    )
    .await?;
    tx.commit().await?;
@@ -1534,7 +1589,7 @@ pub async fn run_wait_result_job_by_hash(

    let hash = script_hash.0;
    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.clone().begin(&authed).await?).into();
-    let path = get_path_for_hash(tx.transaction_mut(), &w_id, hash).await?;
+    let (path, tag) = get_path_and_tag_for_hash(tx.transaction_mut(), &w_id, hash).await?;

    let args = run_query.add_include_headers(headers, args.unwrap_or_default());

@@ -1554,6 +1609,7 @@ pub async fn run_wait_result_job_by_hash(
        false,
        None,
        !run_query.invisible_to_owner.unwrap_or(false),
+        tag,
    )
    .await?;
    tx.commit().await?;
@@ -1601,6 +1657,7 @@ pub async fn run_wait_result_flow_by_path(
        false,
        None,
        !run_query.invisible_to_owner.unwrap_or(false),
+        None,
    )
    .await?;
    tx.commit().await?;
@@ -1615,21 +1672,6 @@ pub async fn run_wait_result_flow_by_path(
    .await
 }

-// a similar function exists on the worker
-pub async fn script_path_to_payload<'c>(
-    script_path: &str,
-    db: &mut Transaction<'c, Postgres>,
-    w_id: &String,
-) -> std::result::Result<JobPayload, Error> {
-    let job_payload = if script_path.starts_with("hub/") {
-        JobPayload::ScriptHub { path: script_path.to_owned() }
-    } else {
-        let script_hash = windmill_common::get_latest_hash_for_path(db, w_id, script_path).await?;
-        JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() }
-    };
-    Ok(job_payload)
-}
-
 async fn run_preview_job(
    authed: Authed,
    Extension(user_db): Extension<UserDB>,
@@ -1664,6 +1706,7 @@ async fn run_preview_job(
        false,
        None,
        true,
+        preview.tag,
    )
    .await?;
    tx.commit().await?;
@@ -1700,6 +1743,7 @@ async fn run_preview_flow_job(
        false,
        None,
        true,
+        None,
    )
    .await?;
    tx.commit().await?;
@@ -1718,7 +1762,7 @@ pub async fn run_job_by_hash(
 ) -> error::Result<(StatusCode, String)> {
    let hash = script_hash.0;
    let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
-    let path = get_path_for_hash(tx.transaction_mut(), &w_id, hash).await?;
+    let (path, tag) = get_path_and_tag_for_hash(tx.transaction_mut(), &w_id, hash).await?;
    let scheduled_for = run_query.get_scheduled_for(tx.transaction_mut()).await?;
    let args = run_query.add_include_headers(headers, args.unwrap_or_default());

@@ -1738,6 +1782,7 @@ pub async fn run_job_by_hash(
        false,
        None,
        !run_query.invisible_to_owner.unwrap_or(false),
+        tag,
    )
    .await?;
    tx.commit().await?;
@@ -1828,6 +1873,9 @@ fn list_completed_jobs_query(
    if let Some(h) = &lq.script_hash {
        sqlb.and_where_eq("script_hash", "?".bind(h));
    }
+    if let Some(t) = &lq.tag {
+        sqlb.and_where_eq("tag", "?".bind(t));
+    }
    if let Some(cb) = &lq.created_by {
        sqlb.and_where_eq("created_by", "?".bind(cb));
    }
@@ -1885,6 +1933,7 @@ pub struct ListCompletedQuery {
    pub args: Option<String>,
    // filter by matching a subset of the result using base64 encoded json subset
    pub result: Option<String>,
+    pub tag: Option<String>,
 }

 async fn list_completed_jobs(
@@ -1930,6 +1979,7 @@ async fn list_completed_jobs(
            "email",
            "visible_to_owner",
            "mem_peak",
+            "tag",
        ],
    )
    .sql()?;
@@ -33,6 +33,7 @@ mod apps;
 mod audit;
 mod capture;
 mod db;
+mod drafts;
 mod favorite;
 mod flows;
 mod folders;
@@ -41,6 +42,7 @@ mod groups;
 mod inputs;
 pub mod jobs;
 mod oauth2;
+mod raw_apps;
 mod resources;
 mod schedule;
 mod scripts;
@@ -50,7 +52,7 @@ mod users;
 mod utils;
 mod variables;
 mod webhook_util;
-mod worker_ping;
+mod workers;
 mod workspaces;

 pub const GIT_VERSION: &str =
@@ -122,6 +124,7 @@ pub async fn run_server(
                        // Reordered alphabetically
                        .nest("/acls", granular_acls::workspaced_service())
                        .nest("/apps", apps::workspaced_service())
+                        .nest("/raw_apps", raw_apps::workspaced_service())
                        .nest("/audit", audit::workspaced_service())
                        .nest("/capture", capture::workspaced_service())
                        .nest("/favorites", favorite::workspaced_service())
@@ -134,6 +137,7 @@ pub async fn run_server(
                        .nest("/resources", resources::workspaced_service())
                        .nest("/schedules", schedule::workspaced_service())
                        .nest("/scripts", scripts::workspaced_service())
+                        .nest("/drafts", drafts::workspaced_service())
                        .nest(
                            "/users",
                            users::workspaced_service().layer(Extension(argon2.clone())),
@@ -146,7 +150,7 @@ pub async fn run_server(
                    "/users",
                    users::global_service().layer(Extension(argon2.clone())),
                )
-                .nest("/workers", worker_ping::global_service())
+                .nest("/workers", workers::global_service())
                .nest("/scripts", scripts::global_service())
                .nest("/flows", flows::global_service())
                .nest("/apps", apps::global_service().layer(cors.clone()))
@@ -29,6 +29,7 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize};
 use sqlx::{Postgres, Transaction};
 use tower_cookies::{Cookie, Cookies};
 use windmill_audit::{audit_log, ActionKind};
+use windmill_common::jobs::JobPayload;
 use windmill_common::users::username_to_permissioned_as;
 use windmill_common::utils::{not_found_if_none, now_from_db};

@@ -44,7 +45,7 @@ use crate::{BASE_URL, HTTP_CLIENT, IS_SECURE, OAUTH_CLIENTS, SLACK_SIGNING_SECRE
 use windmill_common::error::{self, to_anyhow, Error};
 use windmill_common::oauth2::*;

-use windmill_queue::{JobPayload, QueueTransaction};
+use windmill_queue::QueueTransaction;

 use std::{fs, str};

@@ -753,17 +754,20 @@ async fn slack_command(

    if let Some(settings) = settings {
        if let Some(path) = &settings.slack_command_script {
-            let payload = if let Some(path) = path.strip_prefix("flow/") {
-                JobPayload::Flow(path.to_string())
+            let (payload, tag) = if let Some(path) = path.strip_prefix("flow/") {
+                (JobPayload::Flow(path.to_string()), None)
            } else {
                let path = path.strip_prefix("script/").unwrap_or_else(|| path);
-                let script_hash = windmill_common::get_latest_hash_for_path(
+                let (script_hash, tag) = windmill_common::get_latest_deployed_hash_for_path(
                    tx.transaction_mut(),
                    &settings.workspace_id,
                    path,
                )
                .await?;
-                JobPayload::ScriptHash { hash: script_hash, path: path.to_owned() }
+                (
+                    JobPayload::ScriptHash { hash: script_hash, path: path.to_owned() },
+                    tag,
+                )
            };
            let mut map = serde_json::Map::new();
            map.insert("text".to_string(), serde_json::Value::String(form.text));
@@ -788,6 +792,7 @@ async fn slack_command(
                false,
                None,
                true,
+                tag,
            )
            .await?;
            let url = BASE_URL.to_owned();
@@ -0,0 +1,333 @@
+/*
+ * Author: Ruben Fiszel
+ * Copyright: Windmill Labs, Inc 2022
+ * This file and its contents are licensed under the AGPLv3 License.
+ * Please see the included NOTICE for copyright information and
+ * LICENSE-AGPL for a copy of the license.
+ */
+use crate::{
+    db::{UserDB, DB},
+    users::{require_owner_of_path, Authed},
+    webhook_util::{WebhookMessage, WebhookShared},
+};
+use axum::{
+    body,
+    extract::{Extension, Json, Path, Query},
+    response::Response,
+    routing::{delete, get, post},
+    Router,
+};
+use bytes::Bytes;
+use hyper::{header, StatusCode};
+use serde::{Deserialize, Serialize};
+use sql_builder::{bind::Bind, SqlBuilder};
+use sqlx::FromRow;
+use std::str;
+use windmill_audit::{audit_log, ActionKind};
+use windmill_common::{
+    apps::ListAppQuery,
+    error::{Error, JsonResult, Result},
+    utils::{not_found_if_none, paginate, Pagination, StripPath},
+};
+
+pub fn workspaced_service() -> Router {
+    Router::new()
+        .route("/list", get(list_apps))
+        .route("/get_data/:version/*path", get(get_data))
+        .route("/exists/*path", get(exists_app))
+        .route("/update/*path", post(update_app))
+        .route("/delete/*path", delete(delete_app))
+        .route("/create", post(create_app))
+}
+
+#[derive(FromRow, Deserialize, Serialize)]
+pub struct ListableApp {
+    pub path: String,
+    pub workspace_id: String,
+    pub summary: String,
+    pub edited_at: chrono::DateTime<chrono::Utc>,
+    pub extra_perms: serde_json::Value,
+    pub starred: bool,
+    pub version: i32,
+}
+
+#[derive(Deserialize)]
+pub struct CreateApp {
+    pub path: String,
+    pub summary: String,
+    pub value: String,
+}
+
+#[derive(Deserialize)]
+pub struct EditApp {
+    pub path: Option<String>,
+    pub summary: Option<String>,
+    pub value: Option<String>,
+}
+
+async fn list_apps(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path(w_id): Path<String>,
+    Query(pagination): Query<Pagination>,
+    Query(lq): Query<ListAppQuery>,
+) -> JsonResult<Vec<ListableApp>> {
+    let (per_page, offset) = paginate(pagination);
+
+    let mut sqlb = SqlBuilder::select_from("raw_app as app")
+        .fields(&[
+            "app.workspace_id",
+            "app.path",
+            "app.edited_at",
+            "app.summary",
+            "app.extra_perms",
+            "app.version",
+            "favorite.path IS NOT NULL as starred",
+        ])
+        .left()
+        .join("favorite")
+        .on(
+            "favorite.favorite_kind = 'raw_app' AND favorite.workspace_id = app.workspace_id AND favorite.path = app.path AND favorite.usr = ?"
+                .bind(&authed.username),
+        )
+        .order_desc("favorite.path IS NOT NULL")
+        .order_by("app.edited_at", true)
+        .and_where("app.workspace_id = ?".bind(&w_id))
+        .offset(offset)
+        .limit(per_page)
+        .clone();
+
+    if lq.starred_only.unwrap_or(false) {
+        sqlb.and_where_is_not_null("favorite.path");
+    }
+
+    let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
+    let mut tx = user_db.begin(&authed).await?;
+    let rows = sqlx::query_as::<_, ListableApp>(&sql)
+        .fetch_all(&mut tx)
+        .await?;
+
+    tx.commit().await?;
+
+    Ok(Json(rows))
+}
+
+async fn get_data(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, _version, path)): Path<(String, u16, StripPath)>,
+) -> Result<Response> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    let app_o = sqlx::query_scalar!(
+        "SELECT data FROM raw_app
+        WHERE path = $1 AND workspace_id = $2",
+        path.to_owned(),
+        &w_id
+    )
+    .fetch_optional(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    let app = not_found_if_none(app_o, "App", path)?;
+    let res = Response::builder().header(header::CONTENT_TYPE, "text/javascript");
+
+    Ok(res
+        .body(body::boxed(body::Full::from(Bytes::from(app))))
+        .unwrap())
+}
+
+async fn create_app(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(webhook): Extension<WebhookShared>,
+    Path(w_id): Path<String>,
+    Json(app): Json<CreateApp>,
+) -> Result<(StatusCode, String)> {
+    let mut tx = user_db.begin(&authed).await?;
+    if &app.path == "" {
+        return Err(Error::BadRequest("App path cannot be empty".to_string()));
+    }
+
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)",
+        app.path,
+        w_id
+    )
+    .fetch_one(&mut tx)
+    .await?
+    .unwrap_or(false);
+
+    if exists {
+        return Err(Error::BadRequest(format!(
+            "App with path {} already exists",
+            &app.path
+        )));
+    }
+
+    sqlx::query!(
+        "INSERT INTO raw_app
+            (workspace_id, path, summary, extra_perms, data)
+            VALUES ($1, $2, $3, '{}', $4)",
+        w_id,
+        app.path,
+        app.summary,
+        app.value,
+    )
+    .execute(&mut tx)
+    .await?;
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "apps.create",
+        ActionKind::Create,
+        &w_id,
+        Some(&app.path),
+        None,
+    )
+    .await?;
+
+    tx.commit().await?;
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::CreateApp { workspace: w_id, path: app.path.clone() },
+    );
+
+    Ok((StatusCode::CREATED, app.path))
+}
+
+async fn delete_app(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(webhook): Extension<WebhookShared>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> Result<String> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    sqlx::query!(
+        "DELETE FROM raw_app WHERE path = $1 AND workspace_id = $2",
+        path,
+        w_id
+    )
+    .execute(&mut tx)
+    .await?;
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "apps.delete",
+        ActionKind::Delete,
+        &w_id,
+        Some(path),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+    webhook.send_message(
+        w_id.clone().clone(),
+        WebhookMessage::DeleteApp { workspace: w_id, path: path.to_owned() },
+    );
+
+    Ok(format!("app {} deleted", path))
+}
+
+async fn update_app(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Extension(webhook): Extension<WebhookShared>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+    Json(app): Json<EditApp>,
+) -> Result<String> {
+    use sql_builder::prelude::*;
+
+    let path = path.to_path();
+
+    let mut tx = user_db.begin(&authed).await?;
+    let npath = &app.path;
+    let npath = if npath.is_some() || app.summary.is_some() {
+        let mut sqlb = SqlBuilder::update_table("raw_app");
+        sqlb.and_where_eq("path", "?".bind(&path));
+        sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
+
+        if let Some(npath) = &app.path {
+            if npath != path {
+                require_owner_of_path(&authed, path)?;
+
+                let exists = sqlx::query_scalar!(
+                    "SELECT EXISTS(SELECT 1 FROM app WHERE path = $1 AND workspace_id = $2)",
+                    npath,
+                    w_id
+                )
+                .fetch_one(&mut tx)
+                .await?
+                .unwrap_or(false);
+
+                if exists {
+                    return Err(Error::BadRequest(format!(
+                        "App with path {} already exists",
+                        npath
+                    )));
+                }
+            }
+            sqlb.set_str("path", npath);
+        }
+
+        if let Some(nsummary) = &app.summary {
+            sqlb.set_str("summary", nsummary);
+        }
+
+        if let Some(value) = &app.value {
+            sqlb.set_str("data", value);
+            sqlb.set("version", "version + 1");
+        }
+
+        sqlb.returning("path");
+
+        let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
+        let npath_o: Option<String> = sqlx::query_scalar(&sql).fetch_optional(&mut tx).await?;
+        not_found_if_none(npath_o, "Raw App", path)?
+    } else {
+        "".to_string()
+    };
+
+    audit_log(
+        &mut tx,
+        &authed.username,
+        "apps.update",
+        ActionKind::Update,
+        &w_id,
+        Some(&npath),
+        None,
+    )
+    .await?;
+    tx.commit().await?;
+    webhook.send_message(
+        w_id.clone(),
+        WebhookMessage::UpdateApp {
+            workspace: w_id,
+            old_path: path.to_owned(),
+            new_path: npath.clone(),
+        },
+    );
+
+    Ok(format!("app {} updated (npath: {:?})", path, npath))
+}
+
+async fn exists_app(
+    Extension(db): Extension<DB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<bool> {
+    let path = path.to_path();
+    let exists = sqlx::query_scalar!(
+        "SELECT EXISTS(SELECT 1 FROM raw_app WHERE path = $1 AND workspace_id = $2)",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    Ok(Json(exists))
+}
@@ -386,9 +386,8 @@ async fn update_resource(
        if npath != path {
            check_path_conflict(&mut tx, &w_id, &npath).await?;

-            if !authed.is_admin {
-                require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
-            }
+            require_owner_of_path(&authed, path)?;
+
            sqlx::query!(
                "UPDATE variable SET path = $1 WHERE path = $2 AND workspace_id = $3",
                npath,
@@ -23,9 +23,9 @@ use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
    error::{Error, JsonResult, Result},
    schedule::Schedule,
-    utils::{not_found_if_none, paginate, Pagination, StripPath},
+    utils::{not_found_if_none, paginate, Pagination, StripPath}, jobs::JobKind,
 };
-use windmill_queue::{self, schedule::push_scheduled_job, JobKind, QueueTransaction};
+use windmill_queue::{self, schedule::push_scheduled_job, QueueTransaction};

 pub fn workspaced_service() -> Router {
    Router::new()
@@ -32,9 +32,10 @@ use std::{
 use windmill_audit::{audit_log, ActionKind};
 use windmill_common::{
    error::{Error, JsonResult, Result},
+    jobs::JobPayload,
    schedule::Schedule,
    scripts::{
-        to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Script, ScriptHash,
+        to_i64, HubScript, ListScriptQuery, ListableScript, NewScript, Schema, Script, ScriptHash,
        ScriptKind, ScriptLang,
    },
    users::username_to_permissioned_as,
@@ -45,8 +46,32 @@ use windmill_common::{
 use windmill_parser::MainArgSignature;
 use windmill_queue::{self, schedule::push_scheduled_job, QueueTransaction};

+lazy_static::lazy_static! {
+    pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
+        .ok()
+        .map(|x| x.split(',').map(|x| x.to_string()).filter(|x| !x.is_empty()).collect::<Vec<_>>()).unwrap_or_default();
+
+}
+
 const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;

+#[derive(Serialize, sqlx::FromRow)]
+pub struct ScriptWDraft {
+    pub hash: ScriptHash,
+    pub path: String,
+    pub summary: String,
+    pub description: String,
+    pub content: String,
+    pub language: ScriptLang,
+    pub kind: ScriptKind,
+    pub tag: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft: Option<serde_json::Value>,
+    pub schema: Option<Schema>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
+}
+
 pub fn global_service() -> Router {
    Router::new()
        .route(
@@ -73,6 +98,7 @@ pub fn workspaced_service() -> Router {
        .route("/list", get(list_scripts))
        .route("/create", post(create_script))
        .route("/archive/p/*path", post(archive_script_by_path))
+        .route("/get/draft/*path", get(get_script_by_path_w_draft))
        .route("/get/p/*path", get(get_script_by_path))
        .route("/raw/p/*path", get(raw_script_by_path))
        .route("/exists/p/*path", get(exists_script_by_path))
@@ -96,22 +122,18 @@ async fn list_scripts(

    let mut sqlb = SqlBuilder::select_from("script as o")
        .fields(&[
-            "o.workspace_id",
            "hash",
            "o.path",
-            "array_remove(array[parent_hashes[1]], NULL) as parent_hashes",
            "summary",
-            "description",
-            "created_by",
-            "created_at",
+            "COALESCE(draft.created_at, o.created_at) as created_at",
            "archived",
-            "deleted",
-            "is_template",
            "extra_perms",
-            "CASE WHEN lock_error_logs IS NOT NULL THEN 'error' ELSE null END as lock_error_logs",
+            "CASE WHEN lock_error_logs IS NOT NULL THEN true ELSE false END as has_deploy_errors",
            "language",
-            "kind",
            "favorite.path IS NOT NULL as starred",
+            "tag",
+            "draft.path IS NOT NULL as has_draft",
+            "draft_only"
        ])
        .left()
        .join("favorite")
@@ -119,6 +141,11 @@ async fn list_scripts(
            "favorite.favorite_kind = 'script' AND favorite.workspace_id = o.workspace_id AND favorite.path = o.path AND favorite.usr = ?"
                .bind(&authed.username),
        )
+        .left()
+        .join("draft")
+        .on(
+            "draft.path = o.path AND draft.workspace_id = o.workspace_id AND draft.typ = 'script'"
+        )
        .order_desc("favorite.path IS NOT NULL")
        .order_by("created_at", lq.order_desc.unwrap_or(true))
        .and_where("o.workspace_id = ?".bind(&w_id))
@@ -128,7 +155,7 @@ async fn list_scripts(

    if lq.show_archived.unwrap_or(false) {
        sqlb.and_where_eq(
-            "created_at",
+            "o.created_at",
            "(select max(created_at) from script where o.path = path 
            AND workspace_id = ?)"
                .bind(&w_id),
@@ -234,10 +261,20 @@ async fn create_script(
    }
    let parent_hashes_and_perms: Option<ParentInfo> = match (&ns.parent_hash, clashing_script) {
        (None, None) => Ok(None),
-        (None, Some(s)) => Err(Error::BadRequest(format!(
+        (None, Some(s)) if !s.draft_only.unwrap_or(false) => Err(Error::BadRequest(format!(
            "Path conflict for {} with non-archived hash {}",
            &ns.path, &s.hash
        ))),
+        (None, Some(s)) => {
+            sqlx::query!(
+                "DELETE FROM script WHERE hash = $1 AND workspace_id = $2",
+                s.hash.0,
+                &w_id
+            )
+            .execute(&mut tx)
+            .await?;
+            Ok(None)
+        }
        (Some(p_hash), o) => {
            if sqlx::query_scalar!(
                "SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
@@ -272,10 +309,7 @@ async fn create_script(
            let ps = get_script_by_hash_internal(tx.transaction_mut(), &w_id, p_hash).await?;

            if ps.path != ns.path {
-                if !authed.is_admin {
-                    require_owner_of_path(&w_id, &authed.username, &authed.groups, &ps.path, &db)
-                        .await?;
-                }
+                require_owner_of_path(&authed, &ps.path)?;
            }

            let ph = {
@@ -332,8 +366,8 @@ async fn create_script(
    //::text::json is to ensure we use serde_json with preserve order
    sqlx::query!(
        "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \
-         content, created_by, schema, is_template, extra_perms, lock, language, kind) \
-         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14)",
+         content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only) \
+         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16)",
        &w_id,
        &hash.0,
        ns.path,
@@ -348,11 +382,21 @@ async fn create_script(
        lock,
        ns.language: ScriptLang,
        ns.kind.unwrap_or(ScriptKind::Script): ScriptKind,
+        ns.tag,
+        ns.draft_only
    )
    .execute(&mut tx)
    .await?;

    if let Some(p_path) = parent_hashes_and_perms.as_ref().map(|x| x.p_path.clone()) {
+        sqlx::query!(
+            "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'",
+            p_path,
+            &w_id
+        )
+        .execute(&mut tx)
+        .await?;
+
        let schedulables = sqlx::query_as!(
        Schedule,
            "UPDATE schedule SET script_path = $1 WHERE script_path = $2 AND workspace_id = $3 AND is_flow IS false RETURNING *",
@@ -370,6 +414,14 @@ async fn create_script(
                tx = push_scheduled_job(tx, schedule).await?;
            }
        }
+    } else {
+        sqlx::query!(
+            "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'script'",
+            ns.path,
+            &w_id
+        )
+        .execute(&mut tx)
+        .await?;
    }

    if p_hashes.is_some() && !p_hashes.unwrap().is_empty() {
@@ -428,7 +480,7 @@ async fn create_script(
        let (_, new_tx) = windmill_queue::push(
            tx,
            &w_id,
-            windmill_queue::JobPayload::Dependencies { hash, dependencies, language: ns.language },
+            JobPayload::Dependencies { hash, dependencies, language: ns.language },
            serde_json::Map::new(),
            &authed.username,
            &authed.email,
@@ -441,6 +493,7 @@ async fn create_script(
            false,
            None,
            true,
+            None,
        )
        .await?;
        tx = new_tx;
@@ -487,6 +540,31 @@ async fn get_script_by_path(
    Ok(Json(script))
 }

+async fn get_script_by_path_w_draft(
+    authed: Authed,
+    Extension(user_db): Extension<UserDB>,
+    Path((w_id, path)): Path<(String, StripPath)>,
+) -> JsonResult<ScriptWDraft> {
+    let path = path.to_path();
+    let mut tx = user_db.begin(&authed).await?;
+
+    let script_o = sqlx::query_as::<_, ScriptWDraft>(
+        "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, draft.value as draft FROM script LEFT JOIN draft ON 
+         script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
+         WHERE script.path = $1 AND script.workspace_id = $2 \
+         AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \
+         workspace_id = $2)",
+    )
+    .bind(path)
+    .bind(w_id)
+    .fetch_optional(&mut tx)
+    .await?;
+    tx.commit().await?;
+
+    let script = not_found_if_none(script_o, "Script", path)?;
+    Ok(Json(script))
+}
+
 async fn list_paths(
    authed: Authed,
    Extension(user_db): Extension<UserDB>,
@@ -636,6 +714,8 @@ async fn archive_script_by_path(
    let path = path.to_path();
    let mut tx = user_db.begin(&authed).await?;

+    require_owner_of_path(&authed, path)?;
+
    let hash: i64 = sqlx::query_scalar!(
        "UPDATE script SET archived = true WHERE path = $1 AND workspace_id = $2 RETURNING hash",
        path,
@@ -749,7 +829,19 @@ async fn delete_script_by_path(
    let mut tx = user_db.begin(&authed).await?;
    let path = path.to_path();

-    require_admin(authed.is_admin, &authed.username)?;
+    let draft_only = sqlx::query_scalar!(
+        "SELECT draft_only FROM script WHERE path = $1 AND workspace_id = $2",
+        path,
+        w_id
+    )
+    .fetch_one(&db)
+    .await?
+    .unwrap_or(false);
+
+    if !draft_only {
+        require_admin(authed.is_admin, &authed.username)?;
+    }
+
    let script = sqlx::query_scalar!(
        "DELETE FROM script WHERE path = $1 AND workspace_id = $2 RETURNING path",
        path,
@@ -359,7 +359,8 @@ pub struct Authed {
    pub username: String,
    pub is_admin: bool,
    pub groups: Vec<String>,
-    pub folders: Vec<(String, bool)>,
+    // (folder name, can write, is owner)
+    pub folders: Vec<(String, bool, bool)>,
 }

 pub async fn maybe_refresh_folders(path: &str, w_id: &str, authed: Authed, db: &DB) -> Authed {
@@ -369,7 +370,7 @@ pub async fn maybe_refresh_folders(path: &str, w_id: &str, authed: Authed, db: &
    let splitted = path.split('/').collect::<Vec<_>>();
    if splitted.len() >= 2
        && splitted[0] == "f"
-        && !authed.folders.iter().any(|(f, _)| f == splitted[1])
+        && !authed.folders.iter().any(|(f, _, _)| f == splitted[1])
    {
        let name = &authed.username;
        let groups = get_groups_for_user(w_id, name, db)
@@ -509,6 +510,7 @@ pub struct UserInfo {
    pub disabled: bool,
    pub role: Option<String>,
    pub folders: Vec<String>,
+    pub folders_owners: Vec<String>,
 }

 #[derive(FromRow, Serialize)]
@@ -808,9 +810,14 @@ async fn whoami(
            disabled: false,
            role: Some("superadmin".to_string()),
            folders: folders
+                .clone()
                .into_iter()
                .filter_map(|x| if x.1 { Some(x.0) } else { None })
                .collect(),
+            folders_owners: folders
+                .into_iter()
+                .filter_map(|x| if x.2 { Some(x.0) } else { None })
+                .collect(),
        }))
    }
 }
@@ -896,9 +903,14 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo
        disabled: usr.disabled,
        role: usr.role,
        folders: folders
+            .clone()
            .into_iter()
            .filter_map(|x| if x.1 { Some(x.0) } else { None })
            .collect(),
+        folders_owners: folders
+            .into_iter()
+            .filter_map(|x| if x.2 { Some(x.0) } else { None })
+            .collect(),
    }))
 }

@@ -913,47 +925,45 @@ pub async fn get_groups_for_user(w_id: &str, username: &str, db: &DB) -> Result<
    Ok(groups)
 }
 pub async fn is_owner_of_path(
-    Authed { username, is_admin, groups, .. }: Authed,
-    Extension(db): Extension<DB>,
-    Path((w_id, path)): Path<(String, StripPath)>,
+    authed: Authed,
+    Path((_w_id, path)): Path<(String, StripPath)>,
 ) -> JsonResult<bool> {
    let path = path.to_path();
-    if is_admin {
+    if authed.is_admin {
        Ok(Json(true))
    } else {
-        Ok(Json(
-            require_owner_of_path(&w_id, &username, &groups, path, &db)
-                .await
-                .is_ok(),
-        ))
+        Ok(Json(require_owner_of_path(&authed, path).is_ok()))
    }
 }

-pub async fn require_owner_of_path(
-    w_id: &str,
-    username: &str,
-    groups: &Vec<String>,
-    path: &str,
-    db: &DB,
-) -> Result<()> {
+pub fn require_owner_of_path(authed: &Authed, path: &str) -> Result<()> {
+    if authed.is_admin {
+        return Ok(());
+    }
    if !path.is_empty() {
        let splitted = path.split("/").collect::<Vec<&str>>();
        if splitted[0] == "u" {
-            if splitted[1] == username {
-                return Ok(());
+            if splitted[1] == authed.username {
+                Ok(())
            } else {
-                return Err(Error::BadRequest(format!(
+                Err(Error::BadRequest(format!(
                    "only the owner {} is authorized to perform this operation",
                    splitted[1]
-                )));
+                )))
            }
-        } else if splitted[0] == "g" {
-            return crate::groups::require_is_owner(splitted[1], username, groups, w_id, db).await;
        } else if splitted[0] == "f" {
-            return crate::folders::require_is_owner(splitted[1], username, groups, w_id, db).await;
+            crate::folders::require_is_owner(authed, splitted[1])
+        } else {
+            Err(Error::BadRequest(format!(
+                "Not recognized path kind: {}",
+                path
+            )))
        }
+    } else {
+        Err(Error::BadRequest(format!(
+            "Cannot be owner of an empty path"
+        )))
    }
-    Err(Error::BadRequest(format!("not recognized owner kind")))
 }

 async fn whois(
@@ -1034,6 +1044,11 @@ async fn decline_invite(

 lazy_static! {
    pub static ref VALID_USERNAME: Regex = Regex::new(r#"^[a-zA-Z_0-9]+$"#).unwrap();
+
+    pub static ref JOB_RETENTION_SECS: u32 = std::env::var("JOB_RETENTION_SECS")
+    .ok()
+    .and_then(|x| x.parse::<u32>().ok())
+    .unwrap_or(60 * 60 * 24 * 60); // 60 days
 }

 async fn accept_invite(
@@ -1903,16 +1918,24 @@ pub async fn delete_expired_items_perdiodically(
            Err(e) => tracing::error!("Error deleting pip_resolution: {}", e.to_string()),
        }

-        let magic_links_deleted_r: std::result::Result<Vec<String>, _> = sqlx::query_scalar(
-            "DELETE FROM magic_link WHERE expiration <= now()
-        RETURNING concat(substring(token for 10), '*****')",
-        )
-        .fetch_all(db)
-        .await;
+        if *JOB_RETENTION_SECS > 0 {
+            let deleted_jobs = sqlx::query_scalar!(
+                "DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id",
+                *JOB_RETENTION_SECS as i64
+            )
+            .fetch_all(db)
+            .await;

-        match magic_links_deleted_r {
-            Ok(tokens) => tracing::debug!("deleted {} tokens: {:?}", tokens.len(), tokens),
-            Err(e) => tracing::error!("Error deleting token: {}", e.to_string()),
+            match deleted_jobs {
+                Ok(deleted_jobs) => {
+                    tracing::info!(
+                        "deleted {} jobs completed JOB_RETENTION_SECS ago: {:?}",
+                        deleted_jobs.len(),
+                        deleted_jobs
+                    )
+                }
+                Err(e) => tracing::error!("Error deleting jobs: {}", e.to_string()),
+            }
        }

        tokio::select! {
@@ -12,13 +12,6 @@ use crate::{
    users::{maybe_refresh_folders, require_owner_of_path, Authed},
    webhook_util::{WebhookMessage, WebhookShared},
 };
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */

 use axum::{
    extract::{Extension, Path, Query},
@@ -380,9 +373,8 @@ async fn update_variable(
    if let Some(npath) = ns.path {
        if npath != path {
            check_path_conflict(&mut tx, &w_id, &npath).await?;
-            if !authed.is_admin {
-                require_owner_of_path(&w_id, &authed.username, &authed.groups, &path, &db).await?;
-            }
+            require_owner_of_path(&authed, path)?;
+
            let mut v = sqlx::query_scalar!(
                "SELECT value FROM resource  WHERE path = $1 AND workspace_id = $2",
                path,
@@ -21,7 +21,16 @@ use windmill_common::{
 };

 pub fn global_service() -> Router {
-    Router::new().route("/list", get(list_worker_pings))
+    Router::new()
+        .route("/list", get(list_worker_pings))
+        .route("/custom_tags", get(get_custom_tags))
+}
+
+lazy_static::lazy_static! {
+    pub static ref CUSTOM_TAGS: Vec<String> = std::env::var("CUSTOM_TAGS")
+        .ok()
+        .map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<_>>()).unwrap_or_default();
+
 }

 #[derive(FromRow, Serialize, Deserialize)]
@@ -54,3 +63,7 @@ async fn list_worker_pings(
    tx.commit().await?;
    Ok(Json(rows))
 }
+
+async fn get_custom_tags() -> Json<Vec<String>> {
+    Json(CUSTOM_TAGS.clone())
+}
@@ -1165,6 +1165,8 @@ where
                "edited_at",
                "edited_by",
                "archived",
+                "has_draft",
+                "draft_only",
            ] {
                if obj.contains_key(key) {
                    obj.remove(key);
@@ -11,10 +11,16 @@
 //! For users writing scripts to access their infrastructure with firewalls requiring incoming
 //! connections to be from whitelisted IP addresses.

+use std::time::Duration;
+
 use reqwest::Result;

 pub async fn get_ip() -> Result<String> {
-    reqwest::get("https://hub.windmill.dev/getip")
+    reqwest::ClientBuilder::new()
+        .timeout(Duration::from_secs(3))
+        .build()?
+        .get("https://hub.windmill.dev/getip")
+        .send()
        .await?
        .error_for_status()?
        .text()
@@ -33,6 +33,8 @@ pub struct Flow {
    pub archived: bool,
    pub schema: Option<Schema>,
    pub extra_perms: serde_json::Value,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
 }

 #[derive(Serialize)]
@@ -47,6 +49,9 @@ pub struct ListableFlow {
    pub archived: bool,
    pub extra_perms: serde_json::Value,
    pub starred: bool,
+    pub has_draft: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
 }

 #[derive(Deserialize)]
@@ -57,6 +62,7 @@ pub struct NewFlow {
    pub description: String,
    pub value: serde_json::Value,
    pub schema: Option<Schema>,
+    pub draft_only: Option<bool>,
 }

 #[derive(Deserialize, Serialize, Debug, Clone, Default)]
@@ -252,6 +258,8 @@ pub enum FlowModuleValue {
        lock: Option<String>,
        #[serde(skip_serializing_if = "Option::is_none")]
        path: Option<String>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        tag: Option<String>,
        language: ScriptLang,
    },
    Identity,
@@ -1 +1,201 @@
+use serde::{Deserialize, Serialize};
+use sqlx::{Postgres, Transaction};
+use uuid::Uuid;

+use crate::{
+    error,
+    flow_status::FlowStatus,
+    flows::FlowValue,
+    get_latest_deployed_hash_for_path,
+    scripts::{ScriptHash, ScriptLang},
+};
+
+#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
+#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")]
+#[serde(rename_all(serialize = "lowercase"))]
+pub enum JobKind {
+    Script,
+    #[allow(non_camel_case_types)]
+    Script_Hub,
+    Preview,
+    Dependencies,
+    Flow,
+    FlowPreview,
+    Identity,
+    FlowDependencies,
+}
+
+#[derive(Debug, sqlx::FromRow, Serialize, Clone)]
+pub struct QueuedJob {
+    pub workspace_id: String,
+    pub id: Uuid,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub parent_job: Option<Uuid>,
+    pub created_by: String,
+    pub created_at: chrono::DateTime<chrono::Utc>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub started_at: Option<chrono::DateTime<chrono::Utc>>,
+    pub scheduled_for: chrono::DateTime<chrono::Utc>,
+    pub running: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub script_hash: Option<ScriptHash>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub script_path: Option<String>,
+    pub args: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub logs: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub raw_code: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub raw_lock: Option<String>,
+    pub canceled: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub canceled_by: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub canceled_reason: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub last_ping: Option<chrono::DateTime<chrono::Utc>>,
+    pub job_kind: JobKind,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub schedule_path: Option<String>,
+    pub permissioned_as: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub flow_status: Option<serde_json::Value>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub raw_flow: Option<serde_json::Value>,
+    pub is_flow_step: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub language: Option<ScriptLang>,
+    pub same_worker: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub pre_run_error: Option<String>,
+    pub email: String,
+    pub visible_to_owner: bool,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub suspend: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub mem_peak: Option<i32>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub root_job: Option<Uuid>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub leaf_jobs: Option<serde_json::Value>,
+    pub tag: String,
+}
+
+impl QueuedJob {
+    pub fn script_path(&self) -> &str {
+        self.script_path
+            .as_ref()
+            .map(String::as_str)
+            .unwrap_or("tmp/main")
+    }
+}
+
+impl QueuedJob {
+    pub fn parse_raw_flow(&self) -> Option<FlowValue> {
+        self.raw_flow
+            .as_ref()
+            .and_then(|v| serde_json::from_value::<FlowValue>(v.clone()).ok())
+    }
+
+    pub fn parse_flow_status(&self) -> Option<FlowStatus> {
+        self.flow_status
+            .as_ref()
+            .and_then(|v| serde_json::from_value::<FlowStatus>(v.clone()).ok())
+    }
+}
+
+impl Default for QueuedJob {
+    fn default() -> Self {
+        Self {
+            workspace_id: "".to_string(),
+            id: Uuid::default(),
+            parent_job: None,
+            created_by: "".to_string(),
+            created_at: chrono::Utc::now(),
+            started_at: None,
+            scheduled_for: chrono::Utc::now(),
+            running: false,
+            script_hash: None,
+            script_path: None,
+            args: None,
+            logs: None,
+            raw_code: None,
+            raw_lock: None,
+            canceled: false,
+            canceled_by: None,
+            canceled_reason: None,
+            last_ping: None,
+            job_kind: JobKind::Identity,
+            schedule_path: None,
+            permissioned_as: "".to_string(),
+            flow_status: None,
+            raw_flow: None,
+            is_flow_step: false,
+            language: None,
+            same_worker: false,
+            pre_run_error: None,
+            email: "".to_string(),
+            visible_to_owner: false,
+            suspend: None,
+            mem_peak: None,
+            root_job: None,
+            leaf_jobs: None,
+            tag: "deno".to_string(),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum JobPayload {
+    ScriptHub { path: String },
+    ScriptHash { hash: ScriptHash, path: String },
+    Code(RawCode),
+    Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang },
+    FlowDependencies { path: String },
+    Flow(String),
+    RawFlow { value: FlowValue, path: Option<String> },
+    Identity,
+}
+
+#[derive(Clone, Serialize, Deserialize, Debug)]
+pub struct RawCode {
+    pub content: String,
+    pub path: Option<String>,
+    pub language: ScriptLang,
+    pub lock: Option<String>,
+}
+
+type Tag = String;
+
+pub async fn script_path_to_payload<'c>(
+    script_path: &str,
+    db: &mut Transaction<'c, Postgres>,
+    w_id: &String,
+) -> error::Result<(JobPayload, Option<Tag>)> {
+    let (job_payload, tag) = if script_path.starts_with("hub/") {
+        (JobPayload::ScriptHub { path: script_path.to_owned() }, None)
+    } else {
+        let (script_hash, tag) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?;
+        (
+            JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() },
+            tag,
+        )
+    };
+    Ok((job_payload, tag))
+}
+
+pub async fn script_hash_to_tag<'c>(
+    script_hash: &ScriptHash,
+    db: &mut Transaction<'c, Postgres>,
+    w_id: &String,
+) -> error::Result<Option<Tag>> {
+    Ok(sqlx::query_scalar!(
+        "select tag from script where hash = $1 AND workspace_id = $2",
+        script_hash.0,
+        w_id
+    )
+    .fetch_optional(db)
+    .await?
+    .flatten())
+}
@@ -6,7 +6,7 @@
 * LICENSE-AGPL for a copy of the license.
 */

-use std::{net::SocketAddr, sync::Arc};
+use std::net::SocketAddr;

 use error::Error;

@@ -15,6 +15,7 @@ pub mod error;
 pub mod external_ip;
 pub mod flow_status;
 pub mod flows;
+pub mod jobs;
 pub mod more_serde;
 pub mod oauth2;
 pub mod schedule;
@@ -42,11 +43,14 @@ lazy_static::lazy_static! {
    .flatten();
    pub static ref METRICS_ENABLED: bool = METRICS_ADDR.is_some();
    pub static ref BASE_URL: String = std::env::var("BASE_URL").unwrap_or_else(|_| "http://localhost".to_string());
-    pub static ref IS_READY: Arc<std::sync::atomic::AtomicBool> = Arc::new(std::sync::atomic::AtomicBool::new(false));
+    pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
 }

 #[cfg(feature = "tokio")]
-pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::Result<()> {
+pub async fn shutdown_signal(
+    tx: tokio::sync::broadcast::Sender<()>,
+    mut rx: tokio::sync::broadcast::Receiver<()>,
+) -> anyhow::Result<()> {
    use std::io;
    use tokio::signal::unix::SignalKind;

@@ -60,6 +64,9 @@ pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::
    tokio::select! {
        _ = terminate() => {},
        _ = tokio::signal::ctrl_c() => {},
+        _ = rx.recv() => {
+            tracing::info!("shutdown monitor received killpill");
+        },
    }
    println!("signal received, starting graceful shutdown");
    let _ = tx.send(());
@@ -146,23 +153,44 @@ pub async fn connect(
        .map_err(|err| Error::ConnectingToDatabase(err.to_string()))
 }

-// TODO: Move this elsewhere
-pub async fn get_latest_hash_for_path<'c>(
+type Tag = String;
+
+pub async fn get_latest_deployed_hash_for_path<'c>(
    db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
    w_id: &str,
    script_path: &str,
-) -> error::Result<scripts::ScriptHash> {
-    let script_hash_o = sqlx::query_scalar!(
-        "select hash from script where path = $1 AND workspace_id = $2 AND
-    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2) AND
-    deleted = false",
+) -> error::Result<(scripts::ScriptHash, Option<Tag>)> {
+    let r_o = sqlx::query!(
+        "select hash, tag from script where path = $1 AND workspace_id = $2 AND
+    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
+    deleted = false AND archived = false AND lock IS not NULL AND lock_error_logs IS NULL)",
        script_path,
        w_id
    )
    .fetch_optional(db)
    .await?;

-    let script_hash = utils::not_found_if_none(script_hash_o, "ScriptHash", script_path)?;
+    let script = utils::not_found_if_none(r_o, "script", script_path)?;

-    Ok(scripts::ScriptHash(script_hash))
+    Ok((scripts::ScriptHash(script.hash), script.tag))
+}
+
+pub async fn get_latest_hash_for_path<'c>(
+    db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
+    w_id: &str,
+    script_path: &str,
+) -> error::Result<(scripts::ScriptHash, Option<Tag>)> {
+    let r_o = sqlx::query!(
+        "select hash, tag from script where path = $1 AND workspace_id = $2 AND
+    created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
+    deleted = false AND archived = false)",
+        script_path,
+        w_id
+    )
+    .fetch_optional(db)
+    .await?;
+
+    let script = utils::not_found_if_none(r_o, "script", script_path)?;
+
+    Ok((scripts::ScriptHash(script.hash), script.tag))
 }
@@ -136,27 +136,28 @@ pub struct Script {
    pub lock_error_logs: Option<String>,
    pub language: ScriptLang,
    pub kind: ScriptKind,
+    pub tag: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
 }

 #[derive(Serialize)]
 #[cfg_attr(feature = "sqlx", derive(sqlx::FromRow))]
 pub struct ListableScript {
-    pub workspace_id: String,
    pub hash: ScriptHash,
    pub path: String,
-    pub parent_hashes: Option<ScriptHashes>,
    pub summary: String,
-    pub description: String,
-    pub created_by: String,
    pub created_at: chrono::DateTime<chrono::Utc>,
    pub archived: bool,
-    pub deleted: bool,
-    pub is_template: bool,
    pub extra_perms: serde_json::Value,
-    pub lock_error_logs: Option<String>,
    pub language: ScriptLang,
-    pub kind: ScriptKind,
    pub starred: bool,
+    pub tag: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub has_draft: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub draft_only: Option<bool>,
+    pub has_deploy_errors: bool,
 }

 #[derive(Serialize, Deserialize, Debug)]
@@ -186,6 +187,8 @@ pub struct NewScript {
    pub lock: Option<Vec<String>>,
    pub language: ScriptLang,
    pub kind: Option<ScriptKind>,
+    pub tag: Option<String>,
+    pub draft_only: Option<bool>,
 }

 #[derive(Deserialize)]
@@ -34,3 +34,4 @@ cron.workspace = true
 rsmq_async.workspace = true
 tokio.workspace = true
 futures-core.workspace = true
+itertools.workspace = true
@@ -9,8 +9,8 @@
 use std::collections::HashMap;

 use anyhow::Context;
+use itertools::Itertools;
 use reqwest::Client;
-use serde::{Deserialize, Serialize};
 use sqlx::{Pool, Postgres, Transaction};
 use tracing::{instrument, Instrument};
 use ulid::Ulid;
@@ -20,6 +20,7 @@ use windmill_common::{
    error::{self, Error},
    flow_status::{FlowStatus, JobResult, MAX_RETRY_ATTEMPTS, MAX_RETRY_INTERVAL},
    flows::{FlowModule, FlowModuleValue, FlowValue},
+    jobs::{JobKind, JobPayload, QueuedJob, RawCode},
    scripts::{get_full_hub_script_by_path, HubScript, ScriptHash, ScriptLang},
    utils::StripPath,
    METRICS_ENABLED,
@@ -50,6 +51,21 @@ lazy_static::lazy_static! {
    .unwrap();
    pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();

+    pub static ref ACCEPTED_TAGS: Vec<String> = std::env::var("WORKER_TAGS")
+        .ok()
+        .map(|x| x.split(',').map(|x| x.to_string()).collect())
+        .unwrap_or_else(|| vec![
+            "deno".to_string(),
+            "python3".to_string(),
+            "go".to_string(),
+            "bash".to_string(),
+            "dependency".to_string(),
+            "flow".to_string(),
+            "hub".to_string(),
+            "other".to_string()]);
+
+    pub static ref ACCEPTED_TAGS_FILTER: String = format!(" AND ({})",
+        ACCEPTED_TAGS.clone().into_iter().map(|x| format!("(tag = '{x}')")).join(" OR "));
 }

 const MAX_FREE_EXECS: i32 = 1000;
@@ -165,6 +181,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
            None
        }
    } else {
+        let accepted_tags_filter = &*ACCEPTED_TAGS_FILTER;
        /* Jobs can be started if they:
         * - haven't been started before,
         *   running = false
@@ -185,7 +202,9 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Clone>(
                       AND scheduled_for <= now())
                   OR (suspend_until IS NOT NULL
                       AND (   suspend <= 0
-                            OR suspend_until <= now()))) {workspaces_filter}
+                            OR suspend_until <= now()))) 
+                    {workspaces_filter}
+                    {accepted_tags_filter}
                ORDER BY scheduled_for
                FOR UPDATE SKIP LOCKED
                LIMIT 1
@@ -311,6 +330,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
    mut same_worker: bool,
    pre_run_error: Option<&windmill_common::error::Error>,
    visible_to_owner: bool,
+    mut tag: Option<String>,
 ) -> Result<(Uuid, QueueTransaction<'c, R>), Error> {
    let args_json = serde_json::Value::Object(args);
    let job_id: Uuid = Ulid::new().into();
@@ -558,12 +578,34 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(

    let flow_status = raw_flow.as_ref().map(FlowStatus::new);

+    let tag = if job_kind == JobKind::Dependencies || job_kind == JobKind::FlowDependencies {
+        "dependency".to_string()
+    } else if job_kind == JobKind::Flow || job_kind == JobKind::FlowPreview {
+        "flow".to_string()
+    } else if job_kind == JobKind::Identity {
+        // identity is a light script, deno is too
+        "deno".to_string()
+    } else if job_kind == JobKind::Script_Hub {
+        "hub".to_string()
+    } else {
+        if tag == Some("".to_string()) {
+            tag = None;
+        }
+        tag.unwrap_or_else(|| {
+            language
+                .as_ref()
+                .map(|x| x.as_str())
+                .unwrap_or_else(|| "deno")
+                .to_string()
+        })
+    };
+
    let uuid = sqlx::query_scalar!(
        "INSERT INTO queue
            (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, 
                script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, \
-         flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job)
-            VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23) \
+         flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job, tag)
+            VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23, $24) \
         RETURNING id",
        workspace_id,
        job_id,
@@ -587,7 +629,8 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
        pre_run_error.map(|e| e.to_string()),
        email,
        visible_to_owner,
-        root_job
+        root_job,
+        tag
    )
    .fetch_one(&mut tx)
    .await
@@ -655,117 +698,3 @@ pub async fn get_hub_script(
        .await
        .map(|e| e)
 }
-
-#[derive(Debug, sqlx::FromRow, Serialize, Clone)]
-pub struct QueuedJob {
-    pub workspace_id: String,
-    pub id: Uuid,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub parent_job: Option<Uuid>,
-    pub created_by: String,
-    pub created_at: chrono::DateTime<chrono::Utc>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub started_at: Option<chrono::DateTime<chrono::Utc>>,
-    pub scheduled_for: chrono::DateTime<chrono::Utc>,
-    pub running: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub script_hash: Option<ScriptHash>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub script_path: Option<String>,
-    pub args: Option<serde_json::Value>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub logs: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub raw_code: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub raw_lock: Option<String>,
-    pub canceled: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub canceled_by: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub canceled_reason: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub last_ping: Option<chrono::DateTime<chrono::Utc>>,
-    pub job_kind: JobKind,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub schedule_path: Option<String>,
-    pub permissioned_as: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub flow_status: Option<serde_json::Value>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub raw_flow: Option<serde_json::Value>,
-    pub is_flow_step: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub language: Option<ScriptLang>,
-    pub same_worker: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub pre_run_error: Option<String>,
-    pub email: String,
-    pub visible_to_owner: bool,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub suspend: Option<i32>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub mem_peak: Option<i32>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub root_job: Option<Uuid>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub leaf_jobs: Option<serde_json::Value>,
-}
-
-impl QueuedJob {
-    pub fn script_path(&self) -> &str {
-        self.script_path
-            .as_ref()
-            .map(String::as_str)
-            .unwrap_or("tmp/main")
-    }
-}
-
-impl QueuedJob {
-    pub fn parse_raw_flow(&self) -> Option<FlowValue> {
-        self.raw_flow
-            .as_ref()
-            .and_then(|v| serde_json::from_value::<FlowValue>(v.clone()).ok())
-    }
-
-    pub fn parse_flow_status(&self) -> Option<FlowStatus> {
-        self.flow_status
-            .as_ref()
-            .and_then(|v| serde_json::from_value::<FlowStatus>(v.clone()).ok())
-    }
-}
-
-#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
-#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")]
-#[serde(rename_all(serialize = "lowercase"))]
-pub enum JobKind {
-    Script,
-    #[allow(non_camel_case_types)]
-    Script_Hub,
-    Preview,
-    Dependencies,
-    Flow,
-    FlowPreview,
-    Identity,
-    FlowDependencies,
-}
-
-#[derive(Debug, Clone)]
-pub enum JobPayload {
-    ScriptHub { path: String },
-    ScriptHash { hash: ScriptHash, path: String },
-    Code(RawCode),
-    Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang },
-    FlowDependencies { path: String },
-    Flow(String),
-    RawFlow { value: FlowValue, path: Option<String> },
-    Identity,
-}
-
-#[derive(Clone, Serialize, Deserialize, Debug)]
-pub struct RawCode {
-    pub content: String,
-    pub path: Option<String>,
-    pub language: ScriptLang,
-    pub lock: Option<String>,
-}
@@ -6,16 +6,17 @@
 * LICENSE-AGPL for a copy of the license.
 */

-use crate::{push, JobPayload};
+use crate::push;
+use crate::QueueTransaction;
 use sqlx::{query_scalar, Postgres, Transaction};
 use std::str::FromStr;
+use windmill_common::jobs::JobPayload;
 use windmill_common::{
    error::{self, Result},
    schedule::Schedule,
    users::username_to_permissioned_as,
    utils::{now_from_db, StripPath},
 };
-use crate::{QueueTransaction};

 pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
    mut tx: QueueTransaction<'c, R>,
@@ -66,18 +67,19 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
        }
    }

-    let payload = if schedule.is_flow {
-        JobPayload::Flow(schedule.script_path)
+    let (payload, tag) = if schedule.is_flow {
+        (JobPayload::Flow(schedule.script_path), None)
    } else {
-        JobPayload::ScriptHash {
-            hash: windmill_common::get_latest_hash_for_path(
-                tx.transaction_mut(),
-                &schedule.workspace_id,
-                &schedule.script_path,
-            )
-            .await?,
-            path: schedule.script_path,
-        }
+        let (hash, tag) = windmill_common::get_latest_hash_for_path(
+            tx.transaction_mut(),
+            &schedule.workspace_id,
+            &schedule.script_path,
+        )
+        .await?;
+        (
+            JobPayload::ScriptHash { hash, path: schedule.script_path },
+            tag,
+        )
    };

    sqlx::query!(
@@ -104,6 +106,7 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
        false,
        None,
        true,
+        tag,
    )
    .await?;
    Ok(tx) // TODO: Bubble up pushed UUID from here
@@ -0,0 +1,41 @@
+use sqlx::{Pool, Postgres};
+use tokio::{fs::File, io::AsyncReadExt};
+use windmill_common::error::{self, Error};
+use windmill_queue::CLOUD_HOSTED;
+
+use crate::MAX_RESULT_SIZE;
+
+pub async fn read_result(job_dir: &str) -> error::Result<serde_json::Value> {
+    let mut file = File::open(format!("{job_dir}/result.json")).await?;
+    let mut content = "".to_string();
+    file.read_to_string(&mut content).await?;
+    if *CLOUD_HOSTED && content.len() > MAX_RESULT_SIZE {
+        return Err(Error::ExecutionErr("Result is too large for the cloud app (limit 2MB). 
+        If using this script as part of the flow, use the shared folder to pass heavy data between steps.".to_owned()));
+    }
+    serde_json::from_str(&content)
+        .map_err(|e| Error::ExecutionErr(format!("Error parsing result: {e}")))
+}
+
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn set_logs(logs: &str, id: &uuid::Uuid, db: &Pool<Postgres>) {
+    if sqlx::query!(
+        "UPDATE queue SET logs = $1 WHERE id = $2",
+        logs.to_owned(),
+        id
+    )
+    .execute(db)
+    .await
+    .is_err()
+    {
+        tracing::error!(%id, "error updating logs for id {id}")
+    };
+}
+
+pub fn capitalize(s: &str) -> String {
+    let mut c = s.chars();
+    match c.next() {
+        None => String::new(),
+        Some(f) => f.to_uppercase().collect::<String>() + c.as_str(),
+    }
+}
@@ -0,0 +1,521 @@
+#[cfg(feature = "enterprise")]
+use crate::{ROOT_CACHE_DIR, ROOT_TMP_CACHE_DIR, TAR_CACHE_RATE, TAR_PIP_TMP_CACHE_DIR, TMP_DIR};
+#[cfg(feature = "enterprise")]
+use itertools::Itertools;
+#[cfg(feature = "enterprise")]
+use rand::Rng;
+#[cfg(feature = "enterprise")]
+use std::process::Stdio;
+
+#[cfg(feature = "enterprise")]
+use tokio::{process::Command, sync::mpsc::Sender, time::Instant};
+
+#[cfg(feature = "enterprise")]
+use windmill_common::error;
+
+#[cfg(feature = "enterprise")]
+const TAR_CACHE_FILENAME: &str = "denogocache.tar";
+
+#[cfg(feature = "enterprise")]
+pub async fn build_tar_and_push(bucket: &str, folder: String) -> error::Result<()> {
+    tracing::info!("Started building and pushing piptar {folder}");
+    let start = Instant::now();
+    let folder_name = folder.split("/").last().unwrap();
+    let tar_path = format!("{TAR_PIP_TMP_CACHE_DIR}/{folder_name}.tar",);
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "tar",
+        vec!["-c", "-f", &tar_path, "-C", &folder, "."],
+    )
+    .await
+    {
+        tracing::info!("Failed to tar cache. Error: {:?}", e);
+        return Err(e);
+    }
+
+    let tar_metadata = tokio::fs::metadata(&tar_path).await;
+    if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 {
+        tracing::info!("Failed to tar cache: {folder}");
+        return Err(error::Error::ExecutionErr(format!(
+            "Failed to tar cache: {folder}"
+        )));
+    }
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "rclone",
+        vec![
+            "copyto",
+            &tar_path,
+            &format!(":s3,env_auth=true:{bucket}/tar/pip/{folder_name}.tar"),
+            "-v",
+            "--size-only",
+            "--fast-list",
+            "--s3-no-check-bucket",
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed to copy piptar {folder} to bucket. Error: {:?}", e);
+        return Err(e);
+    }
+
+    tracing::info!(
+        "Finished copying piptar {folder} to bucket {bucket} as tar, took: {:?}s. Size of tar: {}",
+        start.elapsed().as_secs(),
+        tar_metadata.unwrap().len()
+    );
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn pull_from_tar(bucket: &str, folder: String) -> error::Result<()> {
+    use tokio::fs::metadata;
+    let folder_name = folder.split("/").last().unwrap();
+
+    tracing::info!("Attempting to pull piptar {folder_name} from bucket");
+
+    let start = Instant::now();
+    let tar_path = format!("tar/pip/{folder_name}.tar");
+    let target = format!("{ROOT_TMP_CACHE_DIR}/{tar_path}.single");
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "rclone",
+        vec![
+            "copyto",
+            &format!(":s3,env_auth=true:{bucket}/{tar_path}"),
+            &target,
+            "-v",
+            "--size-only",
+            "--fast-list",
+        ],
+    )
+    .await
+    {
+        tracing::info!(
+            "Failed to copy tar {folder_name} from bucket. Error: {:?}",
+            e
+        );
+        return Err(e);
+    }
+
+    if metadata(&target).await.is_err() {
+        tracing::info!(
+            "piptar {folder_name} not found in bucket. Took {:?}ms",
+            start.elapsed().as_millis()
+        );
+        return Err(error::Error::ExecutionErr(format!(
+            "tar {folder_name} does not exist in bucket"
+        )));
+    }
+
+    extract_pip_tar(&target, &folder).await?;
+    tokio::fs::remove_file(&target).await?;
+    tracing::info!(
+        "Finished pulling and extracting {folder_name}. Took {:?}ms",
+        start.elapsed().as_millis()
+    );
+
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn cache_global(bucket: &str, tx: Sender<()>) -> error::Result<()> {
+    copy_cache_from_bucket(bucket, tx).await?;
+    copy_cache_to_bucket(bucket).await?;
+
+    // this is to prevent excessive tar upload. 1/100*15min = each worker sync its tar once per day on average
+    if rand::thread_rng().gen_range(0..*TAR_CACHE_RATE) == 0 {
+        copy_cache_to_bucket_as_tar(bucket).await;
+    }
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn copy_cache_from_bucket(bucket: &str, tx: Sender<()>) -> error::Result<()> {
+    tracing::info!("Copying cache from bucket in the background {bucket}");
+    let bucket = bucket.to_string();
+
+    let start = Instant::now();
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "rclone",
+        vec![
+            "copy",
+            &format!(":s3,env_auth=true:{bucket}"),
+            &ROOT_TMP_CACHE_DIR,
+            "--size-only",
+            "--fast-list",
+            "--filter",
+            "- deno/gen/file/**",
+            "--filter",
+            "+ deno/**",
+            "--filter",
+            "+ go/**",
+            "--filter",
+            "+ tar/**",
+            "--filter",
+            "- *",
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed to copy cache from bucket. Error: {:?}", e);
+        return Err(e);
+    }
+
+    tracing::info!(
+        "Finished copying cache from bucket {bucket}, took {:?}s",
+        start.elapsed().as_secs()
+    );
+
+    tx.send(()).await.expect("can send copy cache signal");
+
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn copy_cache_to_bucket(bucket: &str) -> error::Result<()> {
+    tracing::info!("Copying cache to bucket {bucket}");
+    let start = Instant::now();
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "rclone",
+        vec![
+            "copy",
+            &ROOT_TMP_CACHE_DIR,
+            &format!(":s3,env_auth=true:{bucket}"),
+            "--size-only",
+            "--fast-list",
+            "--filter",
+            "- deno/gen/file/**",
+            "--filter",
+            "+ deno/**",
+            "--filter",
+            "+ go/**",
+            "--filter",
+            "- *",
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed to copy cache to bucket. Error: {:?}", e);
+        return Err(e);
+    }
+    tracing::info!(
+        "Finished copying cache to bucket {bucket}, took: {:?}s",
+        start.elapsed().as_secs()
+    );
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn copy_cache_to_bucket_as_tar(bucket: &str) {
+    tracing::info!("Copying cache to bucket {bucket} as tar");
+    let start = Instant::now();
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "tar",
+        vec![
+            "-c",
+            "-f",
+            &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
+            "go",
+            "deno",
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed to tar cache. Error: {:?}", e);
+        return;
+    }
+
+    let tar_metadata =
+        tokio::fs::metadata(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await;
+    if tar_metadata.is_err() || tar_metadata.as_ref().unwrap().len() == 0 {
+        tracing::info!("Failed to tar cache");
+        return;
+    }
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "rclone",
+        vec![
+            "copyto",
+            &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
+            &format!(":s3,env_auth=true:{bucket}/{TAR_CACHE_FILENAME}"),
+            "-v",
+            "--size-only",
+            "--fast-list",
+            "--s3-no-check-bucket",
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed to copy tar to bucket. Error: {:?}", e);
+        return;
+    }
+
+    if let Err(e) =
+        tokio::fs::remove_file(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await
+    {
+        tracing::info!("Failed to remove tar cache. Error: {:?}", e);
+    };
+
+    tracing::info!(
+        "Finished copying cache to bucket {bucket} as tar, took: {:?}s. Size of new tar: {}",
+        start.elapsed().as_secs(),
+        tar_metadata.unwrap().len()
+    );
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn copy_denogo_cache_from_bucket_as_tar(bucket: &str) {
+    use tokio::fs::metadata;
+
+    tracing::info!("Copying denogo cache from bucket {bucket} as tar");
+
+    let start: Instant = Instant::now();
+
+    if let Err(e) = execute_command(
+        ROOT_TMP_CACHE_DIR,
+        "rclone",
+        vec![
+            "copyto",
+            &format!(":s3,env_auth=true:{bucket}/{TAR_CACHE_FILENAME}"),
+            &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
+            "-v",
+            "--size-only",
+            "--fast-list",
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed copying denogo tar from cache. Error: {:?}", e);
+        return;
+    }
+
+    if let Err(e) = execute_command(
+        ROOT_CACHE_DIR,
+        "tar",
+        vec![
+            "-xpvf",
+            &format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}"),
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed to untar denogo. Error: {:?}", e);
+        return;
+    }
+
+    let denogen = format!("{ROOT_TMP_CACHE_DIR}deno/gen/file");
+    if metadata(&denogen).await.is_ok() {
+        let _ = tokio::fs::remove_dir_all(denogen).await;
+    }
+
+    if let Err(e) =
+        tokio::fs::remove_file(format!("{ROOT_TMP_CACHE_DIR}{TAR_CACHE_FILENAME}")).await
+    {
+        tracing::info!("Failed to remove denotar cache. Error: {:?}", e);
+        return;
+    };
+
+    tracing::info!(
+        "Finished copying denogotar from bucket {bucket} as tar, took: {:?}s",
+        start.elapsed().as_secs()
+    );
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn copy_all_piptars_from_bucket(bucket: &str) {
+    tracing::info!("Copying all piptars cache from bucket {bucket}");
+
+    let start = Instant::now();
+
+    if let Err(e) = execute_command(
+        ROOT_CACHE_DIR,
+        "rclone",
+        vec![
+            "copy",
+            &format!(":s3,env_auth=true:{bucket}/tar/pip/"),
+            &TAR_PIP_TMP_CACHE_DIR,
+            "-v",
+            "--size-only",
+            "--fast-list",
+        ],
+    )
+    .await
+    {
+        tracing::info!("Failed transferring all piptars from cache. Error: {:?}", e);
+        return;
+    }
+
+    tracing::info!(
+        "Finished transferring piptars from bucket {bucket} as tar, took: {:?}s",
+        start.elapsed().as_secs()
+    );
+}
+
+// async fn check_if_bucket_syncable(bucket: &str) -> bool {
+//     match Command::new("rclone")
+//         .arg("lsf")
+//         .arg(format!(":s3,env_auth=true:{bucket}/NOSYNC"))
+
+//         .arg("-vv")
+//         .arg("--fast-list")
+//         .stdin(Stdio::null())
+//         .stdout(Stdio::null())
+//         .output()
+//         .await;
+//     return true;
+// }
+
+#[cfg(feature = "enterprise")]
+pub async fn copy_tmp_cache_to_cache() -> error::Result<()> {
+    let start: Instant = Instant::now();
+    execute_command(
+        TMP_DIR,
+        "rclone",
+        vec![
+            "sync",
+            ROOT_TMP_CACHE_DIR,
+            ROOT_CACHE_DIR,
+            "--filter",
+            "- deno/gen/file/**",
+            "--filter",
+            "+ deno/**",
+            "--filter",
+            "+ go/**",
+            "--filter",
+            "- *",
+        ],
+    )
+    .await?;
+
+    tracing::info!(
+        "Finished copying local tmp cache to local cache. Took {}ms",
+        start.elapsed().as_millis(),
+    );
+
+    if let Err(e) = untar_all_piptars().await {
+        tracing::info!("Failed to untar piptars. Error: {:?}", e);
+    }
+
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn untar_all_piptars() -> error::Result<()> {
+    use tokio::fs::{self, metadata};
+
+    use crate::PIP_CACHE_DIR;
+
+    let start: Instant = Instant::now();
+
+    let mut entries = fs::read_dir(TAR_PIP_TMP_CACHE_DIR).await?;
+    while let Some(entry) = entries.next_entry().await? {
+        if let Err(e) = {
+            let entry_path = entry.path();
+            let path = entry_path.to_str().expect("Could not convert path to str");
+            let folder = format!(
+                "{PIP_CACHE_DIR}/{}",
+                path.split('/')
+                    .last()
+                    .unwrap()
+                    .strip_suffix(".tar")
+                    .unwrap()
+            );
+            if metadata(&folder).await.is_ok() {
+                continue;
+            }
+            extract_pip_tar(&path, &folder).await?;
+            Ok(()) as error::Result<()>
+        } {
+            tracing::info!("Failed to extract pip tar. Error: {:?}", e);
+        }
+    }
+
+    tracing::info!(
+        "Finished untarring all piptars. Took {}ms",
+        start.elapsed().as_millis(),
+    );
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn extract_pip_tar(tar: &str, folder: &str) -> error::Result<()> {
+    use tokio::fs;
+
+    let start: Instant = Instant::now();
+    fs::create_dir_all(&folder).await?;
+    if let Err(e) = execute_command(&folder, "tar", vec!["-xpvf", tar]).await {
+        tracing::info!("Failed to untar piptar. Error: {:?}", e);
+        fs::remove_dir_all(&folder).await?;
+        return Err(e);
+    }
+    tracing::info!(
+        "Finished extracting pip tar {folder}. Took {}ms",
+        start.elapsed().as_millis(),
+    );
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn copy_cache_to_tmp_cache() -> error::Result<()> {
+    let start: Instant = Instant::now();
+    execute_command(
+        TMP_DIR,
+        "rclone",
+        vec![
+            "sync",
+            ROOT_CACHE_DIR,
+            ROOT_TMP_CACHE_DIR,
+            "--filter",
+            "- deno/gen/file/**",
+            "--filter",
+            "+ deno/**",
+            "--filter",
+            "+ go/**",
+            "--filter",
+            "- *",
+        ],
+    )
+    .await?;
+    tracing::info!(
+        "Finished copying local cache to local tmp cache. Took {}ms",
+        start.elapsed().as_millis()
+    );
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+pub async fn execute_command(dir: &str, command: &str, args: Vec<&str>) -> error::Result<()> {
+    match Command::new(command)
+        .current_dir(dir)
+        .args(args.clone())
+        .stdin(Stdio::null())
+        .stdout(Stdio::null())
+        .spawn()
+    {
+        Ok(mut h) => {
+            if !h.wait().await.unwrap().success() {
+                return Err(error::Error::ExecutionErr(format!(
+                    "Failed to apply {command} with args: {}",
+                    args.iter().join(" ")
+                )));
+            }
+        }
+        Err(e) => {
+            return Err(error::Error::ExecutionErr(format!(
+                "Failed to apply {command} with args: {}. Error: {e:?}",
+                args.iter().join(" ")
+            )));
+        }
+    }
+    Ok(())
+}
@@ -0,0 +1,373 @@
+use std::process::Stdio;
+
+use itertools::Itertools;
+use tokio::{
+    fs::{DirBuilder, File},
+    io::AsyncReadExt,
+    process::Command,
+};
+use uuid::Uuid;
+use windmill_common::{
+    error::{self, Error},
+    jobs::QueuedJob,
+    utils::calculate_hash,
+};
+use windmill_parser_go::parse_go_imports;
+
+use crate::{
+    common::{capitalize, read_result, set_logs},
+    create_args_and_out_file, get_reserved_variables, handle_child, write_file,
+    AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GO_CACHE_DIR, HOME_ENV,
+    NETRC, NSJAIL_PATH, PATH_ENV,
+};
+
+const GO_REQ_SPLITTER: &str = "//go.sum\n";
+const NSJAIL_CONFIG_RUN_GO_CONTENT: &str = include_str!("../nsjail/run.go.config.proto");
+
+lazy_static::lazy_static! {
+    static ref GO_PATH: String = std::env::var("GO_PATH").unwrap_or_else(|_| "/usr/bin/go".to_string());
+}
+
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn handle_go_job(
+    logs: &mut String,
+    job: &QueuedJob,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    client: &AuthedClientBackgroundTask,
+    inner_content: &str,
+    job_dir: &str,
+    requirements_o: Option<String>,
+    shared_mount: &str,
+    base_internal_url: &str,
+    worker_name: &str,
+) -> Result<serde_json::Value, Error> {
+    //go does not like executing modules at temp root
+    let job_dir = &format!("{job_dir}/go");
+    let (skip_go_mod, skip_tidy) = if let Some(requirements) = requirements_o {
+        gen_go_mod(inner_content, job_dir, &requirements).await?
+    } else {
+        (false, false)
+    };
+
+    logs.push_str("\n\n--- GO DEPENDENCIES SETUP ---\n");
+    set_logs(logs, &job.id, db).await;
+
+    install_go_dependencies(
+        &job.id,
+        inner_content,
+        logs,
+        job_dir,
+        db,
+        true,
+        skip_go_mod,
+        skip_tidy,
+        worker_name,
+        &job.workspace_id,
+    )
+    .await?;
+
+    logs.push_str("\n\n--- GO CODE EXECUTION ---\n");
+    set_logs(logs, &job.id, db).await;
+    let client = &client.get_authed().await;
+    create_args_and_out_file(client, job, job_dir).await?;
+    {
+        let sig = windmill_parser_go::parse_go_sig(&inner_content)?;
+        drop(inner_content);
+
+        const WRAPPER_CONTENT: &str = r#"package main
+
+import (
+    "encoding/json"
+    "os"
+    "fmt"
+    "mymod/inner"
+)
+
+func main() {{
+
+    dat, err := os.ReadFile("args.json")
+    if err != nil {{
+        fmt.Println(err)
+        os.Exit(1)
+    }}
+
+    var req inner.Req
+
+    if err := json.Unmarshal(dat, &req); err != nil {{
+        fmt.Println(err)
+        os.Exit(1)
+    }}
+
+    res, err := inner.Run(req)
+    if err != nil {{
+        fmt.Println(err)
+        os.Exit(1)
+    }}
+    res_json, err := json.Marshal(res)
+    if err != nil {{
+        fmt.Println(err)
+        os.Exit(1)
+    }}
+    f, err := os.OpenFile("result.json", os.O_APPEND|os.O_WRONLY, os.ModeAppend)
+    if err != nil {{
+        fmt.Println(err)
+        os.Exit(1)
+    }}
+    _, err = f.WriteString(string(res_json))
+    if err != nil {{
+        fmt.Println(err)
+        os.Exit(1)
+    }}
+}}"#;
+
+        write_file(job_dir, "main.go", WRAPPER_CONTENT).await?;
+
+        {
+            let spread = &sig
+                .args
+                .clone()
+                .into_iter()
+                .map(|x| format!("req.{}", capitalize(&x.name)))
+                .join(", ");
+            let req_body = &sig
+                .args
+                .into_iter()
+                .map(|x| {
+                    format!(
+                        "{} {} `json:\"{}\"`",
+                        capitalize(&x.name),
+                        windmill_parser_go::otyp_to_string(x.otyp),
+                        x.name
+                    )
+                })
+                .join("\n");
+            let runner_content: String = format!(
+                r#"package inner
+type Req struct {{
+    {req_body}
+}}
+
+func Run(req Req) (interface{{}}, error){{
+    return main({spread})
+}}
+
+"#,
+            );
+            write_file(&format!("{job_dir}/inner"), "runner.go", &runner_content).await?;
+        }
+    }
+    let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
+    reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
+
+    let child = if !*DISABLE_NSJAIL {
+        let _ = write_file(
+            job_dir,
+            "run.config.proto",
+            &NSJAIL_CONFIG_RUN_GO_CONTENT
+                .replace("{JOB_DIR}", job_dir)
+                .replace("{CACHE_DIR}", GO_CACHE_DIR)
+                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
+                .replace("{SHARED_MOUNT}", shared_mount),
+        )
+        .await?;
+        let build_go = Command::new(GO_PATH.as_str())
+            .current_dir(job_dir)
+            .env_clear()
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .env("GOPATH", GO_CACHE_DIR)
+            .env("HOME", HOME_ENV.as_str())
+            .args(vec!["build", "main.go"])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?;
+        handle_child(
+            &job.id,
+            db,
+            logs,
+            build_go,
+            false,
+            worker_name,
+            &job.workspace_id,
+            "go build",
+        )
+        .await?;
+
+        Command::new(NSJAIL_PATH.as_str())
+            .current_dir(job_dir)
+            .env_clear()
+            .envs(reserved_variables)
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .args(vec!["--config", "run.config.proto", "--", "/tmp/go/main"])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?
+    } else {
+        if let Some(ref netrc) = *NETRC {
+            write_file(&HOME_ENV, ".netrc", netrc).await?;
+        }
+        Command::new(GO_PATH.as_str())
+            .current_dir(job_dir)
+            .env_clear()
+            .envs(reserved_variables)
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .env("GOPATH", GO_CACHE_DIR)
+            .env("GOPRIVATE", GOPRIVATE.as_ref().unwrap_or(&String::new()))
+            .env("HOME", HOME_ENV.as_str())
+            .args(vec!["run", "main.go"])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?
+    };
+    handle_child(
+        &job.id,
+        db,
+        logs,
+        child,
+        !*DISABLE_NSJAIL,
+        worker_name,
+        &job.workspace_id,
+        "go run",
+    )
+    .await?;
+    read_result(job_dir).await
+}
+
+async fn gen_go_mod(
+    inner_content: &str,
+    job_dir: &str,
+    requirements: &str,
+) -> error::Result<(bool, bool)> {
+    gen_go_mymod(inner_content, job_dir).await?;
+
+    let md = requirements.split_once(GO_REQ_SPLITTER);
+    if let Some((req, sum)) = md {
+        write_file(job_dir, "go.mod", &req).await?;
+        write_file(job_dir, "go.sum", &sum).await?;
+        Ok((true, true))
+    } else {
+        write_file(job_dir, "go.mod", &requirements).await?;
+        Ok((true, false))
+    }
+}
+
+pub async fn install_go_dependencies(
+    job_id: &Uuid,
+    code: &str,
+    logs: &mut String,
+    job_dir: &str,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    non_dep_job: bool,
+    skip_go_mod: bool,
+    has_sum: bool,
+    worker_name: &str,
+    w_id: &str,
+) -> error::Result<String> {
+    if !skip_go_mod {
+        gen_go_mymod(code, job_dir).await?;
+        let child = Command::new("go")
+            .current_dir(job_dir)
+            .args(vec!["mod", "init", "mymod"])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?;
+
+        handle_child(job_id, db, logs, child, false, worker_name, w_id, "go init").await?;
+    }
+
+    let mut new_lockfile = false;
+
+    let hash = if !has_sum {
+        calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str())
+    } else {
+        "".to_string()
+    };
+
+    let mut skip_tidy = has_sum;
+
+    if !has_sum {
+        if let Some(cached) = sqlx::query_scalar!(
+            "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
+            hash
+        )
+        .fetch_optional(db)
+        .await?
+        {
+            logs.push_str(&format!("\nfound cached resolution"));
+            gen_go_mod(code, job_dir, &cached).await?;
+            skip_tidy = true;
+            new_lockfile = false;
+        } else {
+            new_lockfile = true;
+        }
+    }
+
+    let mod_command = if skip_tidy { "download" } else { "tidy" };
+    let child = Command::new(GO_PATH.as_str())
+        .current_dir(job_dir)
+        .env("GOPATH", GO_CACHE_DIR)
+        .args(vec!["mod", mod_command])
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()?;
+    handle_child(
+        job_id,
+        db,
+        logs,
+        child,
+        false,
+        worker_name,
+        &w_id,
+        &format!("go {mod_command}"),
+    )
+    .await
+    .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
+
+    if (!new_lockfile || has_sum) && non_dep_job {
+        return Ok("".to_string());
+    }
+
+    let mut req_content = "".to_string();
+
+    let mut file = File::open(format!("{job_dir}/go.mod")).await?;
+    file.read_to_string(&mut req_content).await?;
+    req_content.push_str(GO_REQ_SPLITTER);
+    let sum_path = format!("{job_dir}/go.sum");
+    if tokio::fs::metadata(&sum_path).await.is_ok() {
+        let mut file = File::open(sum_path).await?;
+        file.read_to_string(&mut req_content).await?;
+    }
+
+    if non_dep_job {
+        sqlx::query!(
+            "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
+            hash,
+            req_content
+        ).fetch_optional(db).await?;
+
+        return Ok(String::new());
+    } else {
+        Ok(req_content)
+    }
+}
+
+async fn gen_go_mymod(code: &str, job_dir: &str) -> error::Result<()> {
+    let code = if code.trim_start().starts_with("package") {
+        code.to_string()
+    } else {
+        format!("package inner; {code}")
+    };
+
+    let mymod_dir = format!("{job_dir}/inner");
+    DirBuilder::new()
+        .recursive(true)
+        .create(&mymod_dir)
+        .await
+        .expect("could not create go's mymod dir");
+
+    write_file(&mymod_dir, "inner_main.go", &code).await?;
+
+    Ok(())
+}
@@ -10,11 +10,13 @@ use sqlx::{Pool, Postgres};
 use tracing::instrument;
 use uuid::Uuid;
 use windmill_common::{
-    error::Error, flow_status::FlowStatusModule, schedule::Schedule, METRICS_ENABLED,
-};
-use windmill_queue::{
-    delete_job, schedule::get_schedule_opt, JobKind, QueueTransaction, QueuedJob, CLOUD_HOSTED,
+    error::Error,
+    flow_status::FlowStatusModule,
+    jobs::{JobKind, QueuedJob},
+    schedule::Schedule,
+    METRICS_ENABLED,
 };
+use windmill_queue::{delete_job, schedule::get_schedule_opt, QueueTransaction, CLOUD_HOSTED};

 #[instrument(level = "trace", skip_all)]
 pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Send>(
@@ -124,9 +126,10 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
                   , email
                   , visible_to_owner
                   , mem_peak
+                   , tag
                )
            VALUES ($1, $2, $3, $4, $5, $6, COALESCE($26, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE($6, now()))))*1000), $7, $8, $9,\
-                    $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29)
+                    $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $27, $28, $29, $30)
         ON CONFLICT (id) DO UPDATE SET success = $7, result = $11, logs = concat(cj.logs, $12)",
        queued_job.workspace_id,
        queued_job.id,
@@ -156,7 +159,8 @@ pub async fn add_completed_job<R: rsmq_async::RsmqConnection + Clone + Send>(
        duration: Option<i64>,
        queued_job.email,
        queued_job.visible_to_owner,
-        mem_peak
+        mem_peak,
+        queued_job.tag,
    )
    .execute(&mut tx)
    .await
@@ -252,7 +252,7 @@ async function resource(path) {{
            .join(""),
    );
    tracing::debug!("{}", code);
-    let global = context.execute_script("<anon>", code)?;
+    let global = context.execute_script("<anon>", code.into())?;
    let global = context.resolve_value(global).await?;

    let scope = &mut context.handle_scope();
@@ -1,5 +1,9 @@
+mod common;
+mod global_cache;
+mod go_executor;
 mod jobs;
 mod js_eval;
+mod python_executor;
 mod worker;
 mod worker_flow;

@@ -1,92 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-
-// use std::{net::SocketAddr, time::Duration};
-
-// use anyhow::Context;
-// use sqlx::{postgres::PgPoolOptions, Pool, Postgres};
-// use windmill_common::{
-//     error::{self, Error},
-//     utils::rd_string,
-// };
-
-// #[tokio::main]
-// async fn main() -> anyhow::Result<()> {
-//     // dotenv().ok();
-
-//     windmill_common::tracing_init::initialize_tracing();
-
-//     let db = async {
-//         let database_url = std::env::var("DATABASE_URL")
-//             .map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
-
-//         let max_connections = match std::env::var("DATABASE_CONNECTIONS") {
-//             Ok(n) => n.parse::<u32>().context("invalid DATABASE_CONNECTIONS")?,
-//             Err(_) => 10,
-//         };
-
-//         Ok::<Pool<Postgres>, error::Error>(
-//             PgPoolOptions::new()
-//                 .max_connections(max_connections)
-//                 .max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
-//                 .connect(&database_url)
-//                 .await
-//                 .map_err(|err| Error::ConnectingToDatabase(err.to_string()))?,
-//         )
-//     }
-//     .await?;
-
-//     let metrics_addr: Option<SocketAddr> = std::env::var("METRICS_ADDR")
-//         .ok()
-//         .map(|s| {
-//             s.parse::<bool>()
-//                 .map(|b| b.then(|| SocketAddr::from(([0, 0, 0, 0], 8001))))
-//                 .or_else(|_| s.parse::<SocketAddr>().map(Some))
-//         })
-//         .transpose()?
-//         .flatten();
-
-//     let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
-//     let shutdown_signal = windmill_common::shutdown_signal(tx);
-
-//     let workers_f = async {
-//         let instance_name = rd_string(5);
-
-//         let ip = windmill_common::external_ip::get_ip()
-//             .await
-//             .unwrap_or_else(|e| {
-//                 tracing::warn!(error = e.to_string(), "failed to get external IP");
-//                 "unretrievable IP".to_string()
-//             });
-//         let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
-//         windmill_worker::run_worker(
-//             &db.clone(),
-//             &instance_name,
-//             worker_name,
-//             1,
-//             1,
-//             &ip,
-//             rx.resubscribe(),
-//         )
-//         .await;
-//         Ok(()) as anyhow::Result<()>
-//     };
-
-//     let metrics_f = async {
-//         match metrics_addr {
-//             Some(addr) => windmill_common::serve_metrics(addr, rx.resubscribe())
-//                 .await
-//                 .map_err(anyhow::Error::from),
-//             None => Ok(()),
-//         }
-//     };
-
-//     futures::try_join!(shutdown_signal, workers_f, metrics_f)?;
-
-//     Ok(())
-// }
@@ -0,0 +1,584 @@
+use std::process::Stdio;
+
+use itertools::Itertools;
+use regex::Regex;
+use sqlx::{Pool, Postgres};
+use tokio::{
+    fs::{metadata, DirBuilder, File},
+    io::AsyncReadExt,
+    process::Command,
+};
+use uuid::Uuid;
+use windmill_common::{
+    error::{self, Error},
+    jobs::QueuedJob,
+    utils::calculate_hash,
+};
+
+lazy_static::lazy_static! {
+    static ref PYTHON_PATH: String =
+    std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string());
+
+    static ref PIP_INDEX_URL: Option<String> = std::env::var("PIP_INDEX_URL").ok();
+    static ref PIP_EXTRA_INDEX_URL: Option<String> = std::env::var("PIP_EXTRA_INDEX_URL").ok();
+    static ref PIP_TRUSTED_HOST: Option<String> = std::env::var("PIP_TRUSTED_HOST").ok();
+    static ref PIP_LOCAL_DEPENDENCIES: Option<Vec<String>> = {
+        let pip_local_dependencies = std::env::var("PIP_LOCAL_DEPENDENCIES")
+            .ok()
+            .map(|x| x.split(',').map(|x| x.to_string()).collect());
+        if pip_local_dependencies == Some(vec!["".to_string()]) {
+            None
+        } else {
+            pip_local_dependencies
+        }
+    };
+
+    static ref ADDITIONAL_PYTHON_PATHS: Option<Vec<String>> = std::env::var("ADDITIONAL_PYTHON_PATHS")
+        .ok()
+        .map(|x| x.split(':').map(|x| x.to_string()).collect());
+
+    static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap();
+
+}
+
+const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto");
+const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto");
+const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
+
+#[cfg(feature = "enterprise")]
+use crate::global_cache::{build_tar_and_push, pull_from_tar};
+
+use crate::{
+    common::{read_result, set_logs},
+    create_args_and_out_file, get_reserved_variables, handle_child, write_file,
+    AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV,
+    PIP_CACHE_DIR, S3_CACHE_BUCKET,
+};
+
+pub async fn create_dependencies_dir(job_dir: &str) {
+    DirBuilder::new()
+        .recursive(true)
+        .create(&format!("{job_dir}/dependencies"))
+        .await
+        .expect("could not create dependencies dir");
+}
+
+pub async fn pip_compile(
+    job_id: &Uuid,
+    requirements: &str,
+    logs: &mut String,
+    job_dir: &str,
+    db: &Pool<Postgres>,
+    worker_name: &str,
+    w_id: &str,
+) -> error::Result<String> {
+    logs.push_str(&format!("\nresolving dependencies..."));
+    set_logs(logs, job_id, db).await;
+    logs.push_str(&format!("\ncontent of requirements:\n{}", requirements));
+    let req_hash = calculate_hash(&requirements);
+    if let Some(cached) = sqlx::query_scalar!(
+        "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
+        req_hash
+    )
+    .fetch_optional(db)
+    .await?
+    {
+        logs.push_str(&format!("\nfound cached resolution"));
+        return Ok(cached);
+    }
+    let file = "requirements.in";
+    let requirements = if let Some(pip_local_dependencies) = PIP_LOCAL_DEPENDENCIES.as_ref() {
+        let deps = pip_local_dependencies.clone();
+        requirements
+            .lines()
+            .filter(|s| !deps.contains(&s.to_string()))
+            .join("\n")
+    } else {
+        requirements.to_string()
+    };
+    write_file(job_dir, file, &requirements).await?;
+
+    let mut args = vec!["-q", "--no-header", file, "--resolver=backtracking"];
+    if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
+        args.extend(["--extra-index-url", url]);
+    }
+    if let Some(url) = PIP_INDEX_URL.as_ref() {
+        args.extend(["--index-url", url]);
+    }
+    if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
+        args.extend(["--trusted-host", host]);
+    }
+    let child = Command::new("pip-compile")
+        .current_dir(job_dir)
+        .args(args)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()?;
+    handle_child(
+        job_id,
+        db,
+        logs,
+        child,
+        false,
+        worker_name,
+        &w_id,
+        "pip-compile",
+    )
+    .await
+    .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
+    let path_lock = format!("{job_dir}/requirements.txt");
+    let mut file = File::open(path_lock).await?;
+    let mut req_content = "".to_string();
+    file.read_to_string(&mut req_content).await?;
+    let lockfile = req_content
+        .lines()
+        .filter(|x| !x.trim_start().starts_with('#'))
+        .map(|x| x.to_string())
+        .collect::<Vec<String>>()
+        .join("\n");
+    sqlx::query!(
+        "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
+        req_hash,
+        lockfile
+    ).fetch_optional(db).await?;
+    Ok(lockfile)
+}
+
+#[tracing::instrument(level = "trace", skip_all)]
+pub async fn handle_python_job(
+    requirements_o: Option<String>,
+    job_dir: &str,
+    worker_dir: &str,
+    worker_name: &str,
+    job: &QueuedJob,
+    logs: &mut String,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    client: &AuthedClientBackgroundTask,
+    inner_content: &String,
+    shared_mount: &str,
+    base_internal_url: &str,
+) -> windmill_common::error::Result<serde_json::Value> {
+    create_dependencies_dir(job_dir).await;
+
+    let mut additional_python_paths: Vec<String> =
+        ADDITIONAL_PYTHON_PATHS.to_owned().unwrap_or_else(|| vec![]);
+
+    let requirements = match requirements_o {
+        Some(r) => r,
+        None => {
+            let requirements = windmill_parser_py::parse_python_imports(&inner_content)?.join("\n");
+            if requirements.is_empty() {
+                "".to_string()
+            } else {
+                pip_compile(
+                    &job.id,
+                    &requirements,
+                    logs,
+                    job_dir,
+                    db,
+                    worker_name,
+                    &job.workspace_id,
+                )
+                .await
+                .map_err(|e| {
+                    Error::ExecutionErr(format!("pip compile failed: {}", e.to_string()))
+                })?
+            }
+        }
+    };
+
+    if requirements.len() > 0 {
+        additional_python_paths = handle_python_reqs(
+            requirements
+                .split("\n")
+                .filter(|x| !x.starts_with("--"))
+                .collect(),
+            &job.id,
+            &job.workspace_id,
+            logs,
+            db,
+            worker_name,
+            job_dir,
+            worker_dir,
+        )
+        .await?;
+    }
+    logs.push_str("\n\n--- PYTHON CODE EXECUTION ---\n");
+
+    set_logs(logs, &job.id, db).await;
+
+    let relative_imports = RELATIVE_IMPORT_REGEX.is_match(&inner_content);
+
+    let script_path_splitted = &job.script_path().split("/");
+    let dirs_full = script_path_splitted
+        .clone()
+        .take(script_path_splitted.clone().count() - 1)
+        .join("/")
+        .replace("-", "_");
+    let dirs = if dirs_full.len() > 0 {
+        dirs_full
+    } else {
+        "tmp".to_string()
+    };
+    let last = script_path_splitted
+        .clone()
+        .last()
+        .unwrap()
+        .replace("-", "_")
+        .replace(" ", "_")
+        .to_lowercase();
+    let module_dir = format!("{}/{}", job_dir, dirs);
+    tokio::fs::create_dir_all(format!("{module_dir}/")).await?;
+    let _ = write_file(&module_dir, &format!("{last}.py"), inner_content).await?;
+    if relative_imports {
+        let _ = write_file(&job_dir, "loader.py", RELATIVE_PYTHON_LOADER).await?;
+    }
+
+    let sig = windmill_parser_py::parse_python_signature(inner_content)?;
+    let transforms = sig
+        .args
+        .iter()
+        .map(|x| match x.typ {
+            windmill_parser::Typ::Bytes => {
+                let name = &x.name;
+                format!(
+                    "if \"{name}\" in kwargs and kwargs[\"{name}\"] is not None:\n    \
+                                     kwargs[\"{name}\"] = base64.b64decode(kwargs[\"{name}\"])\n",
+                )
+            }
+            windmill_parser::Typ::Datetime => {
+                let name = &x.name;
+                format!(
+                    "if \"{name}\" in kwargs and kwargs[\"{name}\"] is not None:\n    \
+                                     kwargs[\"{name}\"] = datetime.strptime(kwargs[\"{name}\"], \
+                                     '%Y-%m-%dT%H:%M')\n",
+                )
+            }
+            _ => "".to_string(),
+        })
+        .collect::<Vec<String>>()
+        .join("");
+    let client = client.get_authed().await;
+    create_args_and_out_file(&client, job, job_dir).await?;
+
+    let import_loader = if relative_imports {
+        "import loader"
+    } else {
+        ""
+    };
+    let import_base64 = if sig
+        .args
+        .iter()
+        .any(|x| x.typ == windmill_parser::Typ::Bytes)
+    {
+        "import base64"
+    } else {
+        ""
+    };
+    let import_datetime = if sig
+        .args
+        .iter()
+        .any(|x| x.typ == windmill_parser::Typ::Datetime)
+    {
+        "from datetime import datetime"
+    } else {
+        ""
+    };
+    let spread = if sig.star_kwargs {
+        "args = kwargs".to_string()
+    } else {
+        sig.args
+            .into_iter()
+            .map(|x| {
+                let name = &x.name;
+                if x.default.is_none() {
+                    format!("args[\"{name}\"] = kwargs.get(\"{name}\")")
+                } else {
+                    format!(
+                        r#"args["{name}"] = kwargs.get("{name}")
+if args["{name}"] is None:
+    del args["{name}"]"#
+                    )
+                }
+            })
+            .join("\n")
+    };
+
+    let module_dir_dot = dirs.replace("/", ".").replace("-", "_");
+    let wrapper_content: String = format!(
+        r#"
+import json
+{import_loader}
+{import_base64}
+{import_datetime}
+import traceback
+import sys
+from {module_dir_dot} import {last} as inner_script
+
+
+with open("args.json") as f:
+    kwargs = json.load(f, strict=False)
+args = {{}}
+{transforms}
+{spread}
+for k, v in list(args.items()):
+    if v == '<function call>':
+        del args[k]
+
+try:
+    res = inner_script.main(**args)
+    typ = type(res)
+    if typ.__name__ == 'DataFrame':
+        if typ.__module__ == 'pandas.core.frame':
+            res = res.values.tolist()
+        elif typ.__module__ == 'polars.dataframe.frame':
+            res = res.rows()
+    res_json = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '')
+    with open("result.json", 'w') as f:
+        f.write(res_json)
+except Exception as e:
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+    tb = traceback.format_tb(exc_traceback)
+    with open("result.json", 'w') as f:
+        err_json = json.dumps({{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:])  }}, separators=(',', ':'), default=str).replace('\n', '')
+        f.write(err_json)
+        sys.exit(1)
+"#,
+    );
+    write_file(job_dir, "wrapper.py", &wrapper_content).await?;
+
+    let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
+    let additional_python_paths_folders = additional_python_paths.iter().join(":");
+    if !*DISABLE_NSJAIL {
+        let shared_deps = additional_python_paths
+            .into_iter()
+            .map(|pp| {
+                format!(
+                    r#"
+mount {{
+    src: "{pp}"
+    dst: "{pp}"
+    is_bind: true
+    rw: false
+}}
+        "#
+                )
+            })
+            .join("\n");
+        let _ = write_file(
+            job_dir,
+            "run.config.proto",
+            &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT
+                .replace("{JOB_DIR}", job_dir)
+                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
+                .replace("{SHARED_MOUNT}", shared_mount)
+                .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str())
+                .replace("{MAIN}", format!("{dirs}/{last}").as_str())
+                .replace(
+                    "{ADDITIONAL_PYTHON_PATHS}",
+                    additional_python_paths_folders.as_str(),
+                ),
+        )
+        .await?;
+    } else {
+        reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
+    }
+
+    tracing::info!(
+        worker_name = %worker_name,
+        job_id = %job.id,
+        workspace_id = %job.workspace_id,
+        "started python code execution {}",
+        job.id
+    );
+    let child = if !*DISABLE_NSJAIL {
+        Command::new(NSJAIL_PATH.as_str())
+            .current_dir(job_dir)
+            .env_clear()
+            // inject PYTHONPATH here - for some reason I had to do it in nsjail conf
+            .envs(reserved_variables)
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .args(vec![
+                "--config",
+                "run.config.proto",
+                "--",
+                PYTHON_PATH.as_str(),
+                "-u",
+                "-m",
+                "wrapper",
+            ])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?
+    } else {
+        Command::new(PYTHON_PATH.as_str())
+            .current_dir(job_dir)
+            .env_clear()
+            .envs(reserved_variables)
+            .env("PATH", PATH_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .args(vec!["-u", "-m", "wrapper"])
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .spawn()?
+    };
+
+    handle_child(
+        &job.id,
+        db,
+        logs,
+        child,
+        !*DISABLE_NSJAIL,
+        worker_name,
+        &job.workspace_id,
+        "python run",
+    )
+    .await?;
+    read_result(job_dir).await
+}
+
+pub async fn handle_python_reqs(
+    requirements: Vec<&str>,
+    job_id: &Uuid,
+    w_id: &str,
+    logs: &mut String,
+    db: &sqlx::Pool<sqlx::Postgres>,
+    worker_name: &str,
+    job_dir: &str,
+    worker_dir: &str,
+) -> error::Result<Vec<String>> {
+    let mut req_paths: Vec<String> = vec![];
+    let mut vars = vec![("PATH", PATH_ENV.as_str())];
+    if !*DISABLE_NSJAIL {
+        if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
+            vars.push(("EXTRA_INDEX_URL", url));
+        }
+        if let Some(url) = PIP_INDEX_URL.as_ref() {
+            vars.push(("INDEX_URL", url));
+        }
+        if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
+            vars.push(("TRUSTED_HOST", host));
+        }
+        let _ = write_file(
+            job_dir,
+            "download.config.proto",
+            &NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT
+                .replace("{WORKER_DIR}", &worker_dir)
+                .replace("{CACHE_DIR}", PIP_CACHE_DIR)
+                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
+        )
+        .await?;
+    };
+
+    for req in requirements {
+        // todo: handle many reqs
+        let venv_p = format!("{PIP_CACHE_DIR}/{req}");
+        if metadata(&venv_p).await.is_ok() {
+            req_paths.push(venv_p);
+            continue;
+        }
+
+        #[cfg(feature = "enterprise")]
+        if let Some(ref bucket) = *S3_CACHE_BUCKET {
+            sqlx::query_scalar!("UPDATE queue SET last_ping = now() WHERE id = $1", job_id)
+                .execute(db)
+                .await?;
+            if pull_from_tar(bucket, venv_p.clone()).await.is_ok() {
+                req_paths.push(venv_p.clone());
+                continue;
+            }
+        }
+
+        logs.push_str("\n--- PIP INSTALL ---\n");
+        logs.push_str(&format!("\n{req} is being installed for the first time.\n It will be cached for all ulterior uses."));
+
+        tracing::info!(
+            worker_name = %worker_name,
+            job_id = %job_id,
+            workspace_id = %w_id,
+            "started setup python dependencies"
+        );
+
+        let child = if !*DISABLE_NSJAIL {
+            tracing::info!(
+                worker_name = %worker_name,
+                job_id = %job_id,
+                workspace_id = %w_id,
+                "starting nsjail"
+            );
+            let mut vars = vars.clone();
+            let req = req.to_string();
+            vars.push(("REQ", &req));
+            vars.push(("TARGET", &venv_p));
+            Command::new(NSJAIL_PATH.as_str())
+                .current_dir(job_dir)
+                .env_clear()
+                .envs(vars)
+                .args(vec!["--config", "download.config.proto"])
+                .stdout(Stdio::piped())
+                .stderr(Stdio::piped())
+                .spawn()?
+        } else {
+            let mut args = vec![
+                "-m",
+                "pip",
+                "install",
+                &req,
+                "-I",
+                "--no-deps",
+                "--no-color",
+                "--isolated",
+                "--no-warn-conflicts",
+                "--disable-pip-version-check",
+                "-t",
+                venv_p.as_str(),
+            ];
+            if let Some(url) = PIP_EXTRA_INDEX_URL.as_ref() {
+                args.extend(["--extra-index-url", url]);
+            }
+            if let Some(url) = PIP_INDEX_URL.as_ref() {
+                args.extend(["--index-url", url]);
+            }
+            if let Some(host) = PIP_TRUSTED_HOST.as_ref() {
+                args.extend(["--trusted-host", &host]);
+            }
+            Command::new(PYTHON_PATH.as_str())
+                .env_clear()
+                .env("PATH", PATH_ENV.as_str())
+                .args(args)
+                .stdout(Stdio::piped())
+                .stderr(Stdio::piped())
+                .spawn()?
+        };
+
+        let child = handle_child(
+            &job_id,
+            db,
+            logs,
+            child,
+            false,
+            worker_name,
+            &w_id,
+            &format!("pip install {req}"),
+        )
+        .await;
+        tracing::info!(
+            worker_name = %worker_name,
+            job_id = %job_id,
+            workspace_id = %w_id,
+            is_ok = child.is_ok(),
+            "finished setting up python dependencies {}",
+            job_id
+        );
+        child?;
+
+        #[cfg(feature = "enterprise")]
+        if let Some(ref bucket) = *S3_CACHE_BUCKET {
+            let venv_p = venv_p.clone();
+            tokio::spawn(build_tar_and_push(bucket, venv_p));
+        }
+        req_paths.push(venv_p);
+    }
+    Ok(req_paths)
+}
@@ -19,6 +19,9 @@ use tokio::sync::mpsc::Sender;
 use tracing::instrument;
 use uuid::Uuid;
 use windmill_common::flow_status::{FlowStatusModuleWParent, Iterator, JobResult};
+use windmill_common::jobs::{
+    script_hash_to_tag, script_path_to_payload, JobPayload, QueuedJob, RawCode,
+};
 use windmill_common::{
    error::{self, to_anyhow, Error},
    flow_status::{
@@ -30,9 +33,7 @@ use windmill_common::{

 type DB = sqlx::Pool<sqlx::Postgres>;

-use windmill_queue::{
-    canceled_job_to_result, get_queued_job, push, JobPayload, QueueTransaction, QueuedJob, RawCode,
-};
+use windmill_queue::{canceled_job_to_result, get_queued_job, push, QueueTransaction};

 // #[instrument(level = "trace", skip_all)]
 pub async fn update_flow_status_after_job_completion<
@@ -1315,7 +1316,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
        ContinuePayload::ForloopJobs { n, .. } => *n,
    };
    for i in (0..len).into_iter() {
-        let payload = match &job_payloads {
+        let payload_tag = match &job_payloads {
            ContinuePayload::SingleJob(payload) => payload.clone(),
            ContinuePayload::BranchAllJobs(payloads) => payloads[i].clone(),
            ContinuePayload::ForloopJobs { modules, .. } => {
@@ -1324,13 +1325,16 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
                    failure_module.id_append(&format!("{}/{}", status.step, i));
                    fm = Some(failure_module);
                }
-                JobPayload::RawFlow {
-                    value: FlowValue {
-                        modules: (*modules).clone(),
-                        failure_module: fm.clone(),
-                        same_worker: flow.same_worker,
+                JobPayloadWithTag {
+                    payload: JobPayload::RawFlow {
+                        value: FlowValue {
+                            modules: (*modules).clone(),
+                            failure_module: fm.clone(),
+                            same_worker: flow.same_worker,
+                        },
+                        path: Some(format!("{}/loop-{}", flow_job.script_path(), i)),
                    },
-                    path: Some(format!("{}/loop-{}", flow_job.script_path(), i)),
+                    tag: None,
                }
            }
        };
@@ -1370,10 +1374,11 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
        } else {
            flow_job.root_job.or_else(|| Some(flow_job.id))
        };
+
        let (uuid, inner_tx) = push(
            tx,
            &flow_job.workspace_id,
-            payload,
+            payload_tag.payload,
            ok.unwrap_or_else(|| Map::new()),
            &flow_job.created_by,
            &flow_job.email,
@@ -1386,6 +1391,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
            continue_on_same_worker,
            err,
            flow_job.visible_to_owner,
+            payload_tag.tag,
        )
        .await?;
        tx = inner_tx;
@@ -1604,10 +1610,15 @@ enum NextStatus {
    },
 }

+#[derive(Clone)]
+struct JobPayloadWithTag {
+    payload: JobPayload,
+    tag: Option<String>,
+}
 enum ContinuePayload {
-    SingleJob(JobPayload),
+    SingleJob(JobPayloadWithTag),
    ForloopJobs { n: usize, modules: Vec<FlowModule> },
-    BranchAllJobs(Vec<JobPayload>),
+    BranchAllJobs(Vec<JobPayloadWithTag>),
 }

 enum NextFlowTransform {
@@ -1615,22 +1626,6 @@ enum NextFlowTransform {
    Continue(ContinuePayload, NextStatus),
 }

-// a similar function exists on the backend
-// TODO: rewrite this to use an endpoint in the backend directly, instead of checking for hub itself, and then using the API
-async fn script_path_to_payload<'c>(
-    script_path: &str,
-    db: &mut sqlx::Transaction<'c, sqlx::Postgres>,
-    w_id: &String,
-) -> Result<JobPayload, Error> {
-    let job_payload = if script_path.starts_with("hub/") {
-        JobPayload::ScriptHub { path: script_path.to_owned() }
-    } else {
-        let script_hash = windmill_common::get_latest_hash_for_path(db, w_id, script_path).await?;
-        JobPayload::ScriptHash { hash: script_hash, path: script_path.to_owned() }
-    };
-    Ok(job_payload)
-}
-
 async fn compute_next_flow_transform<'c>(
    flow_job: &QueuedJob,
    flow: &FlowValue,
@@ -1647,7 +1642,10 @@ async fn compute_next_flow_transform<'c>(
        FlowModuleValue::Identity => Ok((
            tx,
            NextFlowTransform::Continue(
-                ContinuePayload::SingleJob(JobPayload::Identity),
+                ContinuePayload::SingleJob(JobPayloadWithTag {
+                    payload: JobPayload::Identity,
+                    tag: None,
+                }),
                NextStatus::NextStep,
            ),
        )),
@@ -1656,41 +1654,46 @@ async fn compute_next_flow_transform<'c>(
            Ok((
                tx,
                NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(payload),
+                    ContinuePayload::SingleJob(JobPayloadWithTag { payload, tag: None }),
                    NextStatus::NextStep,
                ),
            ))
        }
        FlowModuleValue::Script { path: script_path, hash: script_hash, .. } => {
-            let payload = if script_hash.is_none() {
+            let (payload, tag) = if script_hash.is_none() {
                script_path_to_payload(script_path, &mut tx, &flow_job.workspace_id).await?
            } else {
-                JobPayload::ScriptHash {
-                    hash: script_hash.clone().unwrap(),
-                    path: script_path.to_owned(),
-                }
+                let hash = script_hash.clone().unwrap();
+                let tag = script_hash_to_tag(&hash, &mut tx, &flow_job.workspace_id).await?;
+                (
+                    JobPayload::ScriptHash { hash, path: script_path.to_owned() },
+                    tag,
+                )
            };
            Ok((
                tx,
                NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(payload),
+                    ContinuePayload::SingleJob(JobPayloadWithTag { payload, tag }),
                    NextStatus::NextStep,
                ),
            ))
        }
-        FlowModuleValue::RawScript { path, content, language, lock, .. } => {
+        FlowModuleValue::RawScript { path, content, language, lock, tag, .. } => {
            let path = path
                .clone()
                .or_else(|| Some(format!("{}/step-{}", flow_job.script_path(), status.step)));
            Ok((
                tx,
                NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(JobPayload::Code(RawCode {
-                        path,
-                        content: content.clone(),
-                        language: language.clone(),
-                        lock: lock.clone(),
-                    })),
+                    ContinuePayload::SingleJob(JobPayloadWithTag {
+                        payload: JobPayload::Code(RawCode {
+                            path,
+                            content: content.clone(),
+                            language: language.clone(),
+                            lock: lock.clone(),
+                        }),
+                        tag: tag.clone(),
+                    }),
                    NextStatus::NextStep,
                ),
            ))
@@ -1788,13 +1791,20 @@ async fn compute_next_flow_transform<'c>(
                    Ok((
                        tx,
                        NextFlowTransform::Continue(
-                            ContinuePayload::SingleJob(JobPayload::RawFlow {
-                                value: FlowValue {
-                                    modules: (*modules).clone(),
-                                    failure_module: fm,
-                                    same_worker: flow.same_worker,
+                            ContinuePayload::SingleJob(JobPayloadWithTag {
+                                payload: JobPayload::RawFlow {
+                                    value: FlowValue {
+                                        modules: (*modules).clone(),
+                                        failure_module: fm,
+                                        same_worker: flow.same_worker,
+                                    },
+                                    path: Some(format!(
+                                        "{}/loop-{}",
+                                        flow_job.script_path(),
+                                        ns.index
+                                    )),
                                },
-                                path: Some(format!("{}/loop-{}", flow_job.script_path(), ns.index)),
+                                tag: None,
                            }),
                            NextStatus::NextLoopIteration(ns),
                        ),
@@ -1867,17 +1877,20 @@ async fn compute_next_flow_transform<'c>(
            Ok((
                tx,
                NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(JobPayload::RawFlow {
-                        value: FlowValue {
-                            modules,
-                            failure_module: fm,
-                            same_worker: flow.same_worker,
+                    ContinuePayload::SingleJob(JobPayloadWithTag {
+                        payload: JobPayload::RawFlow {
+                            value: FlowValue {
+                                modules,
+                                failure_module: fm,
+                                same_worker: flow.same_worker,
+                            },
+                            path: Some(format!(
+                                "{}/branchone-{}",
+                                flow_job.script_path(),
+                                status.step
+                            )),
                        },
-                        path: Some(format!(
-                            "{}/branchone-{}",
-                            flow_job.script_path(),
-                            status.step
-                        )),
+                        tag: None,
                    }),
                    NextStatus::BranchChosen(branch),
                ),
@@ -1907,17 +1920,20 @@ async fn compute_next_flow_transform<'c>(
                                                    .id_append(&format!("{}/{i}", status.step));
                                                fm = Some(failure_module);
                                            }
-                                            JobPayload::RawFlow {
-                                                value: FlowValue {
-                                                    modules: b.modules.clone(),
-                                                    failure_module: fm.clone(),
-                                                    same_worker: flow.same_worker,
+                                            JobPayloadWithTag {
+                                                payload: JobPayload::RawFlow {
+                                                    value: FlowValue {
+                                                        modules: b.modules.clone(),
+                                                        failure_module: fm.clone(),
+                                                        same_worker: flow.same_worker,
+                                                    },
+                                                    path: Some(format!(
+                                                        "{}/branchall-{}",
+                                                        flow_job.script_path(),
+                                                        i
+                                                    )),
                                                },
-                                                path: Some(format!(
-                                                    "{}/branchall-{}",
-                                                    flow_job.script_path(),
-                                                    i
-                                                )),
+                                                tag: None,
                                            }
                                        })
                                        .collect(),
@@ -1978,17 +1994,20 @@ async fn compute_next_flow_transform<'c>(
            Ok((
                tx,
                NextFlowTransform::Continue(
-                    ContinuePayload::SingleJob(JobPayload::RawFlow {
-                        value: FlowValue {
-                            modules,
-                            failure_module: fm.clone(),
-                            same_worker: flow.same_worker,
+                    ContinuePayload::SingleJob(JobPayloadWithTag {
+                        payload: JobPayload::RawFlow {
+                            value: FlowValue {
+                                modules,
+                                failure_module: fm.clone(),
+                                same_worker: flow.same_worker,
+                            },
+                            path: Some(format!(
+                                "{}/branchall-{}",
+                                flow_job.script_path(),
+                                branch_status.branch
+                            )),
                        },
-                        path: Some(format!(
-                            "{}/branchall-{}",
-                            flow_job.script_path(),
-                            branch_status.branch
-                        )),
+                        tag: None,
                    }),
                    NextStatus::NextBranchStep(NextBranch { status: branch_status, flow_jobs }),
                ),
@@ -48,6 +48,9 @@ wmill sync pull
 wmill sync push
 ```

+We recommend using the --yaml option to use yaml instead of json as the encoding
+format. Yaml will be made the default soon.
+
 ### Pushing individual files

 You can push individual resources using
@@ -1,120 +1,65 @@
+// deno-lint-ignore-file no-explicit-any
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
-import { Any, decoverto, model, property } from "./decoverto.ts";
 import {
  AppService,
  AppWithLastVersion,
  colors,
  Command,
  ListableApp,
-  microdiff,
  Policy,
  Table,
 } from "./deps.ts";
 import {
-  Difference,
  GlobalOptions,
-  PushDiffs,
-  Resource,
-  setValueByPath,
+  isSuperset,
+  parseFromFile,
+  removeType,
 } from "./types.ts";

-@model()
-export class AppFile implements Resource, PushDiffs {
-  @property(Any)
+export interface AppFile {
  value: any;
-  @property(() => String)
  summary: string;
-  @property(Any)
  policy: Policy;
+}

-  constructor(value: string, summary: string, policy: Policy) {
-    this.value = value;
-    this.summary = summary;
-    this.policy = policy;
-  }
-  async pushDiffs(
-    workspace: string,
-    remotePath: string,
-    diffs: Difference[]
-  ): Promise<void> {
-    let app: AppWithLastVersion | undefined = undefined;
+export async function pushApp(
+  workspace: string,
+  remotePath: string,
+  app: AppFile | AppWithLastVersion | undefined,
+  newApp: AppFile,
+  raw: boolean
+): Promise<void> {
+  remotePath = removeType(remotePath, "app");
+  if (raw) {
+    // deleting old app if it exists in raw mode
    try {
      app = await AppService.getAppByPath({ workspace, path: remotePath });
-    } catch (e) {}
-
-    if (app) {
-      console.log(
-        colors.bold.yellow(
-          `Applying ${diffs.length} diffs to existing app... ${remotePath}`
-        )
-      );
-      const changeset: {
-        summary?: string | undefined;
-        value?: any;
-        policy?: Policy | undefined;
-      } = {};
-      for (const diff of diffs) {
-        if (
-          diff.type !== "REMOVE" &&
-          diff.path[0] !== "value" &&
-          diff.path[0] !== "policy" &&
-          (diff.path.length !== 1 ||
-            !["summary"].includes(diff.path[0] as string))
-        ) {
-          throw new Error("Invalid app diff with path " + diff.path);
-        }
-        if (diff.type === "CREATE" || diff.type === "CHANGE") {
-          setValueByPath(changeset, diff.path, diff.value);
-        } else if (diff.type === "REMOVE") {
-          setValueByPath(changeset, diff.path, null);
-        }
-      }
-
-      if (
-        (!changeset?.policy ||
-          JSON.stringify(changeset?.policy) == JSON.stringify(app.policy)) &&
-        (!changeset?.value ||
-          JSON.stringify(changeset?.value) == JSON.stringify(app.value)) &&
-        (!changeset?.summary || changeset.summary == app.summary)
-      ) {
-        console.log(
-          colors.yellow(`No changes to push for app ${remotePath}, skipping`)
-        );
-        return;
-      }
-
-      const hasChanges = Object.values(changeset).some(
-        (v) => v !== null && typeof v !== "undefined"
-      );
-      if (!hasChanges) {
-        return;
-      }
-
-      await AppService.updateApp({
-        workspace,
-        path: remotePath,
-        requestBody: changeset,
-      });
-    } else {
-      console.log(colors.yellow.bold("Creating new app..."));
-
-      await AppService.createApp({
-        workspace,
-        requestBody: {
-          path: remotePath,
-          policy: this.policy,
-          summary: this.summary,
-          value: this.value,
-        },
-      });
+    } catch {
+      //ignore
    }
  }
-  async push(workspace: string, remotePath: string): Promise<void> {
-    await this.pushDiffs(
+
+  if (app) {
+    if (isSuperset(newApp, app)) {
+      return;
+    }
+    await AppService.updateApp({
      workspace,
-      remotePath,
-      microdiff({}, this, { cyclesFix: false })
-    );
+      path: remotePath,
+      requestBody: {
+        ...newApp,
+      },
+    });
+  } else {
+    console.log(colors.yellow.bold("Creating new app..."));
+
+    await AppService.createApp({
+      workspace,
+      requestBody: {
+        path: remotePath,
+        ...newApp,
+      },
+    });
  }
 }

@@ -154,19 +99,24 @@ async function push(opts: GlobalOptions, filePath: string) {
  const workspace = await resolveWorkspace(opts);
  await requireLogin(opts);

-  await pushApp(filePath, workspace.workspaceId, remotePath);
-  console.log(colors.bold.underline.green("App pushed"));
-}
+  let app: AppWithLastVersion | undefined = undefined;
+  try {
+    app = await AppService.getAppByPath({
+      workspace: workspace.workspaceId,
+      path: remotePath,
+    });
+  } catch {
+    // app doesn't exist
+  }

-export async function pushApp(
-  filePath: string,
-  workspace: string,
-  remotePath: string
-) {
-  const data = decoverto
-    .type(AppFile)
-    .rawToInstance(await Deno.readTextFile(filePath));
-  await data.push(workspace, remotePath);
+  await pushApp(
+    workspace.workspaceId,
+    remotePath,
+    app,
+    parseFromFile(filePath),
+    false
+  );
+  console.log(colors.bold.underline.green("App pushed"));
 }

 const command = new Command()
@@ -1,5 +1,5 @@
 // deno-lint-ignore-file no-explicit-any
-import { colors, GlobalUserInfo, setClient, UserService } from "./deps.ts";
+import { colors, GlobalUserInfo, log, setClient, UserService } from "./deps.ts";
 import { loginInteractive, tryGetLoginInfo } from "./login.ts";
 import { GlobalOptions } from "./types.ts";
 import {
@@ -53,7 +53,7 @@ export async function resolveWorkspace(
 ): Promise<Workspace> {
  const res = await tryResolveWorkspace(opts);
  if (res.isError) {
-    console.log(colors.red.bold(res.error));
+    log.info(colors.red.bold(res.error));
    return Deno.exit(-1);
  } else {
    return res.value;
@@ -75,7 +75,7 @@ export async function requireLogin(
  try {
    return await UserService.globalWhoami();
  } catch {
-    console.log(
+    log.info(
      "! Could not reach API given existing credentials. Attempting to reauth..."
    );
    const newToken = await loginInteractive(workspace.remote);
@@ -119,7 +119,7 @@ export async function tryResolveVersion(

 export function validatePath(path: string): boolean {
  if (!(path.startsWith("g") || path.startsWith("u") || path.startsWith("f"))) {
-    console.log(
+    log.info(
      colors.red(
        "Given remote path looks invalid. Remote paths are typically of the form <u|g|f>/<username|group|folder>/..."
      )
@@ -1,8 +0,0 @@
-// globally shared decoverto instance
-import { Decoverto } from "npm:decoverto";
-const decoverto = new Decoverto();
-
-// TODO: Properly type FlowModule
-
-export { Any, array, map, MapShape, model, property } from "npm:decoverto";
-export { decoverto };
@@ -1,6 +1,6 @@
 // windmill
-export { setClient } from "https://deno.land/x/windmill@v1.69.3/mod.ts";
-export * from "https://deno.land/x/windmill@v1.69.3/windmill-api/index.ts";
+export { setClient } from "https://deno.land/x/windmill@v1.95.1/mod.ts";
+export * from "https://deno.land/x/windmill@v1.95.1/windmill-api/index.ts";

 // cliffy
 export { Command } from "https://deno.land/x/cliffy@v0.25.7/command/command.ts";
@@ -37,3 +37,4 @@ export { default as microdiff } from "https://deno.land/x/microdiff@v1.3.1/index
 export { default as objectHash } from "https://deno.land/x/object_hash@2.0.3.1/mod.ts";
 export { default as gitignore_parser } from "npm:gitignore-parser";
 export { default as JSZip } from "npm:jszip@3.7.1";
+export * as log from "https://deno.land/std@0.186.0/log/mod.ts";
@@ -1,121 +1,95 @@
 // deno-lint-ignore-file no-explicit-any
-import {
-  Difference,
-  GlobalOptions,
-  PushDiffs,
-  Resource,
-  setValueByPath,
-} from "./types.ts";
+import { GlobalOptions, isSuperset } from "./types.ts";
+import { parse as yamlParse } from "https://deno.land/std@0.184.0/yaml/mod.ts";
+
 import {
  colors,
  Command,
  Flow,
+  FlowModule,
  FlowService,
  JobService,
-  microdiff,
-  OpenFlowWPath,
  Table,
 } from "./deps.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import { resolve, track_job } from "./script.ts";
-import { Any, decoverto, model, property } from "./decoverto.ts";

-// this is effectively "OpenFlow" but a copy as it is accepted by the CLI
-@model()
-export class FlowFile implements Resource, PushDiffs {
-  @property(() => String)
+export interface FlowFile {
  summary: string;
-  @property(() => String)
  description?: string;
-  @property(Any)
  value: any;
-  @property(Any)
  schema?: any;
+}

-  constructor(value: any, summary?: string) {
-    this.summary = summary ?? "";
-    this.value = value;
+const alreadySynced: string[] = [];
+
+export async function pushFlow(
+  workspace: string,
+  remotePath: string,
+  localFlowPath: string,
+  workspaceId: string
+): Promise<void> {
+  if (alreadySynced.includes(localFlowPath)) {
+    return;
+  }
+  alreadySynced.push(localFlowPath);
+  let flow: Flow | undefined = undefined;
+  try {
+    flow = await FlowService.getFlowByPath({
+      workspace: workspaceId,
+      path: remotePath,
+    });
+  } catch {
+    // flow doesn't exist
  }
-  async pushDiffs(
-    workspace: string,
-    remotePath: string,
-    diffs: Difference[]
-  ): Promise<void> {
-    if (
-      await FlowService.existsFlowByPath({
-        workspace: workspace,
-        path: remotePath,
-      })
-    ) {
-      console.log(
-        colors.bold.yellow(
-          `Applying ${diffs.length} diffs to existing flow... ${remotePath}`
-        )
-      );

-      // TODO: Make these optional in backend (not path ofc)
-      const changeset: OpenFlowWPath = {
-        path: remotePath,
-        summary: this.summary,
-        value: this.value,
-        description: this.description, // This is OpenAPIed as optional, but isn't
-        schema: this.schema, // Same
-      };
-      const base_changeset = { ...changeset };
-      for (const diff of diffs) {
-        if (
-          diff.type !== "REMOVE" &&
-          diff.path[0] !== "value" &&
-          (diff.path.length !== 1 ||
-            !["summary", "description", "schema"].includes(
-              diff.path[0] as string
-            ))
-        ) {
-          throw new Error("Invalid flow diff with path " + diff.path);
-        }
-        if (diff.type === "CREATE" || diff.type === "CHANGE") {
-          setValueByPath(changeset, diff.path, diff.value);
-        } else if (diff.type === "REMOVE") {
-          setValueByPath(changeset, diff.path, null);
-        }
-      }
-      const hasChanges = Object.values(changeset).some(
-        (v) => v !== null && typeof v !== "undefined"
-      );
-      if (!hasChanges) {
-        return;
+  if (!localFlowPath.endsWith("/")) {
+    localFlowPath += "/";
+  }
+  const localFlowRaw = await Deno.readTextFile(localFlowPath + "flow.yaml");
+  const localFlow = yamlParse(localFlowRaw) as FlowFile;
+
+  function replaceInlineScripts(modules: FlowModule[]) {
+    modules.forEach((m) => {
+      if (m.value.type == "rawscript") {
+        const path = m.value.content.split(" ")[1];
+        m.value.content = Deno.readTextFileSync(localFlowPath + path);
+      } else if (m.value.type == "forloopflow") {
+        replaceInlineScripts(m.value.modules);
+      } else if (m.value.type == "branchall") {
+        m.value.branches.forEach((b) => replaceInlineScripts(b.modules));
+      } else if (m.value.type == "branchone") {
+        m.value.branches.forEach((b) => replaceInlineScripts(b.modules));
+        replaceInlineScripts(m.value.default);
      }
+    });
+  }

-      const update = {
-        ...changeset,
-        ...base_changeset,
-      };
+  replaceInlineScripts(localFlow.value.modules);

-      await FlowService.updateFlow({
-        workspace: workspace,
-        path: remotePath,
-        requestBody: update,
-      });
-    } else {
-      console.log(colors.bold.yellow("Creating new flow..."));
-      await FlowService.createFlow({
-        workspace: workspace,
-        requestBody: {
-          path: remotePath,
-          summary: this.summary,
-          value: this.value,
-          schema: this.schema,
-          description: this.description,
-        },
-      });
+  if (flow) {
+    if (isSuperset(localFlow, flow)) {
+      console.log(colors.bold.green("Flow is up to date"));
+      return;
    }
-  }
-  async push(workspace: string, remotePath: string): Promise<void> {
-    await this.pushDiffs(
-      workspace,
-      remotePath,
-      microdiff({}, this, { cyclesFix: false })
-    );
+    console.log(colors.bold.yellow(`Updating flow ${remotePath}...`));
+    await FlowService.updateFlow({
+      workspace: workspace,
+      path: remotePath,
+      requestBody: {
+        path: remotePath,
+        ...localFlow,
+      },
+    });
+  } else {
+    console.log(colors.bold.yellow("Creating new flow..."));
+    await FlowService.createFlow({
+      workspace: workspace,
+      requestBody: {
+        path: remotePath,
+        ...localFlow,
+      },
+    });
  }
 }

@@ -128,21 +102,15 @@ async function push(opts: Options, filePath: string, remotePath: string) {
  const workspace = await resolveWorkspace(opts);
  await requireLogin(opts);

-  await pushFlow(filePath, workspace.workspaceId, remotePath);
+  await pushFlow(
+    workspace.workspaceId,
+    remotePath,
+    filePath,
+    workspace.workspaceId
+  );
  console.log(colors.bold.underline.green("Flow pushed"));
 }

-export async function pushFlow(
-  filePath: string,
-  workspace: string,
-  remotePath: string
-) {
-  const data = decoverto
-    .type(FlowFile)
-    .rawToInstance(await Deno.readTextFile(filePath));
-  await data.push(workspace, remotePath);
-}
-
 async function list(opts: GlobalOptions & { showArchived?: boolean }) {
  const workspace = await resolveWorkspace(opts);
  await requireLogin(opts);
@@ -1,135 +1,85 @@
-import { colors, Command, FolderService, microdiff } from "./deps.ts";
+// deno-lint-ignore-file no-explicit-any
+import { colors, Command, Folder, FolderService, log, Table } from "./deps.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
-import {
-  Difference,
-  GlobalOptions,
-  PushDiffs,
-  Resource,
-  setValueByPath,
-} from "./types.ts";
-import {
-  array,
-  decoverto,
-  map,
-  MapShape,
-  model,
-  property,
-} from "./decoverto.ts";
+import { GlobalOptions, isSuperset, parseFromFile } from "./types.ts";

-@model()
-export class FolderFile implements Resource, PushDiffs {
-  @property(array(() => String))
+export interface FolderFile {
  owners: Array<string> | undefined;
-  @property(
-    map(
-      () => String,
-      () => Boolean,
-      { shape: MapShape.Object }
-    )
-  )
  extra_perms: Map<string, boolean> | undefined;
-  @property(() => String)
  display_name: string | undefined;
+}

-  async push(workspace: string, remotePath: string): Promise<void> {
-    if (remotePath.startsWith("/")) {
-      remotePath = remotePath.substring(1);
-    }
-    if (remotePath.startsWith("f/")) {
-      remotePath = remotePath.substring(2);
-    }
+async function list(opts: GlobalOptions) {
+  const workspace = await resolveWorkspace(opts);
+  await requireLogin(opts);

-    await this.pushDiffs(
-      workspace,
-      remotePath,
-      microdiff({}, this, { cyclesFix: false })
-    );
+  const folders = await FolderService.listFolders({
+    workspace: workspace.workspaceId,
+  });
+
+  new Table()
+    .header(["Name", "Owners", "Extra Perms"])
+    .padding(2)
+    .border(true)
+    .body(
+      folders.map((x) => [
+        x.name,
+        x.owners?.join(",") ?? "-",
+        JSON.stringify(x.extra_perms ?? {}),
+      ])
+    )
+    .render();
+}
+
+export async function pushFolder(
+  workspace: string,
+  name: string,
+  folder: Folder | FolderFile | undefined,
+  localFolder: FolderFile,
+  raw: boolean
+): Promise<void> {
+  if (name.startsWith("/")) {
+    name = name.substring(1);
+  }
+  if (name.startsWith("f/")) {
+    name = name.substring(2);
+  }
+  name = name.split("/")[0];
+  log.debug(`Processing local folder ${name}`);
+
+  if (raw) {
+    // deleting old app if it exists in raw mode
+    try {
+      folder = await FolderService.getFolder({ workspace, name });
+      log.debug(`Folder ${name} exists on remote`);
+    } catch {
+      log.debug(`Folder ${name} does not exist on remote`);
+      //ignore
+    }
  }

-  async pushDiffs(
-    workspace: string,
-    remotePath: string,
-    diffs: Difference[]
-  ): Promise<void> {
-    if (remotePath.startsWith("/")) {
-      remotePath = remotePath.substring(1);
-    }
-    if (remotePath.startsWith("f/")) {
-      remotePath = remotePath.substring(2);
-    }
-
-    // TODO: Support this in backend
-    let exists: boolean;
-    try {
-      exists = !!(await FolderService.getFolder({
-        workspace,
-        name: remotePath,
-      }));
-    } catch {
-      exists = false;
-    }
-    if (exists) {
-      console.log(
-        colors.bold.yellow(
-          `Applying ${diffs.length} diffs to existing folder... ${remotePath}`
-        )
-      );
-
-      const changeset: {
-        owners?: string[] | undefined;
-        extra_perms?: any;
-        display_name?: string | undefined;
-      } = {};
-      for (const diff of diffs) {
-        if (
-          diff.type !== "REMOVE" &&
-          (diff.path.length !== 1 ||
-            !["owners", "extra_perms", "display_name"].includes(
-              diff.path[0] as string
-            ))
-        ) {
-          console.log(diff.path);
-          throw new Error("Invalid folder diff with path " + diff.path);
-        }
-        if (diff.type === "CREATE" || diff.type === "CHANGE") {
-          setValueByPath(changeset, diff.path, diff.value);
-        } else if (diff.type === "REMOVE") {
-          setValueByPath(changeset, diff.path, null);
-        }
-      }
-
-      const hasChanges = Object.values(changeset).some(
-        (v) => v !== null && typeof v !== "undefined"
-      );
-      if (!hasChanges) {
-        return;
-      }
-      try {
-        await FolderService.updateFolder({
-          workspace: workspace,
-          name: remotePath,
-          requestBody: {
-            ...changeset,
-            extra_perms: changeset.extra_perms
-              ? Object.fromEntries(this.extra_perms?.entries() ?? [])
-              : undefined,
-          },
-        });
-      } catch (e) {
-        console.error(colors.red.bold(e.body));
-        throw e;
-      }
-    } else {
-      console.log(colors.bold.yellow("Creating new folder: " + remotePath));
-      await FolderService.createFolder({
-        workspace: workspace,
-        requestBody: {
-          name: remotePath,
-          extra_perms: Object.fromEntries(this.extra_perms?.entries() ?? []),
-          owners: this.owners,
-        },
-      });
+  if (folder) {
+    if (isSuperset(localFolder, folder)) {
+      log.debug(`Folder ${name} is up to date`);
+      return;
    }
+    log.debug(`Folder ${name} is not up-to-date, updating...`);
+    await FolderService.updateFolder({
+      workspace: workspace,
+      name: name,
+      requestBody: {
+        ...localFolder,
+      },
+    });
+  } else {
+    console.log(colors.bold.yellow("Creating new folder: " + name));
+    await FolderService.createFolder({
+      workspace: workspace,
+      requestBody: {
+        name: name,
+        ...localFolder,
+      },
+    });
  }
 }

@@ -146,25 +96,21 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) {
    throw new Error("file path must refer to a file.");
  }

-  console.log(colors.bold.yellow("Pushing resource..."));
+  console.log(colors.bold.yellow("Pushing folder..."));

-  await pushFolder(workspace.workspaceId, filePath, remotePath);
-  console.log(colors.bold.underline.green("Resource pushed"));
-}
-
-export async function pushFolder(
-  workspace: string,
-  filePath: string,
-  remotePath: string
-) {
-  const data = decoverto
-    .type(FolderFile)
-    .rawToInstance(await Deno.readTextFile(filePath));
-  data.push(workspace, remotePath);
+  await pushFolder(
+    workspace.workspaceId,
+    remotePath,
+    undefined,
+    parseFromFile(filePath),
+    false
+  );
+  console.log(colors.bold.underline.green("Folder pushed"));
 }

 const command = new Command()
-  .description("resource related commands")
+  .description("folder related commands")
+  .action(list as any)
  .command(
    "push",
    "push a local folder spec. This overrides any remote versions."
@@ -1,14 +1,15 @@
-import { Command } from "./deps.ts";
+// deno-lint-ignore-file no-explicit-any
+import { Command, ResourceService, log } from "./deps.ts";
 import { requireLogin, resolveWorkspace } from "./context.ts";
-import { ResourceTypeFile } from "./resource-type.ts";
+import { pushResourceType } from "./resource-type.ts";
 import { GlobalOptions } from "./types.ts";

 async function pull(opts: GlobalOptions) {
  const workspace = await resolveWorkspace(opts);

  if (workspace.workspaceId !== "admins") {
-    console.log(
-      "Should only sync to admins workspace, but current is not admins.",
+    log.info(
+      "Should only sync to admins workspace, but current is not admins."
    );
    return;
  }
@@ -24,15 +25,12 @@ async function pull(opts: GlobalOptions) {
    created_by: string;
    created_at: Date;
    comments: never[];
-  }[] = await fetch(
-    "https://hub.windmill.dev/resource_types/list",
-    {
-      headers: {
-        "Accept": "application/json",
-        "X-email": userInfo.email,
-      },
+  }[] = await fetch("https://hub.windmill.dev/resource_types/list", {
+    headers: {
+      Accept: "application/json",
+      "X-email": userInfo.email,
    },
-  )
+  })
    .then((r) => r.json())
    .then((list: { id: number; name: string }[]) =>
      list.map((x) =>
@@ -40,9 +38,9 @@ async function pull(opts: GlobalOptions) {
          "https://hub.windmill.dev/resource_types/" + x.id + "/" + x.name,
          {
            headers: {
-              "Accept": "application/json",
+              Accept: "application/json",
            },
-          },
+          }
        )
      )
    )
@@ -50,7 +48,7 @@ async function pull(opts: GlobalOptions) {
    .then((x) =>
      x.map((x) =>
        x.json().catch((e) => {
-          console.log(e);
+          log.info(e);
          return undefined;
        })
      )
@@ -58,14 +56,23 @@ async function pull(opts: GlobalOptions) {
    .then((x) => Promise.all(x))
    .then((x) => x.filter((x) => x).map((x) => x.resource_type));

-  for (
-    const x of list
-  ) {
-    console.log("syncing " + x.name);
-    const f = new ResourceTypeFile();
-    f.description = x.description;
-    f.schema = JSON.parse(x.schema);
-    await f.push(workspace.workspaceId, x.name);
+  const resourceTypes = await ResourceService.listResourceType({
+    workspace: workspace.workspaceId,
+  });
+
+  for (const x of list) {
+    if (resourceTypes.find((y) => y.name === x.name)) {
+      log.info("skipping " + x.name);
+      continue;
+    }
+    log.info("syncing " + x.name);
+    await pushResourceType(
+      workspace.workspaceId,
+      x.name + ".resource-type.json",
+      undefined,
+      x,
+      false
+    );
  }
 }

@@ -1,11 +1,11 @@
 import { GlobalOptions } from "./types.ts";
-import { colors, getAvailablePort, Secret, Select } from "./deps.ts";
-import { open } from 'https://deno.land/x/open/index.ts';
+import { colors, getAvailablePort, log, Secret, Select } from "./deps.ts";
+import { open } from "https://deno.land/x/open/index.ts";

 export async function loginInteractive(remote: string) {
  let token: string | undefined;
  if (
-    await Select.prompt({
+    (await Select.prompt({
      message: "How do you want to login",
      options: [
        {
@@ -17,7 +17,7 @@ export async function loginInteractive(remote: string) {
          value: "t",
        },
      ],
-    }) === "b"
+    })) === "b"
  ) {
    token = await browserLogin(remote);
  } else {
@@ -29,7 +29,7 @@ export async function loginInteractive(remote: string) {

 // deno-lint-ignore require-await
 export async function tryGetLoginInfo(
-  opts: GlobalOptions,
+  opts: GlobalOptions
 ): Promise<string | undefined> {
  if (opts.token) {
    return opts.token;
@@ -39,22 +39,22 @@ export async function tryGetLoginInfo(
 }

 export async function browserLogin(
-  baseUrl: string,
+  baseUrl: string
 ): Promise<string | undefined> {
  const port = await getAvailablePort();
  if (port == undefined) {
-    console.log(colors.red.underline("failed to aquire port"));
+    log.info(colors.red.underline("failed to aquire port"));
    return undefined;
  }

  const server = Deno.listen({ transport: "tcp", port });
-  const url = `${baseUrl}user/cli?port=${port}`
-  console.log(`Login by going to ${url}`);
+  const url = `${baseUrl}user/cli?port=${port}`;
+  log.info(`Login by going to ${url}`);
  try {
-    await open(url)
-    console.log("Opened browser for you");
+    await open(url);
+    log.info("Opened browser for you");
  } catch {
-    console.error(`Failed to open browser, please navigate to ${url}`)
+    console.error(`Failed to open browser, please navigate to ${url}`);
  }
  const firstConnection = await server.accept();
  const httpFirstConnection = Deno.serveHttp(firstConnection);
@@ -63,7 +63,7 @@ export async function browserLogin(
  const token = params.get("token");
  // const _workspace = params.get("workspace");
  await firstRequest?.respondWith(
-    Response.redirect(baseUrl + "user/cli-success", 302),
+    Response.redirect(baseUrl + "user/cli-success", 302)
  );

  setTimeout(() => {
@@ -18,8 +18,16 @@ import folder from "./folder.ts";
 import sync from "./sync.ts";
 import { tryResolveVersion } from "./context.ts";
 import { GlobalOptions } from "./types.ts";
+import * as log from "https://deno.land/std@0.186.0/log/mod.ts";

-export const VERSION = "v1.87.0";
+addEventListener("error", (event) => {
+  if (event.error) {
+    console.error("Error details of: " + event.error.message);
+    console.error(JSON.stringify(event.error, null, 4));
+  }
+});
+
+export const VERSION = "v1.96.3";

 let command: any = new Command()
  .name("wmill")
@@ -28,6 +36,11 @@ let command: any = new Command()
    "--workspace <workspace:string>",
    "Specify the target workspace. This overrides the default workspace."
  )
+  .globalOption("--debug", "Show debug logs")
+  .globalOption(
+    "--show-diffs",
+    "Show diff informations when syncing (may show sensitive informations)"
+  )
  .globalOption(
    "--token <token:string>",
    "Specify an API token. This will override any stored token."
@@ -73,7 +86,26 @@ if (Number.parseInt(VERSION.replace("v", "").replace(".", "")) > 1700) {
  command = command.command("push", push).command("pull", pull);
 }

+export let showDiffs = false;
 try {
+  const LOG_LEVEL = Deno.args.includes("--debug") ? "DEBUG" : "INFO";
+  // const NO_COLORS = Deno.args.includes("--no-colors");
+  showDiffs = Deno.args.includes("--show-diffs");
+
+  log.setup({
+    handlers: {
+      console: new log.handlers.ConsoleHandler(LOG_LEVEL, {
+        formatter: "{msg}",
+      }),
+    },
+    loggers: {
+      default: {
+        level: LOG_LEVEL,
+        handlers: ["console"],
+      },
+    },
+  });
+  log.debug("Debug logging enabled. CLI build against " + VERSION);
  await command.parse(Deno.args);
 } catch (e) {
  if (e.name === "ApiError") {
@@ -30,7 +30,7 @@ export async function downloadZip(
    throw new Error(await zipResponse.text());
  }
  const blob = await zipResponse.blob();
-  return await JSZip.loadAsync(blob);
+  return await JSZip.loadAsync(blob as any);
 }

 async function stub(
@@ -2,14 +2,11 @@
 import { colors, Command } from "./deps.ts";
 import { GlobalOptions } from "./types.ts";

-async function stub(
-  _opts: GlobalOptions,
-  _dir?: string,
-) {
-  console.log(
+async function stub(_opts: GlobalOptions, _dir?: string) {
+  log.info(
    colors.red.underline(
-      'Push is deprecated. Use "sync push --raw" instead. See <TODO_LINK_HERE> for more information.',
-    ),
+      'Push is deprecated. Use "sync push --raw" instead. See <TODO_LINK_HERE> for more information.'
+    )
  );
 }

@@ -1,118 +1,65 @@
 // deno-lint-ignore-file no-explicit-any
 import {
-  Difference,
  GlobalOptions,
-  PushDiffs,
-  Resource as ResourceI,
-  setValueByPath,
+  isSuperset,
+  parseFromFile,
+  removeType,
 } from "./types.ts";
 import { requireLogin, resolveWorkspace } from "./context.ts";
 import {
  colors,
  Command,
-  EditResourceType,
-  microdiff,
+  log,
  ResourceService,
+  ResourceType,
  Table,
 } from "./deps.ts";
-import { Any, decoverto, model, property } from "./decoverto.ts";

-@model()
-export class ResourceTypeFile implements ResourceI, PushDiffs {
-  @property(Any)
+export interface ResourceTypeFile {
  schema?: any;
-  @property(() => String)
  description?: string;
-
-  async push(workspace: string, remotePath: string): Promise<void> {
-    await this.pushDiffs(
-      workspace,
-      remotePath,
-      microdiff({}, this, { cyclesFix: false }),
-    );
-  }
-
-  async pushDiffs(
-    workspace: string,
-    remotePath: string,
-    diffs: Difference[],
-  ): Promise<void> {
-    if (
-      await ResourceService.existsResourceType({
-        workspace: workspace,
-        path: remotePath,
-      })
-    ) {
-      if (
-        (await ResourceService.listResourceType({ workspace })).findIndex((x) =>
-          x.name === remotePath
-        ) === -1
-      ) {
-        console.log(
-          "Resource type " + remotePath +
-          " is already taken for the current workspace, but cannot be updated. Is this a conflict with starter?",
-        );
-        return;
-      }
-      console.log(
-        colors.yellow.bold(
-          `Applying ${diffs.length} diffs to existing resource type... ${remotePath}`,
-        ),
-      );
-      const changeset: EditResourceType = {};
-      for (const diff of diffs) {
-        if (
-          diff.type !== "REMOVE" &&
-          (
-            diff.path.length !== 1 ||
-            !["schema", "description"].includes(diff.path[0] as string)
-          )
-        ) {
-          throw new Error("Invalid resource type diff with path " + diff.path);
-        }
-        if (diff.type === "CREATE" || diff.type === "CHANGE") {
-          setValueByPath(changeset, diff.path, diff.value);
-        } else if (diff.type === "REMOVE") {
-          setValueByPath(changeset, diff.path, null);
-        }
-      }
-
-      const hasChanges = Object.values(changeset).some((v) =>
-        v !== null && typeof v !== "undefined"
-      );
-      if (!hasChanges) {
-        return;
-      }
-
-      await ResourceService.updateResourceType({
-        workspace: workspace,
-        path: remotePath,
-        requestBody: changeset,
-      });
-    } else {
-      console.log(colors.yellow.bold("Creating new resource type..."));
-      await ResourceService.createResourceType({
-        workspace: workspace,
-        requestBody: {
-          name: remotePath,
-          description: this.description,
-          schema: this.schema,
-          workspace_id: workspace,
-        },
-      });
-    }
-  }
 }

 export async function pushResourceType(
  workspace: string,
-  filePath: string,
-  name: string,
-) {
-  const data: ResourceTypeFile = decoverto.type(ResourceTypeFile).rawToInstance(
-    await Deno.readTextFile(filePath),
-  );
-  await data.push(workspace, name);
+  remotePath: string,
+  resource: ResourceTypeFile | ResourceType | undefined,
+  localResource: ResourceTypeFile,
+  raw: boolean
+): Promise<void> {
+  remotePath = removeType(remotePath, "resource-type");
+  if (raw) {
+    try {
+      resource = await ResourceService.getResourceType({
+        workspace: workspace,
+        path: remotePath,
+      });
+    } catch {
+      // resource type doesn't exist
+    }
+  }
+  if (resource) {
+    if (isSuperset(localResource, resource)) {
+      return;
+    }
+
+    await ResourceService.updateResourceType({
+      workspace: workspace,
+      path: remotePath,
+      requestBody: {
+        ...localResource,
+      },
+    });
+  } else {
+    log.info(colors.yellow.bold("Creating new resource type..."));
+    await ResourceService.createResourceType({
+      workspace: workspace,
+      requestBody: {
+        name: remotePath,
+        ...localResource,
+      },
+    });
+  }
 }

 type PushOptions = GlobalOptions;
@@ -124,10 +71,16 @@ async function push(opts: PushOptions, filePath: string, name: string) {
  const workspace = await resolveWorkspace(opts);
  await requireLogin(opts);

-  console.log(colors.bold.yellow("Pushing resource..."));
+  log.info(colors.bold.yellow("Pushing resource..."));

-  await pushResourceType(workspace.workspaceId, filePath, name);
-  console.log(colors.bold.underline.green("Resource pushed"));
+  await pushResourceType(
+    workspace.workspaceId,
+    name,
+    undefined,
+    parseFromFile(filePath),
+    true
+  );
+  log.info(colors.bold.underline.green("Resource pushed"));
 }

 async function list(opts: GlobalOptions) {
@@ -150,7 +103,7 @@ const command = new Command()
  .action(list as any)
  .command(
    "push",
-    "push a local resource spec. This overrides any remote versions.",
+    "push a local resource spec. This overrides any remote versions."
  )
  .arguments("<file_path:string> <name:string>")
  .action(push as any);
@@ -1,130 +1,73 @@
+// deno-lint-ignore-file no-explicit-any
 import {
-  Difference,
  GlobalOptions,
-  PushDiffs,
-  Resource as Resource2,
-  setValueByPath,
+  isSuperset,
+  parseFromFile,
+  removeType,
 } from "./types.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import {
  colors,
  Command,
-  EditResource,
-  microdiff,
+  log,
  Resource,
  ResourceService,
  Table,
 } from "./deps.ts";
-import { Any, decoverto, model, property } from "./decoverto.ts";

-@model()
-export class ResourceFile implements Resource2, PushDiffs {
-  @property(Any)
-  value?: any;
-  @property(() => String)
+export interface ResourceFile {
+  value: any;
  description?: string;
-  @property(() => String)
  resource_type: string;
-  @property(() => Boolean)
  is_oauth?: boolean; // deprecated
-
-  constructor(resource_type: string) {
-    this.resource_type = resource_type;
-  }
-  async pushDiffs(
-    workspace: string,
-    remotePath: string,
-    diffs: Difference[]
-  ): Promise<void> {
-    if (
-      await ResourceService.existsResource({
-        workspace: workspace,
-        path: remotePath,
-      })
-    ) {
-      console.log(
-        colors.yellow.bold(
-          `Applying ${diffs.length} diffs to existing resource... ${remotePath}`
-        )
-      );
-
-      const changeset: EditResource = {
-        path: remotePath, // TODO: Remove this in backend
-      };
-      for (const diff of diffs) {
-        if (diff.path[0] === "is_oauth") {
-          //is_oauth is not updatable
-          continue;
-        }
-        if (
-          diff.type !== "REMOVE" &&
-          diff.path[0] !== "value" &&
-          (diff.path.length !== 1 || diff.path[0] !== "description") &&
-          diff.path[0] !== "resource_type"
-        ) {
-          console.log(
-            colors.red("Invalid variable diff with path " + diff.path)
-          );
-          throw new Error("Invalid folder diff with path " + diff.path);
-        }
-        if (diff.type === "CREATE" || diff.type === "CHANGE") {
-          setValueByPath(changeset, diff.path, diff.value);
-        } else if (diff.type === "REMOVE") {
-          setValueByPath(changeset, diff.path, null);
-        }
-      }
-
-      const hasChanges = Object.values(changeset).some(
-        (v) => v !== null && typeof v !== "undefined"
-      );
-      if (!hasChanges) {
-        return;
-      }
-
-      await ResourceService.updateResource({
-        workspace: workspace,
-        path: remotePath,
-        requestBody: changeset,
-      });
-    } else {
-      if (typeof this.is_oauth !== "undefined") {
-        console.log(
-          colors.yellow(
-            "! is_oauth has been removed in newer versions. Ignoring."
-          )
-        );
-      }
-
-      console.log(colors.yellow.bold("Creating new resource..."));
-      await ResourceService.createResource({
-        workspace: workspace,
-        requestBody: {
-          path: remotePath,
-          resource_type: this.resource_type,
-          value: this.value,
-          description: this.description,
-        },
-      });
-    }
-  }
-  async push(workspace: string, remotePath: string): Promise<void> {
-    await this.pushDiffs(
-      workspace,
-      remotePath,
-      microdiff({}, this, { cyclesFix: false })
-    );
-  }
 }

 export async function pushResource(
  workspace: string,
-  filePath: string,
-  remotePath: string
-) {
-  const data = decoverto
-    .type(ResourceFile)
-    .rawToInstance(await Deno.readTextFile(filePath));
-  await data.push(workspace, remotePath);
+  remotePath: string,
+  resource: ResourceFile | Resource | undefined,
+  localResource: ResourceFile,
+  raw: boolean
+): Promise<void> {
+  remotePath = removeType(remotePath, "resource");
+  if (raw) {
+    try {
+      resource = await ResourceService.getResource({
+        workspace: workspace,
+        path: remotePath,
+      });
+    } catch {
+      // flow doesn't exist
+    }
+  }
+  if (resource) {
+    if (isSuperset(localResource, resource)) {
+      return;
+    }
+
+    await ResourceService.updateResource({
+      workspace: workspace,
+      path: remotePath,
+      requestBody: { ...localResource },
+    });
+  } else {
+    if (localResource.is_oauth) {
+      log.info(
+        colors.yellow(
+          "! is_oauth has been removed in newer versions. Ignoring."
+        )
+      );
+    }
+
+    log.info(colors.yellow.bold("Creating new resource..."));
+    await ResourceService.createResource({
+      workspace: workspace,
+      requestBody: {
+        path: remotePath,
+        ...localResource,
+      },
+    });
+  }
 }

 type PushOptions = GlobalOptions;
@@ -141,10 +84,16 @@ async function push(opts: PushOptions, filePath: string, remotePath: string) {
    throw new Error("file path must refer to a file.");
  }

-  console.log(colors.bold.yellow("Pushing resource..."));
+  log.info(colors.bold.yellow("Pushing resource..."));

-  await pushResource(workspace.workspaceId, filePath, remotePath);
-  console.log(colors.bold.underline.green(`Resource ${remotePath} pushed`));
+  await pushResource(
+    workspace.workspaceId,
+    remotePath,
+    undefined,
+    parseFromFile(filePath),
+    true
+  );
+  log.info(colors.bold.underline.green(`Resource ${remotePath} pushed`));
 }

 async function list(opts: GlobalOptions) {
@@ -1,57 +1,28 @@
 // deno-lint-ignore-file no-explicit-any
-import { GlobalOptions } from "./types.ts";
+import { GlobalOptions, parseFromFile, removeType } from "./types.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import {
  colors,
  Command,
  JobService,
+  log,
+  NewScript,
  readAll,
  Script,
  ScriptService,
  Table,
 } from "./deps.ts";
-import { Any, array, decoverto, model, property } from "./decoverto.ts";
 import { writeAllSync } from "https://deno.land/std@0.176.0/streams/mod.ts";
+import { parse as yamlParse } from "https://deno.land/std@0.184.0/yaml/mod.ts";

-@model()
-export class ScriptFile {
-  @property(() => String)
+export interface ScriptFile {
  parent_hash?: string;
-  @property(() => String)
  summary: string;
-  @property(() => String)
  description: string;
-  @property(Any)
  schema?: any;
-  @property(() => Boolean)
  is_template?: boolean;
-  @property(array(() => String))
  lock?: Array<string>;
-  @property({
-    toInstance: (data) => {
-      if (data == null) return data;
-
-      if (
-        data === "script" ||
-        data === "failure" ||
-        data === "trigger" ||
-        data === "command" ||
-        data === "approvial"
-      ) {
-        return data;
-      }
-
-      throw new Error("Invalid kind " + data);
-    },
-    toPlain: (data) => data,
-  })
-  @property(() => String)
  kind?: "script" | "failure" | "trigger" | "command" | "approval";
-
-  constructor(summary: string, description: string) {
-    this.summary = summary;
-    this.description = description;
-  }
 }

 type PushOptions = GlobalOptions;
@@ -69,7 +40,7 @@ async function push(opts: PushOptions, filePath: string) {
  }
  let contentPath: string;
  let metaPath: string | undefined;
-  if (filePath.endsWith(".script.json")) {
+  if (filePath.endsWith(".script.json") || filePath.endsWith(".script.yaml")) {
    metaPath = filePath;
    contentPath = await findContentFile(filePath);
  } else {
@@ -79,7 +50,7 @@ async function push(opts: PushOptions, filePath: string) {

  await requireLogin(opts);
  await pushScript(metaPath, contentPath, workspace.workspaceId, remotePath);
-  console.log(colors.bold.underline.green(`Script ${remotePath} pushed`));
+  log.info(colors.bold.underline.green(`Script ${remotePath} pushed`));
 }

 export async function handleScriptMetadata(
@@ -87,7 +58,7 @@ export async function handleScriptMetadata(
  workspace: string,
  alreadySynced: string[]
 ): Promise<boolean> {
-  if (path.endsWith(".script.json")) {
+  if (path.endsWith(".script.json") || path.endsWith(".script.yaml")) {
    const contentPath = await findContentFile(path);
    return handleFile(
      contentPath,
@@ -107,14 +78,17 @@ export async function handleFile(
  alreadySynced: string[]
 ): Promise<boolean> {
  if (
-    path.endsWith(".ts") ||
-    path.endsWith(".py") ||
-    path.endsWith(".go") ||
-    path.endsWith(".sh")
+    !path.includes(".inline_script.") &&
+    (path.endsWith(".ts") ||
+      path.endsWith(".py") ||
+      path.endsWith(".go") ||
+      path.endsWith(".sh"))
  ) {
    if (alreadySynced.includes(path)) {
      return true;
    }
+    log.debug(`Processing local script ${path}`);
+
    alreadySynced.push(path);
    const remotePath = path.substring(0, path.length - 3);
    const metaPath = remotePath + ".script.json";
@@ -122,8 +96,16 @@ export async function handleFile(
    try {
      await Deno.stat(metaPath);
      typed = JSON.parse(await Deno.readTextFile(metaPath));
-      typed = decoverto.type(ScriptFile).plainToInstance(typed);
-    } catch {}
+    } catch {
+      const metaPath = remotePath + ".script.yaml";
+      try {
+        await Deno.stat(metaPath);
+        typed = yamlParse(await Deno.readTextFile(metaPath));
+      } catch {
+        // no meta file
+      }
+    }
+
    const language = inferContentTypeFromFilePath(path);

    let remote = undefined;
@@ -132,7 +114,10 @@ export async function handleFile(
        workspace,
        path: remotePath,
      });
-    } catch {}
+      log.debug(`Script ${remotePath} exists on remote`);
+    } catch {
+      log.debug(`Script ${remotePath} does not exist on remote`);
+    }

    if (remote) {
      if (content === remote.content) {
@@ -146,7 +131,7 @@ export async function handleFile(
            remote?.lock == typed.lock?.join("\n") &&
            JSON.stringify(typed.schema) == JSON.stringify(remote.schema))
        ) {
-          console.log(
+          log.info(
            colors.yellow(
              `No change to push for script ${remotePath}, skipping`
            )
@@ -154,13 +139,15 @@ export async function handleFile(
          return true;
        }
      }
-
+      log.info(
+        colors.yellow.bold(`Creating script with a parent ${remotePath}`)
+      );
      await ScriptService.createScript({
        workspace,
        requestBody: {
          content,
          description: typed?.description ?? "",
-          language,
+          language: language as NewScript.language,
          path: remotePath,
          summary: typed?.summary ?? "",
          is_template: typed?.is_template,
@@ -170,18 +157,17 @@ export async function handleFile(
          schema: typed?.schema,
        },
      });
-
-      console.log(
-        colors.yellow.bold(`Creating script with a parent ${remotePath}`)
-      );
    } else {
+      log.info(
+        colors.yellow.bold(`Creating script without parent ${remotePath}`)
+      );
      // no parent hash
      await ScriptService.createScript({
        workspace: workspace,
        requestBody: {
          content,
          description: typed?.description ?? "",
-          language,
+          language: language as NewScript.language,
          path: remotePath,
          summary: typed?.summary ?? "",
          is_template: typed?.is_template,
@@ -191,9 +177,6 @@ export async function handleFile(
          schema: typed?.schema,
        },
      });
-      console.log(
-        colors.yellow.bold(`Creating script without parent ${remotePath}`)
-      );
    }
    return true;
  }
@@ -201,12 +184,19 @@ export async function handleFile(
 }

 export async function findContentFile(filePath: string) {
-  const candidates = [
-    filePath.replace(".script.json", ".ts"),
-    filePath.replace(".script.json", ".py"),
-    filePath.replace(".script.json", ".go"),
-    filePath.replace(".script.json", ".sh"),
-  ];
+  const candidates = filePath.endsWith("script.json")
+    ? [
+        filePath.replace(".script.json", ".ts"),
+        filePath.replace(".script.json", ".py"),
+        filePath.replace(".script.json", ".go"),
+        filePath.replace(".script.json", ".sh"),
+      ]
+    : [
+        filePath.replace(".script.yaml", ".ts"),
+        filePath.replace(".script.yaml", ".py"),
+        filePath.replace(".script.yaml", ".go"),
+        filePath.replace(".script.yaml", ".sh"),
+      ];
  const validCandidates = (
    await Promise.all(
      candidates.map((x) => {
@@ -223,7 +213,7 @@ export async function findContentFile(filePath: string) {
    .map((x) => x.path);
  if (validCandidates.length > 1) {
    throw new Error(
-      "No content path given and more then one candidate found: " +
+      "No content path given and more than one candidate found: " +
        validCandidates.join(", ")
    );
  }
@@ -258,10 +248,10 @@ export async function pushScript(
  workspace: string,
  remotePath: string
 ) {
-  const data = filePath
-    ? decoverto
-        .type(ScriptFile)
-        .rawToInstance(await Deno.readTextFile(filePath))
+  remotePath = removeType(remotePath, "script");
+
+  const data: ScriptFile | undefined = filePath
+    ? parseFromFile(filePath)
    : undefined;
  const content = await Deno.readTextFile(contentPath);

@@ -280,7 +270,7 @@ export async function pushScript(
    }
  }

-  console.log(colors.bold.yellow("Pushing script..."));
+  log.info(colors.bold.yellow(`Pushing script ${remotePath}...`));
  await ScriptService.createScript({
    workspace: workspace,
    requestBody: {
@@ -288,9 +278,9 @@ export async function pushScript(
      summary: data?.summary ?? "",
      content: content,
      description: data?.description ?? "",
-      language: language,
+      language: language as NewScript.language,
      is_template: data?.is_template,
-      kind: data?.kind,
+      kind: data?.kind as NewScript.kind,
      lock: data?.lock,
      parent_hash: parent_hash,
      schema: data?.schema,
@@ -376,7 +366,7 @@ async function run(
            id,
          })
        ).result ?? {};
-      console.log(result);
+      log.info(result);

      break;
    } catch {
@@ -389,16 +379,16 @@ export async function track_job(workspace: string, id: string) {
  try {
    const result = await JobService.getCompletedJob({ workspace, id });

-    console.log(result.logs);
-    console.log();
-    console.log(colors.bold.underline.green("Job Completed"));
-    console.log();
+    log.info(result.logs);
+    log.info("\n");
+    log.info(colors.bold.underline.green("Job Completed"));
+    log.info("\n");
    return;
  } catch {
    /* ignore */
  }

-  console.log(colors.yellow("Waiting for Job " + id + " to start..."));
+  log.info(colors.yellow("Waiting for Job " + id + " to start..."));

  let logOffset = 0;
  let running = false;
@@ -419,7 +409,7 @@ export async function track_job(workspace: string, id: string) {
    } catch {
      retry++;
      if (retry > 3) {
-        console.log("failed to get job updated. skipping log streaming.");
+        log.info("failed to get job updated. skipping log streaming.");
        break;
      }
      continue;
@@ -427,7 +417,7 @@ export async function track_job(workspace: string, id: string) {

    if (!running && updates.running === true) {
      running = true;
-      console.log(colors.green("Job running. Streaming logs..."));
+      log.info(colors.green("Job running. Streaming logs..."));
    }

    if (updates.new_logs) {
@@ -442,9 +432,7 @@ export async function track_job(workspace: string, id: string) {

    if (running && updates.running === false) {
      running = false;
-      console.log(
-        colors.yellow("Job suspended. Waiting for it to continue...")
-      );
+      log.info(colors.yellow("Job suspended. Waiting for it to continue..."));
    }
  }
  await new Promise((resolve, _) => setTimeout(() => resolve(undefined), 1000));
@@ -452,17 +440,17 @@ export async function track_job(workspace: string, id: string) {
  try {
    const final_job = await JobService.getCompletedJob({ workspace, id });
    if ((final_job.logs?.length ?? -1) > logOffset) {
-      console.log(final_job.logs!.substring(logOffset));
+      log.info(final_job.logs!.substring(logOffset));
    }
-    console.log("\n");
+    log.info("\n");
    if (final_job.success) {
-      console.log(colors.bold.underline.green("Job Completed"));
+      log.info(colors.bold.underline.green("Job Completed"));
    } else {
-      console.log(colors.bold.underline.red("Job Completed"));
+      log.info(colors.bold.underline.red("Job Completed"));
    }
-    console.log();
+    log.info("\n");
  } catch {
-    console.log("Job appears to have completed, but no data can be retrieved");
+    log.info("Job appears to have completed, but no data can be retrieved");
  }
 }

@@ -473,10 +461,10 @@ async function show(opts: GlobalOptions, path: string) {
    workspace: workspace.workspaceId,
    path,
  });
-  console.log(colors.underline(s.path));
-  if (s.description) console.log(s.description);
-  console.log("");
-  console.log(s.content);
+  log.info(colors.underline(s.path));
+  if (s.description) log.info(s.description);
+  log.info("");
+  log.info(s.content);
 }

 const command = new Command()
@@ -6,7 +6,6 @@ import {
  ensureDir,
  gitignore_parser,
  JSZip,
-  microdiff,
  path,
  ScriptService,
  FolderService,
@@ -14,29 +13,34 @@ import {
  VariableService,
  AppService,
  FlowService,
+  OpenFlow,
+  FlowModule,
+  RawScript,
+  log,
 } from "./deps.ts";
 import {
-  Difference,
  getTypeStrFromPath,
  GlobalOptions,
-  inferTypeFromPath,
-  setValueByPath,
+  parseFromPath,
+  pushObj,
+  showConflict,
+  showDiff,
 } from "./types.ts";
 import { downloadZip } from "./pull.ts";
-import { FolderFile } from "./folder.ts";
-import { ResourceTypeFile } from "./resource-type.ts";
-import { handleScriptMetadata, ScriptFile } from "./script.ts";
-import { ResourceFile } from "./resource.ts";
-import { FlowFile } from "./flow.ts";
-import { VariableFile } from "./variable.ts";
+
+import { handleScriptMetadata } from "./script.ts";
+
 import { handleFile } from "./script.ts";
 import { equal } from "https://deno.land/x/equal@v1.5.0/mod.ts";
-import * as Diff from "npm:diff";
+import {
+  stringify as yamlStringify,
+  parse as yamlParse,
+} from "https://deno.land/std@0.184.0/yaml/mod.ts";

 type DynFSElement = {
  isDirectory: boolean;
  path: string;
-  getContentBytes(): Promise<Uint8Array>;
+  // getContentBytes(): Promise<Uint8Array>;
  getContentText(): Promise<string>;
  getChildren(): AsyncIterable<DynFSElement>;
 };
@@ -47,35 +51,152 @@ async function FSFSElement(p: string): Promise<DynFSElement> {
      isDirectory: isDir,
      path: localP.substring(p.length + 1),
      async *getChildren(): AsyncIterable<DynFSElement> {
+        if (!isDir) return [];
        for await (const e of Deno.readDir(localP)) {
          yield _internal_element(path.join(localP, e.name), e.isDirectory);
        }
      },
-      async getContentBytes(): Promise<Uint8Array> {
-        return await Deno.readFile(localP);
-      },
+      // async getContentBytes(): Promise<Uint8Array> {
+      //   return await Deno.readFile(localP);
+      // },
      async getContentText(): Promise<string> {
-        return await Deno.readTextFile(localP);
+        const content = await Deno.readTextFile(localP);
+        return content;
      },
    };
  }
  return _internal_element(p, (await Deno.stat(p)).isDirectory);
 }

-function ZipFSElement(zip: JSZip): DynFSElement {
+function prioritizeName(name: string): string {
+  if (name == "id") return "aa";
+  if (name == "type") return "ab";
+  if (name == "summary") return "ad";
+  if (name == "name") return "ae";
+  if (name == "display_name") return "af";
+  if (name == "description") return "ag";
+  if (name == "value") return "ah";
+  if (name == "content") return "ai";
+  if (name == "modules") return "aj";
+  if (name == "failure_module") return "ak";
+  if (name == "input_transforms") return "al";
+  if (name == "lock") return "az";
+
+  return name;
+}
+
+export const yamlOptions = {
+  sortKeys: (a: any, b: any) => {
+    return prioritizeName(a).localeCompare(prioritizeName(b));
+  },
+  noCompatMode: true,
+  noRefs: true,
+};
+
+function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
  function _internal_file(p: string, f: JSZip.JSZipObject): DynFSElement {
+    const isFlow = p.endsWith("flow.json");
+    function transformPath() {
+      if (isFlow) {
+        return p.replace("flow.json", "flow");
+      } else {
+        return useYaml && p.endsWith(".json")
+          ? p.replaceAll(".json", ".yaml")
+          : p;
+      }
+    }
+
+    interface InlineScript {
+      path: string;
+      content: string;
+    }
+    let counter = 0;
+    const seen_names = new Set<string>();
+    function assignPath(
+      summary: string | undefined,
+      language: RawScript.language
+    ): string {
+      let name;
+      if (summary && summary != "" && !seen_names.has(summary)) {
+        name = summary.toLowerCase().replaceAll(" ", "_");
+        seen_names.add(name);
+      } else {
+        name = `inline_script_${counter}`;
+        while (seen_names.has(name)) {
+          counter++;
+          name = `inline_script_${counter}`;
+        }
+        seen_names.add(name);
+      }
+      let ext;
+      if (language == "python3") ext = "py";
+      else if (language == "deno") ext = "ts";
+      else if (language == "go") ext = "go";
+      else if (language == "bash") ext = "sh";
+      return `${name}.inline_script.${ext}`;
+    }
+    function extractInlineScripts(modules: FlowModule[]): InlineScript[] {
+      return modules.flatMap((m) => {
+        if (m.value.type == "rawscript") {
+          const path = assignPath(m.summary, m.value.language);
+          const content = m.value.content;
+          m.value.content = "!inline " + path;
+          return [{ path: path, content: content }];
+        } else if (m.value.type == "forloopflow") {
+          return extractInlineScripts(m.value.modules);
+        } else if (m.value.type == "branchall") {
+          return m.value.branches.flatMap((b) =>
+            extractInlineScripts(b.modules)
+          );
+        } else if (m.value.type == "branchone") {
+          return [
+            ...m.value.branches.flatMap((b) => extractInlineScripts(b.modules)),
+            ...extractInlineScripts(m.value.default),
+          ];
+        } else {
+          return [];
+        }
+      });
+    }
+    const flowPath = transformPath();
    return {
-      isDirectory: false,
-      path: p,
-      // deno-lint-ignore require-yield
+      isDirectory: isFlow,
+      path: flowPath,
      async *getChildren(): AsyncIterable<DynFSElement> {
-        throw new Error("Cannot get children of file");
-      },
-      async getContentBytes(): Promise<Uint8Array> {
-        return await f.async("uint8array");
+        if (isFlow) {
+          const flow: OpenFlow = JSON.parse(await f.async("text"));
+          const inlineScripts = extractInlineScripts(flow.value.modules);
+          for (const s of inlineScripts) {
+            yield {
+              isDirectory: false,
+              path: path.join(flowPath, s.path),
+              async *getChildren() {},
+              // deno-lint-ignore require-await
+              async getContentText() {
+                return s.content;
+              },
+            };
+          }
+
+          yield {
+            isDirectory: false,
+            path: path.join(flowPath, "flow.yaml"),
+            async *getChildren() {},
+            // deno-lint-ignore require-await
+            async getContentText() {
+              return yamlStringify(flow, yamlOptions);
+            },
+          };
+        }
      },
+      // async getContentBytes(): Promise<Uint8Array> {
+      //   return await f.async("uint8array");
+      // },
      async getContentText(): Promise<string> {
-        return await f.async("text");
+        const content = await f.async("text");
+        return useYaml && p.endsWith(".json")
+          ? yamlStringify(JSON.parse(content), yamlOptions)
+          : content;
      },
    };
  }
@@ -95,9 +216,11 @@ function ZipFSElement(zip: JSZip): DynFSElement {
          }
        }
      },
-      async getContentBytes(): Promise<Uint8Array> {
-        throw new Error("Cannot get content of folder");
-      },
+      // // deno-lint-ignore require-await
+      // async getContentBytes(): Promise<Uint8Array> {
+      //   throw new Error("Cannot get content of folder");
+      // },
+      // deno-lint-ignore require-await
      async getContentText(): Promise<string> {
        throw new Error("Cannot get content of folder");
      },
@@ -113,7 +236,7 @@ async function* readDirRecursiveWithIgnore(
  path: string;
  ignored: boolean;
  isDirectory: boolean;
-  getContentBytes(): Promise<Uint8Array>;
+  // getContentBytes(): Promise<Uint8Array>;
  getContentText(): Promise<string>;
 }> {
  const stack: {
@@ -121,7 +244,7 @@ async function* readDirRecursiveWithIgnore(
    isDirectory: boolean;
    ignored: boolean;
    c(): AsyncIterable<DynFSElement>;
-    getContentBytes(): Promise<Uint8Array>;
+    // getContentBytes(): Promise<Uint8Array>;
    getContentText(): Promise<string>;
  }[] = [
    {
@@ -129,9 +252,9 @@ async function* readDirRecursiveWithIgnore(
      ignored: ignore(root.path, root.isDirectory),
      isDirectory: root.isDirectory,
      c: root.getChildren,
-      getContentBytes(): Promise<Uint8Array> {
-        throw undefined;
-      },
+      // getContentBytes(): Promise<Uint8Array> {
+      //   throw undefined;
+      // },
      getContentText(): Promise<string> {
        throw undefined;
      },
@@ -141,13 +264,12 @@ async function* readDirRecursiveWithIgnore(
  while (stack.length > 0) {
    const e = stack.pop()!;
    yield e;
-    if (!e.isDirectory) continue;
    for await (const e2 of e.c()) {
      stack.push({
        path: e2.path,
        ignored: e.ignored || ignore(e2.path, e2.isDirectory),
        isDirectory: e2.isDirectory,
-        getContentBytes: e2.getContentBytes,
+        // getContentBytes: e2.getContentBytes,
        getContentText: e2.getContentText,
        c: e2.getChildren,
      });
@@ -163,27 +285,38 @@ type Change = Added | Deleted | Edit;

 async function elementsToMap(
  els: DynFSElement,
-  ignore: (path: string, isDirectory: boolean) => boolean
+  ignore: (path: string, isDirectory: boolean) => boolean,
+  json: boolean
 ): Promise<{ [key: string]: string }> {
  const map: { [key: string]: string } = {};
  for await (const entry of readDirRecursiveWithIgnore(ignore, els)) {
    if (entry.isDirectory || entry.ignored) continue;
+    if (json && entry.path.endsWith(".yaml")) continue;
+    if (!json && entry.path.endsWith(".json")) continue;
+    if (
+      !["json", "yaml", "go", "sh", "ts", "py"].includes(
+        entry.path.split(".").pop() ?? ""
+      )
+    )
+      continue;
    const content = await entry.getContentText();
    map[entry.path] = content;
  }
  return map;
 }
+
 async function compareDynFSElement(
  els1: DynFSElement,
  els2: DynFSElement | undefined,
-  ignore: (path: string, isDirectory: boolean) => boolean
+  ignore: (path: string, isDirectory: boolean) => boolean,
+  json: boolean
 ): Promise<Change[]> {
  const [m1, m2] = els2
    ? await Promise.all([
-        elementsToMap(els1, ignore),
-        elementsToMap(els2, ignore),
+        elementsToMap(els1, ignore, json),
+        elementsToMap(els2, ignore, json),
      ])
-    : [await elementsToMap(els1, ignore), {}];
+    : [await elementsToMap(els1, ignore, json), {}];

  const changes: Change[] = [];

@@ -192,13 +325,9 @@ async function compareDynFSElement(
      changes.push({ name: "added", path: k, content: v });
    } else if (
      m2[k] != v &&
-      (!k.endsWith(".json") || !equal(JSON.parse(v), JSON.parse(m2[k])))
+      (!k.endsWith(".json") || !equal(JSON.parse(v), JSON.parse(m2[k]))) &&
+      (!k.endsWith(".yaml") || !equal(yamlParse(v), yamlParse(m2[k])))
    ) {
-      // await Deno.writeTextFile("/tmp/k", m2[k])
-      // await Deno.writeTextFile("/tmp/v", v)
-      // console.log(k)
-      // if (k.includes("flow"))
-      //   Deno.exit(1)
      changes.push({ name: "edited", path: k, after: v, before: m2[k] });
    }
  }
@@ -248,7 +377,7 @@ async function ignoreF() {
        (isNotWmillFile(p, isDirectory) || ignore.denies(p))
      );
    };
-  } catch (e) {
+  } catch {
    return (p: string, isDirectory: boolean) =>
      !isWhitelisted(p) && isNotWmillFile(p, isDirectory);
  }
@@ -260,6 +389,7 @@ async function pull(
    yes: boolean;
    failConflicts: boolean;
    plainSecrets?: boolean;
+    json?: boolean;
  }
 ) {
  if (!opts.raw) {
@@ -269,20 +399,28 @@ async function pull(
  const workspace = await resolveWorkspace(opts);
  await requireLogin(opts);

-  console.log(
+  log.info(
    colors.gray(
      "Computing the files to update locally to match remote (taking .wmillignore into account)"
    )
  );
  const remote = ZipFSElement(
-    (await downloadZip(workspace, opts.plainSecrets))!
+    (await downloadZip(workspace, opts.plainSecrets))!,
+    !opts.json
  );
  const local = opts.raw
    ? undefined
-    : await FSFSElement(path.join(Deno.cwd(), opts.raw ? "" : ".wmill"));
-  const changes = await compareDynFSElement(remote, local, await ignoreF());
+    : await FSFSElement(path.join(Deno.cwd(), ".wmill"));
+  const changes = await compareDynFSElement(
+    remote,
+    local,
+    await ignoreF(),
+    opts.json ?? false
+  );

-  console.log(`remote -> local: ${changes.length} changes to apply`);
+  log.info(
+    `remote (${workspace.name}) -> local: ${changes.length} changes to apply`
+  );
  if (changes.length > 0) {
    prettyChanges(changes);
    if (
@@ -297,7 +435,7 @@ async function pull(
    }

    const conflicts = [];
-    console.log(colors.gray(`Applying changes to files ...`));
+    log.info(colors.gray(`Applying changes to files ...`));
    for await (const change of changes) {
      const target = path.join(Deno.cwd(), change.path);
      const stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
@@ -305,7 +443,7 @@ async function pull(
        try {
          const currentLocal = await Deno.readTextFile(target);
          if (currentLocal !== change.before && currentLocal !== change.after) {
-            console.log(
+            log.info(
              colors.red(
                `Conflict detected on ${change.path}\nBoth local and remote have been modified.`
              )
@@ -318,7 +456,7 @@ async function pull(
              });
              continue;
            } else if (opts.yes) {
-              console.log(
+              log.info(
                colors.red(
                  `Override local version with remote since --yes was passed and no --fail-conflicts.`
                )
@@ -334,13 +472,13 @@ async function pull(
              }
            }
          }
-        } catch {}
-        if (!change.path.endsWith(".json")) {
-          console.log(`Editing script content of ${change.path}`);
+        } catch {
+          // ignore
+        }
+        if (!change.path.endsWith(".json") && !change.path.endsWith(".yaml")) {
+          log.info(`Editing script content of ${change.path}`);
        } else {
-          console.log(
-            `Editing ${getTypeStrFromPath(change.path)} ${change.path}`
-          );
+          log.info(`Editing ${getTypeStrFromPath(change.path)} ${change.path}`);
        }
        await Deno.writeTextFile(target, change.after);

@@ -352,9 +490,7 @@ async function pull(
        await ensureDir(path.dirname(target));
        if (!opts.raw) {
          await ensureDir(path.dirname(stateTarget));
-          console.log(
-            `Adding ${getTypeStrFromPath(change.path)} ${change.path}`
-          );
+          log.info(`Adding ${getTypeStrFromPath(change.path)} ${change.path}`);
        }
        await Deno.writeTextFile(target, change.content);
        if (!opts.raw) {
@@ -362,14 +498,14 @@ async function pull(
        }
      } else if (change.name === "deleted") {
        try {
-          console.log(
+          log.info(
            `Deleting ${getTypeStrFromPath(change.path)} ${change.path}`
          );
          await Deno.remove(target);
          if (!opts.raw) {
            await Deno.remove(stateTarget);
          }
-        } catch (e) {
+        } catch {
          if (!opts.raw) {
            await Deno.remove(stateTarget);
          }
@@ -379,11 +515,11 @@ async function pull(
    if (opts.failConflicts) {
      if (conflicts.length > 0) {
        console.error(colors.red(`Conflicts were found`));
-        console.log("Conflicts:");
+        log.info("Conflicts:");
        for (const conflict of conflicts) {
          showConflict(conflict.path, conflict.local, conflict.change.after);
        }
-        console.log(
+        log.info(
          colors.red(`Please resolve theses conflicts manually by either:
  - reverting the content back to its remote (\`wmill pull\` and refuse to preserve local when prompted)
  - pushing the changes with \`wmill push --skip-pull\` to override wmill with all your local changes
@@ -392,49 +528,29 @@ async function pull(
        Deno.exit(1);
      }
    }
-    console.log(
+    log.info(
      colors.green.underline(
        `Done! All ${changes.length} changes applied locally.`
      )
    );
  }
-
-  function showConflict(path: string, local: string, remote: string) {
-    console.log(colors.yellow(`- ${path}`));
-
-    let finalString = "";
-    for (const part of Diff.diffLines(local, remote)) {
-      if (part.removed) {
-        // print red if removed without newline
-        finalString += `\x1b[31m${part.value}\x1b[0m`;
-      } else if (part.added) {
-        // print green if added
-        finalString += `\x1b[32m${part.value}\x1b[0m`;
-      } else {
-        // print white if unchanged
-        finalString += `\x1b[37m${part.value}\x1b[0m`;
-      }
-    }
-    console.log(finalString);
-    console.log("\x1b[31mlocal\x1b[31m - \x1b[32mremote\x1b[32m");
-    console.log();
-  }
 }

 function prettyChanges(changes: Change[]) {
  for (const change of changes) {
    if (change.name === "added") {
-      console.log(
+      log.info(
        colors.green(`+ ${getTypeStrFromPath(change.path)} ` + change.path)
      );
    } else if (change.name === "deleted") {
-      console.log(
+      log.info(
        colors.red(`- ${getTypeStrFromPath(change.path)} ` + change.path)
      );
    } else if (change.name === "edited") {
-      console.log(
+      log.info(
        colors.yellow(`~ ${getTypeStrFromPath(change.path)} ` + change.path)
      );
+      showDiff(change.before, change.after);
    }
  }
 }
@@ -453,10 +569,10 @@ function prettyChanges(changes: Change[]) {
 //       }
 //     }
 //     if (diff.type === "REMOVE" || diff.type === "CHANGE") {
-//       console.log(colors.red("- " + pathString + " = " + diff.oldValue));
+//      log.info(colors.red("- " + pathString + " = " + diff.oldValue));
 //     }
 //     if (diff.type === "CREATE" || diff.type === "CHANGE") {
-//       console.log(colors.green("+ " + pathString + " = " + diff.value));
+//      log.info(colors.green("+ " + pathString + " = " + diff.value));
 //     }
 //   }
 // }
@@ -472,34 +588,45 @@ async function push(
    skipPull: boolean;
    failConflicts: boolean;
    plainSecrets?: boolean;
+    json?: boolean;
  }
 ) {
  if (!opts.raw) {
    if (!opts.skipPull) {
-      console.log(
+      log.info(
        colors.gray("You need to be up-to-date before pushing, pulling first.")
      );
      await pull(opts);
-      console.log(colors.green("Pull done, now pushing."));
-      console.log();
+      log.info(colors.green("Pull done, now pushing."));
+      log.info("\n");
    }
  }

  const workspace = await resolveWorkspace(opts);
  await requireLogin(opts);

-  console.log(
+  log.info(
    colors.gray(
      "Computing the files to update on the remote to match local (taking .wmillignore into account)"
    )
  );
  const remote = opts.raw
    ? undefined
-    : ZipFSElement((await downloadZip(workspace, opts.plainSecrets))!);
+    : ZipFSElement(
+        (await downloadZip(workspace, opts.plainSecrets))!,
+        !opts.json
+      );
  const local = await FSFSElement(path.join(Deno.cwd(), ""));
-  const changes = await compareDynFSElement(local, remote, await ignoreF());
+  const changes = await compareDynFSElement(
+    local,
+    remote,
+    await ignoreF(),
+    opts.json ?? false
+  );

-  console.log(`remote <- local: ${changes.length} changes to apply`);
+  log.info(
+    `remote (${workspace.name}) <- local: ${changes.length} changes to apply`
+  );
  if (changes.length > 0) {
    prettyChanges(changes);
    if (
@@ -511,7 +638,7 @@ async function push(
    ) {
      return;
    }
-    console.log(colors.gray(`Applying changes to files ...`));
+    log.info(colors.gray(`Applying changes to files ...`));
    const alreadySynced: string[] = [];
    for await (const change of changes) {
      const stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
@@ -549,29 +676,28 @@ async function push(
        }
        if (!opts.raw) {
          await ensureDir(path.dirname(stateTarget));
-          console.log(
-            `Editing ${getTypeStrFromPath(change.path)} ${change.path}`
-          );
+          log.info(`Editing ${getTypeStrFromPath(change.path)} ${change.path}`);
        }
-        const obj = inferTypeFromPath(change.path, JSON.parse(change.after));
+        const oldObj = parseFromPath(change.path, change.before);
+        const newObj = parseFromPath(change.path, change.after);

-        const diff = microdiff(
-          inferTypeFromPath(change.path, JSON.parse(change.before)),
-          obj,
-          { cyclesFix: false }
-        );
-        await applyDiff(
+        pushObj(
          workspace.workspaceId,
-          change.path.split(".")[0],
-          obj,
-          diff,
-          opts.plainSecrets
+          change.path,
+          oldObj,
+          newObj,
+          opts.plainSecrets ?? false,
+          opts.raw
        );
+
        if (!opts.raw && stateExists) {
          await Deno.writeTextFile(stateTarget, change.after);
        }
      } else if (change.name === "added") {
-        if (change.path.endsWith(".script.json")) {
+        if (
+          change.path.endsWith(".script.json") ||
+          change.path.endsWith(".script.yaml")
+        ) {
          continue;
        } else if (
          await handleFile(
@@ -585,29 +711,26 @@ async function push(
        }
        if (!opts.raw && stateExists) {
          await ensureDir(path.dirname(stateTarget));
-          console.log(
-            `Adding ${getTypeStrFromPath(change.path)} ${change.path}`
-          );
+          log.info(`Adding ${getTypeStrFromPath(change.path)} ${change.path}`);
        }
-        const obj = inferTypeFromPath(change.path, JSON.parse(change.content));
-        const diff = microdiff({}, obj, { cyclesFix: false });
-        await applyDiff(
+        const obj = parseFromPath(change.path, change.content);
+        pushObj(
          workspace.workspaceId,
-          change.path.split(".")[0],
+          change.path,
+          undefined,
          obj,
-          diff,
-          opts.plainSecrets
+          opts.plainSecrets ?? false,
+          opts.raw
        );
+
        if (!opts.raw && stateExists) {
          await Deno.writeTextFile(stateTarget, change.content);
        }
      } else if (change.name === "deleted") {
-        if (!change.path.includes(".json")) {
+        if (!change.path.includes(".json") && !change.path.includes(".yaml")) {
          continue;
        }
-        console.log(
-          `Deleting ${getTypeStrFromPath(change.path)} ${change.path}`
-        );
+        log.info(`Deleting ${getTypeStrFromPath(change.path)} ${change.path}`);
        const typ = getTypeStrFromPath(change.path);
        const workspaceId = workspace.workspaceId;
        switch (typ) {
@@ -663,52 +786,17 @@ async function push(
        }
        try {
          await Deno.remove(stateTarget);
-        } catch {}
+        } catch {
+          // state target may not exist already
+        }
      }
    }
-    console.log(
+    log.info(
      colors.green.underline(
-        `Done! All ${changes.length} changes pushed to the remote workspace.`
+        `Done! All ${changes.length} changes pushed to the remote workspace ${workspace.workspaceId} named ${workspace.name}.`
      )
    );
  }
-
-  async function applyDiff(
-    workspace: string,
-    remotePath: string,
-    file:
-      | ScriptFile
-      | VariableFile
-      | FlowFile
-      | ResourceFile
-      | ResourceTypeFile
-      | FolderFile,
-    diffs: Difference[],
-    plainSecrets?: boolean
-  ) {
-    if (file instanceof ScriptFile) {
-      throw new Error(
-        "This code path should be unreachable - we should never generate diffs for scripts"
-      );
-    } else if (file instanceof FolderFile) {
-      const parts = remotePath.split("/");
-      if (parts[0] === "f") {
-        remotePath = parts[1];
-      } else {
-        remotePath = parts[0];
-      }
-    }
-    if (diffs.length === 0) {
-      console.log("No diffs to apply to " + remotePath);
-      return;
-    }
-    try {
-      await file.pushDiffs(workspace, remotePath, diffs, plainSecrets);
-    } catch (e) {
-      console.error("Failing to apply diffs to " + remotePath);
-      console.error(JSON.stringify(e));
-    }
-  }
 }

 const command = new Command()
@@ -723,6 +811,8 @@ const command = new Command()
  .option("--yes", "Pull without needing confirmation")
  .option("--raw", "Pull without using state, just overwrite.")
  .option("--plain-secrets", "Pull secrets as plain text")
+  .option("--json", "Use JSON instead of YAML")
+  // deno-lint-ignore no-explicit-any
  .action(pull as any)
  .command("push")
  .description(
@@ -736,6 +826,8 @@ const command = new Command()
  .option("--yes", "Push without needing confirmation")
  .option("--raw", "Push without using state, just overwrite.")
  .option("--plain-secrets", "Push secrets as plain text")
+  .option("--json", "Use JSON instead of YAML")
+  // deno-lint-ignore no-explicit-any
  .action(push as any);

 export default command;
@@ -1,31 +1,21 @@
-import { decoverto } from "./decoverto.ts";
-import { FlowFile } from "./flow.ts";
-import { ResourceTypeFile } from "./resource-type.ts";
-import { ResourceFile } from "./resource.ts";
-import { ScriptFile } from "./script.ts";
-import { VariableFile } from "./variable.ts";
-import { path } from "./deps.ts";
-import { FolderFile } from "./folder.ts";
-import { AppFile } from "./apps.ts";
+// deno-lint-ignore-file no-explicit-any

-// TODO: Remove this & replace with a "pull" that lets the object either pull the remote version or return undefined.
-// Then combine those with diffing, which then gives the new push impl
-export interface Resource {
-  push(
-    workspace: string,
-    remotePath: string,
-    plainSecrets?: boolean
-  ): Promise<void>;
-}
-
-export interface PushDiffs {
-  pushDiffs(
-    workspace: string,
-    remotePath: string,
-    diffs: Difference[],
-    plainSecrets?: boolean
-  ): Promise<void>;
-}
+import { colors, log, path } from "./deps.ts";
+import { pushApp } from "./apps.ts";
+import {
+  parse as yamlParse,
+  stringify as yamlStringify,
+} from "https://deno.land/std@0.184.0/yaml/mod.ts";
+import { equal } from "https://deno.land/x/equal@v1.5.0/equal.ts";
+import { pushFolder } from "./folder.ts";
+import { pushScript } from "./script.ts";
+import { pushFlow } from "./flow.ts";
+import { pushResource } from "./resource.ts";
+import { pushResourceType } from "./resource-type.ts";
+import { pushVariable } from "./variable.ts";
+import * as Diff from "npm:diff";
+import { yamlOptions } from "./sync.ts";
+import { showDiffs } from "./main.ts";

 export interface DifferenceCreate {
  type: "CREATE";
@@ -48,77 +38,111 @@ export interface DifferenceChange {

 export type Difference = DifferenceCreate | DifferenceRemove | DifferenceChange;

-export function setValueByPath(
-  obj: any,
-  path: (string | number)[],
-  value: any
-) {
-  let i;
-  let lastObj = undefined;
-  for (i = 0; i < path.length - 1; i++) {
-    if (!obj) {
-      let oldNewObj;
-      if (typeof path[i] === "number") {
-        oldNewObj = [];
-      } else {
-        oldNewObj = {};
-      }
-      lastObj[path[i - 1]] = oldNewObj;
-      obj = oldNewObj;
-    }
-    lastObj = obj;
-    obj = obj[path[i]];
-  }
-  if (!obj) {
-    let oldNewObj;
-    if (typeof path[i] === "number") {
-      oldNewObj = [];
-    } else {
-      oldNewObj = {};
-    }
-    lastObj[path[i - 1]] = oldNewObj;
-    obj = oldNewObj;
-  }
-  obj[path[i]] = value;
-}
-
 export type GlobalOptions = {
  workspace: string | undefined;
  token: string | undefined;
 };

-export function inferTypeFromPath(
+export function isSuperset(
+  subset: Record<string, any>,
+  superset: Record<string, any>
+): boolean {
+  return Object.keys(subset).every((key) => {
+    const eq = equal(subset[key], superset[key]);
+    if (!eq && showDiffs) {
+      const sub = subset[key];
+      const supers = superset[key];
+      if (!supers) {
+        log.info(`Key ${key} not found in remote`);
+      } else {
+        log.info(`Found diff for ${key}:`);
+        showDiff(
+          yamlStringify(sub, yamlOptions),
+          yamlStringify(supers, yamlOptions)
+        );
+      }
+    }
+    return eq;
+  });
+}
+
+export function showDiff(local: string, remote: string) {
+  let finalString = "";
+  for (const part of Diff.diffLines(local, remote)) {
+    if (part.removed) {
+      // print red if removed without newline
+      finalString += `\x1b[31m${part.value}\x1b[0m`;
+    } else if (part.added) {
+      // print green if added
+      finalString += `\x1b[32m${part.value}\x1b[0m`;
+    } else {
+      let lines = part.value.split("\n");
+
+      if (lines.length > 4) {
+        lines = lines.slice(0, 2);
+        lines.push("...");
+        lines = lines.concat(part.value.split("\n").slice(-2));
+      }
+      // print white if unchanged
+      finalString += `\x1b[37m${lines.join("\n")}\x1b[0m`;
+    }
+  }
+  log.info(finalString);
+}
+
+export function showConflict(path: string, local: string, remote: string) {
+  log.info(colors.yellow(`- ${path}`));
+  showDiff(local, remote);
+  log.info("\x1b[31mlocal\x1b[31m - \x1b[32mremote\x1b[32m");
+  log.info("\n");
+}
+
+export function pushObj(
+  workspace: string,
  p: string,
-  obj: any
-):
-  | ScriptFile
-  | VariableFile
-  | FlowFile
-  | ResourceFile
-  | ResourceTypeFile
-  | FolderFile
-  | AppFile {
+  befObj: any,
+  newObj: any,
+  plainSecrets: boolean,
+  checkForCreate: boolean
+) {
  const typeEnding = getTypeStrFromPath(p);

-  if (typeEnding === "folder") {
-    return decoverto.type(FolderFile).plainToInstance(obj);
+  if (typeEnding === "app") {
+    pushApp(workspace, p, befObj, newObj, checkForCreate);
+  } else if (typeEnding === "folder") {
+    pushFolder(workspace, p, befObj, newObj, checkForCreate);
  } else if (typeEnding === "script") {
-    return decoverto.type(ScriptFile).plainToInstance(obj);
+    pushScript(workspace, p, befObj, newObj);
  } else if (typeEnding === "variable") {
-    return decoverto.type(VariableFile).plainToInstance(obj);
+    pushVariable(workspace, p, befObj, newObj, plainSecrets, checkForCreate);
  } else if (typeEnding === "flow") {
-    return decoverto.type(FlowFile).plainToInstance(obj);
+    const flowName = p.split(".flow/")[0];
+    pushFlow(workspace, flowName, flowName + ".flow", workspace);
  } else if (typeEnding === "resource") {
-    return decoverto.type(ResourceFile).plainToInstance(obj);
+    pushResource(workspace, p, befObj, newObj, checkForCreate);
  } else if (typeEnding === "resource-type") {
-    return decoverto.type(ResourceTypeFile).plainToInstance(obj);
-  } else if (typeEnding === "app") {
-    return decoverto.type(AppFile).plainToInstance(obj);
+    pushResourceType(workspace, p, befObj, newObj, checkForCreate);
  } else {
    throw new Error("infer type unreachable");
  }
 }

+export function parseFromPath(p: string, content: string): any {
+  return p.endsWith(".yaml")
+    ? yamlParse(content)
+    : p.endsWith(".json")
+    ? JSON.parse(content)
+    : content;
+}
+export function parseFromFile(p: string): any {
+  if (p.endsWith(".json")) {
+    return JSON.parse(Deno.readTextFileSync(p));
+  } else if (p.endsWith(".yaml") || p.endsWith(".yml")) {
+    return yamlParse(Deno.readTextFileSync(p));
+  } else {
+    throw new Error("Could not read file " + p);
+  }
+}
 export function getTypeStrFromPath(
  p: string
 ):
@@ -129,6 +153,9 @@ export function getTypeStrFromPath(
  | "resource-type"
  | "folder"
  | "app" {
+  if (p.includes(".flow/")) {
+    return "flow";
+  }
  const parsed = path.parse(p);
  if (
    parsed.ext == ".go" ||
@@ -147,7 +174,6 @@ export function getTypeStrFromPath(
  if (
    typeEnding === "script" ||
    typeEnding === "variable" ||
-    typeEnding === "flow" ||
    typeEnding === "resource" ||
    typeEnding === "resource-type" ||
    typeEnding === "app"
@@ -157,3 +183,13 @@ export function getTypeStrFromPath(
    throw new Error("Could not infer type of path " + JSON.stringify(parsed));
  }
 }
+
+export function removeType(str: string, type: string) {
+  if (
+    !str.endsWith("." + type + ".yaml") &&
+    !str.endsWith("." + type + ".json")
+  ) {
+    throw new Error(str + " does not end with ." + type + ".(yaml|json)");
+  }
+  return str.slice(0, str.length - type.length - 6);
+}
@@ -5,6 +5,7 @@ import {
  colors,
  Command,
  GlobalUserInfo,
+  log,
  passwordGenerator,
  Table,
  UserService,
@@ -36,7 +37,7 @@ async function list(opts: GlobalOptions) {
        x.company ?? "-",
        x.verified ? "true" : "false",
        x.super_admin ? "true" : "false",
-      ]),
+      ])
    )
    .render();
 }
@@ -48,7 +49,7 @@ async function add(
    name?: string;
  },
  email: string,
-  password?: string,
+  password?: string
 ) {
  await requireLogin(opts);
  const password_final = password ?? passwordGenerator("*", 15);
@@ -62,33 +63,34 @@ async function add(
    },
  });
  if (!password) {
-    console.log(colors.red("New Password: " + password_final));
+    log.info(colors.red("New Password: " + password_final));
  }
-  console.log(colors.underline.green("New User Created."));
+  log.info(colors.underline.green("New User Created."));
 }
 async function remove(opts: GlobalOptions, email: string) {
  await requireLogin(opts);

  await UserService.globalUserDelete({ email });
-  console.log(colors.green("Deleted User " + email));
+  log.info(colors.green("Deleted User " + email));
 }

 async function createToken(
-  opts: GlobalOptions & { email: string; password: string },
+  opts: GlobalOptions & { email: string; password: string }
 ) {
  if (opts.email && opts.password) {
-    console.log(
-      "Token: " + await UserService.login({
-        requestBody: {
-          email: opts.email,
-          password: opts.password,
-        },
-      }),
+    log.info(
+      "Token: " +
+        (await UserService.login({
+          requestBody: {
+            email: opts.email,
+            password: opts.password,
+          },
+        }))
    );
  }

  await requireLogin(opts);
-  console.log("Token: " + await UserService.createToken({ requestBody: {} }));
+  log.info("Token: " + (await UserService.createToken({ requestBody: {} })));
 }

 const command = new Command()
@@ -99,7 +101,7 @@ const command = new Command()
  .option("--superadmin", "Specify to make the new user superadmin.")
  .option(
    "--company <company:string>",
-    "Specify to set the company of the new user.",
+    "Specify to set the company of the new user."
  )
  .option("--name <name:string>", "Specify to set the name of the new user.")
  .action(add as any)
@@ -112,14 +114,14 @@ const command = new Command()
    "Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either.",
    {
      depends: ["password"],
-    },
+    }
  )
  .option(
    "--password <password:string>",
    "Specify credentials to use for authentication. This will not be stored. It will only be used to exchange for a token with the API server, which will not be stored either.",
    {
      depends: ["email"],
-    },
+    }
  )
  .action(createToken as any);

@@ -1,21 +1,19 @@
 // deno-lint-ignore-file no-explicit-any
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import {
-  Difference,
  GlobalOptions,
-  PushDiffs,
-  Resource,
-  setValueByPath,
+  isSuperset,
+  parseFromFile,
+  removeType,
 } from "./types.ts";
 import {
  colors,
  Command,
-  EditVariable,
-  microdiff,
+  ListableVariable,
+  log,
  Table,
  VariableService,
 } from "./deps.ts";
-import { decoverto, model, property } from "./decoverto.ts";

 async function list(opts: GlobalOptions) {
  const workspace = await resolveWorkspace(opts);
@@ -40,102 +38,65 @@ async function list(opts: GlobalOptions) {
    .render();
 }

-@model()
-export class VariableFile implements Resource, PushDiffs {
-  @property(() => String)
+export interface VariableFile {
  value: string;
-  @property(() => Boolean)
  is_secret: boolean;
-  @property(() => String)
  description: string;
-  @property(() => Number)
  account?: number;
-  @property(() => Boolean)
  is_oauth?: boolean;
+}

-  constructor(value: string, is_secret: boolean, description: string) {
-    this.value = value;
-    this.is_secret = is_secret;
-    this.description = description;
-  }
-  async pushDiffs(
-    workspace: string,
-    remotePath: string,
-    diffs: Difference[],
-    plainSecrets?: boolean
-  ): Promise<void> {
-    if (await VariableService.existsVariable({ workspace, path: remotePath })) {
-      console.log(
-        colors.bold.yellow(
-          `Applying ${diffs.length} diffs to existing variable... ${remotePath}`
-        )
-      );
-      const changeset: EditVariable = {};
-      for (const diff of diffs) {
-        if (
-          diff.type !== "REMOVE" &&
-          (diff.path.length !== 1 ||
-            ![
-              "path",
-              "value",
-              "is_secret",
-              "description",
-              "account",
-              "is_oauth",
-            ].includes(diff.path[0] as string))
-        ) {
-          console.log(
-            colors.red("Invalid variable diff with path " + diff.path)
-          );
-          throw new Error("Invalid variable diff with path " + diff.path);
-        }
-        if (diff.type === "CREATE" || diff.type === "CHANGE") {
-          setValueByPath(changeset, diff.path, diff.value);
-        } else if (diff.type === "REMOVE") {
-          setValueByPath(changeset, diff.path, null);
-        }
-      }
+export async function pushVariable(
+  workspace: string,
+  remotePath: string,
+  variable: VariableFile | ListableVariable | undefined,
+  localVariable: VariableFile,
+  plainSecrets: boolean,
+  raw: boolean
+): Promise<void> {
+  remotePath = removeType(remotePath, "variable");
+  log.debug(`Processing local variable ${remotePath}`);

-      const hasChanges = Object.values(changeset).some(
-        (v) => v !== null && typeof v !== "undefined"
-      );
-      if (!hasChanges) {
-        return;
-      }
-
-      await VariableService.updateVariable({
-        workspace,
+  if (raw) {
+    try {
+      variable = await VariableService.getVariable({
+        workspace: workspace,
        path: remotePath,
-        alreadyEncrypted: !plainSecrets,
-        requestBody: changeset,
-      });
-    } else {
-      console.log(colors.yellow.bold("Creating new variable..."));
-      await VariableService.createVariable({
-        workspace,
-        alreadyEncrypted: !plainSecrets,
-        requestBody: {
-          path: remotePath,
-          description: this.description,
-          is_secret: this.is_secret,
-          value: this.value,
-          account: this.account,
-          is_oauth: this.is_oauth,
-        },
+        decryptSecret: plainSecrets,
      });
+      log.debug(`Variable ${remotePath} exists on remote`);
+    } catch {
+      log.debug(`Variable ${remotePath} does not exist on remote`);
    }
  }
-  async push(
-    workspace: string,
-    remotePath: string,
-    plainSecrets?: boolean
-  ): Promise<void> {
-    await this.pushDiffs(
+
+  if (variable) {
+    if (isSuperset(localVariable, variable)) {
+      log.debug(`Variable ${remotePath} is up-to-date`);
+      return;
+    }
+    log.debug(`Variable ${remotePath} is not up-to-date, updating`);
+
+    await VariableService.updateVariable({
      workspace,
-      remotePath,
-      microdiff({}, this, { cyclesFix: false }),
-      plainSecrets
-    );
+      path: remotePath,
+      alreadyEncrypted: !plainSecrets,
+      requestBody: {
+        ...localVariable,
+        is_secret:
+          localVariable.is_secret && !variable.is_secret ? true : undefined,
+      },
+    });
+  } else {
+    log.info(colors.yellow.bold(`Creating new variable ${remotePath}...`));
+    await VariableService.createVariable({
+      workspace,
+      alreadyEncrypted: !plainSecrets,
+      requestBody: {
+        path: remotePath,
+        ...localVariable,
+      },
+    });
  }
 }

@@ -156,27 +117,17 @@ async function push(
    throw new Error("file path must refer to a file.");
  }

-  console.log(colors.bold.yellow("Pushing variable..."));
+  log.info(colors.bold.yellow("Pushing variable..."));

  await pushVariable(
    workspace.workspaceId,
-    filePath,
    remotePath,
-    opts.plainSecrets
+    undefined,
+    parseFromFile(filePath),
+    opts.plainSecrets,
+    true
  );
-  console.log(colors.bold.underline.green(`Variable ${remotePath} pushed`));
-}
-
-export async function pushVariable(
-  workspace: string,
-  filePath: string,
-  remotePath: string,
-  plainSecrets: boolean
-) {
-  const data = decoverto
-    .type(VariableFile)
-    .rawToInstance(await Deno.readTextFile(filePath));
-  await data.push(workspace, remotePath, plainSecrets);
+  log.info(colors.bold.underline.green(`Variable ${remotePath} pushed`));
 }

 const command = new Command()
@@ -9,7 +9,7 @@
          "lock": "",
          "path": null,
          "type": "rawscript",
-          "content": "// import * as wmill from \"https://deno.land/x/windmill@v1.69.3/mod.ts\"\n\nexport async function main(x: string) {\n  console.log(\"Hello from Deno! The argument x is \" + x);\n  return x;\n}\n",
+          "content": "// import * as wmill from \"https://deno.land/x/windmill@v1.69.3/mod.ts\"\n\nexport async function main(x: string) {\n log.info(\"Hello from Deno! The argument x is \" + x);\n  return x;\n}\n",
          "language": "deno",
          "input_transforms": {
            "x": {
@@ -7,36 +7,19 @@ import {
  Command,
  DelimiterStream,
  Input,
+  log,
  setClient,
  Table,
  UserService,
  WorkspaceService,
 } from "./deps.ts";
-import { decoverto, model, property } from "./decoverto.ts";
 import { requireLogin } from "./context.ts";

-@model()
-export class Workspace {
-  @property(() => String)
+export interface Workspace {
  remote: string;
-  @property(() => String)
  workspaceId: string;
-  @property(() => String)
  name: string;
-  @property(() => String)
  token: string;
-
-  constructor(
-    remote: string,
-    workspaceId: string,
-    name: string,
-    token: string
-  ) {
-    this.remote = remote;
-    this.workspaceId = workspaceId;
-    this.name = name;
-    this.token = token;
-  }
 }

 function makeWorkspaceStream(
@@ -52,7 +35,7 @@ function makeWorkspaceStream(
            if (line.length <= 2) {
              return;
            }
-            const workspace = decoverto.type(Workspace).rawToInstance(line);
+            const workspace = JSON.parse(line) as Workspace;
            workspace.remote = new URL(workspace.remote).toString(); // add trailing slash in all cases!
            controller.enqueue(workspace);
          } catch {
@@ -140,12 +123,12 @@ async function list(opts: GlobalOptions) {
    )
    .render();

-  console.log("Active: " + colors.green.bold(activeName || "none"));
+  log.info("Active: " + colors.green.bold(activeName || "none"));
 }

 async function switchC(opts: GlobalOptions, workspaceName: string) {
  if (opts.workspace) {
-    console.log(
+    log.info(
      colors.red.bold(
        "! Workspace needs to be specified as positional argument, not as option."
      )
@@ -155,14 +138,14 @@ async function switchC(opts: GlobalOptions, workspaceName: string) {

  const all = await allWorkspaces();
  if (all.findIndex((x) => x.name === workspaceName) === -1) {
-    console.log(
+    log.info(
      colors.red.bold(
        `! This workspace profile ${workspaceName} does not exist locally.`
      )
    );
-    console.log("available workspace profiles:");
+    log.info("available workspace profiles:");
    for (const w of all) {
-      console.log("  - " + w.name);
+      log.info("  - " + w.name);
    }
    return;
  }
@@ -184,7 +167,7 @@ export async function add(
  remote: string | undefined
 ) {
  if (opts.workspace) {
-    console.log(
+    log.info(
      colors.red.bold(
        "! Workspace needs to be specified as positional argument, not as option."
      )
@@ -239,14 +222,14 @@ export async function add(
      requestBody: { id: workspaceId },
    });
  } catch (e) {
-    console.log(
+    log.info(
      colors.red.bold("! Credentials or instance is invalid. Aborting.")
    );
    throw e;
  }
  if (opts.create) {
    if (!alreadyExists) {
-      console.log(
+      log.info(
        colors.yellow(
          `Workspace at id ${workspaceId} on ${remote} does not exist. Creating...`
        )
@@ -260,17 +243,17 @@ export async function add(
      });
    }
  } else if (!alreadyExists) {
-    console.log(
+    log.info(
      colors.red.bold(
        `! Workspace at id ${workspaceId} on ${remote} does not exist. Re-run with --create to create it. Aborting.`
      )
    );
-    console.log(
+    log.info(
      "On that instance and with those credentials, the workspaces that you can access are:"
    );
    const workspaces = await WorkspaceService.listWorkspaces();
    for (const workspace of workspaces) {
-      console.log(`- ${workspace.id} (name: ${workspace.name})`);
+      log.info(`- ${workspace.id} (name: ${workspace.name})`);
    }
    Deno.exit(1);
  }
@@ -288,7 +271,7 @@ export async function add(
    (await getRootStore()) + "/activeWorkspace",
    workspaceName
  );
-  console.log(
+  log.info(
    colors.green.underline(
      `Added workspace ${workspaceName} for ${workspaceId} on ${remote}!`
    )
@@ -317,16 +300,16 @@ export async function removeWorkspace(
  const orgWorkspaces = await allWorkspaces();
  if (orgWorkspaces.findIndex((x) => x.name === name) === -1) {
    if (!silent) {
-      console.log(
+      log.info(
        colors.red.bold(`! Workspace profile ${name} does not exist locally`)
      );
-      console.log("available workspace profiles:");
+      log.info("available workspace profiles:");
      await list(opts);
    }
    return;
  }
  if (silent) {
-    console.log(colors.yellow(`Replacing existing workspace ${name}`));
+    log.info(colors.yellow(`Replacing existing workspace ${name}`));
  }

  await Deno.writeTextFile(
@@ -338,9 +321,7 @@ export async function removeWorkspace(
  );

  if (!silent) {
-    console.log(
-      colors.green.underline(`Succesfully removed workspace ${name}!`)
-    );
+    log.info(colors.green.underline(`Succesfully removed workspace ${name}!`));
  }
 }

@@ -350,9 +331,9 @@ async function remove(_opts: GlobalOptions, name: string) {

 async function whoami(_opts: GlobalOptions) {
  await requireLogin(_opts);
-  console.log(await UserService.globalWhoami());
+  log.info(await UserService.globalWhoami());
  const activeName = await getActiveWorkspaceName(_opts);
-  console.log("Active: " + colors.green.bold(activeName || "none"));
+  log.info("Active: " + colors.green.bold(activeName || "none"));
 }

 const command = new Command()
@@ -1 +1 @@
-VITE_DEFAULT_WEBHOOK_TYPE=async 
+VITE_DEFAULT_WEBHOOK_TYPE=async
@@ -7,4 +7,5 @@ node_modules
 CaddyfileRemoteRuben
 tests-out/
 storageState.json
-.env.production
+.env.production
+dist/
@@ -1,12 +0,0 @@
-http://localhost {  
-  bind {$ADDRESS}
-  reverse_proxy /api/* http://localhost:8000
-  reverse_proxy /* http://localhost:3000
-  reverse_proxy /ws/* http://localhost:3001
-}
-
-https://localhost {  
-  bind {$ADDRESS}
-  reverse_proxy /api/* http://localhost:8000
-  reverse_proxy /ws/* http://localhost:3001
-}
@@ -1,12 +0,0 @@
-{
-  auto_https off
-}
-
-:8080 {  
-  bind {$ADDRESS}
-  reverse_proxy /api/* http://localhost:8000
-  reverse_proxy /* http://localhost:3000
-  reverse_proxy /ws/* http://lsp:3001 {
-    lb_policy header "Authorization"
-  }
-}
@@ -1,14 +0,0 @@
-http://localhost {  
-  bind {$ADDRESS}
-  reverse_proxy /api/* https://app.windmill.dev {
-    	header_up Host {http.reverse_proxy.upstream.hostport}
-  }
-  reverse_proxy /* http://localhost:3000
-}
-
-
-https://localhost {  
-  bind {$ADDRESS}
-  reverse_proxy /ws/* http://localhost:3001
-}
-
@@ -1,17 +0,0 @@
-http://localhost {  
-  bind {$ADDRESS}
-  reverse_proxy /api/* https://app.windmill.dev {
-    	header_up Host {http.reverse_proxy.upstream.hostport}
-      header_down Set-Cookie windmill.dev localhost
-  }
-  reverse_proxy /* http://localhost:3000
-}
-
-
-https://localhost {  
-  bind {$ADDRESS}
-  reverse_proxy /ws/* https://app.windmill.dev {
-    	header_up Host {http.reverse_proxy.upstream.hostport}
-  }
-}
-
@@ -1,25 +1,41 @@
 # Developing

-## Starting The Development Server
+- install the dependencies with `npm install` (or `pnpm install` or `yarn`)
+- generate the windmill client:

-Once you've created a project and installed dependencies with `npm install` (or
-`pnpm install` or `yarn`), start a development server:
+```bash
+npm run generate-backend-client
+## on mac use
+npm run generate-backend-client-mac
+```
+
+Once the dependencies are installed, just start the dev server:

 ```bash
 npm run dev
 ```

-To have the whole stack backing your dev environment, 3 solutions:
+The default proxy is setup to use the remote backend: <https://app.windmill.dev>.
+
+You can configure another proxy to use like so:
+
+```bash
+REMOTE=http://127.0.0.1:8000 REMOTE_LSP=http://127.0.0.1:3001 npm run dev
+```
+
+## Use a Local backend

 ### 1. Backend is run by docker

-In the root folder:
-
 ```bash
 docker build . -t windmill
 docker compose up db windmill_server windmill_worker
 ```

+```bash
+REMOTE=http://localhost REMOTE_LSP=http://localhost npm run dev
+```
+
 ### 2. Backend is run by cargo

 **Prerequisites**
@@ -28,20 +44,20 @@ docker compose up db windmill_server windmill_worker
 - Install llvm

  **On OSX:**
+
  ```bash
  brew install llvm caddy gsed
-  
+
  # make LLVM tools available on PATH
  echo 'export PATH="/opt/homebrew/opt/llvm/bin:$PATH"' >> ~/.zshrc
-  
+
  # now, restart your shell. You should now have the `lld` binary on your PATH.
  ```
+
 - To test that you have Rust and Cargo installed run `cargo --version`

- In your terminal, go to the backend directory and run `cargo build`
- Run `cargo run`
+**Known issue on M1 Mac while running `cargorun`**

-**Known issue on M1 Mac while running `cargo build`**
 - You may encounter `linking with cc failed` build time error.
 - To solve this run:
  ```bash
@@ -49,34 +65,6 @@ docker compose up db windmill_server windmill_worker
  source ~/.zshrc
  ```

-
-**Do a Frontend Build**
-
-In order to run the backend, you need to have a frontend build inside `frontend/build/`.
-
-Otherwise, `cargo run` will break.
-
-So, in the frontend folder, run:
-
-```bash
-# !!! on OSX, you are not allowed to use the system SED, but you need to use GNU SED.
-# !!! thus, in `frontend/package.json`, replace all `sed` occurences with `gsed`.
-
-# prerequisite for build
-npm run generate-backend-client
-
-npm run build
-# now, you'll have a `frontend/build` folder.
-```
-
-**Known issue while running `npm run build`**
- You may encounter `FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory` error.
- To solve this run:
-```bash
-export NODE_OPTIONS=--max_old_space_size=8096
-```
- run `npm run build` again
-
 In the root folder:

 ```bash
@@ -89,38 +77,16 @@ In the backend folder:
 DATABASE_URL=postgres://postgres:changeme@127.0.0.1:5433/windmill?sslmode=disable cargo run
 ```

-You can now access [http://127.0.0.1:8000](http://127.0.0.1:8000).
-
-### In both cases
-
 In the frontend folder:

 ```bash
-sudo caddy run --config ./Caddyfile
+REMOTE=http://127.0.0.1:8000 REMOTE_LSP=http://127.0.0.1:3001 npm run dev
 ```

-(sudo is required to bind port 80 and 443)
-
-and then go to <http://localhost>
-
-### 3. Backend is run by remote!
-
-```bash
-sudo caddy run --config ./CaddyfileRemote
-```
-
-and then go to <http://localhost>
-
 ## Building

 ```bash
-npm run build
-```
-
-## Generating the backend client automatically
-
-```bash
-npm run generate-backend-client
+NODE_OPTIONS=--max_old_space_size=8096 npm run build
 ```

 ## Formatting
@@ -146,13 +112,13 @@ Recommended config for VS Code:

 - turn _format on save_ on

-
 ## Building

 The project is built with [SvelteKit](https://kit.svelte.dev/) and uses as output static files.
 There are others adapters for sveltekit, but we use the static adapter.

 To build the frontend as static assets, use:
+
 ```
 npm run build
 ```
@@ -160,10 +126,11 @@ npm run build
 The output is in the `build` folder.

 The default build assume you serve every non static files as the 200.html file which is catchall. If you prefer a normal layout, you can use:
+
 ```
 NOTCATCHALL=true npm run build
 ```
+
 which will generate an index.html and allow you to serve the frontend with any static server.

 Env variables used for build are set in .env file. See [https://vitejs.dev/guide/env-and-mode.html#env-files](https://vitejs.dev/guide/env-and-mode.html#env-files) for more details.
-
@@ -1,6 +1,6 @@
 {
 	"name": "windmill",
-	"version": "1.87.0",
+	"version": "1.96.3",
 	"scripts": {
 		"dev": "vite dev",
 		"build": "vite build",
@@ -9,49 +9,50 @@
 		"check:watch": "svelte-check --tsconfig ./tsconfig.json --watch",
 		"lint": "prettier --ignore-path .gitignore --check --plugin-search-dir=. . && eslint --ignore-path .gitignore .",
 		"format": "prettier --ignore-path .gitignore --write --plugin-search-dir=. .",
-		"package": "svelte-package && cd package && rm README.md && rm README_DEV.md && sed -i -e 's/windmill/windmill-components/g' package.json",
+		"package": "svelte-package -o package",
 		"generate-backend-client": "openapi --input ../backend/windmill-api/openapi.yaml --output ./src/lib/gen --useOptions && sed -i '213 i \\    request.referrerPolicy = \"no-referrer\"\n' src/lib/gen/core/request.ts",
+		"generate-backend-client-mac": "openapi --input ../backend/windmill-api/openapi.yaml --output ./src/lib/gen --useOptions",
 		"pretest": "tsc --incremental -p tests/tsconfig.json",
 		"test": "playwright test --config=tests-out/playwright.config.js"
 	},
 	"devDependencies": {
-		"@playwright/test": "^1.31.1",
+		"@playwright/test": "^1.33.0",
 		"@rgossiaux/svelte-headlessui": "^1.0.2",
-		"@sveltejs/adapter-static": "^2.0.1",
-		"@sveltejs/kit": "^1.0.0-next.589",
-		"@sveltejs/package": "^1.0.2",
+		"@sveltejs/adapter-static": "^2.0.2",
+		"@sveltejs/kit": "^1.16.2",
+		"@sveltejs/package": "^2.0.0",
 		"@tailwindcss/forms": "^0.5.3",
 		"@tailwindcss/typography": "^0.5.8",
 		"@types/d3": "^7.4.0",
 		"@types/d3-zoom": "^3.0.2",
-		"@types/lodash": "^4.14.191",
-		"@types/node": "^18.11.18",
+		"@types/lodash": "^4.14.194",
+		"@types/node": "^18.16.3",
 		"@types/vscode": "~1.77.0",
-		"@typescript-eslint/eslint-plugin": "^5.49.0",
-		"@typescript-eslint/parser": "^5.48.0",
-		"@zerodevx/svelte-toast": "^0.8.1",
+		"@typescript-eslint/eslint-plugin": "^5.59.5",
+		"@typescript-eslint/parser": "^5.59.1",
+		"@zerodevx/svelte-toast": "^0.9.3",
 		"autoprefixer": "^10.4.13",
-		"cssnano": "^6.0.0",
+		"cssnano": "^6.0.1",
 		"d3-dag": "^0.11.5",
-		"eslint": "^8.38.0",
+		"eslint": "^8.40.0",
 		"eslint-config-prettier": "^8.6.0",
 		"eslint-plugin-svelte3": "^4.0.0",
-		"monaco-editor-workers": "0.36.0",
+		"monaco-editor-workers": "0.37.0",
 		"ol": "^7.2.2",
 		"openapi-typescript-codegen": "^0.23.0",
 		"path-browserify": "^1.0.1",
 		"pdfjs-dist": "^3.5.141",
-		"postcss": "^8.4.18",
+		"postcss": "^8.4.23",
 		"postcss-load-config": "^4.0.1",
-		"prettier": "^2.8.3",
+		"prettier": "^2.8.8",
 		"prettier-plugin-svelte": "^2.9.0",
 		"simple-svelte-autocomplete": "^2.5.1",
 		"style-to-object": "^0.4.1",
-		"stylelint-config-recommended": "^9.0.0",
+		"stylelint-config-recommended": "^12.0.0",
 		"svelte": "^3.55.1",
 		"svelte-awesome": "^3.2.0",
 		"svelte-awesome-color-picker": "^2.4.1",
-		"svelte-check": "^3.0.2",
+		"svelte-check": "^3.2.0",
 		"svelte-highlight": "^7.2.1",
 		"svelte-multiselect": "^8.6.0",
 		"svelte-overlay": "^1.4.1",
@@ -59,43 +60,196 @@
 		"svelte-preprocess": "^5.0.1",
 		"svelte-range-slider-pips": "^2.1.1",
 		"svelte-splitpanes": "^0.7.3",
-		"svelte2tsx": "^0.6.11",
-		"tailwindcss": "^3.2.2",
+		"svelte2tsx": "^0.6.14",
+		"tailwindcss": "^3.3.2",
 		"tslib": "^2.4.1",
-		"typescript": "^4.9.3",
-		"vite": "^4.0.4",
+		"typescript": "^5.0.4",
+		"vite": "^4.3.3",
 		"yootils": "^0.3.1"
 	},
 	"type": "module",
 	"dependencies": {
+		"@aws-crypto/sha256-js": "^4.0.0",
 		"@fortawesome/free-brands-svg-icons": "^6.2.1",
 		"@fortawesome/free-solid-svg-icons": "^6.2.1",
 		"@leeoniya/ufuzzy": "^1.0.6",
 		"@popperjs/core": "^2.11.6",
 		"@redocly/json-to-json-schema": "^0.0.1",
-		"@tanstack/svelte-table": "^8.7.6",
+		"@tanstack/svelte-table": "^8.9.1",
 		"ag-grid-svelte": "^0.1.4",
 		"chart.js": "^3.9.1",
 		"chartjs-adapter-date-fns": "^3.0.0",
 		"chartjs-plugin-zoom": "^2.0.0",
 		"d3-zoom": "^3.0.0",
 		"date-fns": "^2.29.3",
-		"fast-equals": "^4.0.3",
-		"highlight.js": "^11.7.0",
+		"fast-equals": "^5.0.1",
+		"highlight.js": "^11.8.0",
 		"lodash": "^4.17.21",
-		"lucide-svelte": "^0.115.0",
+		"lucide-svelte": "^0.214.0",
 		"monaco-editor": "0.36.1",
 		"monaco-languageclient": "5.0.1",
 		"svelte-autosize": "^1.0.1",
 		"svelte-chartjs": "^3.1.0",
+		"svelte-dnd-action": "^0.9.22",
 		"svelte-portal": "^2.2.0",
 		"svelte-select": "^5.6.0",
 		"svelte-timezone-picker": "^2.0.3",
-		"tailwind-merge": "^1.9.1",
+		"tailwind-merge": "^1.12.0",
 		"vscode-languageclient": "^8.1.0",
 		"vscode-ws-jsonrpc": "3.0.0"
 	},
 	"peerDependencies": {
+		"@sveltejs/kit": "^1.16.2",
 		"svelte": "^3.55.1"
+	},
+	"exports": {
+		"./package.json": "./package.json",
+		".": {
+			"types": "./dist/index.d.ts",
+			"svelte": "./dist/index.js"
+		},
+		"./assets/app.css": "./package/assets/app.css",
+		"./components/IconedResourceType.svelte": {
+			"types": "./package/components/IconedResourceType.svelte.d.ts",
+			"svelte": "./package/components/IconedResourceType.svelte",
+			"default": "./package/components/IconedResourceType.svelte"
+		},
+		"./components/common/drawer/Drawer.svelte": {
+			"types": "./package/components/common/drawer/Drawer.svelte.d.ts",
+			"svelte": "./package/components/common/drawer/Drawer.svelte",
+			"default": "./package/components/common/drawer/Drawer.svelte"
+		},
+		"./components/common/drawer/DrawerContent.svelte": {
+			"types": "./package/components/common/drawer/DrawerContent.svelte.d.ts",
+			"svelte": "./package/components/common/drawer/DrawerContent.svelte",
+			"default": "./package/components/common/drawer/DrawerContent.svelte"
+		},
+		"./components/common/button/Button.svelte": {
+			"types": "./package/components/common/button/Button.svelte.d.ts",
+			"svelte": "./package/components/common/button/Button.svelte",
+			"default": "./package/components/common/button/Button.svelte"
+		},
+		"./components/RadioButton.svelte": {
+			"types": "./package/components/RadioButton.svelte.d.ts",
+			"svelte": "./package/components/RadioButton.svelte",
+			"default": "./package/components/RadioButton.svelte"
+		},
+		"./components/common/tabs/Tabs.svelte": {
+			"types": "./package/components/common/tabs/Tabs.svelte.d.ts",
+			"svelte": "./package/components/common/tabs/Tabs.svelte",
+			"default": "./package/components/common/tabs/Tabs.svelte"
+		},
+		"./components/common/tabs/Tab.svelte": {
+			"types": "./package/components/common/tabs/Tab.svelte.d.ts",
+			"svelte": "./package/components/common/tabs/Tab.svelte",
+			"default": "./package/components/common/tabs/Tab.svelte"
+		},
+		"./components/common/alert/Alert.svelte": {
+			"types": "./package/components/common/alert/Alert.svelte.d.ts",
+			"svelte": "./package/components/common/alert/Alert.svelte",
+			"default": "./package/components/common/alert/Alert.svelte"
+		},
+		"./components/apps/editor/AppPreview.svelte": {
+			"types": "./package/components/apps/editor/AppPreview.svelte.d.ts",
+			"svelte": "./package/components/apps/editor/AppPreview.svelte",
+			"default": "./package/components/apps/editor/AppPreview.svelte"
+		},
+		"./components/FlowViewer.svelte": {
+			"types": "./package/components/FlowViewer.svelte.d.ts",
+			"svelte": "./package/components/FlowViewer.svelte",
+			"default": "./package/components/FlowViewer.svelte"
+		},
+		"./components/SchemaViewer.svelte": {
+			"types": "./package/components/SchemaViewer.svelte.d.ts",
+			"svelte": "./package/components/SchemaViewer.svelte",
+			"default": "./package/components/SchemaViewer.svelte"
+		},
+		"./components/SchemaEditor.svelte": {
+			"types": "./package/components/SchemaEditor.svelte.d.ts",
+			"svelte": "./package/components/SchemaEditor.svelte",
+			"default": "./package/components/SchemaEditor.svelte"
+		},
+		"./common": {
+			"types": "./package/common.d.ts",
+			"default": "./package/common.js"
+		},
+		"./components/icons": {
+			"types": "./package/components/icons/index.d.ts",
+			"svelte": "./package/components/icons/index.js",
+			"default": "./package/components/icons/index.js"
+		},
+		"./components/apps/inputType": {
+			"types": "./package/components/apps/inputType.d.ts",
+			"default": "./package/components/apps/inputType.js"
+		},
+		"./components/apps/types": {
+			"types": "./package/components/apps/types.d.ts",
+			"default": "./package/components/apps/types.js"
+		},
+		"./components/apps/editor/inlineScriptsPanel/utils": {
+			"types": "./package/components/apps/editor/inlineScriptsPanel/utils.d.ts",
+			"default": "./package/components/apps/editor/inlineScriptsPanel/utils.js"
+		}
+	},
+	"files": [
+		"dist",
+		"package"
+	],
+	"license": "Apache-2.0",
+	"svelte": "./dist/index.js",
+	"typesVersions": {
+		">4.0": {
+			"components/IconedResourceType.svelte": [
+				"./package/components/IconedResourceType.svelte.d.ts"
+			],
+			"components/common/drawer/Drawer.svelte": [
+				"./package/components/common/drawer/Drawer.svelte.d.ts"
+			],
+			"components/common/drawer/DrawerContent.svelte": [
+				"./package/components/common/drawer/DrawerContent.svelte.d.ts"
+			],
+			"components/common/button/Button.svelte": [
+				"./package/components/common/button/Button.svelte.d.ts"
+			],
+			"components/RadioButton.svelte": [
+				"./package/components/RadioButton.svelte.d.ts"
+			],
+			"components/common/tabs/Tabs.svelte": [
+				"./package/components/common/tabs/Tabs.svelte.d.ts"
+			],
+			"components/common/tabs/Tab.svelte": [
+				"./package/components/common/tabs/Tab.svelte.d.ts"
+			],
+			"components/common/alert/Alert.svelte": [
+				"./package/components/common/alert/Alert.svelte.d.ts"
+			],
+			"components/apps/editor/AppPreview.svelte": [
+				"./package/components/apps/editor/AppPreview.svelte.d.ts"
+			],
+			"components/FlowViewer.svelte": [
+				"./package/components/FlowViewer.svelte.d.ts"
+			],
+			"components/SchemaViewer.svelte": [
+				"./package/components/SchemaViewer.svelte.d.ts"
+			],
+			"components/SchemaEditor.svelte": [
+				"./package/components/SchemaEditor.svelte.d.ts"
+			],
+			"common": [
+				"./package/common.d.ts"
+			],
+			"components/icons": [
+				"./package/components/icons/index.d.ts"
+			],
+			"components/apps/inputType": [
+				"./package/components/apps/inputType.d.ts"
+			],
+			"components/apps/types": [
+				"./package/components/apps/types.d.ts"
+			],
+			"components/apps/editor/inlineScriptsPanel/utils": [
+				"./package/components/apps/editor/inlineScriptsPanel/utils.d.ts"
+			]
+		}
 	}
 }
@@ -0,0 +1,4 @@
+npm run package
+sed -i -e 's/windmill/windmill-components/g' package.json
+npm publish
+sed -i -e 's/windmill-components/windmill/g' package.json
@@ -1,2 +1,10 @@
 /// <reference types="@sveltejs/kit" />

+declare type Item = import('svelte-dnd-action').Item
+declare type DndEvent<ItemType = Item> = import('svelte-dnd-action').DndEvent<ItemType>
+declare namespace svelte.JSX {
+	interface HTMLAttributes<T> {
+		onconsider?: (event: CustomEvent<DndEvent<ItemType>> & { target: EventTarget & T }) => void
+		onfinalize?: (event: CustomEvent<DndEvent<ItemType>> & { target: EventTarget & T }) => void
+	}
+}
@@ -8,7 +8,7 @@ export type SupportedLanguage = Script.language

 export interface SchemaProperty {
 	type: string | undefined
-	description: string
+	description?: string
 	pattern?: string
 	default?: any
 	enum?: string[]
@@ -2,7 +2,12 @@
 	const apiTokenApps: Record<string, { img?: string; instructions: string[]; key?: string }> = {
 		airtable: {
 			img: '/airtable_connect.png',
-			instructions: ['Click on the top-right avatar', 'Click on Account', 'Find "Api"']
+			instructions: [
+				'Click to <a href="https://airtable.com/create/tokens" target="_blank" rel=”noopener noreferrer”>https://airtable.com/create/tokens</a>',
+				'Click on "Create new token"',
+				'Set a name, specify the scopes or the access level and click on "Create token"',
+				'Copy the token'
+			]
 		},
 		discord_webhook: {
 			img: '/discord_webhook.png',
@@ -456,7 +461,7 @@
 				>
 			{/if}
 		{/if}
-		<div slot="actions">
+		<div slot="actions" class="flex gap-1">
 			{#if step > 1 && !no_back}
 				<Button variant="border" on:click={back}>Back</Button>
 			{/if}
--- a/Show More
+++ b/Show More