feat: Typescript support for scripts (alpha)

* typescript support

* frontend

* type inference

* type inference

* v0 works

* v0 typescript

* v0 typescript

* deno-client v0

* deno-client v0

* build_deno

* rm autogenerated files

* test workflow

* test workflow

* test workflow

* test workflow

* test workflow

* test workflow

* test workflow

* test workflow

* test workflow

* test workflow

* test workflow

* test workflow

* test workflow

* test workflow

* test workflow

* test workflow

* test workflow

* test workflow

* on tags

* createResource

* createResource

* createResource2

* typescript support

* templates

* include version
This commit is contained in:
Ruben Fiszel
2022-05-17 20:42:05 +02:00
committed by GitHub
parent 000f4aab8f
commit 51a4cd359e
35 changed files with 1566 additions and 389 deletions
+49
View File
@@ -0,0 +1,49 @@
name: Publish deno-client
on:
push:
tags:
- "v*"
env:
repo: windmill-deno-client
jobs:
build_deno_and_push_to_repo:
runs-on: ubuntu-latest
container: openapitools/openapi-generator-cli:v6.0.0-beta
steps:
- uses: actions/checkout@v3
- name: generate_deno
run: |
cd deno-client
rm .gitignore
./generate.sh
- name: Pushes to another repository
id: push_directory
uses: cpina/github-action-push-to-another-repository@devel
env:
API_TOKEN_GITHUB: ${{ secrets.DENO_PAT }}
with:
source-directory: deno-client/
destination-github-username: ${{ github.repository_owner }}
destination-repository-name: ${{ env.repo }}
user-email: ruben@windmill.dev
commit-message: See ORIGIN_COMMIT from $GITHUB_REF
target-branch: main
tag_repo:
needs: [build_deno_and_push_to_repo]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
repository: ${{ github.repository_owner }}/${{ env.repo }}
token: ${{ secrets.DENO_PAT }}
path: ./client
- name: Push client
run: |
cd ./client
git config --global user.email "ruben@windmill.dev"
git config --global user.name "rubenfiszel[bot]"
git tag -a ${{ github.ref_name }} -m "${{ github.ref_name }}"
git push --tags
+468 -9
View File
@@ -2,6 +2,16 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "Inflector"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3"
dependencies = [
"lazy_static",
"regex",
]
[[package]]
name = "adler"
version = "1.0.2"
@@ -87,6 +97,20 @@ dependencies = [
"term",
]
[[package]]
name = "ast_node"
version = "0.7.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc4c00309ed1c8104732df4a5fa9acc3b796b6f8531dfbd5ce0078c86f997244"
dependencies = [
"darling",
"pmutil",
"proc-macro2",
"quote",
"swc_macros_common",
"syn",
]
[[package]]
name = "async-lock"
version = "2.5.0"
@@ -255,6 +279,15 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dea908e7347a8c64e378c17e30ef880ad73e3b4498346b055c2c00ea342f3179"
[[package]]
name = "better_scoped_tls"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b73e8ecdec39e98aa3b19e8cd0b8ed8f77ccb86a6b0b2dc7cd86d105438a2123"
dependencies = [
"scoped-tls",
]
[[package]]
name = "bit-set"
version = "0.5.2"
@@ -606,6 +639,41 @@ dependencies = [
"subtle 1.0.0",
]
[[package]]
name = "darling"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim 0.9.3",
"syn",
]
[[package]]
name = "darling_macro"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72"
dependencies = [
"darling_core",
"quote",
"syn",
]
[[package]]
name = "data-encoding"
version = "2.3.2"
@@ -618,6 +686,15 @@ version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f578e8e2c440e7297e008bb5486a3a8a194775224bbc23729b0dbdfaeebf162e"
[[package]]
name = "debug_unreachable"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a032eac705ca39214d169f83e3d3da290af06d8d1d344d1baad2fd002dca4b3"
dependencies = [
"unreachable",
]
[[package]]
name = "deno_core"
version = "0.110.0"
@@ -725,7 +802,7 @@ dependencies = [
"lazy_static",
"regex",
"serde",
"strsim",
"strsim 0.10.0",
]
[[package]]
@@ -782,6 +859,18 @@ dependencies = [
"syn",
]
[[package]]
name = "enum_kind"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78b940da354ae81ef0926c5eaa428207b8f4f091d3956c891dfbd124162bed99"
dependencies = [
"pmutil",
"proc-macro2",
"swc_macros_common",
"syn",
]
[[package]]
name = "event-listener"
version = "2.5.2"
@@ -880,6 +969,18 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "from_variant"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0951635027ca477be98f8774abd6f0345233439d63f307e47101acb40c7cc63d"
dependencies = [
"pmutil",
"proc-macro2",
"swc_macros_common",
"syn",
]
[[package]]
name = "fslock"
version = "0.1.8"
@@ -1402,6 +1503,12 @@ dependencies = [
"tokio-native-tls",
]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "idna"
version = "0.2.3"
@@ -1472,6 +1579,19 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c"
[[package]]
name = "is-macro"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94b2c46692aee0d1b3aad44e781ac0f0e7db42ef27adaa0a877b627040019813"
dependencies = [
"Inflector",
"pmutil",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "itertools"
version = "0.8.2"
@@ -1550,7 +1670,7 @@ dependencies = [
"serde",
"serde_derive",
"sha2 0.8.2",
"string_cache",
"string_cache 0.7.5",
"term",
"unicode-xid 0.1.0",
]
@@ -1593,6 +1713,79 @@ dependencies = [
"webpki-roots 0.22.3",
]
[[package]]
name = "lexical"
version = "6.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccd3e434c16f0164124ade12dcdee324fcc3dafb1cad0c7f1d8c2451a1aa6886"
dependencies = [
"lexical-core",
]
[[package]]
name = "lexical-core"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92912c4af2e7d9075be3e5e3122c4d7263855fa6cce34fbece4dd08e5884624d"
dependencies = [
"lexical-parse-float",
"lexical-parse-integer",
"lexical-util",
"lexical-write-float",
"lexical-write-integer",
]
[[package]]
name = "lexical-parse-float"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f518eed87c3be6debe6d26b855c97358d8a11bf05acec137e5f53080f5ad2dd8"
dependencies = [
"lexical-parse-integer",
"lexical-util",
"static_assertions",
]
[[package]]
name = "lexical-parse-integer"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afc852ec67c6538bbb2b9911116a385b24510e879a69ab516e6a151b15a79168"
dependencies = [
"lexical-util",
"static_assertions",
]
[[package]]
name = "lexical-util"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c72a9d52c5c4e62fa2cdc2cb6c694a39ae1382d9c2a17a466f18e272a0930eb1"
dependencies = [
"static_assertions",
]
[[package]]
name = "lexical-write-float"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a89ec1d062e481210c309b672f73a0567b7855f21e7d2fae636df44d12e97f9"
dependencies = [
"lexical-util",
"lexical-write-integer",
"static_assertions",
]
[[package]]
name = "lexical-write-integer"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "094060bd2a7c2ff3a16d5304a6ae82727cb3cc9d1c70f813cc73f744c319337e"
dependencies = [
"lexical-util",
"static_assertions",
]
[[package]]
name = "libc"
version = "0.2.123"
@@ -1853,6 +2046,18 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-bigint"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f"
dependencies = [
"autocfg 1.1.0",
"num-integer",
"num-traits",
"serde",
]
[[package]]
name = "num-integer"
version = "0.1.44"
@@ -2055,17 +2260,36 @@ version = "0.7.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662"
dependencies = [
"phf_shared",
"phf_shared 0.7.24",
"rand 0.6.5",
]
[[package]]
name = "phf_generator"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"
dependencies = [
"phf_shared 0.10.0",
"rand 0.8.5",
]
[[package]]
name = "phf_shared"
version = "0.7.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0"
dependencies = [
"siphasher",
"siphasher 0.2.3",
]
[[package]]
name = "phf_shared"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
dependencies = [
"siphasher 0.3.10",
]
[[package]]
@@ -2112,6 +2336,17 @@ version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
[[package]]
name = "pmutil"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3894e5d549cccbe44afecf72922f277f603cd4bb0219c8342631ef18fffbe004"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "ppv-lite86"
version = "0.2.16"
@@ -2569,6 +2804,12 @@ dependencies = [
"walkdir",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustls"
version = "0.19.1"
@@ -2612,7 +2853,7 @@ dependencies = [
"lalrpop",
"lalrpop-util",
"log",
"num-bigint",
"num-bigint 0.2.6",
"num-traits",
"unic-emoji-char",
"unic-ucd-ident",
@@ -2644,6 +2885,12 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "scoped-tls"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
[[package]]
name = "scopeguard"
version = "1.1.0"
@@ -2837,6 +3084,12 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac"
[[package]]
name = "siphasher"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
[[package]]
name = "slab"
version = "0.4.6"
@@ -3008,6 +3261,12 @@ dependencies = [
"tokio-rustls 0.22.0",
]
[[package]]
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "string_cache"
version = "0.7.5"
@@ -3016,32 +3275,71 @@ checksum = "89c058a82f9fd69b1becf8c274f412281038877c553182f1d02eb027045a2d67"
dependencies = [
"lazy_static",
"new_debug_unreachable",
"phf_shared",
"phf_shared 0.7.24",
"precomputed-hash",
"serde",
"string_cache_codegen",
"string_cache_codegen 0.4.4",
"string_cache_shared",
]
[[package]]
name = "string_cache"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08"
dependencies = [
"new_debug_unreachable",
"once_cell",
"parking_lot 0.12.0",
"phf_shared 0.10.0",
"precomputed-hash",
"serde",
]
[[package]]
name = "string_cache_codegen"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6"
dependencies = [
"phf_generator",
"phf_shared",
"phf_generator 0.7.24",
"phf_shared 0.7.24",
"proc-macro2",
"quote",
"string_cache_shared",
]
[[package]]
name = "string_cache_codegen"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988"
dependencies = [
"phf_generator 0.10.0",
"phf_shared 0.10.0",
"proc-macro2",
"quote",
]
[[package]]
name = "string_cache_shared"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc"
[[package]]
name = "string_enum"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f584cc881e9e5f1fd6bf827b0444aa94c30d8fe6378cf241071b5f5700b2871f"
dependencies = [
"pmutil",
"proc-macro2",
"quote",
"swc_macros_common",
"syn",
]
[[package]]
name = "stringprep"
version = "0.1.2"
@@ -3052,6 +3350,12 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "strsim"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c"
[[package]]
name = "strsim"
version = "0.10.0"
@@ -3070,6 +3374,125 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
[[package]]
name = "swc_atoms"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba8735ce37e421749498e038955abc1135eec6a4af0b54a173e55d2e5542d472"
dependencies = [
"string_cache 0.8.4",
"string_cache_codegen 0.5.2",
]
[[package]]
name = "swc_common"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5988fd245a86176271951c089f9144b7b48014bc1212872f9bd194f5f0f9dfd"
dependencies = [
"ahash",
"ast_node",
"better_scoped_tls",
"cfg-if 1.0.0",
"debug_unreachable",
"either",
"from_variant",
"num-bigint 0.4.3",
"once_cell",
"rustc-hash",
"serde",
"siphasher 0.3.10",
"string_cache 0.8.4",
"swc_eq_ignore_macros",
"swc_visit",
"tracing",
"unicode-width",
"url",
]
[[package]]
name = "swc_ecma_ast"
version = "0.78.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ed68ad13e4489f309ffed9d302337d7c8bde11d00b8b275b7aa7fda4da035bf"
dependencies = [
"is-macro",
"num-bigint 0.4.3",
"serde",
"string_enum",
"swc_atoms",
"swc_common",
"unicode-id",
]
[[package]]
name = "swc_ecma_parser"
version = "0.104.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efb97dc6efc95313dedc5158055cc811da77395ef7b54be61948b5ad097a3671"
dependencies = [
"either",
"enum_kind",
"lexical",
"num-bigint 0.4.3",
"serde",
"smallvec",
"swc_atoms",
"swc_common",
"swc_ecma_ast",
"tracing",
"typed-arena",
"unicode-id",
]
[[package]]
name = "swc_eq_ignore_macros"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c8f200a2eaed938e7c1a685faaa66e6d42fa9e17da5f62572d3cbc335898f5e"
dependencies = [
"pmutil",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "swc_macros_common"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5dca3f08d02da4684c3373150f7c045128f81ea00f0c434b1b012bc65a6cce3"
dependencies = [
"pmutil",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "swc_visit"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5c639379dd2a8a0221fa1e12fafbdd594ba53a0cace6560054da52409dfcc1a"
dependencies = [
"either",
"swc_visit_macros",
]
[[package]]
name = "swc_visit_macros"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3b9b72892df873972549838bf84d6c56234c7502148a7e23b5a3da6e0fedfb8"
dependencies = [
"Inflector",
"pmutil",
"proc-macro2",
"quote",
"swc_macros_common",
"syn",
]
[[package]]
name = "syn"
version = "1.0.91"
@@ -3607,6 +4030,12 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
[[package]]
name = "typed-arena"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0685c84d5d54d1c26f7d3eb96cd41550adb97baed141a761cf335d3d33bcd0ae"
[[package]]
name = "typenum"
version = "1.15.0"
@@ -3691,6 +4120,12 @@ version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f"
[[package]]
name = "unicode-id"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69fe8d9274f490a36442acb4edfd0c4e473fdfc6a8b5cd32f28a0235761aedbe"
[[package]]
name = "unicode-normalization"
version = "0.1.19"
@@ -3706,6 +4141,12 @@ version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99"
[[package]]
name = "unicode-width"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
[[package]]
name = "unicode-xid"
version = "0.1.0"
@@ -3730,6 +4171,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87d6678d7916394abad0d4b19df4d3802e1fd84abd7d701f39b75ee71b9e8cf1"
[[package]]
name = "unreachable"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91"
dependencies = [
"void",
]
[[package]]
name = "untrusted"
version = "0.7.1"
@@ -3796,6 +4246,12 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "void"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
[[package]]
name = "walkdir"
version = "2.3.2"
@@ -4066,6 +4522,9 @@ dependencies = [
"slack-http-verifier",
"sql-builder",
"sqlx",
"swc_common",
"swc_ecma_ast",
"swc_ecma_parser",
"tempfile",
"thiserror",
"time 0.3.9",
+3
View File
@@ -55,6 +55,9 @@ regex = "^1"
deno_core = "^0"
indexmap = "~1.6.2"
async-recursion = "^1"
swc_common = "^0"
swc_ecma_parser = "^0"
swc_ecma_ast = "^0"
sqlx = { version = "^0", features = ["macros", "offline", "migrate", "uuid", "json", "chrono", "postgres", "runtime-tokio-rustls"]}
dotenv = "^0"
@@ -0,0 +1,12 @@
-- Add down migration script here
DROP TYPE SCRIPT_LANG;
ALTER TABLE script
DROP COLUMN language SCRIPT_LANG;
ALTER TABLE queue
DROP COLUMN language SCRIPT_LANG;
ALTER TABLE completed_job
DROP COLUMN language SCRIPT_LANG;
@@ -0,0 +1,11 @@
-- Add up migration script here
CREATE TYPE SCRIPT_LANG AS ENUM ('python3', 'deno');
ALTER TABLE script
ADD COLUMN language SCRIPT_LANG NOT NULL DEFAULT 'python3';
ALTER TABLE queue
ADD COLUMN language SCRIPT_LANG NOT NULL DEFAULT 'python3';
ALTER TABLE completed_job
ADD COLUMN language SCRIPT_LANG NOT NULL DEFAULT 'python3';
+45 -5
View File
@@ -1317,11 +1317,16 @@ paths:
type: array
items:
type: string
language:
type: string
enum: [python3, deno]
required:
- path
- summary
- description
- content
- language
responses:
"201":
description: script created
@@ -1330,14 +1335,35 @@ paths:
schema:
type: string
/scripts/tojsonschema:
/scripts/python/tojsonschema:
post:
summary: inspect code to infer jsonschema of arguments
operationId: toJsonschema
summary: inspect python code to infer jsonschema of arguments
operationId: pythonToJsonschema
tags:
- script
requestBody:
description: code with the main function
description: python code with the main function
required: true
content:
application/json:
schema:
type: string
responses:
"200":
description: parsed args
content:
application/json:
schema:
$ref: "#/components/schemas/MainArgSignature"
/scripts/deno/tojsonschema:
post:
summary: inspect deno code to infer jsonschema of arguments
operationId: denoToJsonschema
tags:
- script
requestBody:
description: deno code with the main function
required: true
content:
application/json:
@@ -2654,6 +2680,9 @@ components:
type: string
lock_error_logs:
type: string
language:
type: string
enum: [python3, deno]
required:
- hash
- path
@@ -2665,10 +2694,11 @@ components:
- deleted
- is_template
- extra_perms
- language
ScriptArgs:
type: object
additionalProperties: true
additionalProperties: {}
QueuedJob:
type: object
@@ -2729,6 +2759,9 @@ components:
$ref: "#/components/schemas/FlowValue"
is_flow_step:
type: boolean
language:
type: string
enum: [python3, deno]
required:
- id
- running
@@ -2796,6 +2829,9 @@ components:
$ref: "#/components/schemas/FlowValue"
is_flow_step:
type: boolean
language:
type: string
enum: [python3, deno]
required:
- id
- created_by
@@ -3057,10 +3093,14 @@ components:
type: string
args:
$ref: "#/components/schemas/ScriptArgs"
language:
type: string
enum: [python3, deno]
required:
- content
- args
- language
CreateResource:
type: object
+92 -70
View File
@@ -436,6 +436,64 @@
]
}
},
"2420cb110a116dfbc6b8658a6d2d35db60c6aadd157488cb72eb9116ae7e9f54": {
"query": "INSERT INTO queue\n (workspace_id, id, parent_job, created_by, permissioned_as, scheduled_for, \n script_hash, script_path, raw_code, args, job_kind, schedule_path, raw_flow, flow_status, is_flow_step, language)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) RETURNING id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Varchar",
"Uuid",
"Uuid",
"Varchar",
"Varchar",
"Timestamptz",
"Int8",
"Varchar",
"Text",
"Jsonb",
{
"Custom": {
"name": "job_kind",
"kind": {
"Enum": [
"script",
"preview",
"flow",
"dependencies",
"flowpreview"
]
}
}
},
"Varchar",
"Jsonb",
"Jsonb",
"Bool",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno"
]
}
}
}
]
},
"nullable": [
false
]
}
},
"255aafff962738317f3227ae4eb871830d89b4c12c73d8dbabe6836da124e54d": {
"query": "select path from script where hash = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
"describe": {
@@ -1902,53 +1960,6 @@
"nullable": []
}
},
"a151ceeddfd4a2825d4528542d3adc5c0a6947573558a2fd62429ba5da369617": {
"query": "INSERT INTO queue\n (workspace_id, id, parent_job, created_by, permissioned_as, scheduled_for, \n script_hash, script_path, raw_code, args, job_kind, schedule_path, raw_flow, flow_status, is_flow_step)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Varchar",
"Uuid",
"Uuid",
"Varchar",
"Varchar",
"Timestamptz",
"Int8",
"Varchar",
"Text",
"Jsonb",
{
"Custom": {
"name": "job_kind",
"kind": {
"Enum": [
"script",
"preview",
"flow",
"dependencies",
"flowpreview"
]
}
}
},
"Varchar",
"Jsonb",
"Jsonb",
"Bool"
]
},
"nullable": [
false
]
}
},
"a1d46b44718a63d6ce5a9054d493dadbffb205500dc8fb55e9816bcdb613e0d5": {
"query": "DELETE FROM queue WHERE schedule_path = $1",
"describe": {
@@ -2300,6 +2311,40 @@
]
}
},
"be33c6eb702c149044650d49b3c50493d7538d590be3f4ff6242fea85c57c667": {
"query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Int8",
"Varchar",
"Int8Array",
"Text",
"Text",
"Text",
"Varchar",
"Text",
"Bool",
"Jsonb",
"Text",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno"
]
}
}
}
]
},
"nullable": []
}
},
"bf1d8e043338867e1da1ed236ff6c85a566d5fd58d4b0d5c3a10454513811ba3": {
"query": "UPDATE workspace_settings\n SET slack_team_id = null, slack_name = null WHERE workspace_id = $1",
"describe": {
@@ -2802,29 +2847,6 @@
]
}
},
"f12e710a0c2b2e13b98fd522028b6af8be74a9126a8207aa2974b47fd36e1845": {
"query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Int8",
"Varchar",
"Int8Array",
"Text",
"Text",
"Text",
"Varchar",
"Text",
"Bool",
"Jsonb",
"Text"
]
},
"nullable": []
}
},
"f325a1262084bd3468e12dc8bcc289a96536f172b679af54dd0fbc82d4d7c987": {
"query": "DELETE FROM usr_to_group WHERE usr = $1 AND group_ = $2 AND workspace_id = $3",
"describe": {
+33 -10
View File
@@ -12,6 +12,7 @@ use sqlx::{query_scalar, Postgres, Transaction};
use std::collections::HashMap;
use crate::js_eval::eval_timeout;
use crate::scripts::ScriptLang;
use crate::users::create_token_for_owner;
use crate::{
audit::{audit_log, ActionKind},
@@ -83,6 +84,7 @@ pub struct QueuedJob {
pub flow_status: Option<serde_json::Value>,
pub raw_flow: Option<serde_json::Value>,
pub is_flow_step: bool,
pub language: ScriptLang,
}
#[derive(Debug, sqlx::FromRow, Serialize)]
@@ -111,6 +113,7 @@ struct CompletedJob {
flow_status: Option<serde_json::Value>,
raw_flow: Option<serde_json::Value>,
is_flow_step: bool,
language: ScriptLang,
}
#[derive(Deserialize, Clone, Copy)]
@@ -264,6 +267,7 @@ async fn run_preview_job(
JobPayload::Code(RawCode {
content: preview.content,
path: preview.path,
language: preview.language,
}),
preview.args,
&authed.username,
@@ -417,6 +421,7 @@ async fn list_jobs(
"permissioned_as",
"flow_status",
"is_flow_step",
"language",
],
);
let sqlc = list_completed_jobs_query(
@@ -450,6 +455,7 @@ async fn list_jobs(
"permissioned_as",
"flow_status",
"is_flow_step",
"language",
],
);
let sql = format!(
@@ -564,6 +570,7 @@ async fn list_completed_jobs(
"null as flow_status",
"null as raw_flow",
"is_flow_step",
"language",
],
)
.sql()?;
@@ -857,6 +864,7 @@ struct UnifiedJob {
permissioned_as: String,
flow_status: Option<serde_json::Value>,
is_flow_step: bool,
language: ScriptLang,
}
impl From<UnifiedJob> for Job {
@@ -887,6 +895,7 @@ impl From<UnifiedJob> for Job {
flow_status: uj.flow_status,
raw_flow: None,
is_flow_step: uj.is_flow_step,
language: uj.language,
}),
"QueuedJob" => Job::QueuedJob(QueuedJob {
workspace_id: uj.workspace_id,
@@ -912,6 +921,7 @@ impl From<UnifiedJob> for Job {
flow_status: uj.flow_status,
raw_flow: None,
is_flow_step: uj.is_flow_step,
language: uj.language,
}),
t => panic!("job type {} not valid", t),
}
@@ -925,6 +935,7 @@ struct CancelJob {
pub struct RawCode {
content: String,
path: Option<String>,
language: ScriptLang,
}
#[derive(Deserialize)]
@@ -932,6 +943,7 @@ struct Preview {
content: String,
path: Option<String>,
args: Option<Map<String, Value>>,
language: ScriptLang,
}
#[derive(Deserialize)]
@@ -1004,22 +1016,32 @@ pub async fn push<'c>(
}
}
let (script_hash, script_path, raw_code, job_kind, raw_flow) = match job_payload {
let (script_hash, script_path, raw_code, job_kind, raw_flow, language) = match job_payload {
JobPayload::ScriptHash { hash, path } => {
(Some(hash.0), Some(path), None, JobKind::Script, None)
}
JobPayload::Code(RawCode { content, path }) => {
(None, path, Some(content), JobKind::Preview, None)
(Some(hash.0), Some(path), None, JobKind::Script, None, None)
}
JobPayload::Code(RawCode {
content,
path,
language,
}) => (
None,
path,
Some(content),
JobKind::Preview,
None,
Some(language),
),
JobPayload::Dependencies { hash, dependencies } => (
Some(hash.0),
None,
Some(dependencies.join("\n")),
JobKind::Dependencies,
None,
Some(ScriptLang::Python3),
),
JobPayload::RawFlow { value, path } => {
(None, path, None, JobKind::FlowPreview, Some(value))
(None, path, None, JobKind::FlowPreview, Some(value), None)
}
JobPayload::Flow(flow) => {
let value_json = sqlx::query_scalar!("SELECT value FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
@@ -1032,7 +1054,7 @@ pub async fn push<'c>(
"could not convert json to flow for {flow}: {err:?}"
))
})?;
(None, Some(flow), None, JobKind::Flow, Some(value))
(None, Some(flow), None, JobKind::Flow, Some(value), None)
}
};
@@ -1046,8 +1068,8 @@ pub async fn push<'c>(
let uuid = sqlx::query_scalar!(
"INSERT INTO queue
(workspace_id, id, parent_job, created_by, permissioned_as, scheduled_for,
script_hash, script_path, raw_code, args, job_kind, schedule_path, raw_flow, flow_status, is_flow_step)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING id",
script_hash, script_path, raw_code, args, job_kind, schedule_path, raw_flow, flow_status, is_flow_step, language)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) RETURNING id",
workspace_id,
job_id,
parent_job,
@@ -1062,7 +1084,8 @@ pub async fn push<'c>(
schedule_path,
raw_flow.map(|f| serde_json::json!(f)),
flow_status.map(|f| serde_json::json!(f)),
is_flow_step
is_flow_step,
language: ScriptLang
)
.fetch_one(&mut tx)
.await?;
+154 -10
View File
@@ -25,7 +25,7 @@ pub struct MainArgSignature {
pub args: Vec<Arg>,
}
#[derive(Serialize)]
#[derive(Serialize, Clone)]
#[serde(rename_all(serialize = "lowercase"))]
pub enum Typ {
Str,
@@ -39,7 +39,7 @@ pub enum Typ {
Unknown,
}
#[derive(Serialize)]
#[derive(Serialize, Clone)]
pub struct Arg {
pub name: String,
pub typ: Typ,
@@ -47,7 +47,7 @@ pub struct Arg {
pub has_default: bool,
}
pub fn parse_signature(code: &str) -> error::Result<MainArgSignature> {
pub fn parse_python_signature(code: &str) -> error::Result<MainArgSignature> {
let ast = parser::parse_program(code)
.map_err(|e| error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())))?
.statements;
@@ -115,6 +115,136 @@ pub fn parse_signature(code: &str) -> error::Result<MainArgSignature> {
}
}
use swc_common::sync::Lrc;
use swc_common::{FileName, SourceMap};
use swc_ecma_ast::{
AssignPat, BindingIdent, Decl, ExportDecl, FnDecl, Ident, ModuleDecl, ModuleItem, Pat, Stmt,
TsKeywordTypeKind, TsType,
};
use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsConfig};
pub fn parse_deno_signature(code: &str) -> error::Result<MainArgSignature> {
let cm: Lrc<SourceMap> = Default::default();
let fm = cm.new_source_file(FileName::Custom("test.ts".into()), code.into());
let lexer = Lexer::new(
// We want to parse ecmascript
Syntax::Typescript(TsConfig::default()),
// EsVersion defaults to es5
Default::default(),
StringInput::from(&*fm),
None,
);
let mut parser = Parser::new_from(lexer);
let mut err_s = "".to_string();
for e in parser.take_errors() {
err_s += &e.into_kind().msg().to_string();
}
let ast = parser
.parse_module()
.map_err(|e| {
error::Error::ExecutionErr(format!("impossible to parse module: {err_s}\n{e:?}"))
})?
.body;
// println!("{ast:?}");
let params = ast.into_iter().find_map(|x| match x {
ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl {
decl:
Decl::Fn(FnDecl {
ident:
Ident {
span: _,
sym,
optional: _,
},
declare: _,
function,
}),
span: _,
})) if &sym.to_string() == "main" => Some(function.params),
_ => None,
});
if let Some(params) = params {
Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args: params
.into_iter()
.map(|x| match x.pat {
Pat::Ident(ident) => {
let (name, typ) = binding_ident_to_arg(&ident)?;
Ok(Arg {
name,
typ,
default: None,
has_default: false,
})
}
Pat::Assign(AssignPat {
span: _,
left,
right,
type_ann: _,
}) => {
let (name, typ) =
left.as_ident().map(binding_ident_to_arg).ok_or_else(|| {
error::Error::ExecutionErr(format!(
"Arg {left:?} has unexepected syntax"
))
})??;
Ok(Arg {
name,
typ,
default: serde_json::to_value(right)
.map_err(|e| error::Error::ExecutionErr(e.to_string()))?
.as_object()
.and_then(|x| x.get("value").to_owned())
.cloned(),
has_default: true,
})
}
_ => Err(error::Error::ExecutionErr(format!(
"Arg {x:?} has unexepected syntax"
))),
})
.collect::<Result<Vec<Arg>, error::Error>>()?,
})
} else {
Err(error::Error::ExecutionErr(
"main function was not findable (expected to find 'export main function(...)'"
.to_string(),
))
}
}
fn binding_ident_to_arg(
BindingIdent { id, type_ann }: &BindingIdent,
) -> anyhow::Result<(String, Typ)> {
Ok((
id.sym.to_string(),
type_ann
.as_ref()
.map(|x| match &*x.type_ann {
TsType::TsKeywordType(t) => match t.kind {
TsKeywordTypeKind::TsObjectKeyword => Typ::Dict,
TsKeywordTypeKind::TsBooleanKeyword => Typ::Bool,
TsKeywordTypeKind::TsBigIntKeyword => Typ::Int,
TsKeywordTypeKind::TsNumberKeyword => Typ::Float,
TsKeywordTypeKind::TsStringKeyword => Typ::Str,
_ => Typ::Unknown,
},
// TODO: we can do better here and extract the inner type of array
TsType::TsArrayType(_) => Typ::List,
_ => Typ::Unknown,
})
.unwrap_or(Typ::Unknown),
))
}
const STDIMPORTS: [&str; 301] = [
"__future__",
"_abc",
@@ -468,7 +598,7 @@ fn to_value(et: &ExpressionType) -> Option<serde_json::Value> {
}
}
pub fn parse_imports(code: &str) -> error::Result<Vec<String>> {
pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
let find_requirements = code
.lines()
.find_position(|x| x.starts_with("#requirements:"));
@@ -527,7 +657,7 @@ mod tests {
use super::*;
#[test]
fn test_parse_sig() -> anyhow::Result<()> {
fn test_parse_python_sig() -> anyhow::Result<()> {
//let code = "print(2 + 3, fd=sys.stderr)";
let code = "
@@ -540,13 +670,13 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt
return {\"len\": len(name), \"splitted\": name.split() }
";
println!("{}", serde_json::to_string(&parse_signature(code)?)?);
println!("{}", serde_json::to_string(&parse_python_signature(code)?)?);
Ok(())
}
#[test]
fn test_parse_imports() -> anyhow::Result<()> {
fn test_parse_python_imports() -> anyhow::Result<()> {
//let code = "print(2 + 3, fd=sys.stderr)";
let code = "
@@ -559,14 +689,14 @@ def main():
pass
";
let r = parse_imports(code)?;
let r = parse_python_imports(code)?;
println!("{}", serde_json::to_string(&r)?);
assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
Ok(())
}
#[test]
fn test_parse_imports2() -> anyhow::Result<()> {
fn test_parse_python_imports2() -> anyhow::Result<()> {
//let code = "print(2 + 3, fd=sys.stderr)";
let code = "
#requirements:
@@ -583,10 +713,24 @@ def main():
pass
";
let r = parse_imports(code)?;
let r = parse_python_imports(code)?;
println!("{}", serde_json::to_string(&r)?);
assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
Ok(())
}
#[test]
fn test_parse_deno_sig() -> anyhow::Result<()> {
let code = "
export function main(test1: string, test2: string = \"burkina\") {
console.log(42)
}
";
println!("{}", serde_json::to_string(&parse_deno_signature(code)?)?);
Ok(())
}
}
+35 -7
View File
@@ -35,7 +35,12 @@ use std::{
const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;
pub fn global_service() -> Router {
Router::new().route("/tojsonschema", post(parse_code_to_jsonschema))
Router::new()
.route(
"/python/tojsonschema",
post(parse_python_code_to_jsonschema),
)
.route("/deno/tojsonschema", post(parse_deno_code_to_jsonschema))
}
pub fn workspaced_service() -> Router {
@@ -50,6 +55,13 @@ pub fn workspaced_service() -> Router {
.route("/deployment_status/h/:hash", get(get_deployment_status))
}
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Hash)]
#[sqlx(type_name = "SCRIPT_LANG", rename_all = "lowercase")]
#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
pub enum ScriptLang {
Deno,
Python3,
}
#[derive(sqlx::Type, PartialEq, Debug, Hash, Clone, Copy)]
#[sqlx(transparent)]
pub struct ScriptHash(pub i64);
@@ -113,6 +125,7 @@ pub struct Script {
pub extra_perms: serde_json::Value,
pub lock: Option<String>,
pub lock_error_logs: Option<String>,
pub language: ScriptLang,
}
#[derive(Serialize, Deserialize, sqlx::Type, Debug)]
@@ -138,6 +151,7 @@ pub struct NewScript {
pub schema: Option<Schema>,
pub is_template: Option<bool>,
pub lock: Option<Vec<String>>,
pub language: ScriptLang,
}
#[derive(Deserialize)]
@@ -181,6 +195,7 @@ async fn list_scripts(
"extra_perms",
"null as lock",
"CASE WHEN lock_error_logs IS NOT NULL THEN 'error' ELSE null END as lock_error_logs",
"language",
])
.order_by("created_at", lq.order_desc.unwrap_or(true))
.and_where("workspace_id = ? OR workspace_id = 'starter'".bind(&w_id))
@@ -344,10 +359,16 @@ async fn create_script(
.map(|v| v.1.clone())
.unwrap_or(json!({}));
let lock = if ns.language == ScriptLang::Deno {
Some("".to_string())
} else {
ns.lock.as_ref().map(|x| x.join("\n"))
};
//::text::json is to ensure we use serde_json with preserve order
sqlx::query!(
"INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, \
created_by, schema, is_template, extra_perms, lock) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12)",
created_by, schema, is_template, extra_perms, lock, language) VALUES \
($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13)",
&w_id,
&hash.0,
ns.path,
@@ -359,13 +380,14 @@ async fn create_script(
ns.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
ns.is_template.unwrap_or(false),
extra_perms,
ns.lock.as_ref().map(|x| x.join("\n"))
lock,
ns.language: ScriptLang
)
.execute(&mut tx)
.await?;
let mut tx = if ns.lock.is_none() {
let dependencies = parser::parse_imports(&ns.content)?;
let mut tx = if ns.lock.is_none() && ns.language == ScriptLang::Python3 {
let dependencies = parser::parse_python_imports(&ns.content)?;
let (_, tx) = jobs::push(
tx,
&w_id,
@@ -593,10 +615,16 @@ async fn delete_script_by_hash(
Ok(Json(script))
}
async fn parse_code_to_jsonschema(
async fn parse_python_code_to_jsonschema(
Json(code): Json<String>,
) -> JsonResult<parser::MainArgSignature> {
parser::parse_signature(&code).map(Json)
parser::parse_python_signature(&code).map(Json)
}
async fn parse_deno_code_to_jsonschema(
Json(code): Json<String>,
) -> JsonResult<parser::MainArgSignature> {
parser::parse_deno_signature(&code).map(Json)
}
pub fn to_i64(s: &str) -> Result<i64> {
+219 -101
View File
@@ -24,7 +24,7 @@ use crate::{
QueuedJob,
},
parser::{self, Typ},
scripts::ScriptHash,
scripts::{ScriptHash, ScriptLang},
users::{create_token_for_owner, get_email_from_username},
variables,
};
@@ -44,11 +44,14 @@ use tokio::sync::mpsc;
const TMP_DIR: &str = "/tmp/windmill";
const PIP_CACHE_DIR: &str = "/tmp/windmill/cache/pip";
const DENO_CACHE_DIR: &str = "/tmp/windmill/cache/deno";
const NUM_SECS_ENV_CHECK: u64 = 15;
const INCLUDE_DEPS_SH_CONTENT: &str = include_str!("../../nsjail/download_deps.sh");
const NSJAIL_CONFIG_DOWNLOAD_CONTENT: &str = include_str!("../../nsjail/download.config.proto");
const NSJAIL_CONFIG_RUN_CONTENT: &str = include_str!("../../nsjail/run.config.proto");
const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str =
include_str!("../../nsjail/run.python3.config.proto");
const NSJAIL_CONFIG_RUN_DENO_CONTENT: &str = include_str!("../../nsjail/run.deno.config.proto");
pub async fn run_worker(
db: &DB,
@@ -66,17 +69,13 @@ pub async fn run_worker(
let worker_dir = format!("{TMP_DIR}/{worker_name}");
tracing::debug!(worker_dir = %worker_dir, worker_name = %worker_name, "Creating worker dir");
DirBuilder::new()
.recursive(true)
.create(&worker_dir)
.await
.expect("could not create initial worker dir");
DirBuilder::new()
.recursive(true)
.create(&PIP_CACHE_DIR)
.await
.expect("could not create initial worker dir");
for x in [&worker_dir, PIP_CACHE_DIR, DENO_CACHE_DIR] {
DirBuilder::new()
.recursive(true)
.create(x)
.await
.expect("could not create initial worker dir");
}
let _ = write_file(&worker_dir, "download_deps.sh", INCLUDE_DEPS_SH_CONTENT).await;
@@ -289,6 +288,7 @@ async fn handle_job(
.expect("could not create initial job dir");
let mut status: Result<ExitStatus, Error>;
if matches!(job.job_kind, JobKind::Dependencies) {
let requirements = job
.raw_code
@@ -344,12 +344,17 @@ async fn handle_job(
.await?;
}
} else {
let (inner_content, requirements_o) = if matches!(job.job_kind, JobKind::Preview) {
let (inner_content, requirements_o, language) = if matches!(job.job_kind, JobKind::Preview)
{
let code = (job.raw_code.as_ref().unwrap_or(&"no raw code".to_owned())).to_owned();
let reqs = parser::parse_imports(&code)?.join("\n");
(code, Some(reqs))
let reqs = if job.language == ScriptLang::Python3 {
Some(parser::parse_python_imports(&code)?.join("\n"))
} else {
None
};
(code, reqs, job.language.to_owned())
} else {
sqlx::query_as::<_, (String, Option<String>)>("SELECT content, lock FROM script WHERE hash = $1 AND (workspace_id = $2 OR workspace_id = 'starter')")
sqlx::query_as::<_, (String, Option<String>, ScriptLang)>("SELECT content, lock, language FROM script WHERE hash = $1 AND (workspace_id = $2 OR workspace_id = 'starter')")
.bind(&job.script_hash.unwrap_or(ScriptHash(0)).0)
.bind(&job.workspace_id)
.fetch_optional(db)
@@ -357,69 +362,76 @@ async fn handle_job(
.ok_or_else(|| Error::InternalErr(format!("expected content and lock")))?
};
let requirements =
requirements_o.ok_or_else(|| Error::InternalErr(format!("lockfile missing")))?;
match language {
ScriptLang::Python3 => {
let requirements = requirements_o
.ok_or_else(|| Error::InternalErr(format!("lockfile missing")))?;
let _ = write_file(
&job_dir,
"download.config.proto",
&NSJAIL_CONFIG_DOWNLOAD_CONTENT
.replace("{JOB_DIR}", &job_dir)
.replace("{WORKER_DIR}", &worker_dir)
.replace("{CACHE_DIR}", PIP_CACHE_DIR),
)
.await?;
let _ = write_file(&job_dir, "requirements.txt", &requirements).await?;
let _ = write_file(
&job_dir,
"download.config.proto",
&NSJAIL_CONFIG_DOWNLOAD_CONTENT
.replace("{JOB_DIR}", &job_dir)
.replace("{WORKER_DIR}", &worker_dir)
.replace("{CACHE_DIR}", PIP_CACHE_DIR),
)
.await?;
let _ = write_file(&job_dir, "requirements.txt", &requirements).await?;
let child = Command::new("nsjail")
.current_dir(&job_dir)
.args(vec!["--config", "download.config.proto"])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
let child = Command::new("nsjail")
.current_dir(&job_dir)
.args(vec!["--config", "download.config.proto"])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
logs.push_str("\n--- DEPENDENCIES INSTALL ---\n");
status = handle_child(job, db, &mut logs, &mut last_line, timeout, child).await;
if status.is_ok() {
logs.push_str("\n\n--- CODE EXECUTION ---\n");
logs.push_str("\n--- PIP DEPENDENCIES INSTALL ---\n");
status = handle_child(job, db, &mut logs, &mut last_line, timeout, child).await;
set_logs(logs, job.id, db).await;
if status.is_ok() {
logs.push_str("\n\n--- PTHON CODE EXECUTION ---\n");
let _ = write_file(&job_dir, "inner.py", &inner_content).await?;
set_logs(logs, job.id, db).await;
let sig = crate::parser::parse_signature(&inner_content)?;
let transforms = sig.args.into_iter().map(|x| match x.typ {
let _ = write_file(&job_dir, "inner.py", &inner_content).await?;
let sig = crate::parser::parse_python_signature(&inner_content)?;
let transforms = sig.args.into_iter().map(|x| match x.typ {
Typ::Bytes => format!("if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n kwargs[\"{}\"] = base64.b64decode(kwargs[\"{}\"])\n", x.name, x.name, x.name, x.name),
Typ::Datetime => format!("if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n kwargs[\"{}\"] = datetime.strptime(kwargs[\"{}\"], '%Y-%m-%dT%H:%M')\n", x.name, x.name, x.name, x.name),
_ => "".to_string()
}).collect::<Vec<String>>().join("");
let tx = db.begin().await?;
let tx = db.begin().await?;
let token = create_token_for_owner(
&db,
&job.workspace_id,
&job.permissioned_as,
crate::users::NewToken {
label: Some("ephemeral-script".to_string()),
expiration: Some(
chrono::Utc::now() + chrono::Duration::seconds((timeout * 2).into()),
),
},
&job.created_by,
)
.await?;
let token = create_token_for_owner(
&db,
&job.workspace_id,
&job.permissioned_as,
crate::users::NewToken {
label: Some("ephemeral-script".to_string()),
expiration: Some(
chrono::Utc::now()
+ chrono::Duration::seconds((timeout * 2).into()),
),
},
&job.created_by,
)
.await?;
let args = if let Some(args) = &job.args {
Some(transform_json_value(&token, &job.workspace_id, base_url, args.clone()).await)
} else {
None
};
let ser_args = serde_json::to_string(&args)
.map_err(|e| Error::ExecutionErr(e.to_string()))?
.replace("\\\"", "\\\\\"");
let wrapper_content: String = format!(
r#"
let args = if let Some(args) = &job.args {
Some(
transform_json_value(&token, &job.workspace_id, base_url, args.clone())
.await,
)
} else {
None
};
let ser_args = serde_json::to_string(&args)
.map_err(|e| Error::ExecutionErr(e.to_string()))?
.replace("\\\"", "\\\\\"");
let wrapper_content: String = format!(
r#"
import json
import base64
from datetime import datetime
@@ -443,44 +455,150 @@ print()
print("result:")
print(res_json)
"#,
);
write_file(&job_dir, "main.py", &wrapper_content).await?;
);
write_file(&job_dir, "main.py", &wrapper_content).await?;
tx.commit().await?;
let reserved_variables = variables::get_reserved_variables(
&job.workspace_id,
&token,
&get_email_from_username(&job.created_by, db)
.await?
.unwrap_or_else(|| "nosuitable@email.xyz".to_string()),
&job.created_by,
&job.id.to_string(),
)
.into_iter()
.map(|rv| (rv.name, rv.value));
tx.commit().await?;
let reserved_variables = variables::get_reserved_variables(
&job.workspace_id,
&token,
&get_email_from_username(&job.created_by, db)
.await?
.unwrap_or_else(|| "nosuitable@email.xyz".to_string()),
&job.created_by,
&job.id.to_string(),
)
.into_iter()
.map(|rv| (rv.name, rv.value));
let _ = write_file(
&job_dir,
"run.config.proto",
&NSJAIL_CONFIG_RUN_PYTHON3_CONTENT.replace("{JOB_DIR}", &job_dir),
)
.await?;
let _ = write_file(
&job_dir,
"run.config.proto",
&NSJAIL_CONFIG_RUN_CONTENT.replace("{JOB_DIR}", &job_dir),
)
.await?;
let child = Command::new("nsjail")
.current_dir(&job_dir)
.envs(reserved_variables)
.args(vec![
"--config",
"run.config.proto",
"--",
"/usr/local/bin/python3",
"-u",
"/tmp/main.py",
])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
status = handle_child(job, db, &mut logs, &mut last_line, timeout, child).await;
}
}
ScriptLang::Deno => {
logs.push_str("\n\n--- DENO CODE EXECUTION ---\n");
let child = Command::new("nsjail")
.current_dir(&job_dir)
.envs(reserved_variables)
.args(vec![
"--config",
set_logs(logs, job.id, db).await;
let _ = write_file(&job_dir, "inner.ts", &inner_content).await?;
let sig = crate::parser::parse_deno_signature(&inner_content)?;
// let transforms = sig.args.clone().into_iter().map(|x| match x.typ {
// Typ::Bytes => format!("if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n kwargs[\"{}\"] = base64.b64decode(kwargs[\"{}\"])\n", x.name, x.name, x.name, x.name),
// Typ::Datetime => format!("if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n kwargs[\"{}\"] = datetime.strptime(kwargs[\"{}\"], '%Y-%m-%dT%H:%M')\n", x.name, x.name, x.name, x.name),
// _ => "".to_string()
// }).collect::<Vec<String>>().join("");
let tx = db.begin().await?;
let token = create_token_for_owner(
&db,
&job.workspace_id,
&job.permissioned_as,
crate::users::NewToken {
label: Some("ephemeral-script".to_string()),
expiration: Some(
chrono::Utc::now() + chrono::Duration::seconds((timeout * 2).into()),
),
},
&job.created_by,
)
.await?;
let args = if let Some(args) = &job.args {
Some(
transform_json_value(&token, &job.workspace_id, base_url, args.clone())
.await,
)
} else {
None
};
let ser_args = serde_json::to_string(&args)
.map_err(|e| Error::ExecutionErr(e.to_string()))?
.replace("\\\"", "\\\\\"");
let spread = sig.args.into_iter().map(|x| x.name).join(",");
let wrapper_content: String = format!(
r#"
import {{ main }} from "./inner.ts";
const {{{spread}}}= JSON.parse(`{ser_args}`);
async function run() {{
let res: any = await main({spread});
if (res == undefined) {{
res = {{}}
}}
if (typeof res !== 'object') {{
res = {{ res1: res }}
}}
const res_json = JSON.stringify(res);
console.log();
console.log("result:");
console.log(res_json);
}}
run();
"#,
);
write_file(&job_dir, "main.ts", &wrapper_content).await?;
tx.commit().await?;
let reserved_variables = variables::get_reserved_variables(
&job.workspace_id,
&token,
&get_email_from_username(&job.created_by, db)
.await?
.unwrap_or_else(|| "nosuitable@email.xyz".to_string()),
&job.created_by,
&job.id.to_string(),
)
.into_iter()
.map(|rv| (rv.name, rv.value));
let _ = write_file(
&job_dir,
"run.config.proto",
"--",
"/usr/local/bin/python3",
"-u",
"/tmp/main.py",
])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
status = handle_child(job, db, &mut logs, &mut last_line, timeout, child).await;
&NSJAIL_CONFIG_RUN_DENO_CONTENT
.replace("{JOB_DIR}", &job_dir)
.replace("{CACHE_DIR}", DENO_CACHE_DIR),
)
.await?;
let child = Command::new("nsjail")
.current_dir(&job_dir)
.envs(reserved_variables)
.args(vec![
"--config",
"run.config.proto",
"--",
"/usr/bin/deno",
"run",
"--v8-flags=--max-heap-size=2048",
"-A",
"/tmp/main.ts",
])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
status = handle_child(job, db, &mut logs, &mut last_line, timeout, child).await;
}
}
}
tokio::fs::remove_dir_all(job_dir).await?;
BIN
View File
Binary file not shown.
+1
View File
@@ -0,0 +1 @@
windmill-api
+5
View File
@@ -0,0 +1,5 @@
{
"deno.enable": true,
"deno.lint": true,
"deno.unstable": true
}
+5
View File
@@ -0,0 +1,5 @@
#!/bin/bash
set -e
/usr/local/bin/docker-entrypoint.sh generate -i ../backend/openapi.yaml -g typescript --additional-properties platform=deno -o windmill-api
sed -i 's/this\.type = "Job";//' windmill-api/models/Job.ts
+46
View File
@@ -0,0 +1,46 @@
import { ResourceApi, VariableApi, ServerConfiguration } from './windmill-api/index.ts'
import { createConfiguration, type Configuration as Configuration } from './windmill-api/configuration.ts'
export * from './windmill-api/index.ts'
export function createConf(): Configuration & { workspace_id: string } {
const token = Deno.env.get("WM_TOKEN") ?? 'no_token'
return {
...createConfiguration({
baseServer: new ServerConfiguration(Deno.env.get("BASE_URL") ?? 'http://localhost:8000/api', {}),
authMethods: { bearerAuth: { tokenProvider: { getToken() { return token } } } },
}), workspace_id: Deno.env.get("WM_WORKSPACE") ?? 'no_workspace'
}
}
export async function getResource(path: string): Promise<any> {
const conf = createConf()
const resource = await new ResourceApi(conf).getResource(conf.workspace_id, path)
return await transformLeaves(resource.value)
}
export async function getVariable(path: string): Promise<string | undefined> {
const conf = createConf()
const variable = await new VariableApi(conf).getVariable(conf.workspace_id, path)
return variable.value
}
export async function transformLeaves(d: { [key: string]: any }): Promise<{ [key: string]: any }> {
for (const k in d) {
d[k] = await _transformLeaf(d[k])
}
return d
}
const VAR_RESOURCE_PREFIX = "$var:"
async function _transformLeaf(v: any): Promise<any> {
if (typeof v === 'object') {
return transformLeaves(v)
}
else if (typeof v === 'string' && v.startsWith(VAR_RESOURCE_PREFIX)) {
const varName = v.substring(VAR_RESOURCE_PREFIX.length)
return await getVariable(varName)
} else {
return v
}
}
+7
View File
@@ -0,0 +1,7 @@
{
"$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json",
"spaces": 2,
"generator-cli": {
"version": "6.0.0-beta"
}
}
+4
View File
@@ -5,4 +5,8 @@
bind {$ADDRESS}
reverse_proxy /api/* http://localhost:8000
reverse_proxy /* http://localhost:3000
reverse_proxy /ws/* http://localhost:3001 {
lb_policy header "Authorization"
}
}
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "windmill",
"version": "1.6.0",
"version": "1.6.1",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "windmill",
"version": "1.6.0",
"version": "1.6.1",
"dependencies": {
"@codingame/monaco-jsonrpc": "^0.3.1",
"@codingame/monaco-languageclient": "^0.17.0",
+15 -11
View File
@@ -1,12 +1,22 @@
import type { Schema, SchemaProperty } from './common'
import { ScriptService } from './gen'
import { ScriptService, type MainArgSignature } from './gen'
import { sendUserToast } from './utils'
export async function inferArgs(code: string, schema: Schema): Promise<void> {
export async function inferArgs(language: "python3" | "deno", code: string, schema: Schema): Promise<void> {
try {
const inferedSchema = await ScriptService.toJsonschema({
requestBody: code
})
let inferedSchema: MainArgSignature
if (language == "python3") {
inferedSchema = await ScriptService.pythonToJsonschema({
requestBody: code
})
} else if (language == "deno") {
inferedSchema = await ScriptService.denoToJsonschema({
requestBody: code
})
} else {
return
}
schema.required = []
const oldProperties = Object.assign({}, schema.properties)
schema.properties = {}
@@ -30,12 +40,6 @@ export async function inferArgs(code: string, schema: Schema): Promise<void> {
}
}
function array_move<T>(arr: T[], fromIndex: number, toIndex: number) {
var element = arr[fromIndex]
arr.splice(fromIndex, 1)
arr.splice(toIndex, 0, element)
}
function pythonToJsonSchemaType(t: string, s: SchemaProperty): void {
if (t === 'int') {
s.type = 'integer'
+64 -36
View File
@@ -12,7 +12,9 @@
let divEl: HTMLDivElement | null = null
let editor: monaco.editor.IStandaloneCodeEditor
let monaco
export let lang = 'python'
export let deno = false
export let lang = deno ? 'typescript' : 'python'
export let code: string
export let readOnly = false
export let hash: string = (Math.random() + 1).toString(36).substring(2)
@@ -76,7 +78,7 @@
export async function reloadWebsocket() {
closeWebsockets()
if (lang == 'python') {
if (lang == 'python' || deno) {
// install Monaco language client services
const { MonacoLanguageClient, CloseAction, ErrorAction, createConnection } = await import(
'@codingame/monaco-languageclient'
@@ -86,7 +88,7 @@
return new MonacoLanguageClient({
name: name,
clientOptions: {
documentSelector: ['python'],
documentSelector: deno ? ['typescript'] : ['python'],
errorHandler: {
error: () => ErrorAction.Shutdown,
closed: () => CloseAction.Restart
@@ -95,7 +97,18 @@
isTrusted: true
},
// workspaceFolder: { uri: Uri.parse(`/tmp/${name}`), name: 'tmp', index: 0 },
initializationOptions
initializationOptions,
middleware: {
workspace: {
configuration: (params, token, configuration) => {
return [
{
enable: true
}
]
}
}
}
},
connectionProvider: {
get: (errorHandler, closeHandler) => {
@@ -132,28 +145,38 @@
console.error(`connection to ${name} language server failed`)
}
}
connectToLanguageServer(`wss://${$page.url.host}/ws/pyright`, 'pyright', {
executionEnvironments: [
{
root: '/tmp/pyright',
pythonVersion: '3.7',
pythonPlatform: 'platform',
extraPaths: []
}
]
})
connectToLanguageServer(`wss://${$page.url.host}/ws/black`, 'black', {
formatters: {
black: {
command: 'black',
args: ['--quiet', '-']
if (deno) {
connectToLanguageServer(`ws://${$page.url.host}/ws/deno`, 'deno', {
deno: {
enable: true,
lint: true
}
},
formatFiletypes: {
python: 'black'
}
})
})
} else {
connectToLanguageServer(`wss://${$page.url.host}/ws/pyright`, 'pyright', {
executionEnvironments: [
{
root: '/tmp/pyright',
pythonVersion: '3.7',
pythonPlatform: 'platform',
extraPaths: []
}
]
})
connectToLanguageServer(`wss://${$page.url.host}/ws/black`, 'black', {
formatters: {
black: {
command: 'black',
args: ['--quiet', '-']
}
},
formatFiletypes: {
python: 'black'
}
})
}
}
}
@@ -232,15 +255,20 @@
}
if (lang == 'typescript') {
// compiler options
monaco.languages.typescript.typescriptDefaults.setCompilerOptions({
target: monaco.languages.typescript.ScriptTarget.ES6,
allowNonTsExtensions: true,
noLib: true
})
if (deno) {
monaco.languages.typescript.typescriptDefaults.setDiagnosticsOptions({
diagnosticCodesToIgnore: [2691]
})
} else {
// compiler options
monaco.languages.typescript.typescriptDefaults.setCompilerOptions({
target: monaco.languages.typescript.ScriptTarget.ES6,
allowNonTsExtensions: true,
noLib: true
})
monaco.languages.typescript.typescriptDefaults.addExtraLib(
`
monaco.languages.typescript.typescriptDefaults.addExtraLib(
`
/**
* get variable (including secret) at path
* @param {string} path - path of the variable (e.g: g/all/pretty_secret)
@@ -276,11 +304,11 @@ export const previous_result: any;
*/
export const params: any;
`,
'file:///node_modules/@types/windmill/index.d.ts'
)
'file:///node_modules/@types/windmill/index.d.ts'
)
}
}
if (lang == 'python') {
if (lang == 'python' || deno) {
const { MonacoServices } = await import('@codingame/monaco-languageclient')
MonacoServices.install(monaco)
@@ -1,10 +1,8 @@
<script lang="ts">
import { workspaceStore } from '../../stores'
import { createEventDispatcher } from 'svelte'
import Modal from '../../routes/components/Modal.svelte'
import { Group, GroupService, UserService } from '../../gen'
import { type Group, GroupService, UserService } from '../../gen'
import AutoComplete from 'simple-svelte-autocomplete'
import PageHeader from './PageHeader.svelte'
import TableCustom from './TableCustom.svelte'
@@ -1,3 +1,51 @@
<script context="module">
const PYTHON_INIT_CODE = `import os
import wmill
from datetime import datetime
# Our webeditor includes a syntax, type checker through a language server running pyright
# and the autoformatter Black in our servers. Use Cmd/Ctrl + S to autoformat the code.
# Beware that the code is only saved when you click Save and not across reload.
# You can however navigate to any steps safely.
"""
The client is used to interact with windmill itself through its standard API.
One can explore the methods available through autocompletion of \`client.XXX\`.
Only the most common methods are included for ease of use. Request more as
feedback if you feel you are missing important ones.
"""
def main(name: str = "Nicolas Bourbaki",
age: int = 42,
obj: dict = {"even": "dicts"},
l: list = ["or", "lists!"],
file_: bytes = bytes(0),
dtime: datetime = datetime.now()):
"""A main function is required for the script to be able to accept arguments.
Types are recommended."""
print(f"Hello World and a warm welcome especially to {name}")
print("and its acolytes..", age, obj, l, len(file_), dtime)
# retrieve variables, including secrets by querying the windmill platform.
# secret fetching is audited by windmill.
secret = wmill.get_variable("g/all/pretty_secret")
print(f"The env variable at \`g_all/pretty_secret\`: {secret}")
# interact with the windmill platform to get the version
version = wmill.get_version()
# fetch reserved variables as environment variables
user = os.environ.get("WM_USERNAME")
# the return value is then parsed and can be retrieved by other scripts conveniently
return {"version": version, "splitted": name.split(), "user": user}
`
const DENO_INIT_CODE = `
// only do the following import if you require your script to interact with the windmill
// for instance to get a variable or resource
// import * as wmill from 'https://deno.land/x/windmill@v${__pkg__.version}/index.ts'
export function main(x: string, y: string = 'default arg') {
// let x = wmill.getVariable('u/user/foo');
// let y = wmill.getResource('u/user/foo')
return { foo: x }
}
`
</script>
<script lang="ts">
import { ScriptService, type Script } from '../../gen'
@@ -12,6 +60,7 @@
import ScriptSchema from './ScriptSchema.svelte'
import { inferArgs } from '../../infer'
import Required from './Required.svelte'
import RadioButton from './RadioButton.svelte'
let editor: ScriptEditor
let scriptSchema: ScriptSchema
@@ -25,6 +74,13 @@
history.replaceState({}, '', $page.url)
}
if (script.content == '') {
initContent(script.language)
}
function initContent(lang: string) {
script.content = lang == 'deno' ? DENO_INIT_CODE : PYTHON_INIT_CODE
}
async function editScript(): Promise<void> {
try {
const newHash = await ScriptService.createScript({
@@ -36,7 +92,8 @@
content: script.content,
parent_hash: script.hash != '' ? script.hash : undefined,
schema: script.schema,
is_template: script.is_template
is_template: script.is_template,
language: script.language
}
})
sendUserToast(`Success! New script version created with hash ${newHash}`)
@@ -59,7 +116,7 @@
}
async function inferSchema() {
await inferArgs(script.content, script.schema)
await inferArgs(script.language, script.content, script.schema)
}
async function changeStep(step: number) {
@@ -151,6 +208,19 @@
<a href="https://docs.windmill.dev/docs/reference/namespaces">docs</a>
</div>
</Path>
<h3 class="text-gray-700 pb-1 border-b">Language</h3>
<div class="max-w-md">
<RadioButton
label="Language"
small={true}
options={[
['Python 3.10', 'python3'],
['Typescript (Deno) WIP', 'deno']
]}
on:change={(e) => initContent(e.detail)}
bind:value={script.language}
/>
</div>
<h3 class="text-gray-700 pb-1 border-b">Metadata</h3>
<label class="block ">
@@ -211,6 +281,7 @@
bind:schema={script.schema}
path={script.path}
bind:code={script.content}
lang={script.language}
/>
</div>
{:else if step === 3}
@@ -29,7 +29,7 @@
import { check } from 'svelte-awesome/icons'
import Modal from './Modal.svelte'
import { Highlight } from 'svelte-highlight'
import { json, python } from 'svelte-highlight/src/languages'
import { json, python, typescript } from 'svelte-highlight/src/languages'
import github from 'svelte-highlight/src/styles/github'
import ItemPicker from './ItemPicker.svelte'
import VariableEditor from './VariableEditor.svelte'
@@ -40,18 +40,20 @@
import { VSplitPane } from 'svelte-split-pane'
import SchemaForm from './SchemaForm.svelte'
import DisplayResult from './DisplayResult.svelte'
import type { Preview } from '../../gen/models/Preview'
// Exported
export let schema: Schema = emptySchema()
export let code: string
export let path: string | undefined
export let lang: Preview.language
// Control Editor layout
export let viewPreview = true
export let previewTab: 'logs' | 'input' | 'output' | 'history' | 'last_save' = 'logs'
let websocketAlive = { pyright: false, black: false }
let websocketAlive = { pyright: false, black: false, deno: false }
// Internal state
let editor: Editor
@@ -107,7 +109,8 @@
requestBody: {
path,
content: editor.getCode(),
args: args
args: args,
language: lang
}
})
previewJob = undefined
@@ -169,7 +172,7 @@
isDefault.push(k)
}
})
await inferArgs(editor.getCode(), schema)
await inferArgs(lang, editor.getCode(), schema)
schema = schema
isDefault.forEach((key) => (args[key] = schema.properties[key].default))
@@ -267,7 +270,11 @@
{modalViewerContent}
</pre>
{:else if modalViewerMode === 'code'}
<Highlight language={python} code={modalViewerContent} />
{#if lang == 'python3'}
<Highlight language={python} code={modalViewerContent} />
{:else if lang == 'deno'}
<Highlight language={typescript} code={modalViewerContent} />
{/if}
{/if}
</div></Modal
>
@@ -276,16 +283,29 @@
bind:this={variablePicker}
pickCallback={(path, name) => {
if (!path) {
if (!getEditor().getCode().includes('import os')) {
getEditor().insertAtBeginning('import os\n')
if (lang == 'deno') {
getEditor().insertAtCursor(`Deno.env.get('${name}')`)
} else {
if (!getEditor().getCode().includes('import os')) {
getEditor().insertAtBeginning('import os\n')
}
getEditor().insertAtCursor(`os.environ.get("${name}")`)
}
getEditor().insertAtCursor(`os.environ.get("${name}")`)
sendUserToast(`${name} inserted at cursor`)
} else {
if (!getEditor().getCode().includes('import wmill')) {
getEditor().insertAtBeginning('import wmill\n')
if (lang == 'deno') {
if (!getEditor().getCode().includes('import * as wmill from')) {
getEditor().insertAtBeginning(
`import * as wmill from 'https://deno.land/x/windmill@v${__pkg__.version}/index.ts'\n`
)
}
getEditor().insertAtCursor(`wmill.getVariable('${path}'')`)
} else {
if (!getEditor().getCode().includes('import wmill')) {
getEditor().insertAtBeginning('import wmill\n')
}
getEditor().insertAtCursor(`wmill.get_variable("${path}")`)
}
getEditor().insertAtCursor(`wmill.get_variable("${path}")`)
sendUserToast(`${name} inserted at cursor`)
}
}}
@@ -312,7 +332,19 @@
<ItemPicker
bind:this={resourcePicker}
pickCallback={(path, _) => {
getEditor().insertAtCursor(`client.get_resource("${path}")`)
if (lang == 'deno') {
if (!getEditor().getCode().includes('import * as wmill from')) {
getEditor().insertAtBeginning(
`import * as wmill from 'https://deno.land/x/windmill@v${__pkg__.version}/index.ts'\n`
)
}
getEditor().insertAtCursor(`wmill.getResource('${path}'')`)
} else {
if (!getEditor().getCode().includes('import wmill')) {
getEditor().insertAtBeginning('import wmill\n')
}
getEditor().insertAtCursor(`wmill.get_resource("${path}")`)
}
sendUserToast(`${path} inserted at cursor`)
}}
itemName="Resource"
@@ -369,7 +401,6 @@
}}
>Resource picker <Icon data={faSearch} scale={0.7} />
</button>
<button
class="default-button-secondary font-semibold py-px text-xs mr-2 align-middle max-h-8"
on:click|stopPropagation={() => {
@@ -384,9 +415,14 @@
editor.reloadWebsocket()
}}
>
Reload assistants (status: <span
class={websocketAlive.pyright ? 'text-green-600' : 'text-red-600'}>pyright</span
> <span class={websocketAlive.black ? 'text-green-600' : 'text-red-600'}> black</span>)
Reload assistants (status: {#if lang == 'deno'}<span
class={websocketAlive.deno ? 'text-green-600' : 'text-red-600'}>deno</span
>{:else if lang == 'python3'}<span
class={websocketAlive.pyright ? 'text-green-600' : 'text-red-600'}>pyright</span
>
<span class={websocketAlive.black ? 'text-green-600' : 'text-red-600'}>
black</span
>{/if})
</button>
</div>
</div>
@@ -404,6 +440,7 @@
localStorage.setItem(path ?? 'last_save', code)
}}
class="h-full"
deno={lang == 'deno'}
automaticLayout={true}
/>
</div>
@@ -10,7 +10,8 @@
import RadioButton from './RadioButton.svelte'
import Modal from './Modal.svelte'
import { Highlight } from 'svelte-highlight'
import { python } from 'svelte-highlight/src/languages'
import { python, typescript } from 'svelte-highlight/src/languages'
import github from 'svelte-highlight/src/styles/github'
export let scriptPath: string | undefined = undefined
@@ -21,12 +22,17 @@
let itemPicker: ItemPicker
let modalViewer: Modal
let code: string = ''
let lang: 'deno' | 'python3' | undefined
const dispatch = createEventDispatcher()
async function getScript() {
code = (await ScriptService.getScriptByPath({ workspace: $workspaceStore!, path: scriptPath! }))
.content
const script = await ScriptService.getScriptByPath({
workspace: $workspaceStore!,
path: scriptPath!
})
code = script.content
lang = script.language
}
async function loadItems(isFlow: boolean): Promise<void> {
@@ -103,6 +109,10 @@
<Modal bind:this={modalViewer}>
<div slot="title">Script {scriptPath}</div>
<div slot="content">
<Highlight language={python} {code} />
{#if lang == 'python3'}
<Highlight language={python} {code} />
{:else if lang == 'deno'}
<Highlight language={typescript} {code} />
{/if}
</div>
</Modal>
+8 -5
View File
@@ -32,7 +32,7 @@
import DisplayResult from '../components/DisplayResult.svelte'
import Highlight from 'svelte-highlight'
import python from 'svelte-highlight/src/languages/python'
import { python, typescript } from 'svelte-highlight/src/languages'
import github from 'svelte-highlight/src/styles/github'
import { userStore, workspaceStore } from '../../stores'
import CenteredPage from '../components/CenteredPage.svelte'
@@ -420,10 +420,13 @@
class="text-xs overflow-auto max-h-96 w-full p-3">{#if viewTab == 'logs'}{#if job && 'logs' in job && job.logs}{job.logs}
{:else if job}No logs are available yet
{:else}Loading...{/if}
{:else if viewTab == 'code'}{#if job && 'raw_code' in job && job.raw_code}<Highlight
language={python}
code={job.raw_code}
/>
{:else if viewTab == 'code'}
{#if job && 'raw_code' in job && job.raw_code}
{#if job.language == 'python3'}
<Highlight language={python} code={job.raw_code} />
{:else if job.language == 'deno'}
<Highlight language={typescript} code={job.raw_code} />
{/if}
{:else if job}No code is available
{:else}Loading...{/if}
{:else if job && 'result' in job && job.result}<DisplayResult result={job.result} />
+2 -1
View File
@@ -214,7 +214,7 @@
</p>
{:else}
<div class="grid md:grid-cols-2 gap-4 sm:grid-cols-1 2xl:grid-cols-3">
{#each scripts as { summary, path, hash, extra_perms, canWrite, lock_error_logs }}
{#each scripts as { summary, path, hash, language, extra_perms, canWrite, lock_error_logs }}
<div
class="flex flex-col justify-between script max-w-lg overflow-visible shadow-sm shadow-blue-100 border border-gray-200 bg-gray-50 py-2"
>
@@ -233,6 +233,7 @@
<div class="flex flex-row pl-6 pr-2 mt-2">
<div class="mr-3 w-full">
<SharedBadge {canWrite} extraPerms={extra_perms} />
<Badge twBgColor="bg-blue-200">{language}</Badge>
{#if lock_error_logs}<Badge
twBgColor="bg-red-200"
tooltip="The script was not deployed due to an error during deployment. See more details about the error on the script page."
+5 -75
View File
@@ -5,81 +5,10 @@
import { workspaceStore } from '../../stores'
import ScriptBuilder from '../components/ScriptBuilder.svelte'
import type { Schema } from '../../common'
import { sendUserToast } from '../../utils'
import { emptySchema, sendUserToast } from '../../utils'
// Default
let schema: Schema = {
$schema: 'https://json-schema.org/draft/2020-12/schema',
properties: {
age: {
default: 42,
description: '',
type: 'integer'
},
l: {
default: ['or', 'lists!'],
description: '',
type: 'array'
},
name: {
default: 'Nicolas Bourbaki',
description: '',
type: 'string'
},
obj: {
default: {
even: 'dicts'
},
description: '',
type: 'object'
}
},
required: [],
type: 'object'
}
let code = `import os
import wmill
from datetime import datetime
# Our webeditor includes a syntax, type checker through a language server running pyright
# and the autoformatter Black in our servers. Use Cmd/Ctrl + S to autoformat the code.
# Beware that the code is only saved when you click Save and not across reload.
# You can however navigate to any steps safely.
"""
The client is used to interact with windmill itself through its standard API.
One can explore the methods available through autocompletion of \`client.XXX\`.
Only the most common methods are included for ease of use. Request more as
feedback if you feel you are missing important ones.
"""
def main(name: str = "Nicolas Bourbaki",
age: int = 42,
obj: dict = {"even": "dicts"},
l: list = ["or", "lists!"],
file_: bytes = bytes(0),
dtime: datetime = datetime.now()):
"""A main function is required for the script to be able to accept arguments.
Types are recommended."""
print(f"Hello World and a warm welcome especially to {name}")
print("and its acolytes..", age, obj, l, len(file_), dtime)
# retrieve variables, including secrets by querying the windmill platform.
# secret fetching is audited by windmill.
secret = wmill.get_variable("g/all/pretty_secret")
print(f"The env variable at \`g_all/pretty_secret\`: {secret}")
# interact with the windmill platform to get the version
version = wmill.get_version()
# fetch reserved variables as environment variables
user = os.environ.get("WM_USERNAME")
# the return value is then parsed and can be retrieved by other scripts conveniently
return {"version": version, "splitted": name.split(), "user": user}
`
let schema: Schema = emptySchema()
$: templatePath = $page.url.searchParams.get('template')
@@ -92,14 +21,15 @@ def main(name: str = "Nicolas Bourbaki",
hash: '',
path: '',
summary: '',
content: code,
content: '',
schema: schema,
created_by: '',
created_at: '',
archived: false,
deleted: false,
is_template: false,
extra_perms: {}
extra_perms: {},
language: 'python3'
}
async function loadTemplate(): Promise<void> {
@@ -8,39 +8,28 @@
const initialState = $page.url.searchParams.get('state')
let scriptLoadedFromUrl = initialState != undefined ? JSON.parse(atob(initialState)) : undefined
let script: Script = {
hash: $page.params.hash,
path: '',
summary: '',
content: '',
created_by: '',
created_at: '',
archived: false,
deleted: false,
is_template: false,
extra_perms: {}
}
let script: Script | undefined
let initialPath: string = ''
let scriptBuilder: ScriptBuilder
async function loadScript(): Promise<void> {
script =
scriptLoadedFromUrl != undefined && scriptLoadedFromUrl.hash == script.hash
scriptLoadedFromUrl != undefined && scriptLoadedFromUrl.hash == $page.params.hash
? scriptLoadedFromUrl
: await ScriptService.getScriptByHash({
workspace: $workspaceStore!,
hash: script.hash
hash: $page.params.hash
})
initialPath = script.path
scriptBuilder.setCode(script)
initialPath = script!.path
}
$: {
if ($workspaceStore && scriptBuilder) {
if ($workspaceStore) {
loadScript()
}
}
</script>
<ScriptBuilder bind:this={scriptBuilder} {initialPath} {script} />
{#if script}
<ScriptBuilder {initialPath} {script} />
{/if}
@@ -14,7 +14,7 @@
faSpinner
} from '@fortawesome/free-solid-svg-icons'
import Highlight from 'svelte-highlight'
import python from 'svelte-highlight/src/languages/python'
import { typescript, python } from 'svelte-highlight/src/languages'
import github from 'svelte-highlight/src/styles/github'
import Tooltip from '../../components/Tooltip.svelte'
@@ -327,7 +327,11 @@
</div>
<div>
<h3 class="text-gray-700 pb-1 mb-3 border-b">Code</h3>
<Highlight language={python} code={script.content} />
{#if script.language == 'python3'}
<Highlight language={python} code={script.content} />
{:else if script.language == 'deno'}
<Highlight language={typescript} code={script.content} />
{/if}
</div>
<div>
<h3 class="text-gray-700 pb-1 mb-3 border-b">Dependencies lock file</h3>
@@ -18,7 +18,7 @@
script = await ScriptService.getScriptByHash({ workspace: $workspaceStore!, hash })
if (script.schema == undefined) {
script.schema = emptySchema()
inferArgs(script.content, script.schema)
inferArgs(script.language, script.content, script.schema)
script = script
}
} else {
+12
View File
@@ -2,6 +2,15 @@ import preprocess from 'svelte-preprocess'
import adapter from '@sveltejs/adapter-static'
const ppath = process.env.PREVIEW_PATH
import { readFileSync } from 'fs';
import { fileURLToPath } from 'url';
const file = fileURLToPath(new URL('package.json', import.meta.url));
const json = readFileSync(file, 'utf8');
const version = JSON.parse(json)
/** @type {import('@sveltejs/kit').Config} */
const config = {
// Consult https://github.com/sveltejs/svelte-preprocess
@@ -17,6 +26,9 @@ const config = {
base: ppath ? ppath : ''
},
vite: {
define: {
'__pkg__': version
},
ssr: {
noExternal: ['dayjs']
},
+5
View File
@@ -71,10 +71,15 @@ class DiagnosticLS(LanguageServerWebSocketHandler):
procargs = ["diagnostic-languageserver", "--stdio"]
class DenoLS(LanguageServerWebSocketHandler):
procargs = ["deno", "lsp"]
if __name__ == "__main__":
app = web.Application([
(r"/ws/pyright", PyrightLS),
(r"/ws/black", DiagnosticLS),
(r"/ws/deno", DenoLS),
])
app.listen(3001, address="0.0.0.0")
ioloop.IOLoop.current().start()
+108
View File
@@ -0,0 +1,108 @@
name: "deno run script"
mode: ONCE
hostname: "deno"
log_level: ERROR
time_limit: 300
rlimit_as: 16000
rlimit_cpu: 1000
rlimit_fsize: 1024
rlimit_nofile: 64
cwd: "/tmp"
clone_newnet: false
keep_caps: false
keep_env: true
mount_proc: true
mount {
src: "/bin"
dst: "/bin"
is_bind: true
}
mount {
src: "/lib"
dst: "/lib"
is_bind: true
}
mount {
src: "/lib64"
dst: "/lib64"
is_bind: true
}
mount {
src: "/usr"
dst: "/usr"
is_bind: true
}
mount {
src: "/dev/null"
dst: "/dev/null"
is_bind: true
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
}
mount {
src: "{JOB_DIR}/inner.ts"
dst: "/tmp/inner.ts"
is_bind: true
}
mount {
src: "{JOB_DIR}/main.ts"
dst: "/tmp/main.ts"
is_bind: true
}
mount {
src: "/etc/ssl"
dst: "/etc/ssl"
is_bind: true
}
mount {
src: "/etc/resolv.conf"
dst: "/etc/resolv.conf"
is_bind: true
}
mount {
src: "/dev/random"
dst: "/dev/random"
is_bind: true
}
mount {
src: "{CACHE_DIR}"
dst: "/tmp/.cache/deno"
is_bind: true
rw: true
mandatory: false
}
iface_no_lo: true
envar: "DENO_DIR=/tmp/.cache/deno"
envar: "NO_COLOR=true"