feat: migrate ts parser to wasm (#1686)

* migrate ts parser to wasm

* add wasm

* revert py
This commit is contained in:
Ruben Fiszel
2023-06-06 09:16:19 +02:00
parent 0079eb0d5b
commit 1f8910d730
27 changed files with 929 additions and 247 deletions
-1
View File
@@ -29,7 +29,6 @@ FROM mcr.microsoft.com/vscode/devcontainers/rust:bullseye
RUN apt update \
&& apt-get install -y \
lld \
python3 \
libprotobuf-dev \
libnl-route-3-dev \
+1 -1
View File
@@ -26,7 +26,7 @@ RUN apt-get update && apt-get install -y git libssl-dev pkg-config
RUN apt-get -y update \
&& apt-get install -y \
curl lld
curl
ENV SQLX_OFFLINE=true
+3 -3
View File
@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm
RUN apt-get -y update \
&& apt-get install -y \
curl lld nodejs npm
curl nodejs npm
RUN rustup component add rustfmt
@@ -52,13 +52,13 @@ RUN mkdir /backend
COPY /backend/windmill-api/openapi.yaml /backend/windmill-api/openapi.yaml
COPY /openflow.openapi.yaml /openflow.openapi.yaml
COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh
RUN cd /backend/windmill-api && . ./build_openapi.sh
COPY /backend/parsers/windmill-parser-py-wasm/pkg/ /backend/windmill-parser-py-wasm/pkg/
RUN npm run generate-backend-client
ENV NODE_OPTIONS "--max-old-space-size=8192"
RUN npm run build
RUN npm run check
FROM rust_base AS planner
+3 -4
View File
@@ -390,15 +390,14 @@ running options.
3. Install deno and python3, have the bins at `/usr/bin/deno` and
`/usr/local/bin/python3`
4. Install [caddy](https://caddyserver.com)
5. Install the [lld linker](https://lld.llvm.org/)
6. Go to `frontend/`:
5. Go to `frontend/`:
1. `npm install`, `npm run generate-backend-client` then `npm run dev`
2. In another shell `npm run build` otherwise the backend will not find the
`frontend/build` folder and will crash
3. In another shell `sudo caddy run --config Caddyfile`
7. Go to `backend/`:
6. Go to `backend/`:
`DATABASE_URL=<DATABASE_URL_TO_YOUR_WINDMILL_DB> RUST_LOG=info cargo run`
8. Et voilà, windmill should be available at `http://localhost/`
7. Et voilà, windmill should be available at `http://localhost/`
## Contributors
+1 -4
View File
@@ -1,10 +1,7 @@
[build]
rustflags = [
"--cfg",
"tokio_unstable",
"-C",
"link-arg=-fuse-ld=lld",
"-Clink-arg=-Wl,--no-rosegment",
"tokio_unstable"
]
incremental = true
+6 -1
View File
@@ -1,4 +1,9 @@
{
"python.analysis.typeCheckingMode": "basic",
"rust-analyzer.linkedProjects": ["./windmill-common/Cargo.toml"]
"rust-analyzer.linkedProjects": [
"./windmill-common/Cargo.toml",
"./parsers/windmill-parser-ts-wasm/Cargo.toml",
"./parsers/windmill-parser-ts-wasm/Cargo.toml",
"./parsers/windmill-parser-ts-wasm/Cargo.toml"
]
}
+58 -2
View File
@@ -778,6 +778,16 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "console_error_panic_hook"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc"
dependencies = [
"cfg-if",
"wasm-bindgen",
]
[[package]]
name = "const-oid"
version = "0.9.2"
@@ -3488,6 +3498,17 @@ dependencies = [
"serde_json",
]
[[package]]
name = "serde-wasm-bindgen"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b4c031cd0d9014307d82b8abf653c0290fbdaeb4c02d00c63cf52f728628bf"
dependencies = [
"js-sys",
"serde",
"wasm-bindgen",
]
[[package]]
name = "serde_bytes"
version = "0.11.9"
@@ -5007,6 +5028,30 @@ version = "0.2.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93"
[[package]]
name = "wasm-bindgen-test"
version = "0.3.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9e636f3a428ff62b3742ebc3c70e254dfe12b8c2b469d688ea59cdd4abcf502"
dependencies = [
"console_error_panic_hook",
"js-sys",
"scoped-tls",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-bindgen-test-macro",
]
[[package]]
name = "wasm-bindgen-test-macro"
version = "0.3.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f18c1fad2f7c4958e7bcce014fa212f59a65d5e3721d0f77e6c0b27ede936ba3"
dependencies = [
"proc-macro2",
"quote",
]
[[package]]
name = "wasm-streams"
version = "0.2.3"
@@ -5306,15 +5351,26 @@ name = "windmill-parser-ts"
version = "1.109.1"
dependencies = [
"anyhow",
"deno_core",
"serde-wasm-bindgen",
"serde_json",
"swc_common",
"swc_ecma_ast",
"swc_ecma_parser",
"windmill-common",
"wasm-bindgen",
"windmill-parser",
]
[[package]]
name = "windmill-parser-ts-wasm"
version = "1.109.1"
dependencies = [
"serde_json",
"wasm-bindgen",
"wasm-bindgen-test",
"windmill-parser",
"windmill-parser-ts",
]
[[package]]
name = "windmill-queue"
version = "1.109.1"
+4 -1
View File
@@ -14,6 +14,7 @@ members = [
"./windmill-api-client",
"./parsers/windmill-parser",
"./parsers/windmill-parser-ts",
"./parsers/windmill-parser-ts-wasm",
"./parsers/windmill-parser-go",
"./parsers/windmill-parser-py",
]
@@ -164,4 +165,6 @@ rsmq_async = { version = "5.1.5" }
gosyn = "0.2.2"
bytes = "1.4.0"
gethostname = "0.4.3"
wasm-bindgen = "0.2"
serde-wasm-bindgen = "0.4"
wasm-bindgen-test = "0.3.0"
@@ -0,0 +1,19 @@
[package]
name = "windmill-parser-ts-wasm"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
crate-type = ["cdylib"]
name = "windmill_parser_ts_wasm"
path = "./src/lib.rs"
[dev-dependencies]
windmill-parser.workspace = true
wasm-bindgen-test.workspace = true
[dependencies]
windmill-parser-ts.workspace = true
wasm-bindgen.workspace = true
serde_json.workspace = true
@@ -0,0 +1,6 @@
{
"name": "windmill-parser-ts-wasm",
"lockfileVersion": 3,
"requires": true,
"packages": {}
}
@@ -0,0 +1 @@
{}
@@ -0,0 +1,12 @@
{
"name": "windmill-parser-ts-wasm",
"version": "1.109.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-parser-ts-wasm",
"version": "1.109.1"
}
}
}
@@ -0,0 +1,17 @@
{
"name": "windmill-parser-ts-wasm",
"collaborators": [
"Ruben Fiszel <ruben@windmill.dev>"
],
"version": "1.109.1",
"files": [
"windmill_parser_ts_wasm_bg.wasm",
"windmill_parser_ts_wasm.js",
"windmill_parser_ts_wasm.d.ts"
],
"module": "windmill_parser_ts_wasm.js",
"types": "windmill_parser_ts_wasm.d.ts",
"sideEffects": [
"./snippets/*"
]
}
@@ -0,0 +1,40 @@
/* tslint:disable */
/* eslint-disable */
/**
* @param {string} code
* @returns {string}
*/
export function parse_deno_wasm(code: string): string;
export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
export interface InitOutput {
readonly memory: WebAssembly.Memory;
readonly parse_deno_wasm: (a: number, b: number, c: number) => void;
readonly __wbindgen_malloc: (a: number) => number;
readonly __wbindgen_realloc: (a: number, b: number, c: number) => number;
readonly __wbindgen_add_to_stack_pointer: (a: number) => number;
readonly __wbindgen_free: (a: number, b: number) => void;
readonly __wbindgen_exn_store: (a: number) => void;
}
export type SyncInitInput = BufferSource | WebAssembly.Module;
/**
* Instantiates the given `module`, which can either be bytes or
* a precompiled `WebAssembly.Module`.
*
* @param {SyncInitInput} module
*
* @returns {InitOutput}
*/
export function initSync(module: SyncInitInput): InitOutput;
/**
* If `module_or_path` is {RequestInfo} or {URL}, makes a request and
* for everything else, calls `WebAssembly.instantiate` directly.
*
* @param {InitInput | Promise<InitInput>} module_or_path
*
* @returns {Promise<InitOutput>}
*/
export default function __wbg_init (module_or_path?: InitInput | Promise<InitInput>): Promise<InitOutput>;
@@ -0,0 +1,487 @@
let wasm;
const heap = new Array(128).fill(undefined);
heap.push(undefined, null, true, false);
function getObject(idx) { return heap[idx]; }
let heap_next = heap.length;
function dropObject(idx) {
if (idx < 132) return;
heap[idx] = heap_next;
heap_next = idx;
}
function takeObject(idx) {
const ret = getObject(idx);
dropObject(idx);
return ret;
}
let WASM_VECTOR_LEN = 0;
let cachedUint8Memory0 = null;
function getUint8Memory0() {
if (cachedUint8Memory0 === null || cachedUint8Memory0.byteLength === 0) {
cachedUint8Memory0 = new Uint8Array(wasm.memory.buffer);
}
return cachedUint8Memory0;
}
const cachedTextEncoder = (typeof TextEncoder !== 'undefined' ? new TextEncoder('utf-8') : { encode: () => { throw Error('TextEncoder not available') } } );
const encodeString = (typeof cachedTextEncoder.encodeInto === 'function'
? function (arg, view) {
return cachedTextEncoder.encodeInto(arg, view);
}
: function (arg, view) {
const buf = cachedTextEncoder.encode(arg);
view.set(buf);
return {
read: arg.length,
written: buf.length
};
});
function passStringToWasm0(arg, malloc, realloc) {
if (realloc === undefined) {
const buf = cachedTextEncoder.encode(arg);
const ptr = malloc(buf.length) >>> 0;
getUint8Memory0().subarray(ptr, ptr + buf.length).set(buf);
WASM_VECTOR_LEN = buf.length;
return ptr;
}
let len = arg.length;
let ptr = malloc(len) >>> 0;
const mem = getUint8Memory0();
let offset = 0;
for (; offset < len; offset++) {
const code = arg.charCodeAt(offset);
if (code > 0x7F) break;
mem[ptr + offset] = code;
}
if (offset !== len) {
if (offset !== 0) {
arg = arg.slice(offset);
}
ptr = realloc(ptr, len, len = offset + arg.length * 3) >>> 0;
const view = getUint8Memory0().subarray(ptr + offset, ptr + len);
const ret = encodeString(arg, view);
offset += ret.written;
}
WASM_VECTOR_LEN = offset;
return ptr;
}
function isLikeNone(x) {
return x === undefined || x === null;
}
let cachedInt32Memory0 = null;
function getInt32Memory0() {
if (cachedInt32Memory0 === null || cachedInt32Memory0.byteLength === 0) {
cachedInt32Memory0 = new Int32Array(wasm.memory.buffer);
}
return cachedInt32Memory0;
}
function addHeapObject(obj) {
if (heap_next === heap.length) heap.push(heap.length + 1);
const idx = heap_next;
heap_next = heap[idx];
heap[idx] = obj;
return idx;
}
let cachedFloat64Memory0 = null;
function getFloat64Memory0() {
if (cachedFloat64Memory0 === null || cachedFloat64Memory0.byteLength === 0) {
cachedFloat64Memory0 = new Float64Array(wasm.memory.buffer);
}
return cachedFloat64Memory0;
}
const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } );
if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); };
function getStringFromWasm0(ptr, len) {
ptr = ptr >>> 0;
return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
}
let cachedBigInt64Memory0 = null;
function getBigInt64Memory0() {
if (cachedBigInt64Memory0 === null || cachedBigInt64Memory0.byteLength === 0) {
cachedBigInt64Memory0 = new BigInt64Array(wasm.memory.buffer);
}
return cachedBigInt64Memory0;
}
function debugString(val) {
// primitive types
const type = typeof val;
if (type == 'number' || type == 'boolean' || val == null) {
return `${val}`;
}
if (type == 'string') {
return `"${val}"`;
}
if (type == 'symbol') {
const description = val.description;
if (description == null) {
return 'Symbol';
} else {
return `Symbol(${description})`;
}
}
if (type == 'function') {
const name = val.name;
if (typeof name == 'string' && name.length > 0) {
return `Function(${name})`;
} else {
return 'Function';
}
}
// objects
if (Array.isArray(val)) {
const length = val.length;
let debug = '[';
if (length > 0) {
debug += debugString(val[0]);
}
for(let i = 1; i < length; i++) {
debug += ', ' + debugString(val[i]);
}
debug += ']';
return debug;
}
// Test for built-in
const builtInMatches = /\[object ([^\]]+)\]/.exec(toString.call(val));
let className;
if (builtInMatches.length > 1) {
className = builtInMatches[1];
} else {
// Failed to match the standard '[object ClassName]'
return toString.call(val);
}
if (className == 'Object') {
// we're a user defined class or Object
// JSON.stringify avoids problems with cycles, and is generally much
// easier than looping through ownProperties of `val`.
try {
return 'Object(' + JSON.stringify(val) + ')';
} catch (_) {
return 'Object';
}
}
// errors
if (val instanceof Error) {
return `${val.name}: ${val.message}\n${val.stack}`;
}
// TODO we could test for more things here, like `Set`s and `Map`s.
return className;
}
/**
* @param {string} code
* @returns {string}
*/
export function parse_deno_wasm(code) {
let deferred2_0;
let deferred2_1;
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len0 = WASM_VECTOR_LEN;
wasm.parse_deno_wasm(retptr, ptr0, len0);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
wasm.__wbindgen_free(deferred2_0, deferred2_1);
}
}
function handleError(f, args) {
try {
return f.apply(this, args);
} catch (e) {
wasm.__wbindgen_exn_store(addHeapObject(e));
}
}
async function __wbg_load(module, imports) {
if (typeof Response === 'function' && module instanceof Response) {
if (typeof WebAssembly.instantiateStreaming === 'function') {
try {
return await WebAssembly.instantiateStreaming(module, imports);
} catch (e) {
if (module.headers.get('Content-Type') != 'application/wasm') {
console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e);
} else {
throw e;
}
}
}
const bytes = await module.arrayBuffer();
return await WebAssembly.instantiate(bytes, imports);
} else {
const instance = await WebAssembly.instantiate(module, imports);
if (instance instanceof WebAssembly.Instance) {
return { instance, module };
} else {
return instance;
}
}
}
function __wbg_get_imports() {
const imports = {};
imports.wbg = {};
imports.wbg.__wbindgen_object_drop_ref = function(arg0) {
takeObject(arg0);
};
imports.wbg.__wbg_eval_8130c2f52f1a6d39 = function(arg0, arg1) {
const ret = eval(getStringFromWasm0(arg0, arg1));
return addHeapObject(ret);
};
imports.wbg.__wbindgen_string_get = function(arg0, arg1) {
const obj = getObject(arg1);
const ret = typeof(obj) === 'string' ? obj : undefined;
var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
var len1 = WASM_VECTOR_LEN;
getInt32Memory0()[arg0 / 4 + 1] = len1;
getInt32Memory0()[arg0 / 4 + 0] = ptr1;
};
imports.wbg.__wbindgen_boolean_get = function(arg0) {
const v = getObject(arg0);
const ret = typeof(v) === 'boolean' ? (v ? 1 : 0) : 2;
return ret;
};
imports.wbg.__wbindgen_is_bigint = function(arg0) {
const ret = typeof(getObject(arg0)) === 'bigint';
return ret;
};
imports.wbg.__wbindgen_bigint_from_i64 = function(arg0) {
const ret = arg0;
return addHeapObject(ret);
};
imports.wbg.__wbindgen_jsval_eq = function(arg0, arg1) {
const ret = getObject(arg0) === getObject(arg1);
return ret;
};
imports.wbg.__wbindgen_number_get = function(arg0, arg1) {
const obj = getObject(arg1);
const ret = typeof(obj) === 'number' ? obj : undefined;
getFloat64Memory0()[arg0 / 8 + 1] = isLikeNone(ret) ? 0 : ret;
getInt32Memory0()[arg0 / 4 + 0] = !isLikeNone(ret);
};
imports.wbg.__wbindgen_is_object = function(arg0) {
const val = getObject(arg0);
const ret = typeof(val) === 'object' && val !== null;
return ret;
};
imports.wbg.__wbindgen_in = function(arg0, arg1) {
const ret = getObject(arg0) in getObject(arg1);
return ret;
};
imports.wbg.__wbindgen_bigint_from_u64 = function(arg0) {
const ret = BigInt.asUintN(64, arg0);
return addHeapObject(ret);
};
imports.wbg.__wbindgen_error_new = function(arg0, arg1) {
const ret = new Error(getStringFromWasm0(arg0, arg1));
return addHeapObject(ret);
};
imports.wbg.__wbindgen_jsval_loose_eq = function(arg0, arg1) {
const ret = getObject(arg0) == getObject(arg1);
return ret;
};
imports.wbg.__wbg_get_7303ed2ef026b2f5 = function(arg0, arg1) {
const ret = getObject(arg0)[arg1 >>> 0];
return addHeapObject(ret);
};
imports.wbg.__wbg_length_820c786973abdd8a = function(arg0) {
const ret = getObject(arg0).length;
return ret;
};
imports.wbg.__wbindgen_is_function = function(arg0) {
const ret = typeof(getObject(arg0)) === 'function';
return ret;
};
imports.wbg.__wbg_next_f4bc0e96ea67da68 = function(arg0) {
const ret = getObject(arg0).next;
return addHeapObject(ret);
};
imports.wbg.__wbg_next_ec061e48a0e72a96 = function() { return handleError(function (arg0) {
const ret = getObject(arg0).next();
return addHeapObject(ret);
}, arguments) };
imports.wbg.__wbg_done_b6abb27d42b63867 = function(arg0) {
const ret = getObject(arg0).done;
return ret;
};
imports.wbg.__wbg_value_2f4ef2036bfad28e = function(arg0) {
const ret = getObject(arg0).value;
return addHeapObject(ret);
};
imports.wbg.__wbg_iterator_7c7e58f62eb84700 = function() {
const ret = Symbol.iterator;
return addHeapObject(ret);
};
imports.wbg.__wbg_get_f53c921291c381bd = function() { return handleError(function (arg0, arg1) {
const ret = Reflect.get(getObject(arg0), getObject(arg1));
return addHeapObject(ret);
}, arguments) };
imports.wbg.__wbg_call_557a2f2deacc4912 = function() { return handleError(function (arg0, arg1) {
const ret = getObject(arg0).call(getObject(arg1));
return addHeapObject(ret);
}, arguments) };
imports.wbg.__wbg_isArray_04e59fb73f78ab5b = function(arg0) {
const ret = Array.isArray(getObject(arg0));
return ret;
};
imports.wbg.__wbg_instanceof_ArrayBuffer_ef2632aa0d4bfff8 = function(arg0) {
let result;
try {
result = getObject(arg0) instanceof ArrayBuffer;
} catch {
result = false;
}
const ret = result;
return ret;
};
imports.wbg.__wbg_isSafeInteger_2088b01008075470 = function(arg0) {
const ret = Number.isSafeInteger(getObject(arg0));
return ret;
};
imports.wbg.__wbg_entries_13e011453776468f = function(arg0) {
const ret = Object.entries(getObject(arg0));
return addHeapObject(ret);
};
imports.wbg.__wbg_buffer_55ba7a6b1b92e2ac = function(arg0) {
const ret = getObject(arg0).buffer;
return addHeapObject(ret);
};
imports.wbg.__wbg_new_09938a7d020f049b = function(arg0) {
const ret = new Uint8Array(getObject(arg0));
return addHeapObject(ret);
};
imports.wbg.__wbg_set_3698e3ca519b3c3c = function(arg0, arg1, arg2) {
getObject(arg0).set(getObject(arg1), arg2 >>> 0);
};
imports.wbg.__wbg_length_0aab7ffd65ad19ed = function(arg0) {
const ret = getObject(arg0).length;
return ret;
};
imports.wbg.__wbg_instanceof_Uint8Array_1349640af2da2e88 = function(arg0) {
let result;
try {
result = getObject(arg0) instanceof Uint8Array;
} catch {
result = false;
}
const ret = result;
return ret;
};
imports.wbg.__wbindgen_bigint_get_as_i64 = function(arg0, arg1) {
const v = getObject(arg1);
const ret = typeof(v) === 'bigint' ? v : undefined;
getBigInt64Memory0()[arg0 / 8 + 1] = isLikeNone(ret) ? BigInt(0) : ret;
getInt32Memory0()[arg0 / 4 + 0] = !isLikeNone(ret);
};
imports.wbg.__wbindgen_debug_string = function(arg0, arg1) {
const ret = debugString(getObject(arg1));
const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
const len1 = WASM_VECTOR_LEN;
getInt32Memory0()[arg0 / 4 + 1] = len1;
getInt32Memory0()[arg0 / 4 + 0] = ptr1;
};
imports.wbg.__wbindgen_throw = function(arg0, arg1) {
throw new Error(getStringFromWasm0(arg0, arg1));
};
imports.wbg.__wbindgen_memory = function() {
const ret = wasm.memory;
return addHeapObject(ret);
};
return imports;
}
function __wbg_init_memory(imports, maybe_memory) {
}
function __wbg_finalize_init(instance, module) {
wasm = instance.exports;
__wbg_init.__wbindgen_wasm_module = module;
cachedBigInt64Memory0 = null;
cachedFloat64Memory0 = null;
cachedInt32Memory0 = null;
cachedUint8Memory0 = null;
return wasm;
}
function initSync(module) {
if (wasm !== undefined) return wasm;
const imports = __wbg_get_imports();
__wbg_init_memory(imports);
if (!(module instanceof WebAssembly.Module)) {
module = new WebAssembly.Module(module);
}
const instance = new WebAssembly.Instance(module, imports);
return __wbg_finalize_init(instance, module);
}
async function __wbg_init(input) {
if (wasm !== undefined) return wasm;
if (typeof input === 'undefined') {
input = new URL('windmill_parser_ts_wasm_bg.wasm', import.meta.url);
}
const imports = __wbg_get_imports();
if (typeof input === 'string' || (typeof Request === 'function' && input instanceof Request) || (typeof URL === 'function' && input instanceof URL)) {
input = fetch(input);
}
__wbg_init_memory(imports);
const { instance, module } = await __wbg_load(await input, imports);
return __wbg_finalize_init(instance, module);
}
export { initSync }
export default __wbg_init;
@@ -0,0 +1,9 @@
/* tslint:disable */
/* eslint-disable */
export const memory: WebAssembly.Memory;
export function parse_deno_wasm(a: number, b: number, c: number): void;
export function __wbindgen_malloc(a: number): number;
export function __wbindgen_realloc(a: number, b: number, c: number): number;
export function __wbindgen_add_to_stack_pointer(a: number): number;
export function __wbindgen_free(a: number, b: number): void;
export function __wbindgen_exn_store(a: number): void;
@@ -0,0 +1,13 @@
use serde_json;
use wasm_bindgen::prelude::*;
use windmill_parser_ts::parse_deno_signature;
#[wasm_bindgen]
pub fn parse_deno_wasm(code: &str) -> String {
let r = parse_deno_signature(code, false);
if let Ok(r) = r {
return serde_json::to_string(&r).unwrap();
} else {
return "{\"type\": \"Invalid\"}".to_string();
}
}
@@ -0,0 +1,185 @@
use serde_json::json;
use wasm_bindgen_test::wasm_bindgen_test;
use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ};
use windmill_parser_ts::parse_deno_signature;
#[wasm_bindgen_test]
fn test_parse_deno_sig() -> Result<(), String> {
let code = "
export function main(test1?: string, test2: string = \"burkina\",
test3: wmill.Resource<'postgres'>, b64: Base64, ls: Base64[],
email: Email, literal: \"test\", literal_union: \"test\" | \"test2\",
opt_type?: string | null, opt_type_union: string | null, opt_type_union_union2: string | undefined,
min_object: {a: string, b: number}) {
console.log(42)
}
";
assert_eq!(
parse_deno_signature(code, false)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![
Arg {
otyp: None,
name: "test1".to_string(),
typ: Typ::Str(None),
default: None,
has_default: true
},
Arg {
otyp: None,
name: "test2".to_string(),
typ: Typ::Str(None),
default: Some(json!("burkina")),
has_default: true
},
Arg {
otyp: None,
name: "test3".to_string(),
typ: Typ::Resource("postgres".to_string()),
default: None,
has_default: false
},
Arg {
otyp: None,
name: "b64".to_string(),
typ: Typ::Bytes,
default: None,
has_default: false
},
Arg {
otyp: None,
name: "ls".to_string(),
typ: Typ::List(Box::new(Typ::Bytes)),
default: None,
has_default: false
},
Arg {
otyp: None,
name: "email".to_string(),
typ: Typ::Email,
default: None,
has_default: false
},
Arg {
otyp: None,
name: "literal".to_string(),
typ: Typ::Str(Some(vec!["test".to_string()])),
default: None,
has_default: false
},
Arg {
otyp: None,
name: "literal_union".to_string(),
typ: Typ::Str(Some(vec!["test".to_string(), "test2".to_string()])),
default: None,
has_default: false
},
Arg {
otyp: None,
name: "opt_type".to_string(),
typ: Typ::Str(None),
default: None,
has_default: true
},
Arg {
otyp: None,
name: "opt_type_union".to_string(),
typ: Typ::Str(None),
default: None,
has_default: true
},
Arg {
otyp: None,
name: "opt_type_union_union2".to_string(),
typ: Typ::Str(None),
default: None,
has_default: true
},
Arg {
otyp: None,
name: "min_object".to_string(),
typ: Typ::Object(vec![
ObjectProperty { key: "a".to_string(), typ: Box::new(Typ::Str(None)) },
ObjectProperty { key: "b".to_string(), typ: Box::new(Typ::Float) }
]),
default: None,
has_default: false
}
]
}
);
Ok(())
}
#[wasm_bindgen_test]
fn test_parse_deno_sig_implicit_types() -> Result<(), String> {
let code = "
export function main(test2 = \"burkina\",
bool = true,
float = 4.2,
int = 42,
ls = [\"test\"],
min_object = {a: \"test\", b: 42}) {
console.log(42)
}
";
assert_eq!(
parse_deno_signature(code, false)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![
Arg {
otyp: None,
name: "test2".to_string(),
typ: Typ::Str(None),
default: Some(json!("burkina")),
has_default: true
},
Arg {
otyp: None,
name: "bool".to_string(),
typ: Typ::Bool,
default: Some(json!(true)),
has_default: true
},
Arg {
otyp: None,
name: "float".to_string(),
typ: Typ::Float,
default: Some(json!(4.2)),
has_default: true
},
Arg {
otyp: None,
name: "int".to_string(),
typ: Typ::Int,
default: Some(json!(42)),
has_default: true
},
Arg {
otyp: None,
name: "ls".to_string(),
typ: Typ::List(Box::new(Typ::Str(None))),
default: Some(json!(["test"])),
has_default: true
},
Arg {
otyp: None,
name: "min_object".to_string(),
typ: Typ::Object(vec![
ObjectProperty { key: "a".to_string(), typ: Box::new(Typ::Str(None)) },
ObjectProperty { key: "b".to_string(), typ: Box::new(Typ::Int) }
]),
default: Some(json!({"a": "test", "b": 42})),
has_default: true
}
]
}
);
Ok(())
}
@@ -4,14 +4,17 @@ version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser_ts"
path = "./src/lib.rs"
[target.'cfg(target_arch = "wasm32")'.dependencies]
wasm-bindgen.workspace = true
serde-wasm-bindgen.workspace = true
[dependencies]
windmill-parser.workspace = true
windmill-common.workspace = true
deno_core.workspace = true
swc_common.workspace = true
swc_ecma_parser.workspace = true
swc_ecma_ast.workspace = true
+29 -220
View File
@@ -5,9 +5,8 @@
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use deno_core::{serde_v8, v8, JsRuntime, RuntimeOptions};
// use deno_core::{serde_v8, v8, JsRuntime, RuntimeOptions};
use serde_json::Value;
use windmill_common::error;
use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectProperty, Typ};
use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Span, Spanned};
@@ -19,7 +18,10 @@ use swc_ecma_ast::{
};
use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsConfig};
pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> error::Result<MainArgSignature> {
#[cfg(target_arch = "wasm32")]
use wasm_bindgen::prelude::*;
pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> Result<MainArgSignature, String> {
let cm: Lrc<SourceMap> = Default::default();
let fm = cm.new_source_file(FileName::Custom("main.ts".into()), code.into());
let lexer = Lexer::new(
@@ -40,11 +42,7 @@ pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> error::Result<MainAr
let ast = parser
.parse_module()
.map_err(|_| {
error::Error::ExecutionErr(format!(
"Error while parsing code, it is invalid typescript"
))
})?
.map_err(|_| format!("Error while parsing code, it is invalid typescript"))?
.body;
// println!("{ast:?}");
@@ -63,18 +61,18 @@ pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> error::Result<MainAr
args: params
.into_iter()
.map(|x| parse_param(x, &cm, skip_dflt))
.collect::<Result<Vec<Arg>, error::Error>>()?,
.collect::<Result<Vec<Arg>, String>>()?,
};
Ok(r)
} else {
Err(error::Error::ExecutionErr(
Err(
"main function was not findable (expected to find 'export function main(...)'"
.to_string(),
))
)
}
}
fn parse_param(x: Param, cm: &Lrc<SourceMap>, skip_dflt: bool) -> error::Result<Arg> {
fn parse_param(x: Param, cm: &Lrc<SourceMap>, skip_dflt: bool) -> Result<Arg, String> {
let r = match x.pat {
Pat::Ident(ident) => {
let (name, typ, nullable) = binding_ident_to_arg(&ident);
@@ -89,11 +87,11 @@ fn parse_param(x: Param, cm: &Lrc<SourceMap>, skip_dflt: bool) -> error::Result<
Pat::Assign(AssignPat { left, right, .. }) => {
let (name, mut typ, _nullable) =
left.as_ident().map(binding_ident_to_arg).ok_or_else(|| {
error::Error::ExecutionErr(format!(
format!(
"parameter syntax unsupported: `{}`",
cm.span_to_snippet(left.span())
.unwrap_or_else(|_| cm.span_to_string(left.span()))
))
)
})?;
let dflt = if skip_dflt {
@@ -122,11 +120,11 @@ fn parse_param(x: Param, cm: &Lrc<SourceMap>, skip_dflt: bool) -> error::Result<
}
Ok(Arg { otyp: None, name, typ, default: dflt, has_default: true })
}
_ => Err(error::Error::ExecutionErr(format!(
_ => Err(format!(
"parameter syntax unsupported: `{}`",
cm.span_to_snippet(x.span())
.unwrap_or_else(|_| cm.span_to_string(x.span()))
))),
)),
};
r
}
@@ -143,6 +141,7 @@ fn eval_span(span: Span, cm: &Lrc<SourceMap>) -> Option<Value> {
None => None,
}
}
fn binding_ident_to_arg(BindingIdent { id, type_ann }: &BindingIdent) -> (String, Typ, bool) {
let (typ, nullable) = type_ann
.as_ref()
@@ -266,211 +265,21 @@ fn tstype_to_typ(ts_type: &TsType) -> (Typ, bool) {
}
}
#[cfg(target_arch = "wasm32")]
#[wasm_bindgen]
extern "C" {
pub fn eval(s: &str) -> JsValue;
pub fn alert(s: &str);
}
#[cfg(target_arch = "wasm32")]
pub fn eval_sync(code: &str) -> Result<serde_json::Value, String> {
let mut context = JsRuntime::new(RuntimeOptions::default());
let code = format!("let x = {}; x", code);
let res = context.execute_script("<anon>", code.into());
match res {
Ok(global) => {
let scope = &mut context.handle_scope();
let local = v8::Local::new(scope, global);
let deserialized_value = serde_v8::from_v8::<serde_json::Value>(scope, local);
match deserialized_value {
Ok(value) => Ok(value),
Err(err) => Err(format!("Cannot deserialize value: {:?}", err)),
}
}
Err(err) => Err(format!("Evaling error: {:?}", err)),
}
serde_wasm_bindgen::from_value(eval(format!("let x = {}; x", code).as_str()))
.map_err(|err| format!("Cannot deserialize value: {:?}", err))
}
#[cfg(test)]
mod tests {
use serde_json::json;
// Note this useful idiom: importing names from outer (for mod tests) scope.
use super::*;
#[test]
fn test_parse_deno_sig() -> anyhow::Result<()> {
let code = "
export function main(test1?: string, test2: string = \"burkina\",
test3: wmill.Resource<'postgres'>, b64: Base64, ls: Base64[],
email: Email, literal: \"test\", literal_union: \"test\" | \"test2\",
opt_type?: string | null, opt_type_union: string | null, opt_type_union_union2: string | undefined,
min_object: {a: string, b: number}) {
console.log(42)
}
";
assert_eq!(
parse_deno_signature(code, false)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![
Arg {
otyp: None,
name: "test1".to_string(),
typ: Typ::Str(None),
default: None,
has_default: true
},
Arg {
otyp: None,
name: "test2".to_string(),
typ: Typ::Str(None),
default: Some(json!("burkina")),
has_default: true
},
Arg {
otyp: None,
name: "test3".to_string(),
typ: Typ::Resource("postgres".to_string()),
default: None,
has_default: false
},
Arg {
otyp: None,
name: "b64".to_string(),
typ: Typ::Bytes,
default: None,
has_default: false
},
Arg {
otyp: None,
name: "ls".to_string(),
typ: Typ::List(Box::new(Typ::Bytes)),
default: None,
has_default: false
},
Arg {
otyp: None,
name: "email".to_string(),
typ: Typ::Email,
default: None,
has_default: false
},
Arg {
otyp: None,
name: "literal".to_string(),
typ: Typ::Str(Some(vec!["test".to_string()])),
default: None,
has_default: false
},
Arg {
otyp: None,
name: "literal_union".to_string(),
typ: Typ::Str(Some(vec!["test".to_string(), "test2".to_string()])),
default: None,
has_default: false
},
Arg {
otyp: None,
name: "opt_type".to_string(),
typ: Typ::Str(None),
default: None,
has_default: true
},
Arg {
otyp: None,
name: "opt_type_union".to_string(),
typ: Typ::Str(None),
default: None,
has_default: true
},
Arg {
otyp: None,
name: "opt_type_union_union2".to_string(),
typ: Typ::Str(None),
default: None,
has_default: true
},
Arg {
otyp: None,
name: "min_object".to_string(),
typ: Typ::Object(vec![
ObjectProperty { key: "a".to_string(), typ: Box::new(Typ::Str(None)) },
ObjectProperty { key: "b".to_string(), typ: Box::new(Typ::Float) }
]),
default: None,
has_default: false
}
]
}
);
Ok(())
}
#[test]
fn test_parse_deno_sig_implicit_types() -> anyhow::Result<()> {
let code = "
export function main(test2 = \"burkina\",
bool = true,
float = 4.2,
int = 42,
ls = [\"test\"],
min_object = {a: \"test\", b: 42}) {
console.log(42)
}
";
assert_eq!(
parse_deno_signature(code, false)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![
Arg {
otyp: None,
name: "test2".to_string(),
typ: Typ::Str(None),
default: Some(json!("burkina")),
has_default: true
},
Arg {
otyp: None,
name: "bool".to_string(),
typ: Typ::Bool,
default: Some(json!(true)),
has_default: true
},
Arg {
otyp: None,
name: "float".to_string(),
typ: Typ::Float,
default: Some(json!(4.2)),
has_default: true
},
Arg {
otyp: None,
name: "int".to_string(),
typ: Typ::Int,
default: Some(json!(42)),
has_default: true
},
Arg {
otyp: None,
name: "ls".to_string(),
typ: Typ::List(Box::new(Typ::Str(None))),
default: Some(json!(["test"])),
has_default: true
},
Arg {
otyp: None,
name: "min_object".to_string(),
typ: Typ::Object(vec![
ObjectProperty { key: "a".to_string(), typ: Box::new(Typ::Str(None)) },
ObjectProperty { key: "b".to_string(), typ: Box::new(Typ::Int) }
]),
default: Some(json!({"a": "test", "b": 42})),
has_default: true
}
]
}
);
Ok(())
}
#[cfg(not(target_arch = "wasm32"))]
pub fn eval_sync(code: &str) -> Result<serde_json::Value, String> {
panic!("eval_sync is only available in wasm32")
}
+3 -1
View File
@@ -922,7 +922,9 @@ async fn parse_python_code_to_jsonschema(Json(code): Json<String>) -> Json<SigPa
}
async fn parse_deno_code_to_jsonschema(Json(code): Json<String>) -> Json<SigParsing> {
result_to_sig_parsing(windmill_parser_ts::parse_deno_signature(&code, false))
result_to_sig_parsing(
windmill_parser_ts::parse_deno_signature(&code, false).map_err(|e| Error::ExecutionErr(e)),
)
}
async fn parse_go_code_to_jsonschema(Json(code): Json<String>) -> Json<SigParsing> {
result_to_sig_parsing(windmill_parser_go::parse_go_sig(&code))
+2 -1
View File
@@ -1512,7 +1512,8 @@ async fn handle_deno_job(
let write_wrapper_f = async {
// let mut start = Instant::now();
let args = windmill_parser_ts::parse_deno_signature(inner_content, true)?.args;
let args = windmill_parser_ts::parse_deno_signature(inner_content, true)
.map_err(|e| Error::BadRequest(e))?.args;
let dates = args.iter().enumerate().filter_map(|(i, x)| if matches!(x.typ, Typ::Datetime) {
Some(i)
} else {
+2 -2
View File
@@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y git libssl-dev pkg-config npm
RUN apt-get -y update \
&& apt-get install -y \
curl lld nodejs npm
curl nodejs npm
RUN rustup component add rustfmt
@@ -53,11 +53,11 @@ COPY /backend/windmill-api/openapi.yaml /backend/windmill-api/openapi.yaml
COPY /openflow.openapi.yaml /openflow.openapi.yaml
COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh
RUN cd /backend/windmill-api && . ./build_openapi.sh
COPY /backend/parsers/windmill-parser-py-wasm/pkg/ /backend/windmill-parser-py-wasm/pkg/
RUN npm run generate-backend-client
ENV NODE_OPTIONS "--max-old-space-size=8192"
RUN npm run build
RUN npm run check
+16
View File
@@ -36,6 +36,7 @@
"svelte-timezone-picker": "^2.0.3",
"tailwind-merge": "^1.12.0",
"vscode-ws-jsonrpc": "3.0.0",
"windmill-parser-ts-wasm": "file:../backend/parsers/windmill-parser-ts-wasm/pkg",
"y-monaco": "^0.1.4",
"y-websocket": "^1.5.0",
"yjs": "^13.6.1"
@@ -97,6 +98,14 @@
"svelte": "^3.59.1"
}
},
"../../../usr/lib/node_modules/windmill-parser-ts-wasm": {
"version": "1.109.1",
"extraneous": true
},
"../backend/parsers/windmill-parser-ts-wasm/pkg": {
"name": "windmill-parser-ts-wasm",
"version": "1.109.1"
},
"node_modules/@alloc/quick-lru": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz",
@@ -7690,6 +7699,10 @@
"string-width": "^1.0.2 || 2 || 3 || 4"
}
},
"node_modules/windmill-parser-ts-wasm": {
"resolved": "../backend/parsers/windmill-parser-ts-wasm/pkg",
"link": true
},
"node_modules/word-wrap": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
@@ -13383,6 +13396,9 @@
"string-width": "^1.0.2 || 2 || 3 || 4"
}
},
"windmill-parser-ts-wasm": {
"version": "file:../backend/parsers/windmill-parser-ts-wasm/pkg"
},
"word-wrap": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
+2 -1
View File
@@ -98,7 +98,8 @@
"vscode-ws-jsonrpc": "3.0.0",
"y-monaco": "^0.1.4",
"y-websocket": "^1.5.0",
"yjs": "^13.6.1"
"yjs": "^13.6.1",
"windmill-parser-ts-wasm": "file:../backend/parsers/windmill-parser-ts-wasm/pkg"
},
"peerDependencies": {
"@sveltejs/kit": "^1.20.1",
+5 -3
View File
@@ -3,6 +3,10 @@ import { get, writable } from 'svelte/store'
import type { Schema, SchemaProperty } from './common.js'
import { sortObject } from './utils.js'
import { tick } from 'svelte'
import init, { parse_deno_wasm } from 'windmill-parser-ts-wasm'
import wasmUrl from 'windmill-parser-ts-wasm/windmill_parser_ts_wasm_bg.wasm?url'
init(wasmUrl)
const loadSchemaLastRun = writable<[string | undefined, MainArgSignature | undefined]>(undefined)
@@ -24,9 +28,7 @@ export async function inferArgs(
requestBody: code
})
} else if (language == 'deno') {
inferedSchema = await ScriptService.denoToJsonschema({
requestBody: code
})
inferedSchema = JSON.parse(parse_deno_wasm(code))
} else if (language == 'go') {
inferedSchema = await ScriptService.goToJsonschema({
requestBody: code