diff --git a/backend/parsers/windmill-parser-wasm/build.sh b/backend/parsers/windmill-parser-wasm/build.sh old mode 100644 new mode 100755 index 0ac5a6fde0..044f92d93b --- a/backend/parsers/windmill-parser-wasm/build.sh +++ b/backend/parsers/windmill-parser-wasm/build.sh @@ -1,4 +1,4 @@ #!/bin/bash set -eou pipefail -deno task wasmbuild --out ../../../cli/wasm/ +deno task wasmbuild --out ../../../cli/wasm/ -p windmill-parser-wasm diff --git a/backend/parsers/windmill-parser-wasm/deno.json b/backend/parsers/windmill-parser-wasm/deno.json index 0764e447d0..abdd5e39d1 100644 --- a/backend/parsers/windmill-parser-wasm/deno.json +++ b/backend/parsers/windmill-parser-wasm/deno.json @@ -1,5 +1,5 @@ { - "tasks": { - "wasmbuild": "deno run -A https://deno.land/x/wasmbuild@0.15.4/main.ts" - } - } \ No newline at end of file + "tasks": { + "wasmbuild": "deno run -A jsr:@deno/wasmbuild@0.17.2" + } +} diff --git a/cli/wasm/windmill_parser_wasm.generated.d.ts b/cli/wasm/windmill_parser_wasm.generated.d.ts new file mode 100644 index 0000000000..23c9054422 --- /dev/null +++ b/cli/wasm/windmill_parser_wasm.generated.d.ts @@ -0,0 +1,120 @@ +// deno-lint-ignore-file +// deno-fmt-ignore-file + +export interface InstantiateResult { + instance: WebAssembly.Instance; + exports: { + parse_deno: typeof parse_deno; + parse_outputs: typeof parse_outputs; + parse_ts_imports: typeof parse_ts_imports; + parse_bash: typeof parse_bash; + parse_powershell: typeof parse_powershell; + parse_go: typeof parse_go; + parse_python: typeof parse_python; + parse_sql: typeof parse_sql; + parse_mysql: typeof parse_mysql; + parse_bigquery: typeof parse_bigquery; + parse_snowflake: typeof parse_snowflake; + parse_mssql: typeof parse_mssql; + parse_db_resource: typeof parse_db_resource; + parse_graphql: typeof parse_graphql; + parse_php: typeof parse_php + }; +} + +/** Gets if the Wasm module has been instantiated. */ +export function isInstantiated(): boolean; + +/** Options for instantiating a Wasm instance. */ +export interface InstantiateOptions { + /** Optional url to the Wasm file to instantiate. */ + url?: URL; + /** Callback to decompress the raw Wasm file bytes before instantiating. */ + decompress?: (bytes: Uint8Array) => Uint8Array; +} + +/** Instantiates an instance of the Wasm module returning its functions. +* @remarks It is safe to call this multiple times and once successfully +* loaded it will always return a reference to the same object. */ +export function instantiate(opts?: InstantiateOptions): Promise; + +/** Instantiates an instance of the Wasm module along with its exports. + * @remarks It is safe to call this multiple times and once successfully + * loaded it will always return a reference to the same object. */ +export function instantiateWithInstance(opts?: InstantiateOptions): Promise; + +/** +* @param {string} code +* @returns {string} +*/ +export function parse_deno(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_outputs(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_ts_imports(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_bash(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_powershell(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_go(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_python(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_sql(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_mysql(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_bigquery(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_snowflake(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_mssql(code: string): string; +/** +* @param {string} code +* @returns {string | undefined} +*/ +export function parse_db_resource(code: string): string | undefined; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_graphql(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_php(code: string): string; diff --git a/cli/wasm/windmill_parser_wasm.generated.js b/cli/wasm/windmill_parser_wasm.generated.js index a233901797..3866aad19a 100644 --- a/cli/wasm/windmill_parser_wasm.generated.js +++ b/cli/wasm/windmill_parser_wasm.generated.js @@ -1,7 +1,10 @@ // @generated file from wasmbuild -- do not edit +// @ts-nocheck: generated // deno-lint-ignore-file // deno-fmt-ignore-file -// source-hash: f38890e80d35ec2affe2a260e12308f96bcbf771 +/// + +// source-hash: ae1bfe27a90c5656a723fb7ced38f4857a5d6931 let wasm; const heap = new Array(128).fill(undefined); @@ -80,6 +83,7 @@ function passStringToWasm0(arg, malloc, realloc) { const ret = encodeString(arg, view); offset += ret.written; + ptr = realloc(ptr, len, offset, 1) >>> 0; } WASM_VECTOR_LEN = offset; @@ -531,6 +535,33 @@ export function parse_mssql(code) { } } +/** + * @param {string} code + * @returns {string | undefined} + */ +export function parse_db_resource(code) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passStringToWasm0( + code, + wasm.__wbindgen_malloc, + wasm.__wbindgen_realloc, + ); + const len0 = WASM_VECTOR_LEN; + wasm.parse_db_resource(retptr, ptr0, len0); + var r0 = getInt32Memory0()[retptr / 4 + 0]; + var r1 = getInt32Memory0()[retptr / 4 + 1]; + let v2; + if (r0 !== 0) { + v2 = getStringFromWasm0(r0, r1).slice(); + wasm.__wbindgen_free(r0, r1 * 1, 1); + } + return v2; + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } +} + /** * @param {string} code * @returns {string} @@ -558,6 +589,33 @@ export function parse_graphql(code) { } } +/** + * @param {string} code + * @returns {string} + */ +export function parse_php(code) { + let deferred2_0; + let deferred2_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passStringToWasm0( + code, + wasm.__wbindgen_malloc, + wasm.__wbindgen_realloc, + ); + const len0 = WASM_VECTOR_LEN; + wasm.parse_php(retptr, ptr0, len0); + var r0 = getInt32Memory0()[retptr / 4 + 0]; + var r1 = getInt32Memory0()[retptr / 4 + 1]; + deferred2_0 = r0; + deferred2_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + function handleError(f, args) { try { return f.apply(this, args); @@ -585,6 +643,10 @@ const imports = { getInt32Memory0()[arg0 / 4 + 1] = len1; getInt32Memory0()[arg0 / 4 + 0] = ptr1; }, + __wbg_eval_c74b339326c2fc56: function (arg0, arg1) { + const ret = eval(getStringFromWasm0(arg0, arg1)); + return addHeapObject(ret); + }, __wbindgen_error_new: function (arg0, arg1) { const ret = new Error(getStringFromWasm0(arg0, arg1)); return addHeapObject(ret); @@ -598,18 +660,6 @@ const imports = { const ret = typeof (getObject(arg0)) === "bigint"; return ret; }, - __wbindgen_bigint_from_i64: function (arg0) { - const ret = arg0; - return addHeapObject(ret); - }, - __wbindgen_jsval_eq: function (arg0, arg1) { - const ret = getObject(arg0) === getObject(arg1); - return ret; - }, - __wbindgen_bigint_from_u64: function (arg0) { - const ret = BigInt.asUintN(64, arg0); - return addHeapObject(ret); - }, __wbindgen_number_get: function (arg0, arg1) { const obj = getObject(arg1); const ret = typeof obj === "number" ? obj : undefined; @@ -625,19 +675,27 @@ const imports = { const ret = getObject(arg0) in getObject(arg1); return ret; }, - __wbg_eval_c330e1bec5b1705c: function (arg0, arg1) { - const ret = eval(getStringFromWasm0(arg0, arg1)); + __wbindgen_bigint_from_i64: function (arg0) { + const ret = arg0; + return addHeapObject(ret); + }, + __wbindgen_jsval_eq: function (arg0, arg1) { + const ret = getObject(arg0) === getObject(arg1); + return ret; + }, + __wbindgen_bigint_from_u64: function (arg0) { + const ret = BigInt.asUintN(64, arg0); return addHeapObject(ret); }, __wbindgen_jsval_loose_eq: function (arg0, arg1) { const ret = getObject(arg0) == getObject(arg1); return ret; }, - __wbg_get_f01601b5a68d10e3: function (arg0, arg1) { + __wbg_get_bd8e338fbd5f5cc8: function (arg0, arg1) { const ret = getObject(arg0)[arg1 >>> 0]; return addHeapObject(ret); }, - __wbg_length_1009b1af0c481d7b: function (arg0) { + __wbg_length_cd7af8117672b8b8: function (arg0) { const ret = getObject(arg0).length; return ret; }, @@ -645,45 +703,45 @@ const imports = { const ret = typeof (getObject(arg0)) === "function"; return ret; }, - __wbg_next_9b877f231f476d01: function (arg0) { + __wbg_next_40fc327bfc8770e6: function (arg0) { const ret = getObject(arg0).next; return addHeapObject(ret); }, - __wbg_next_6529ee0cca8d57ed: function () { + __wbg_next_196c84450b364254: function () { return handleError(function (arg0) { const ret = getObject(arg0).next(); return addHeapObject(ret); }, arguments); }, - __wbg_done_5fe336b092d60cf2: function (arg0) { + __wbg_done_298b57d23c0fc80c: function (arg0) { const ret = getObject(arg0).done; return ret; }, - __wbg_value_0c248a78fdc8e19f: function (arg0) { + __wbg_value_d93c65011f51a456: function (arg0) { const ret = getObject(arg0).value; return addHeapObject(ret); }, - __wbg_iterator_db7ca081358d4fb2: function () { + __wbg_iterator_2cee6dadfd956dfa: function () { const ret = Symbol.iterator; return addHeapObject(ret); }, - __wbg_get_7b48513de5dc5ea4: function () { + __wbg_get_e3c254076557e348: function () { return handleError(function (arg0, arg1) { const ret = Reflect.get(getObject(arg0), getObject(arg1)); return addHeapObject(ret); }, arguments); }, - __wbg_call_90c26b09837aba1c: function () { + __wbg_call_27c0f87801dedf93: function () { return handleError(function (arg0, arg1) { const ret = getObject(arg0).call(getObject(arg1)); return addHeapObject(ret); }, arguments); }, - __wbg_isArray_74fb723e24f76012: function (arg0) { + __wbg_isArray_2ab64d95e09ea0ae: function (arg0) { const ret = Array.isArray(getObject(arg0)); return ret; }, - __wbg_instanceof_ArrayBuffer_e7d53d51371448e2: function (arg0) { + __wbg_instanceof_ArrayBuffer_836825be07d4c9d2: function (arg0) { let result; try { result = getObject(arg0) instanceof ArrayBuffer; @@ -693,30 +751,40 @@ const imports = { const ret = result; return ret; }, - __wbg_isSafeInteger_f93fde0dca9820f8: function (arg0) { + __wbg_instanceof_Map_87917e0a7aaf4012: function (arg0) { + let result; + try { + result = getObject(arg0) instanceof Map; + } catch (_) { + result = false; + } + const ret = result; + return ret; + }, + __wbg_isSafeInteger_f7b04ef02296c4d2: function (arg0) { const ret = Number.isSafeInteger(getObject(arg0)); return ret; }, - __wbg_entries_9e2e2aa45aa5094a: function (arg0) { + __wbg_entries_95cc2c823b285a09: function (arg0) { const ret = Object.entries(getObject(arg0)); return addHeapObject(ret); }, - __wbg_buffer_a448f833075b71ba: function (arg0) { + __wbg_buffer_12d079cc21e14bdb: function (arg0) { const ret = getObject(arg0).buffer; return addHeapObject(ret); }, - __wbg_new_8f67e318f15d7254: function (arg0) { + __wbg_new_63b92bc8671ed464: function (arg0) { const ret = new Uint8Array(getObject(arg0)); return addHeapObject(ret); }, - __wbg_set_2357bf09366ee480: function (arg0, arg1, arg2) { + __wbg_set_a47bac70306a19a7: function (arg0, arg1, arg2) { getObject(arg0).set(getObject(arg1), arg2 >>> 0); }, - __wbg_length_1d25fa9e4ac21ce7: function (arg0) { + __wbg_length_c20a40f15020d68a: function (arg0) { const ret = getObject(arg0).length; return ret; }, - __wbg_instanceof_Uint8Array_bced6f43aed8c1aa: function (arg0) { + __wbg_instanceof_Uint8Array_2b3bbecd033d19f6: function (arg0) { let result; try { result = getObject(arg0) instanceof Uint8Array; @@ -753,47 +821,119 @@ const imports = { }, }; -import { Loader } from "https://deno.land/x/wasmbuild@0.15.4/loader.ts"; -import { cacheToLocalDir } from "https://deno.land/x/wasmbuild@0.15.4/cache.ts"; +class WasmBuildLoader { + #options; + #lastLoadPromise; + #instantiated; -const loader = new Loader({ + constructor(options) { + this.#options = options; + } + + get instance() { + return this.#instantiated?.instance; + } + + get module() { + return this.#instantiated?.module; + } + + load( + url, + decompress, + ) { + if (this.#instantiated) { + return Promise.resolve(this.#instantiated); + } else if (this.#lastLoadPromise == null) { + this.#lastLoadPromise = (async () => { + try { + this.#instantiated = await this.#instantiate(url, decompress); + return this.#instantiated; + } finally { + this.#lastLoadPromise = undefined; + } + })(); + } + return this.#lastLoadPromise; + } + + async #instantiate(url, decompress) { + const imports = this.#options.imports; + if (this.#options.cache != null && url.protocol !== "file:") { + try { + const result = await this.#options.cache( + url, + decompress ?? ((bytes) => bytes), + ); + if (result instanceof URL) { + url = result; + decompress = undefined; // already decompressed + } else if (result != null) { + return WebAssembly.instantiate(result, imports); + } + } catch { + // ignore if caching ever fails (ex. when on deploy) + } + } + + const isFile = url.protocol === "file:"; + + // make file urls work in Node via dnt + const isNode = globalThis.process?.versions?.node != null; + if (isFile && typeof Deno !== "object") { + throw new Error( + "Loading local files are not supported in this environment", + ); + } + if (isNode && isFile) { + // the deno global will be shimmed by dnt + const wasmCode = await Deno.readFile(url); + return WebAssembly.instantiate( + decompress ? decompress(wasmCode) : wasmCode, + imports, + ); + } + + switch (url.protocol) { + case "file:": + case "https:": + case "http:": { + const wasmResponse = await fetchWithRetries(url); + if (decompress) { + const wasmCode = new Uint8Array(await wasmResponse.arrayBuffer()); + return WebAssembly.instantiate(decompress(wasmCode), imports); + } + if ( + isFile || + wasmResponse.headers.get("content-type")?.toLowerCase() + .startsWith("application/wasm") + ) { + return WebAssembly.instantiateStreaming(wasmResponse, imports); + } else { + return WebAssembly.instantiate( + await wasmResponse.arrayBuffer(), + imports, + ); + } + } + default: + throw new Error(`Unsupported protocol: ${url.protocol}`); + } + } +} +const isNodeOrDeno = typeof Deno === "object" || + (typeof process !== "undefined" && process.versions != null && + process.versions.node != null); + +const loader = new WasmBuildLoader({ imports, - cache: cacheToLocalDir, + cache: isNodeOrDeno ? cacheToLocalDir : undefined, }); -/** - * Decompression callback - * - * @callback DecompressCallback - * @param {Uint8Array} compressed - * @return {Uint8Array} decompressed - */ -/** - * Options for instantiating a Wasm instance. - * @typedef {Object} InstantiateOptions - * @property {URL=} url - Optional url to the Wasm file to instantiate. - * @property {DecompressCallback=} decompress - Callback to decompress the - * raw Wasm file bytes before instantiating. - */ - -/** Instantiates an instance of the Wasm module returning its functions. - * @remarks It is safe to call this multiple times and once successfully - * loaded it will always return a reference to the same object. - * @param {InstantiateOptions=} opts - */ export async function instantiate(opts) { return (await instantiateWithInstance(opts)).exports; } -/** Instantiates an instance of the Wasm module along with its exports. - * @remarks It is safe to call this multiple times and once successfully - * loaded it will always return a reference to the same object. - * @param {InstantiateOptions=} opts - * @returns {Promise<{ - * instance: WebAssembly.Instance; - * exports: { parse_deno: typeof parse_deno; parse_outputs: typeof parse_outputs; parse_ts_imports: typeof parse_ts_imports; parse_bash: typeof parse_bash; parse_powershell: typeof parse_powershell; parse_go: typeof parse_go; parse_python: typeof parse_python; parse_sql: typeof parse_sql; parse_mysql: typeof parse_mysql; parse_bigquery: typeof parse_bigquery; parse_snowflake: typeof parse_snowflake; parse_mssql: typeof parse_mssql; parse_graphql: typeof parse_graphql } - * }>} - */ export async function instantiateWithInstance(opts) { const { instance } = await loader.load( opts?.url ?? new URL("windmill_parser_wasm_bg.wasm", import.meta.url), @@ -822,11 +962,177 @@ function getWasmInstanceExports() { parse_bigquery, parse_snowflake, parse_mssql, + parse_db_resource, parse_graphql, + parse_php, }; } -/** Gets if the Wasm module has been instantiated. */ export function isInstantiated() { return loader.instance != null; } +export async function cacheToLocalDir(url, decompress) { + const localPath = await getUrlLocalPath(url); + if (localPath == null) { + return undefined; + } + if (!await exists(localPath)) { + const fileBytes = decompress(new Uint8Array(await getUrlBytes(url))); + try { + await Deno.writeFile(localPath, fileBytes); + } catch { + // ignore and return the wasm bytes + return fileBytes; + } + } + return toFileUrl(localPath); +} +async function getUrlLocalPath(url) { + try { + const dataDirPath = await getInitializedLocalDataDirPath(); + const hash = await getUrlHash(url); + return `${dataDirPath}/${hash}.wasm`; + } catch { + return undefined; + } +} +async function getInitializedLocalDataDirPath() { + const dataDir = localDataDir(); + if (dataDir == null) { + throw new Error(`Could not find local data directory.`); + } + const dirPath = `${dataDir}/deno-wasmbuild`; + await ensureDir(dirPath); + return dirPath; +} +async function exists(filePath) { + try { + await Deno.lstat(filePath); + return true; + } catch (error) { + if (error instanceof Deno.errors.NotFound) { + return false; + } + throw error; + } +} +async function ensureDir(dir) { + try { + const fileInfo = await Deno.lstat(dir); + if (!fileInfo.isDirectory) { + throw new Error(`Path was not a directory '${dir}'`); + } + } catch (err) { + if (err instanceof Deno.errors.NotFound) { + // if dir not exists. then create it. + await Deno.mkdir(dir, { recursive: true }); + return; + } + throw err; + } +} +async function getUrlHash(url) { + // Taken from MDN: https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest + const hashBuffer = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(url.href), + ); + // convert buffer to byte array + const hashArray = Array.from(new Uint8Array(hashBuffer)); + // convert bytes to hex string + const hashHex = hashArray + .map((b) => b.toString(16).padStart(2, "0")) + .join(""); + return hashHex; +} +async function getUrlBytes(url) { + const response = await fetchWithRetries(url); + return await response.arrayBuffer(); +} +// the below is extracted from deno_std/path +const WHITESPACE_ENCODINGS = { + "\u0009": "%09", + "\u000A": "%0A", + "\u000B": "%0B", + "\u000C": "%0C", + "\u000D": "%0D", + "\u0020": "%20", +}; +function encodeWhitespace(string) { + return string.replaceAll(/[\s]/g, (c) => { + return WHITESPACE_ENCODINGS[c] ?? c; + }); +} +function toFileUrl(path) { + return Deno.build.os === "windows" + ? windowsToFileUrl(path) + : posixToFileUrl(path); +} +function posixToFileUrl(path) { + const url = new URL("file:///"); + url.pathname = encodeWhitespace( + path.replace(/%/g, "%25").replace(/\\/g, "%5C"), + ); + return url; +} +function windowsToFileUrl(path) { + const [, hostname, pathname] = path.match( + /^(?:[/\\]{2}([^/\\]+)(?=[/\\](?:[^/\\]|$)))?(.*)/, + ); + const url = new URL("file:///"); + url.pathname = encodeWhitespace(pathname.replace(/%/g, "%25")); + if (hostname != null && hostname != "localhost") { + url.hostname = hostname; + if (!url.hostname) { + throw new TypeError("Invalid hostname."); + } + } + return url; +} +export async function fetchWithRetries(url, maxRetries = 5) { + let sleepMs = 250; + let iterationCount = 0; + while (true) { + iterationCount++; + try { + const res = await fetch(url); + if (res.ok || iterationCount > maxRetries) { + return res; + } + } catch (err) { + if (iterationCount > maxRetries) { + throw err; + } + } + console.warn(`Failed fetching. Retrying in ${sleepMs}ms...`); + await new Promise((resolve) => setTimeout(resolve, sleepMs)); + sleepMs = Math.min(sleepMs * 2, 10000); + } +} +// MIT License - Copyright (c) justjavac. +// https://github.com/justjavac/deno_dirs/blob/e8c001bbef558f08fd486d444af391729b0b8068/data_local_dir/mod.ts +function localDataDir() { + switch (Deno.build.os) { + case "linux": { + const xdg = Deno.env.get("XDG_DATA_HOME"); + if (xdg) { + return xdg; + } + const home = Deno.env.get("HOME"); + if (home) { + return `${home}/.local/share`; + } + break; + } + case "darwin": { + const home = Deno.env.get("HOME"); + if (home) { + return `${home}/Library/Application Support`; + } + break; + } + case "windows": + return Deno.env.get("LOCALAPPDATA") ?? undefined; + } + return undefined; +} diff --git a/cli/wasm/windmill_parser_wasm_bg.wasm b/cli/wasm/windmill_parser_wasm_bg.wasm index 3b447b45aa..f2f34e37a2 100644 Binary files a/cli/wasm/windmill_parser_wasm_bg.wasm and b/cli/wasm/windmill_parser_wasm_bg.wasm differ diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 0000000000..e69de29bb2