starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-11 16:09:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: update parsers for CLI

Browse Source
This commit is contained in:
Ruben Fiszel
2024-07-22 20:25:56 +02:00
parent a5627572ea
commit f65ccc07de
6 changed files with 498 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/parsers/windmill-parser-wasm/build.sh
Regular → Executable
+1 -1
View File
@@ -1,4 +1,4 @@
#!/bin/bash
set -eou pipefail
deno task wasmbuild --out ../../../cli/wasm/
deno task wasmbuild --out ../../../cli/wasm/ -p windmill-parser-wasm
backend/parsers/windmill-parser-wasm/deno.json
+4 -4
View File
@@ -1,5 +1,5 @@
{
"tasks": {
"wasmbuild": "deno run -A https://deno.land/x/wasmbuild@0.15.4/main.ts"
}
}
"tasks": {
"wasmbuild": "deno run -A jsr:@deno/wasmbuild@0.17.2"
}
}
cli/wasm/windmill_parser_wasm.generated.d.ts
Vendored
+120
View File
@@ -0,0 +1,120 @@
// deno-lint-ignore-file
// deno-fmt-ignore-file
export interface InstantiateResult {
instance: WebAssembly.Instance;
exports: {
parse_deno: typeof parse_deno;
parse_outputs: typeof parse_outputs;
parse_ts_imports: typeof parse_ts_imports;
parse_bash: typeof parse_bash;
parse_powershell: typeof parse_powershell;
parse_go: typeof parse_go;
parse_python: typeof parse_python;
parse_sql: typeof parse_sql;
parse_mysql: typeof parse_mysql;
parse_bigquery: typeof parse_bigquery;
parse_snowflake: typeof parse_snowflake;
parse_mssql: typeof parse_mssql;
parse_db_resource: typeof parse_db_resource;
parse_graphql: typeof parse_graphql;
parse_php: typeof parse_php
};
}
/** Gets if the Wasm module has been instantiated. */
export function isInstantiated(): boolean;
/** Options for instantiating a Wasm instance. */
export interface InstantiateOptions {
/** Optional url to the Wasm file to instantiate. */
url?: URL;
/** Callback to decompress the raw Wasm file bytes before instantiating. */
decompress?: (bytes: Uint8Array) => Uint8Array;
}
/** Instantiates an instance of the Wasm module returning its functions.
* @remarks It is safe to call this multiple times and once successfully
* loaded it will always return a reference to the same object. */
export function instantiate(opts?: InstantiateOptions): Promise<InstantiateResult["exports"]>;
/** Instantiates an instance of the Wasm module along with its exports.
* @remarks It is safe to call this multiple times and once successfully
* loaded it will always return a reference to the same object. */
export function instantiateWithInstance(opts?: InstantiateOptions): Promise<InstantiateResult>;
/**
* @param {string} code
* @returns {string}
*/
export function parse_deno(code: string): string;
/**
* @param {string} code
* @returns {string}
*/
export function parse_outputs(code: string): string;
/**
* @param {string} code
* @returns {string}
*/
export function parse_ts_imports(code: string): string;
/**
* @param {string} code
* @returns {string}
*/
export function parse_bash(code: string): string;
/**
* @param {string} code
* @returns {string}
*/
export function parse_powershell(code: string): string;
/**
* @param {string} code
* @returns {string}
*/
export function parse_go(code: string): string;
/**
* @param {string} code
* @returns {string}
*/
export function parse_python(code: string): string;
/**
* @param {string} code
* @returns {string}
*/
export function parse_sql(code: string): string;
/**
* @param {string} code
* @returns {string}
*/
export function parse_mysql(code: string): string;
/**
* @param {string} code
* @returns {string}
*/
export function parse_bigquery(code: string): string;
/**
* @param {string} code
* @returns {string}
*/
export function parse_snowflake(code: string): string;
/**
* @param {string} code
* @returns {string}
*/
export function parse_mssql(code: string): string;
/**
* @param {string} code
* @returns {string | undefined}
*/
export function parse_db_resource(code: string): string | undefined;
/**
* @param {string} code
* @returns {string}
*/
export function parse_graphql(code: string): string;
/**
* @param {string} code
* @returns {string}
*/
export function parse_php(code: string): string;
cli/wasm/windmill_parser_wasm.generated.js
+373 -67
View File
@@ -1,7 +1,10 @@
// @generated file from wasmbuild -- do not edit
// @ts-nocheck: generated
// deno-lint-ignore-file
// deno-fmt-ignore-file
// source-hash: f38890e80d35ec2affe2a260e12308f96bcbf771
/// <reference types="./windmill_parser_wasm.generated.d.ts" />
// source-hash: ae1bfe27a90c5656a723fb7ced38f4857a5d6931
let wasm;
const heap = new Array(128).fill(undefined);
@@ -80,6 +83,7 @@ function passStringToWasm0(arg, malloc, realloc) {
const ret = encodeString(arg, view);
offset += ret.written;
ptr = realloc(ptr, len, offset, 1) >>> 0;
}
WASM_VECTOR_LEN = offset;
@@ -531,6 +535,33 @@ export function parse_mssql(code) {
}
}
/**
* @param {string} code
* @returns {string | undefined}
*/
export function parse_db_resource(code) {
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(
code,
wasm.__wbindgen_malloc,
wasm.__wbindgen_realloc,
);
const len0 = WASM_VECTOR_LEN;
wasm.parse_db_resource(retptr, ptr0, len0);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
let v2;
if (r0 !== 0) {
v2 = getStringFromWasm0(r0, r1).slice();
wasm.__wbindgen_free(r0, r1 * 1, 1);
}
return v2;
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
}
}
/**
* @param {string} code
* @returns {string}
@@ -558,6 +589,33 @@ export function parse_graphql(code) {
}
}
/**
* @param {string} code
* @returns {string}
*/
export function parse_php(code) {
let deferred2_0;
let deferred2_1;
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(
code,
wasm.__wbindgen_malloc,
wasm.__wbindgen_realloc,
);
const len0 = WASM_VECTOR_LEN;
wasm.parse_php(retptr, ptr0, len0);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
}
}
function handleError(f, args) {
try {
return f.apply(this, args);
@@ -585,6 +643,10 @@ const imports = {
getInt32Memory0()[arg0 / 4 + 1] = len1;
getInt32Memory0()[arg0 / 4 + 0] = ptr1;
},
__wbg_eval_c74b339326c2fc56: function (arg0, arg1) {
const ret = eval(getStringFromWasm0(arg0, arg1));
return addHeapObject(ret);
},
__wbindgen_error_new: function (arg0, arg1) {
const ret = new Error(getStringFromWasm0(arg0, arg1));
return addHeapObject(ret);
@@ -598,18 +660,6 @@ const imports = {
const ret = typeof (getObject(arg0)) === "bigint";
return ret;
},
__wbindgen_bigint_from_i64: function (arg0) {
const ret = arg0;
return addHeapObject(ret);
},
__wbindgen_jsval_eq: function (arg0, arg1) {
const ret = getObject(arg0) === getObject(arg1);
return ret;
},
__wbindgen_bigint_from_u64: function (arg0) {
const ret = BigInt.asUintN(64, arg0);
return addHeapObject(ret);
},
__wbindgen_number_get: function (arg0, arg1) {
const obj = getObject(arg1);
const ret = typeof obj === "number" ? obj : undefined;
@@ -625,19 +675,27 @@ const imports = {
const ret = getObject(arg0) in getObject(arg1);
return ret;
},
__wbg_eval_c330e1bec5b1705c: function (arg0, arg1) {
const ret = eval(getStringFromWasm0(arg0, arg1));
__wbindgen_bigint_from_i64: function (arg0) {
const ret = arg0;
return addHeapObject(ret);
},
__wbindgen_jsval_eq: function (arg0, arg1) {
const ret = getObject(arg0) === getObject(arg1);
return ret;
},
__wbindgen_bigint_from_u64: function (arg0) {
const ret = BigInt.asUintN(64, arg0);
return addHeapObject(ret);
},
__wbindgen_jsval_loose_eq: function (arg0, arg1) {
const ret = getObject(arg0) == getObject(arg1);
return ret;
},
__wbg_get_f01601b5a68d10e3: function (arg0, arg1) {
__wbg_get_bd8e338fbd5f5cc8: function (arg0, arg1) {
const ret = getObject(arg0)[arg1 >>> 0];
return addHeapObject(ret);
},
__wbg_length_1009b1af0c481d7b: function (arg0) {
__wbg_length_cd7af8117672b8b8: function (arg0) {
const ret = getObject(arg0).length;
return ret;
},
@@ -645,45 +703,45 @@ const imports = {
const ret = typeof (getObject(arg0)) === "function";
return ret;
},
__wbg_next_9b877f231f476d01: function (arg0) {
__wbg_next_40fc327bfc8770e6: function (arg0) {
const ret = getObject(arg0).next;
return addHeapObject(ret);
},
__wbg_next_6529ee0cca8d57ed: function () {
__wbg_next_196c84450b364254: function () {
return handleError(function (arg0) {
const ret = getObject(arg0).next();
return addHeapObject(ret);
}, arguments);
},
__wbg_done_5fe336b092d60cf2: function (arg0) {
__wbg_done_298b57d23c0fc80c: function (arg0) {
const ret = getObject(arg0).done;
return ret;
},
__wbg_value_0c248a78fdc8e19f: function (arg0) {
__wbg_value_d93c65011f51a456: function (arg0) {
const ret = getObject(arg0).value;
return addHeapObject(ret);
},
__wbg_iterator_db7ca081358d4fb2: function () {
__wbg_iterator_2cee6dadfd956dfa: function () {
const ret = Symbol.iterator;
return addHeapObject(ret);
},
__wbg_get_7b48513de5dc5ea4: function () {
__wbg_get_e3c254076557e348: function () {
return handleError(function (arg0, arg1) {
const ret = Reflect.get(getObject(arg0), getObject(arg1));
return addHeapObject(ret);
}, arguments);
},
__wbg_call_90c26b09837aba1c: function () {
__wbg_call_27c0f87801dedf93: function () {
return handleError(function (arg0, arg1) {
const ret = getObject(arg0).call(getObject(arg1));
return addHeapObject(ret);
}, arguments);
},
__wbg_isArray_74fb723e24f76012: function (arg0) {
__wbg_isArray_2ab64d95e09ea0ae: function (arg0) {
const ret = Array.isArray(getObject(arg0));
return ret;
},
__wbg_instanceof_ArrayBuffer_e7d53d51371448e2: function (arg0) {
__wbg_instanceof_ArrayBuffer_836825be07d4c9d2: function (arg0) {
let result;
try {
result = getObject(arg0) instanceof ArrayBuffer;
@@ -693,30 +751,40 @@ const imports = {
const ret = result;
return ret;
},
__wbg_isSafeInteger_f93fde0dca9820f8: function (arg0) {
__wbg_instanceof_Map_87917e0a7aaf4012: function (arg0) {
let result;
try {
result = getObject(arg0) instanceof Map;
} catch (_) {
result = false;
}
const ret = result;
return ret;
},
__wbg_isSafeInteger_f7b04ef02296c4d2: function (arg0) {
const ret = Number.isSafeInteger(getObject(arg0));
return ret;
},
__wbg_entries_9e2e2aa45aa5094a: function (arg0) {
__wbg_entries_95cc2c823b285a09: function (arg0) {
const ret = Object.entries(getObject(arg0));
return addHeapObject(ret);
},
__wbg_buffer_a448f833075b71ba: function (arg0) {
__wbg_buffer_12d079cc21e14bdb: function (arg0) {
const ret = getObject(arg0).buffer;
return addHeapObject(ret);
},
__wbg_new_8f67e318f15d7254: function (arg0) {
__wbg_new_63b92bc8671ed464: function (arg0) {
const ret = new Uint8Array(getObject(arg0));
return addHeapObject(ret);
},
__wbg_set_2357bf09366ee480: function (arg0, arg1, arg2) {
__wbg_set_a47bac70306a19a7: function (arg0, arg1, arg2) {
getObject(arg0).set(getObject(arg1), arg2 >>> 0);
},
__wbg_length_1d25fa9e4ac21ce7: function (arg0) {
__wbg_length_c20a40f15020d68a: function (arg0) {
const ret = getObject(arg0).length;
return ret;
},
__wbg_instanceof_Uint8Array_bced6f43aed8c1aa: function (arg0) {
__wbg_instanceof_Uint8Array_2b3bbecd033d19f6: function (arg0) {
let result;
try {
result = getObject(arg0) instanceof Uint8Array;
@@ -753,47 +821,119 @@ const imports = {
},
};
import { Loader } from "https://deno.land/x/wasmbuild@0.15.4/loader.ts";
import { cacheToLocalDir } from "https://deno.land/x/wasmbuild@0.15.4/cache.ts";
class WasmBuildLoader {
#options;
#lastLoadPromise;
#instantiated;
const loader = new Loader({
constructor(options) {
this.#options = options;
}
get instance() {
return this.#instantiated?.instance;
}
get module() {
return this.#instantiated?.module;
}
load(
url,
decompress,
) {
if (this.#instantiated) {
return Promise.resolve(this.#instantiated);
} else if (this.#lastLoadPromise == null) {
this.#lastLoadPromise = (async () => {
try {
this.#instantiated = await this.#instantiate(url, decompress);
return this.#instantiated;
} finally {
this.#lastLoadPromise = undefined;
}
})();
}
return this.#lastLoadPromise;
}
async #instantiate(url, decompress) {
const imports = this.#options.imports;
if (this.#options.cache != null && url.protocol !== "file:") {
try {
const result = await this.#options.cache(
url,
decompress ?? ((bytes) => bytes),
);
if (result instanceof URL) {
url = result;
decompress = undefined; // already decompressed
} else if (result != null) {
return WebAssembly.instantiate(result, imports);
}
} catch {
// ignore if caching ever fails (ex. when on deploy)
}
}
const isFile = url.protocol === "file:";
// make file urls work in Node via dnt
const isNode = globalThis.process?.versions?.node != null;
if (isFile && typeof Deno !== "object") {
throw new Error(
"Loading local files are not supported in this environment",
);
}
if (isNode && isFile) {
// the deno global will be shimmed by dnt
const wasmCode = await Deno.readFile(url);
return WebAssembly.instantiate(
decompress ? decompress(wasmCode) : wasmCode,
imports,
);
}
switch (url.protocol) {
case "file:":
case "https:":
case "http:": {
const wasmResponse = await fetchWithRetries(url);
if (decompress) {
const wasmCode = new Uint8Array(await wasmResponse.arrayBuffer());
return WebAssembly.instantiate(decompress(wasmCode), imports);
}
if (
isFile ||
wasmResponse.headers.get("content-type")?.toLowerCase()
.startsWith("application/wasm")
) {
return WebAssembly.instantiateStreaming(wasmResponse, imports);
} else {
return WebAssembly.instantiate(
await wasmResponse.arrayBuffer(),
imports,
);
}
}
default:
throw new Error(`Unsupported protocol: ${url.protocol}`);
}
}
}
const isNodeOrDeno = typeof Deno === "object" ||
(typeof process !== "undefined" && process.versions != null &&
process.versions.node != null);
const loader = new WasmBuildLoader({
imports,
cache: cacheToLocalDir,
cache: isNodeOrDeno ? cacheToLocalDir : undefined,
});
/**
* Decompression callback
*
* @callback DecompressCallback
* @param {Uint8Array} compressed
* @return {Uint8Array} decompressed
*/
/**
* Options for instantiating a Wasm instance.
* @typedef {Object} InstantiateOptions
* @property {URL=} url - Optional url to the Wasm file to instantiate.
* @property {DecompressCallback=} decompress - Callback to decompress the
* raw Wasm file bytes before instantiating.
*/
/** Instantiates an instance of the Wasm module returning its functions.
* @remarks It is safe to call this multiple times and once successfully
* loaded it will always return a reference to the same object.
* @param {InstantiateOptions=} opts
*/
export async function instantiate(opts) {
return (await instantiateWithInstance(opts)).exports;
}
/** Instantiates an instance of the Wasm module along with its exports.
* @remarks It is safe to call this multiple times and once successfully
* loaded it will always return a reference to the same object.
* @param {InstantiateOptions=} opts
* @returns {Promise<{
* instance: WebAssembly.Instance;
* exports: { parse_deno: typeof parse_deno; parse_outputs: typeof parse_outputs; parse_ts_imports: typeof parse_ts_imports; parse_bash: typeof parse_bash; parse_powershell: typeof parse_powershell; parse_go: typeof parse_go; parse_python: typeof parse_python; parse_sql: typeof parse_sql; parse_mysql: typeof parse_mysql; parse_bigquery: typeof parse_bigquery; parse_snowflake: typeof parse_snowflake; parse_mssql: typeof parse_mssql; parse_graphql: typeof parse_graphql }
* }>}
*/
export async function instantiateWithInstance(opts) {
const { instance } = await loader.load(
opts?.url ?? new URL("windmill_parser_wasm_bg.wasm", import.meta.url),
@@ -822,11 +962,177 @@ function getWasmInstanceExports() {
parse_bigquery,
parse_snowflake,
parse_mssql,
parse_db_resource,
parse_graphql,
parse_php,
};
}
/** Gets if the Wasm module has been instantiated. */
export function isInstantiated() {
return loader.instance != null;
}
export async function cacheToLocalDir(url, decompress) {
const localPath = await getUrlLocalPath(url);
if (localPath == null) {
return undefined;
}
if (!await exists(localPath)) {
const fileBytes = decompress(new Uint8Array(await getUrlBytes(url)));
try {
await Deno.writeFile(localPath, fileBytes);
} catch {
// ignore and return the wasm bytes
return fileBytes;
}
}
return toFileUrl(localPath);
}
async function getUrlLocalPath(url) {
try {
const dataDirPath = await getInitializedLocalDataDirPath();
const hash = await getUrlHash(url);
return `${dataDirPath}/${hash}.wasm`;
} catch {
return undefined;
}
}
async function getInitializedLocalDataDirPath() {
const dataDir = localDataDir();
if (dataDir == null) {
throw new Error(`Could not find local data directory.`);
}
const dirPath = `${dataDir}/deno-wasmbuild`;
await ensureDir(dirPath);
return dirPath;
}
async function exists(filePath) {
try {
await Deno.lstat(filePath);
return true;
} catch (error) {
if (error instanceof Deno.errors.NotFound) {
return false;
}
throw error;
}
}
async function ensureDir(dir) {
try {
const fileInfo = await Deno.lstat(dir);
if (!fileInfo.isDirectory) {
throw new Error(`Path was not a directory '${dir}'`);
}
} catch (err) {
if (err instanceof Deno.errors.NotFound) {
// if dir not exists. then create it.
await Deno.mkdir(dir, { recursive: true });
return;
}
throw err;
}
}
async function getUrlHash(url) {
// Taken from MDN: https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest
const hashBuffer = await crypto.subtle.digest(
"SHA-256",
new TextEncoder().encode(url.href),
);
// convert buffer to byte array
const hashArray = Array.from(new Uint8Array(hashBuffer));
// convert bytes to hex string
const hashHex = hashArray
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
return hashHex;
}
async function getUrlBytes(url) {
const response = await fetchWithRetries(url);
return await response.arrayBuffer();
}
// the below is extracted from deno_std/path
const WHITESPACE_ENCODINGS = {
"\u0009": "%09",
"\u000A": "%0A",
"\u000B": "%0B",
"\u000C": "%0C",
"\u000D": "%0D",
"\u0020": "%20",
};
function encodeWhitespace(string) {
return string.replaceAll(/[\s]/g, (c) => {
return WHITESPACE_ENCODINGS[c] ?? c;
});
}
function toFileUrl(path) {
return Deno.build.os === "windows"
? windowsToFileUrl(path)
: posixToFileUrl(path);
}
function posixToFileUrl(path) {
const url = new URL("file:///");
url.pathname = encodeWhitespace(
path.replace(/%/g, "%25").replace(/\\/g, "%5C"),
);
return url;
}
function windowsToFileUrl(path) {
const [, hostname, pathname] = path.match(
/^(?:[/\\]{2}([^/\\]+)(?=[/\\](?:[^/\\]|$)))?(.*)/,
);
const url = new URL("file:///");
url.pathname = encodeWhitespace(pathname.replace(/%/g, "%25"));
if (hostname != null && hostname != "localhost") {
url.hostname = hostname;
if (!url.hostname) {
throw new TypeError("Invalid hostname.");
}
}
return url;
}
export async function fetchWithRetries(url, maxRetries = 5) {
let sleepMs = 250;
let iterationCount = 0;
while (true) {
iterationCount++;
try {
const res = await fetch(url);
if (res.ok || iterationCount > maxRetries) {
return res;
}
} catch (err) {
if (iterationCount > maxRetries) {
throw err;
}
}
console.warn(`Failed fetching. Retrying in ${sleepMs}ms...`);
await new Promise((resolve) => setTimeout(resolve, sleepMs));
sleepMs = Math.min(sleepMs * 2, 10000);
}
}
// MIT License - Copyright (c) justjavac.
// https://github.com/justjavac/deno_dirs/blob/e8c001bbef558f08fd486d444af391729b0b8068/data_local_dir/mod.ts
function localDataDir() {
switch (Deno.build.os) {
case "linux": {
const xdg = Deno.env.get("XDG_DATA_HOME");
if (xdg) {
return xdg;
}
const home = Deno.env.get("HOME");
if (home) {
return `${home}/.local/share`;
}
break;
}
case "darwin": {
const home = Deno.env.get("HOME");
if (home) {
return `${home}/Library/Application Support`;
}
break;
}
case "windows":
return Deno.env.get("LOCALAPPDATA") ?? undefined;
}
return undefined;
}
cli/wasm/windmill_parser_wasm_bg.wasm
BIN
View File
Binary file not shown.
frontend/Dockerfile
View File