From 5fd36052fa882e1c47f0d96cd2e4b371233a4160 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 26 Aug 2023 11:13:56 +0200 Subject: [PATCH] feat: add lockfile for deno + use npm module for deno for windmill-client --- backend/windmill-api/src/scripts.rs | 3 +- backend/windmill-worker/Cargo.toml | 1 - backend/windmill-worker/src/deno_executor.rs | 54 ++++++++++++++++++- backend/windmill-worker/src/worker.rs | 10 ++-- .../src/lib/components/DisplayResult.svelte | 2 +- frontend/src/lib/components/EditorBar.svelte | 26 +++------ .../apps/editor/component/default-codes.ts | 46 +++------------- frontend/src/lib/script_helpers.ts | 19 +++++-- .../(logged)/flows/get/[...path]/+page.svelte | 10 +++- .../scripts/get/[...hash]/+page.svelte | 8 +-- typescript-client/client.ts | 51 +++++++++++------- 11 files changed, 138 insertions(+), 92 deletions(-) diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 93bf5c7a91..919daed481 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -347,7 +347,8 @@ async fn create_script( let lock = if !(ns.language == ScriptLang::Python3 || ns.language == ScriptLang::Go - || ns.language == ScriptLang::Bun) + || ns.language == ScriptLang::Bun + || ns.language == ScriptLang::Deno) { Some(String::new()) } else { diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 91e581feba..103718342b 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -10,7 +10,6 @@ path = "src/lib.rs" [features] default = [] -deno-lock = [] enterprise = ["windmill-queue/enterprise", "dep:gcp_auth", "dep:jsonwebtoken", "dep:pem", "dep:sha2"] [dependencies] diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 5883ae4cd8..53e9f8f162 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -1,6 +1,7 @@ use std::{collections::HashMap, process::Stdio}; use itertools::Itertools; +use uuid::Uuid; use crate::{ common::{ @@ -10,7 +11,7 @@ use crate::{ AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, NPM_CONFIG_REGISTRY, PATH_ENV, }; -use tokio::process::Command; +use tokio::{fs::File, io::AsyncReadExt, process::Command}; use windmill_common::{error::Result, BASE_URL}; use windmill_common::{ error::{self}, @@ -62,8 +63,53 @@ fn get_common_deno_proc_envs(token: &str, base_internal_url: &str) -> HashMap, + w_id: &str, + worker_name: &str, +) -> error::Result { + let _ = write_file(job_dir, "main.ts", code).await?; + + let child = Command::new(DENO_PATH.as_str()) + .current_dir(job_dir) + .args(vec![ + "cache", + "--unstable", + "--lock=lock.json", + "--lock-write", + "main.ts", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + + handle_child( + job_id, + db, + logs, + child, + false, + worker_name, + w_id, + "deno cache", + None, + ) + .await?; + + let path_lock = format!("{job_dir}/lock.json"); + let mut file = File::open(path_lock).await?; + let mut req_content = "".to_string(); + file.read_to_string(&mut req_content).await?; + Ok(req_content) +} + #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_deno_job( + requirements_o: Option, logs: &mut String, job: &QueuedJob, db: &sqlx::Pool, @@ -210,6 +256,12 @@ run().catch(async (e) => {{ args.push(&import_map_path); args.push(&reload); args.push("--unstable"); + if let Some(reqs) = requirements_o { + if !reqs.is_empty() { + let _ = write_file(job_dir, "lock.json", &reqs).await?; + args.push("--lock=lock.json"); + } + } if let Some(deno_flags) = DENO_FLAGS.as_ref() { for flag in deno_flags { args.push(flag); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index fe7b50a05d..8a149271aa 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -56,7 +56,7 @@ use windmill_queue::{add_completed_job, add_completed_job_error,IDLE_WORKERS}; use crate::{ worker_flow::{ handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress, - }, python_executor::{create_dependencies_dir, pip_compile, handle_python_job, handle_python_reqs}, common::{read_result, set_logs, write_file, transform_json_value}, go_executor::{handle_go_job, install_go_dependencies}, js_eval::{transpile_ts, eval_fetch_timeout}, pg_executor::do_postgresql, mysql_executor::do_mysql, graphql_executor::do_graphql, bun_executor::{handle_bun_job, gen_lockfile}, bash_executor::{ANSI_ESCAPE_RE, handle_powershell_job, handle_bash_job}, deno_executor::handle_deno_job, + }, python_executor::{create_dependencies_dir, pip_compile, handle_python_job, handle_python_reqs}, common::{read_result, set_logs, write_file, transform_json_value}, go_executor::{handle_go_job, install_go_dependencies}, js_eval::{transpile_ts, eval_fetch_timeout}, pg_executor::do_postgresql, mysql_executor::do_mysql, graphql_executor::do_graphql, bun_executor::{handle_bun_job, gen_lockfile}, bash_executor::{ANSI_ESCAPE_RE, handle_powershell_job, handle_bash_job}, deno_executor::{handle_deno_job, generate_deno_lock}, }; #[cfg(feature = "enterprise")] @@ -1393,6 +1393,7 @@ mount {{ } Some(ScriptLang::Deno) => { handle_deno_job( + requirements_o, logs, job, db, @@ -1831,7 +1832,9 @@ async fn handle_app_dependency_job( } else { Ok(()) } - } +} + + async fn capture_dependency_job( @@ -1891,8 +1894,7 @@ async fn capture_dependency_job( .await } ScriptLang::Deno => { - Ok(String::new()) - // generate_deno_lock(job_id, job_raw_code, logs, job_dir, db, timeout).await + generate_deno_lock(job_id, job_raw_code, logs, job_dir, db, w_id, worker_name).await }, ScriptLang::Bun => { let _ = write_file(job_dir, "main.ts", job_raw_code).await?; diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index 10f14635ad..2407b563fe 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -134,7 +134,7 @@ {/if} {#if typeof result == 'object' && Object.keys(result).length > 0}
The result keys are: {truncate(Object.keys(result).join(', '), 50)} {#if !disableExpand}
diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 4ac9787c0f..73c74e1bc7 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -113,8 +113,6 @@ codeObj = await getScriptByPath(e.detail.path ?? '') } - let version = __pkg__.version - const dispatch = createEventDispatcher() function compile(schema: Schema) { @@ -257,19 +255,15 @@ if (!editor) return if (lang == 'deno') { if (!editor.getCode().includes('import * as wmill from')) { - editor.insertAtBeginning( - `import * as wmill from 'https://deno.land/x/windmill@v${version}/mod.ts'\n` - ) + editor.insertAtBeginning(`import * as wmill from "npm:windmill-client@1"\n`) } editor.insertAtCursor(`(await wmill.getVariable('${path}'))`) } else if (lang === 'bun') { const code = editor.getCode() - if (!code.includes(`import { getVariable } from "windmill-client@${__pkg__.version}`)) { - editor.insertAtBeginning( - `import { getVariable } from "windmill-client@${__pkg__.version}"\n` - ) + if (!code.includes(`import * as wmill from`)) { + editor.insertAtBeginning(`import * as wmill from "windmill-client"\n`) } - editor.insertAtCursor(`(await getVariable('${path}'))`) + editor.insertAtCursor(`(await wmill.getVariable('${path}'))`) } else if (lang == 'python3') { if (!editor.getCode().includes('import wmill')) { editor.insertAtBeginning('import wmill\n') @@ -314,19 +308,15 @@ if (!editor) return if (lang == 'deno') { if (!editor.getCode().includes('import * as wmill from')) { - editor.insertAtBeginning( - `import * as wmill from 'https://deno.land/x/windmill@v${version}/mod.ts'\n` - ) + editor.insertAtBeginning(`import * as wmill from "npm:windmill-client@1"\n`) } editor.insertAtCursor(`(await wmill.getResource('${path}'))`) } else if (lang === 'bun') { const code = editor.getCode() - if (!code.includes(`import { getResource } from "windmill-client@${__pkg__.version}`)) { - editor.insertAtBeginning( - `import { getResource } from "windmill-client@${__pkg__.version}"\n` - ) + if (!code.includes(`import * as wmill from`)) { + editor.insertAtBeginning(`import * as wmill from "windmill-client"\n`) } - editor.insertAtCursor(`(await getResource('${path}'))`) + editor.insertAtCursor(`(await wmill.getResource('${path}'))`) } else if (lang == 'python3') { if (!editor.getCode().includes('import wmill')) { editor.insertAtBeginning('import wmill\n') diff --git a/frontend/src/lib/components/apps/editor/component/default-codes.ts b/frontend/src/lib/components/apps/editor/component/default-codes.ts index 9f2490e9dc..746311c818 100644 --- a/frontend/src/lib/components/apps/editor/component/default-codes.ts +++ b/frontend/src/lib/components/apps/editor/component/default-codes.ts @@ -38,7 +38,7 @@ export const DEFAULT_CODES: Partial< "age": 84 } ]`, - pgsql: `import { pgSql } from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts"; + pgsql: `import { pgSql } from "npm:windmill-client@1"; type Postgresql = object @@ -75,7 +75,7 @@ export async function main(db: Postgresql) { "age": 84 } ]`, - pgsql: `import { pgSql } from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts"; + pgsql: `import { pgSql } from "npm:windmill-client@1"; type Postgresql = object @@ -132,7 +132,7 @@ export async function main(db: Postgresql) { "<3" ] }`, - pgsql: `import { pgSql } from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts"; + pgsql: `import { pgSql } from "npm:windmill-client@1"; type Postgresql = object @@ -206,7 +206,7 @@ export async function main(db: Postgresql) { "y": { "field": "b", "type": "quantitative" }, }, }`, - pgsql: `import { pgSql } from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts"; + pgsql: `import { pgSql } from "npm:windmill-client@1"; type Postgresql = object @@ -253,7 +253,7 @@ export async function main(Postgresqlstgresql) { } } }`, - pgsql: `import { pgSql } from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts"; + pgsql: `import { pgSql } from "npm:windmill-client@1"; type Postgresql = object @@ -300,7 +300,7 @@ export async function main(Postgresqlstgresql) { "<3" ] }`, - pgsql: `import { pgSql } from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts"; + pgsql: `import { pgSql } from "npm:windmill-client@1"; type Postgresql = object @@ -356,7 +356,7 @@ export async function main(db: Postgresql) { "backgroundColor": "orange" } ]`, - pgsql: `import { pgSql } from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts"; + pgsql: `import { pgSql } from "npm:windmill-client@1"; type Postgresql = object @@ -466,37 +466,7 @@ export async function main(db: Postgresql) { ], "backgroundColor": "orange" } - ]`, - pgsql: `import { pgSql } from "https://deno.land/x/windmill@v1.88.1/mod.ts"; - -type Postgresql = object - -export async function main(db: Postgresql) { - try { - const query = await pgSql(db)\`SELECT * FROM demo;\`; - const rows = query.rows.map((row, i) => ({ - x: new Date(Date.now() - (i * 1000 * 60 * 60 * 24)).toISOString(), - y: row['0'] - })) - return [ - { - label: "foo", - data: rows, - backgroundColor: "rgb(255, 12, 137)" - }, - { - label: "bar", - data: rows.map(({x, y}) => ({ - x, - y: y * 2 - })), - backgroundColor: "orange" - } - ]; - } catch(e) { - return []; - } -}` + ]` }, iconcomponent: { deno: `export async function main() { diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 769293592b..2893a4783e 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -36,8 +36,8 @@ export async function main() { export const DENO_INIT_CODE = `// Ctrl/CMD+. to cache dependencies on imports hover. -// import { toWords } from "npm:number-to-words@1" -// import * as wmill from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts" +// Deno uses "npm:" prefix to import from npm (https://deno.land/manual@v1.36.3/node/npm_specifiers) +// import * as wmill from "npm:windmill-client@1" // fill the type, or use the +Resource type to get a type-safe reference to a resource // type Postgresql = object @@ -111,7 +111,7 @@ func main(message string, name string) (interface{}, error) { } ` -export const DENO_INIT_CODE_CLEAR = `// import * as wmill from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts" +export const DENO_INIT_CODE_CLEAR = `// import * as wmill from "npm:windmill-client@1" export async function main(x: string) { return x @@ -206,7 +206,7 @@ dflt="\${2:-default value}" echo "Hello $msg" ` -export const DENO_INIT_CODE_TRIGGER = `import * as wmill from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts" +export const DENO_INIT_CODE_TRIGGER = `import * as wmill from "npm:windmill-client@1" export async function main() { @@ -251,7 +251,13 @@ func main() (interface{}, error) { } ` -export const DENO_INIT_CODE_APPROVAL = `import * as wmill from "https://deno.land/x/windmill@v1.99.0/mod.ts" +export const DENO_INIT_CODE_APPROVAL = `import * as wmill from "npm:windmill-client@1" + +export async function main(approver?: string) { + return wmill.getResumeEndpoints(approver) +}` + +export const BUN_INIT_CODE_APPROVAL = `import * as wmill from "windmill-client@1" export async function main(approver?: string) { return wmill.getResumeEndpoints(approver) @@ -363,6 +369,9 @@ export function initialCode( } else if (language == 'graphql') { return GRAPHQL_INIT_CODE } else if (language == 'bun') { + if (kind === 'approval') { + return BUN_INIT_CODE_APPROVAL + } if (subkind === 'flow') { return BUN_INIT_CODE_CLEAR } diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index 334d83120a..2be07bea39 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -16,7 +16,7 @@ import Urlize from '$lib/components/Urlize.svelte' import DeployWorkspaceDrawer from '$lib/components/DeployWorkspaceDrawer.svelte' import SavedInputs from '$lib/components/SavedInputs.svelte' - import { FolderOpen, Archive, Trash, Server, Share } from 'lucide-svelte' + import { FolderOpen, Archive, Trash, Server, Share, Badge, Loader2 } from 'lucide-svelte' import DetailPageHeader from '$lib/components/details/DetailPageHeader.svelte' import WebhooksPanel from '$lib/components/details/WebhooksPanel.svelte' @@ -35,6 +35,8 @@ let path = $page.params.path let shareModal: ShareModal + let deploymentInProgress = false + $: cliCommand = `wmill flow run ${flow?.path} -d '${JSON.stringify(args)}'` $: { @@ -265,6 +267,12 @@ Edited by {flow.edited_by} + {#if deploymentInProgress} + + + Deployment in progress + + {/if} {#if flow.archived}
This flow was archived diff --git a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte index 6e7a36f2d5..915b3f161d 100644 --- a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte @@ -532,7 +532,7 @@ Code - Lock file + Lockfile Inputs @@ -548,7 +548,7 @@ -
+
{#if script?.lock} - 
{script.lock}
+ 
{script.lock}
{:else}

There is no lock file for this script diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 651ecb50ae..05dc7666da 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -1,5 +1,6 @@ import { ResourceService, VariableService } from "./index"; import { OpenAPI } from "./index"; +import { JobService } from "./src"; export { AdminService, @@ -243,25 +244,37 @@ export async function databaseUrlFromResource(path: string): Promise { return `postgresql://${resource.user}:${resource.password}@${resource.host}:${resource.port}/${resource.dbname}?sslmode=${resource.sslmode}`; } -// /** -// * Get URLs needed for resuming a flow after this step -// * @param approver approver name -// * @returns approval page UI URL, resume and cancel API URLs for resumeing the flow -// */ -// export async function getResumeUrls(approver?: string): Promise<{ -// approvalPage: string; -// resume: string; -// cancel: string; -// }> { -// const nonce = Math.floor(Math.random() * 4294967295); -// const workspace = getWorkspace(); -// return await JobService.getResumeUrls({ -// workspace, -// resumeId: nonce, -// approver, -// id: process.env.get("WM_JOB_ID") ?? "NO_JOB_ID", -// }); -// } +/** + * Get URLs needed for resuming a flow after this step + * @param approver approver name + * @returns approval page UI URL, resume and cancel API URLs for resumeing the flow + */ +export async function getResumeUrls(approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> { + const nonce = Math.floor(Math.random() * 4294967295); + !clientSet && setClient(); + const workspace = getWorkspace(); + return await JobService.getResumeUrls({ + workspace, + resumeId: nonce, + approver, + id: getEnv("WM_JOB_ID") ?? "NO_JOB_ID", + }); +} + +/** + * @deprecated use getResumeUrls instead + */ +export function getResumeEndpoints(approver?: string): Promise<{ + approvalPage: string; + resume: string; + cancel: string; +}> { + return getResumeUrls(approver); +} export function base64ToUint8Array(data: string): Uint8Array { return Uint8Array.from(atob(data), (c) => c.charCodeAt(0));