starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-24 08:01:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add lockfile for deno + use npm module for deno for windmill-client

Browse Source
This commit is contained in:
Ruben Fiszel
2023-08-26 11:13:56 +02:00
parent 837b6cf76e
commit 5fd36052fa
11 changed files with 138 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/windmill-api/src/scripts.rs
+2 -1
View File
@@ -347,7 +347,8 @@ async fn create_script(
let lock = if !(ns.language == ScriptLang::Python3
|| ns.language == ScriptLang::Go
|| ns.language == ScriptLang::Bun)
|| ns.language == ScriptLang::Bun
|| ns.language == ScriptLang::Deno)
{
Some(String::new())
} else {
backend/windmill-worker/Cargo.toml
-1
View File
@@ -10,7 +10,6 @@ path = "src/lib.rs"
[features]
default = []
deno-lock = []
enterprise = ["windmill-queue/enterprise", "dep:gcp_auth", "dep:jsonwebtoken", "dep:pem", "dep:sha2"]
[dependencies]
backend/windmill-worker/src/deno_executor.rs
+53 -1
View File
@@ -1,6 +1,7 @@
use std::{collections::HashMap, process::Stdio};
use itertools::Itertools;
use uuid::Uuid;
use crate::{
common::{
@@ -10,7 +11,7 @@ use crate::{
AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, NPM_CONFIG_REGISTRY,
PATH_ENV,
};
use tokio::process::Command;
use tokio::{fs::File, io::AsyncReadExt, process::Command};
use windmill_common::{error::Result, BASE_URL};
use windmill_common::{
error::{self},
@@ -62,8 +63,53 @@ fn get_common_deno_proc_envs(token: &str, base_internal_url: &str) -> HashMap<St
return deno_envs;
}
pub async fn generate_deno_lock(
job_id: &Uuid,
code: &str,
logs: &mut String,
job_dir: &str,
db: &sqlx::Pool<sqlx::Postgres>,
w_id: &str,
worker_name: &str,
) -> error::Result<String> {
let _ = write_file(job_dir, "main.ts", code).await?;
let child = Command::new(DENO_PATH.as_str())
.current_dir(job_dir)
.args(vec![
"cache",
"--unstable",
"--lock=lock.json",
"--lock-write",
"main.ts",
])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
handle_child(
job_id,
db,
logs,
child,
false,
worker_name,
w_id,
"deno cache",
None,
)
.await?;
let path_lock = format!("{job_dir}/lock.json");
let mut file = File::open(path_lock).await?;
let mut req_content = "".to_string();
file.read_to_string(&mut req_content).await?;
Ok(req_content)
}
#[tracing::instrument(level = "trace", skip_all)]
pub async fn handle_deno_job(
requirements_o: Option<String>,
logs: &mut String,
job: &QueuedJob,
db: &sqlx::Pool<sqlx::Postgres>,
@@ -210,6 +256,12 @@ run().catch(async (e) => {{
args.push(&import_map_path);
args.push(&reload);
args.push("--unstable");
if let Some(reqs) = requirements_o {
if !reqs.is_empty() {
let _ = write_file(job_dir, "lock.json", &reqs).await?;
args.push("--lock=lock.json");
}
}
if let Some(deno_flags) = DENO_FLAGS.as_ref() {
for flag in deno_flags {
args.push(flag);
backend/windmill-worker/src/worker.rs
+6 -4
View File
@@ -56,7 +56,7 @@ use windmill_queue::{add_completed_job, add_completed_job_error,IDLE_WORKERS};
use crate::{
worker_flow::{
handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress,
}, python_executor::{create_dependencies_dir, pip_compile, handle_python_job, handle_python_reqs}, common::{read_result, set_logs, write_file, transform_json_value}, go_executor::{handle_go_job, install_go_dependencies}, js_eval::{transpile_ts, eval_fetch_timeout}, pg_executor::do_postgresql, mysql_executor::do_mysql, graphql_executor::do_graphql, bun_executor::{handle_bun_job, gen_lockfile}, bash_executor::{ANSI_ESCAPE_RE, handle_powershell_job, handle_bash_job}, deno_executor::handle_deno_job,
}, python_executor::{create_dependencies_dir, pip_compile, handle_python_job, handle_python_reqs}, common::{read_result, set_logs, write_file, transform_json_value}, go_executor::{handle_go_job, install_go_dependencies}, js_eval::{transpile_ts, eval_fetch_timeout}, pg_executor::do_postgresql, mysql_executor::do_mysql, graphql_executor::do_graphql, bun_executor::{handle_bun_job, gen_lockfile}, bash_executor::{ANSI_ESCAPE_RE, handle_powershell_job, handle_bash_job}, deno_executor::{handle_deno_job, generate_deno_lock},
};
#[cfg(feature = "enterprise")]
@@ -1393,6 +1393,7 @@ mount {{
}
Some(ScriptLang::Deno) => {
handle_deno_job(
requirements_o,
logs,
job,
db,
@@ -1831,7 +1832,9 @@ async fn handle_app_dependency_job(
} else {
Ok(())
}
}
}
async fn capture_dependency_job(
@@ -1891,8 +1894,7 @@ async fn capture_dependency_job(
.await
}
ScriptLang::Deno => {
Ok(String::new())
// generate_deno_lock(job_id, job_raw_code, logs, job_dir, db, timeout).await
generate_deno_lock(job_id, job_raw_code, logs, job_dir, db, w_id, worker_name).await
},
ScriptLang::Bun => {
let _ = write_file(job_dir, "main.ts", job_raw_code).await?;
frontend/src/lib/components/DisplayResult.svelte
+1 -1
View File
@@ -134,7 +134,7 @@
</div>
{/if}
{#if typeof result == 'object' && Object.keys(result).length > 0}<div
class="mb-2 w-full text-sm relative"
class="mb-2 w-full min-w-[400px] text-sm relative"
>The result keys are: <b>{truncate(Object.keys(result).join(', '), 50)}</b>
{#if !disableExpand}
<div class="text-tertiary text-xs absolute top-5.5 right-0 inline-flex gap-2">
frontend/src/lib/components/EditorBar.svelte
+8 -18
View File
@@ -113,8 +113,6 @@
codeObj = await getScriptByPath(e.detail.path ?? '')
}
let version = __pkg__.version
const dispatch = createEventDispatcher()
function compile(schema: Schema) {
@@ -257,19 +255,15 @@
if (!editor) return
if (lang == 'deno') {
if (!editor.getCode().includes('import * as wmill from')) {
editor.insertAtBeginning(
`import * as wmill from 'https://deno.land/x/windmill@v${version}/mod.ts'\n`
)
editor.insertAtBeginning(`import * as wmill from "npm:windmill-client@1"\n`)
}
editor.insertAtCursor(`(await wmill.getVariable('${path}'))`)
} else if (lang === 'bun') {
const code = editor.getCode()
if (!code.includes(`import { getVariable } from "windmill-client@${__pkg__.version}`)) {
editor.insertAtBeginning(
`import { getVariable } from "windmill-client@${__pkg__.version}"\n`
)
if (!code.includes(`import * as wmill from`)) {
editor.insertAtBeginning(`import * as wmill from "windmill-client"\n`)
}
editor.insertAtCursor(`(await getVariable('${path}'))`)
editor.insertAtCursor(`(await wmill.getVariable('${path}'))`)
} else if (lang == 'python3') {
if (!editor.getCode().includes('import wmill')) {
editor.insertAtBeginning('import wmill\n')
@@ -314,19 +308,15 @@
if (!editor) return
if (lang == 'deno') {
if (!editor.getCode().includes('import * as wmill from')) {
editor.insertAtBeginning(
`import * as wmill from 'https://deno.land/x/windmill@v${version}/mod.ts'\n`
)
editor.insertAtBeginning(`import * as wmill from "npm:windmill-client@1"\n`)
}
editor.insertAtCursor(`(await wmill.getResource('${path}'))`)
} else if (lang === 'bun') {
const code = editor.getCode()
if (!code.includes(`import { getResource } from "windmill-client@${__pkg__.version}`)) {
editor.insertAtBeginning(
`import { getResource } from "windmill-client@${__pkg__.version}"\n`
)
if (!code.includes(`import * as wmill from`)) {
editor.insertAtBeginning(`import * as wmill from "windmill-client"\n`)
}
editor.insertAtCursor(`(await getResource('${path}'))`)
editor.insertAtCursor(`(await wmill.getResource('${path}'))`)
} else if (lang == 'python3') {
if (!editor.getCode().includes('import wmill')) {
editor.insertAtBeginning('import wmill\n')
frontend/src/lib/components/apps/editor/component/default-codes.ts
+8 -38
View File
@@ -38,7 +38,7 @@ export const DEFAULT_CODES: Partial<
"age": 84
}
]`,
pgsql: `import { pgSql } from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts";
pgsql: `import { pgSql } from "npm:windmill-client@1";
type Postgresql = object
@@ -75,7 +75,7 @@ export async function main(db: Postgresql) {
"age": 84
}
]`,
pgsql: `import { pgSql } from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts";
pgsql: `import { pgSql } from "npm:windmill-client@1";
type Postgresql = object
@@ -132,7 +132,7 @@ export async function main(db: Postgresql) {
"<3"
]
}`,
pgsql: `import { pgSql } from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts";
pgsql: `import { pgSql } from "npm:windmill-client@1";
type Postgresql = object
@@ -206,7 +206,7 @@ export async function main(db: Postgresql) {
"y": { "field": "b", "type": "quantitative" },
},
}`,
pgsql: `import { pgSql } from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts";
pgsql: `import { pgSql } from "npm:windmill-client@1";
type Postgresql = object
@@ -253,7 +253,7 @@ export async function main(Postgresqlstgresql) {
}
}
}`,
pgsql: `import { pgSql } from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts";
pgsql: `import { pgSql } from "npm:windmill-client@1";
type Postgresql = object
@@ -300,7 +300,7 @@ export async function main(Postgresqlstgresql) {
"<3"
]
}`,
pgsql: `import { pgSql } from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts";
pgsql: `import { pgSql } from "npm:windmill-client@1";
type Postgresql = object
@@ -356,7 +356,7 @@ export async function main(db: Postgresql) {
"backgroundColor": "orange"
}
]`,
pgsql: `import { pgSql } from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts";
pgsql: `import { pgSql } from "npm:windmill-client@1";
type Postgresql = object
@@ -466,37 +466,7 @@ export async function main(db: Postgresql) {
],
"backgroundColor": "orange"
}
]`,
pgsql: `import { pgSql } from "https://deno.land/x/windmill@v1.88.1/mod.ts";
type Postgresql = object
export async function main(db: Postgresql) {
try {
const query = await pgSql(db)\`SELECT * FROM demo;\`;
const rows = query.rows.map((row, i) => ({
x: new Date(Date.now() - (i * 1000 * 60 * 60 * 24)).toISOString(),
y: row['0']
}))
return [
{
label: "foo",
data: rows,
backgroundColor: "rgb(255, 12, 137)"
},
{
label: "bar",
data: rows.map(({x, y}) => ({
x,
y: y * 2
})),
backgroundColor: "orange"
}
];
} catch(e) {
return [];
}
}`
]`
},
iconcomponent: {
deno: `export async function main() {
frontend/src/lib/script_helpers.ts
+14 -5
View File
@@ -36,8 +36,8 @@ export async function main() {
export const DENO_INIT_CODE = `// Ctrl/CMD+. to cache dependencies on imports hover.
// import { toWords } from "npm:number-to-words@1"
// import * as wmill from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts"
// Deno uses "npm:" prefix to import from npm (https://deno.land/manual@v1.36.3/node/npm_specifiers)
// import * as wmill from "npm:windmill-client@1"
// fill the type, or use the +Resource type to get a type-safe reference to a resource
// type Postgresql = object
@@ -111,7 +111,7 @@ func main(message string, name string) (interface{}, error) {
}
`
export const DENO_INIT_CODE_CLEAR = `// import * as wmill from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts"
export const DENO_INIT_CODE_CLEAR = `// import * as wmill from "npm:windmill-client@1"
export async function main(x: string) {
return x
@@ -206,7 +206,7 @@ dflt="\${2:-default value}"
echo "Hello $msg"
`
export const DENO_INIT_CODE_TRIGGER = `import * as wmill from "https://deno.land/x/windmill@v${__pkg__.version}/mod.ts"
export const DENO_INIT_CODE_TRIGGER = `import * as wmill from "npm:windmill-client@1"
export async function main() {
@@ -251,7 +251,13 @@ func main() (interface{}, error) {
}
`
export const DENO_INIT_CODE_APPROVAL = `import * as wmill from "https://deno.land/x/windmill@v1.99.0/mod.ts"
export const DENO_INIT_CODE_APPROVAL = `import * as wmill from "npm:windmill-client@1"
export async function main(approver?: string) {
return wmill.getResumeEndpoints(approver)
}`
export const BUN_INIT_CODE_APPROVAL = `import * as wmill from "windmill-client@1"
export async function main(approver?: string) {
return wmill.getResumeEndpoints(approver)
@@ -363,6 +369,9 @@ export function initialCode(
} else if (language == 'graphql') {
return GRAPHQL_INIT_CODE
} else if (language == 'bun') {
if (kind === 'approval') {
return BUN_INIT_CODE_APPROVAL
}
if (subkind === 'flow') {
return BUN_INIT_CODE_CLEAR
}
frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte
+9 -1
View File
@@ -16,7 +16,7 @@
import Urlize from '$lib/components/Urlize.svelte'
import DeployWorkspaceDrawer from '$lib/components/DeployWorkspaceDrawer.svelte'
import SavedInputs from '$lib/components/SavedInputs.svelte'
import { FolderOpen, Archive, Trash, Server, Share } from 'lucide-svelte'
import { FolderOpen, Archive, Trash, Server, Share, Badge, Loader2 } from 'lucide-svelte'
import DetailPageHeader from '$lib/components/details/DetailPageHeader.svelte'
import WebhooksPanel from '$lib/components/details/WebhooksPanel.svelte'
@@ -35,6 +35,8 @@
let path = $page.params.path
let shareModal: ShareModal
let deploymentInProgress = false
$: cliCommand = `wmill flow run ${flow?.path} -d '${JSON.stringify(args)}'`
$: {
@@ -265,6 +267,12 @@
Edited <TimeAgo date={flow.edited_at ?? ''} /> by {flow.edited_by}
</span>
{#if deploymentInProgress}
<Badge color="yellow">
<Loader2 size={12} class="inline animate-spin mr-1" />
Deployment in progress
</Badge>
{/if}
{#if flow.archived}
<div class="" />
<Alert type="error" title="Archived">This flow was archived</Alert>
frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte
+5 -3
View File
@@ -532,7 +532,7 @@
<Tabs selected="code">
<Tab value="code" size="xs">Code</Tab>
<Tab value="dependencies" size="xs">Lock file</Tab>
<Tab value="dependencies" size="xs">Lockfile</Tab>
<Tab value="arguments" size="xs">
<span class="inline-flex items-center gap-1">
Inputs
@@ -548,7 +548,7 @@
</Tab>
<svelte:fragment slot="content">
<TabContent value="code">
<div class="p-2">
<div class="p-2 w-full overflow-auto">
<HighlightCode
language={script.language}
code={script.content}
@@ -559,7 +559,9 @@
<TabContent value="dependencies">
<div class="">
{#if script?.lock}
<pre class="bg-surface-secondary text-sm p-2 h-full">{script.lock}</pre>
<pre class="bg-surface-secondary text-sm p-2 h-full overflow-auto w-full"
>{script.lock}</pre
>
{:else}
<p class="bg-surface-secondary text-sm p-2">
There is no lock file for this script
typescript-client/client.ts
+32 -19
View File
@@ -1,5 +1,6 @@
import { ResourceService, VariableService } from "./index";
import { OpenAPI } from "./index";
import { JobService } from "./src";
export {
AdminService,
@@ -243,25 +244,37 @@ export async function databaseUrlFromResource(path: string): Promise<string> {
return `postgresql://${resource.user}:${resource.password}@${resource.host}:${resource.port}/${resource.dbname}?sslmode=${resource.sslmode}`;
}
// /**
// * Get URLs needed for resuming a flow after this step
// * @param approver approver name
// * @returns approval page UI URL, resume and cancel API URLs for resumeing the flow
// */
// export async function getResumeUrls(approver?: string): Promise<{
// approvalPage: string;
// resume: string;
// cancel: string;
// }> {
// const nonce = Math.floor(Math.random() * 4294967295);
// const workspace = getWorkspace();
// return await JobService.getResumeUrls({
// workspace,
// resumeId: nonce,
// approver,
// id: process.env.get("WM_JOB_ID") ?? "NO_JOB_ID",
// });
// }
/**
* Get URLs needed for resuming a flow after this step
* @param approver approver name
* @returns approval page UI URL, resume and cancel API URLs for resumeing the flow
*/
export async function getResumeUrls(approver?: string): Promise<{
approvalPage: string;
resume: string;
cancel: string;
}> {
const nonce = Math.floor(Math.random() * 4294967295);
!clientSet && setClient();
const workspace = getWorkspace();
return await JobService.getResumeUrls({
workspace,
resumeId: nonce,
approver,
id: getEnv("WM_JOB_ID") ?? "NO_JOB_ID",
});
}
/**
* @deprecated use getResumeUrls instead
*/
export function getResumeEndpoints(approver?: string): Promise<{
approvalPage: string;
resume: string;
cancel: string;
}> {
return getResumeUrls(approver);
}
export function base64ToUint8Array(data: string): Uint8Array {
return Uint8Array.from(atob(data), (c) => c.charCodeAt(0));