From fbcac1e0a6eb4e51e991e76f9b30a3de6fdeca39 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 6 Aug 2024 01:02:30 +0200 Subject: [PATCH] support tarbundle in vscode extension --- backend/windmill-api/src/jobs.rs | 59 ++++++++++++++------- backend/windmill-common/src/scripts.rs | 1 + backend/windmill-worker/src/global_cache.rs | 2 +- backend/windmill-worker/src/worker.rs | 37 +++++++------ cli/script.ts | 2 +- frontend/src/lib/components/Dev.svelte | 20 +++++-- 6 files changed, 77 insertions(+), 44 deletions(-) diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 622a1bb0ad..f5ece1b195 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -184,7 +184,10 @@ pub fn workspaced_service() -> Router { .layer(ce_headers.clone()), ) .route("/run/preview", post(run_preview_script)) - .route("/run/preview_bundle", post(run_bundle_preview_script)) + .route( + "/run/preview_bundle", + post(run_bundle_preview_script).layer(axum::extract::DefaultBodyLimit::disable()), + ) .route("/add_batch_jobs/:n", post(add_batch_jobs)) .route("/run/preview_flow", post(run_preview_flow_job)) .route( @@ -2611,6 +2614,7 @@ enum PreviewKind { Http, Noop, Bundle, + Tarbundle, } #[derive(Deserialize)] @@ -3810,6 +3814,8 @@ async fn run_bundle_preview_script( Query(run_query): Query, mut multipart: axum::extract::Multipart, ) -> error::Result<(StatusCode, String)> { + use windmill_common::scripts::PREVIEW_IS_TAR_CODEBASE_HASH; + check_license_key_valid().await?; check_scopes(&authed, || format!("runscript"))?; @@ -3822,9 +3828,12 @@ async fn run_bundle_preview_script( let mut job_id = None; let mut tx = None; let mut uploaded = false; + let mut is_tar = false; + while let Some(field) = multipart.next_field().await.unwrap() { let name = field.name().unwrap().to_string(); - let data = field.bytes().await.unwrap(); + let data = field.bytes().await; + let data = data.map_err(to_anyhow)?; if name == "preview" { let preview: Preview = serde_json::from_slice(&data).map_err(to_anyhow)?; @@ -3836,27 +3845,33 @@ async fn run_bundle_preview_script( let args = preview.args.unwrap_or_default(); + is_tar = match preview.kind { + Some(PreviewKind::Tarbundle) => true, + _ => false, + }; + + // tracing::info!("is_tar 1: {is_tar}"); // hmap.insert("") let (uuid, ntx) = push( &db, ltx, &w_id, - match preview.kind { - Some(PreviewKind::Identity) => JobPayload::Identity, - Some(PreviewKind::Noop) => JobPayload::Noop, - _ => JobPayload::Code(RawCode { - hash: Some(PREVIEW_IS_CODEBASE_HASH), - content: preview.content.unwrap_or_default(), - path: preview.path, - language: preview.language.unwrap_or(ScriptLang::Deno), - lock: preview.lock, - concurrent_limit: None, // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here - concurrency_time_window_s: None, // TODO(gbouv): same as above - cache_ttl: None, - dedicated_worker: preview.dedicated_worker, - custom_concurrency_key: None, - }), - }, + JobPayload::Code(RawCode { + hash: if is_tar { + Some(PREVIEW_IS_TAR_CODEBASE_HASH) + } else { + Some(PREVIEW_IS_CODEBASE_HASH) + }, + content: preview.content.unwrap_or_default(), + path: preview.path, + language: preview.language.unwrap_or(ScriptLang::Deno), + lock: preview.lock, + concurrent_limit: None, // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here + concurrency_time_window_s: None, // TODO(gbouv): same as above + cache_ttl: None, + dedicated_worker: preview.dedicated_worker, + custom_concurrency_key: None, + }), PushArgs::from(&args), authed.display_username(), &authed.email, @@ -3881,7 +3896,7 @@ async fn run_bundle_preview_script( tx = Some(ntx); } if name == "file" { - let id = job_id + let mut id = job_id .as_ref() .ok_or_else(|| { Error::BadRequest( @@ -3890,6 +3905,12 @@ async fn run_bundle_preview_script( })? .to_string(); + // tracing::info!("is_tar 2: {is_tar}"); + + if is_tar { + id = format!("{}.tar", id); + } + uploaded = true; if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index ac49ca3746..86c506f467 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -133,6 +133,7 @@ impl Display for ScriptKind { } pub const PREVIEW_IS_CODEBASE_HASH: i64 = -42; +pub const PREVIEW_IS_TAR_CODEBASE_HASH: i64 = -43; #[derive(Serialize, sqlx::FromRow)] pub struct Script { diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 2abce202cb..12584b905d 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -111,7 +111,7 @@ pub async fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> { tracing::info!("Failed to untar to {folder}. Error: {:?}", e); fs::remove_dir_all(&folder).await?; return Err(error::Error::ExecutionErr(format!( - "Failed to untar piptar {folder}" + "Failed to untar tar {folder}" ))); } tracing::info!( diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 96bc14417a..e274f1790d 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -7,8 +7,7 @@ */ use windmill_common::{ - auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms, JWT_SECRET}, - worker::{get_windmill_memory_usage, get_worker_memory_usage, TMP_DIR}, + auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms, JWT_SECRET}, scripts::PREVIEW_IS_TAR_CODEBASE_HASH, worker::{get_windmill_memory_usage, get_worker_memory_usage, TMP_DIR} }; use anyhow::{Context, Result}; @@ -2792,23 +2791,23 @@ async fn handle_code_execution_job( envs, codebase, } = match job.job_kind { - JobKind::Preview => ContentReqLangEnvs { - content: job - .raw_code - .clone() - .unwrap_or_else(|| "no raw code".to_owned()), - lockfile: job.raw_lock.clone(), - language: job.language.to_owned(), - envs: None, - codebase: if job - .script_hash - .is_some_and(|y| y.0 == PREVIEW_IS_CODEBASE_HASH) - { - Some(job.id.to_string()) - } else { - None - }, - }, + JobKind::Preview => { + let codebase = match job.script_hash.map(|x| x.0) { + Some(PREVIEW_IS_CODEBASE_HASH) => Some(job.id.to_string()), + Some(PREVIEW_IS_TAR_CODEBASE_HASH) => Some(format!("{}.tar", job.id)), + _ => None, + }; + + ContentReqLangEnvs { + content: job + .raw_code + .clone() + .unwrap_or_else(|| "no raw code".to_owned()), + lockfile: job.raw_lock.clone(), + language: job.language.to_owned(), + envs: None, + codebase + }}, JobKind::Script_Hub => { get_hub_script_content_and_requirements(job.script_path.clone(), db).await? } diff --git a/cli/script.ts b/cli/script.ts index ba39bae292..4e130be7ea 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -47,7 +47,7 @@ import { } from "./conf.ts"; import { SyncCodebase, listSyncCodebases } from "./codebase.ts"; import fs from "node:fs"; -import { Tarball } from "npm:@ayonli/jsext/archive"; +import { type Tarball } from "npm:@ayonli/jsext/archive"; export interface ScriptFile { parent_hash?: string; diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index 4d211f6eb2..55a6bb80a6 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -196,11 +196,12 @@ replaceScript(event.data) } else if (event.data.type == 'testBundle') { if (event.data.id == lastBundleCommandId) { - testBundle(event.data.file) + testBundle(event.data.file, event.data.isTar) } else { sendUserToast(`Bundle received ${lastBundleCommandId} was obsolete, ignoring`, true) } } else if (event.data.type == 'testBundleError') { + loadingCodebaseButton = false sendUserToast( typeof event.data.error == 'object' ? JSON.stringify(event.data.error) : event.data.error, true @@ -244,7 +245,7 @@ window.parent?.postMessage({ type: 'refresh' }, '*') }) - async function testBundle(file: string) { + async function testBundle(file: string, isTar: boolean) { testJobLoader?.abstractRun(async () => { try { const form = new FormData() @@ -252,14 +253,25 @@ 'preview', JSON.stringify({ content: currentScript?.content, - kind: 'bundle', + kind: isTar ? 'tarbundle' : 'bundle', path: currentScript?.path, args, language: currentScript?.language, tag: currentScript?.tag }) ) - form.append('file', file) + // sendUserToast(JSON.stringify(file)) + if (isTar) { + var array: number[] = [] + file = atob(file) + for (var i = 0; i < file.length; i++) { + array.push(file.charCodeAt(i)) + } + let blob = new Blob([new Uint8Array(array)], { type: 'application/octet-stream' }) + form.append('file', blob) + } else { + form.append('file', file) + } const url = '/api/w/' + workspace + '/jobs/run/preview_bundle'