starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-22 00:01:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

support tarbundle in vscode extension

Browse Source
This commit is contained in:
Ruben Fiszel
2024-08-06 01:02:30 +02:00
parent 1c83876775
commit fbcac1e0a6
6 changed files with 77 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/windmill-api/src/jobs.rs
+40 -19
View File
@@ -184,7 +184,10 @@ pub fn workspaced_service() -> Router {
.layer(ce_headers.clone()),
)
.route("/run/preview", post(run_preview_script))
.route("/run/preview_bundle", post(run_bundle_preview_script))
.route(
"/run/preview_bundle",
post(run_bundle_preview_script).layer(axum::extract::DefaultBodyLimit::disable()),
)
.route("/add_batch_jobs/:n", post(add_batch_jobs))
.route("/run/preview_flow", post(run_preview_flow_job))
.route(
@@ -2611,6 +2614,7 @@ enum PreviewKind {
Http,
Noop,
Bundle,
Tarbundle,
}
#[derive(Deserialize)]
@@ -3810,6 +3814,8 @@ async fn run_bundle_preview_script(
Query(run_query): Query<RunJobQuery>,
mut multipart: axum::extract::Multipart,
) -> error::Result<(StatusCode, String)> {
use windmill_common::scripts::PREVIEW_IS_TAR_CODEBASE_HASH;
check_license_key_valid().await?;
check_scopes(&authed, || format!("runscript"))?;
@@ -3822,9 +3828,12 @@ async fn run_bundle_preview_script(
let mut job_id = None;
let mut tx = None;
let mut uploaded = false;
let mut is_tar = false;
while let Some(field) = multipart.next_field().await.unwrap() {
let name = field.name().unwrap().to_string();
let data = field.bytes().await.unwrap();
let data = field.bytes().await;
let data = data.map_err(to_anyhow)?;
if name == "preview" {
let preview: Preview = serde_json::from_slice(&data).map_err(to_anyhow)?;
@@ -3836,27 +3845,33 @@ async fn run_bundle_preview_script(
let args = preview.args.unwrap_or_default();
is_tar = match preview.kind {
Some(PreviewKind::Tarbundle) => true,
_ => false,
};
// tracing::info!("is_tar 1: {is_tar}");
// hmap.insert("")
let (uuid, ntx) = push(
&db,
ltx,
&w_id,
match preview.kind {
Some(PreviewKind::Identity) => JobPayload::Identity,
Some(PreviewKind::Noop) => JobPayload::Noop,
_ => JobPayload::Code(RawCode {
hash: Some(PREVIEW_IS_CODEBASE_HASH),
content: preview.content.unwrap_or_default(),
path: preview.path,
language: preview.language.unwrap_or(ScriptLang::Deno),
lock: preview.lock,
concurrent_limit: None, // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here
concurrency_time_window_s: None, // TODO(gbouv): same as above
cache_ttl: None,
dedicated_worker: preview.dedicated_worker,
custom_concurrency_key: None,
}),
},
JobPayload::Code(RawCode {
hash: if is_tar {
Some(PREVIEW_IS_TAR_CODEBASE_HASH)
} else {
Some(PREVIEW_IS_CODEBASE_HASH)
},
content: preview.content.unwrap_or_default(),
path: preview.path,
language: preview.language.unwrap_or(ScriptLang::Deno),
lock: preview.lock,
concurrent_limit: None, // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here
concurrency_time_window_s: None, // TODO(gbouv): same as above
cache_ttl: None,
dedicated_worker: preview.dedicated_worker,
custom_concurrency_key: None,
}),
PushArgs::from(&args),
authed.display_username(),
&authed.email,
@@ -3881,7 +3896,7 @@ async fn run_bundle_preview_script(
tx = Some(ntx);
}
if name == "file" {
let id = job_id
let mut id = job_id
.as_ref()
.ok_or_else(|| {
Error::BadRequest(
@@ -3890,6 +3905,12 @@ async fn run_bundle_preview_script(
})?
.to_string();
// tracing::info!("is_tar 2: {is_tar}");
if is_tar {
id = format!("{}.tar", id);
}
uploaded = true;
if let Some(os) = windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS
backend/windmill-common/src/scripts.rs
+1
View File
@@ -133,6 +133,7 @@ impl Display for ScriptKind {
}
pub const PREVIEW_IS_CODEBASE_HASH: i64 = -42;
pub const PREVIEW_IS_TAR_CODEBASE_HASH: i64 = -43;
#[derive(Serialize, sqlx::FromRow)]
pub struct Script {
backend/windmill-worker/src/global_cache.rs
+1 -1
View File
@@ -111,7 +111,7 @@ pub async fn extract_tar(tar: bytes::Bytes, folder: &str) -> error::Result<()> {
tracing::info!("Failed to untar to {folder}. Error: {:?}", e);
fs::remove_dir_all(&folder).await?;
return Err(error::Error::ExecutionErr(format!(
"Failed to untar piptar {folder}"
"Failed to untar tar {folder}"
)));
}
tracing::info!(
backend/windmill-worker/src/worker.rs
+18 -19
View File
@@ -7,8 +7,7 @@
*/
use windmill_common::{
auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms, JWT_SECRET},
worker::{get_windmill_memory_usage, get_worker_memory_usage, TMP_DIR},
auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms, JWT_SECRET}, scripts::PREVIEW_IS_TAR_CODEBASE_HASH, worker::{get_windmill_memory_usage, get_worker_memory_usage, TMP_DIR}
};
use anyhow::{Context, Result};
@@ -2792,23 +2791,23 @@ async fn handle_code_execution_job(
envs,
codebase,
} = match job.job_kind {
JobKind::Preview => ContentReqLangEnvs {
content: job
.raw_code
.clone()
.unwrap_or_else(|| "no raw code".to_owned()),
lockfile: job.raw_lock.clone(),
language: job.language.to_owned(),
envs: None,
codebase: if job
.script_hash
.is_some_and(|y| y.0 == PREVIEW_IS_CODEBASE_HASH)
{
Some(job.id.to_string())
} else {
None
},
},
JobKind::Preview => {
let codebase = match job.script_hash.map(|x| x.0) {
Some(PREVIEW_IS_CODEBASE_HASH) => Some(job.id.to_string()),
Some(PREVIEW_IS_TAR_CODEBASE_HASH) => Some(format!("{}.tar", job.id)),
_ => None,
};
ContentReqLangEnvs {
content: job
.raw_code
.clone()
.unwrap_or_else(|| "no raw code".to_owned()),
lockfile: job.raw_lock.clone(),
language: job.language.to_owned(),
envs: None,
codebase
}},
JobKind::Script_Hub => {
get_hub_script_content_and_requirements(job.script_path.clone(), db).await?
}
cli/script.ts
+1 -1
View File
@@ -47,7 +47,7 @@ import {
} from "./conf.ts";
import { SyncCodebase, listSyncCodebases } from "./codebase.ts";
import fs from "node:fs";
import { Tarball } from "npm:@ayonli/jsext/archive";
import { type Tarball } from "npm:@ayonli/jsext/archive";
export interface ScriptFile {
parent_hash?: string;
frontend/src/lib/components/Dev.svelte
+16 -4
View File
@@ -196,11 +196,12 @@
replaceScript(event.data)
} else if (event.data.type == 'testBundle') {
if (event.data.id == lastBundleCommandId) {
testBundle(event.data.file)
testBundle(event.data.file, event.data.isTar)
} else {
sendUserToast(`Bundle received ${lastBundleCommandId} was obsolete, ignoring`, true)
}
} else if (event.data.type == 'testBundleError') {
loadingCodebaseButton = false
sendUserToast(
typeof event.data.error == 'object' ? JSON.stringify(event.data.error) : event.data.error,
true
@@ -244,7 +245,7 @@
window.parent?.postMessage({ type: 'refresh' }, '*')
})
async function testBundle(file: string) {
async function testBundle(file: string, isTar: boolean) {
testJobLoader?.abstractRun(async () => {
try {
const form = new FormData()
@@ -252,14 +253,25 @@
'preview',
JSON.stringify({
content: currentScript?.content,
kind: 'bundle',
kind: isTar ? 'tarbundle' : 'bundle',
path: currentScript?.path,
args,
language: currentScript?.language,
tag: currentScript?.tag
})
)
form.append('file', file)
// sendUserToast(JSON.stringify(file))
if (isTar) {
var array: number[] = []
file = atob(file)
for (var i = 0; i < file.length; i++) {
array.push(file.charCodeAt(i))
}
let blob = new Blob([new Uint8Array(array)], { type: 'application/octet-stream' })
form.append('file', blob)
} else {
form.append('file', file)
}
const url = '/api/w/' + workspace + '/jobs/run/preview_bundle'