From fcfad69195a776d4958c035dca16100368966a66 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:32:17 +0300 Subject: [PATCH 1/4] feat: Migrate to bun.lock (#5112) * bun: Migrate to bun.lock (In backwards compatible way) Read more: https://bun.sh/blog/bun-lock-text-lockfile * Clean up * More clean up * Mount bun.lock to jailed process --- .../nsjail/run.bun.config.proto | 6 + backend/windmill-worker/src/bun_executor.rs | 103 +++++++++++------- 2 files changed, 68 insertions(+), 41 deletions(-) diff --git a/backend/windmill-worker/nsjail/run.bun.config.proto b/backend/windmill-worker/nsjail/run.bun.config.proto index 9fbb90ecc1..751452251d 100644 --- a/backend/windmill-worker/nsjail/run.bun.config.proto +++ b/backend/windmill-worker/nsjail/run.bun.config.proto @@ -77,6 +77,12 @@ mount { mandatory: false } +mount { + src: "{JOB_DIR}/bun.lock" + dst: "/tmp/{LANG}/bun.lock" + is_bind: true + mandatory: false +} mount { src: "{JOB_DIR}/wrapper.mjs" diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 5f62720b44..dfe9c5b328 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -62,24 +62,37 @@ const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js"); const NSJAIL_CONFIG_RUN_BUN_CONTENT: &str = include_str!("../nsjail/run.bun.config.proto"); +pub const BUN_LOCK_SPLIT: &str = "\n//bun.lock\n"; pub const BUN_LOCKB_SPLIT: &str = "\n//bun.lockb\n"; +pub const BUN_LOCK_SPLIT_WINDOWS: &str = "\r\n//bun.lock\r\n"; pub const BUN_LOCKB_SPLIT_WINDOWS: &str = "\r\n//bun.lockb\r\n"; pub const EMPTY_FILE: &str = ""; -fn split_lockfile(lockfile: &str) -> (&str, Option<&str>, bool) { - if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT) { +/// Returns (package.json, bun.lock(b), is_empty, is_binary) +fn split_lockfile(lockfile: &str) -> (&str, Option<&str>, bool, bool) { + if let Some(index) = lockfile.find(BUN_LOCK_SPLIT) { + // Split using "\n//bun.lock\n" + let (before, after_with_sep) = lockfile.split_at(index); + let after = &after_with_sep[BUN_LOCK_SPLIT.len()..]; + (before, Some(after), after == EMPTY_FILE, false) + } else if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT) { // Split using "\n//bun.lockb\n" let (before, after_with_sep) = lockfile.split_at(index); let after = &after_with_sep[BUN_LOCKB_SPLIT.len()..]; - (before, Some(after), after == EMPTY_FILE) + (before, Some(after), after == EMPTY_FILE, true) + } else if let Some(index) = lockfile.find(BUN_LOCK_SPLIT_WINDOWS) { + // Split using "\r\n//bun.lock\r\n" + let (before, after_with_sep) = lockfile.split_at(index); + let after = &after_with_sep[BUN_LOCK_SPLIT_WINDOWS.len()..]; + (before, Some(after), after == EMPTY_FILE, false) } else if let Some(index) = lockfile.find(BUN_LOCKB_SPLIT_WINDOWS) { // Split using "\r\n//bun.lockb\r\n" let (before, after_with_sep) = lockfile.split_at(index); let after = &after_with_sep[BUN_LOCKB_SPLIT_WINDOWS.len()..]; - (before, Some(after), after == EMPTY_FILE) + (before, Some(after), after == EMPTY_FILE, true) } else { - (lockfile, None, false) + (lockfile, None, false, false) } } @@ -199,18 +212,18 @@ pub async fn gen_bun_lockfile( } if !npm_mode { #[cfg(any(target_os = "linux", target_os = "macos"))] - content.push_str(BUN_LOCKB_SPLIT); + content.push_str(BUN_LOCK_SPLIT); #[cfg(target_os = "windows")] - content.push_str(BUN_LOCKB_SPLIT_WINDOWS); + content.push_str(BUN_LOCK_SPLIT_WINDOWS); { - let file = format!("{job_dir}/bun.lockb"); + let file = format!("{job_dir}/bun.lock"); if !empty_deps && tokio::fs::metadata(&file).await.is_ok() { let mut file = File::open(&file).await?; - let mut buf = vec![]; - file.read_to_end(&mut buf).await?; - content.push_str(&base64::engine::general_purpose::STANDARD.encode(&buf)); + let mut buf = String::default(); + file.read_to_string(&mut buf).await?; + content.push_str(&buf); } else { content.push_str(&EMPTY_FILE); } @@ -277,7 +290,7 @@ pub async fn install_bun_lockfile( .env_clear() .envs(PROXY_ENVS.clone()) .envs(common_bun_proc_envs) - .args(vec!["install"]) + .args(vec!["install", "--save-text-lockfile"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -785,26 +798,30 @@ async fn compute_bundle_local_and_remote_path( } pub async fn prepare_job_dir(reqs: &str, job_dir: &str) -> Result<()> { - let (pkg, lock, empty) = split_lockfile(reqs); + let (pkg, lock, empty, is_binary) = split_lockfile(reqs); let _ = write_file(job_dir, "package.json", pkg)?; if !empty { if let Some(lock) = lock { - let _ = write_lockb(lock, job_dir).await?; + let _ = write_lock(lock, job_dir, is_binary).await?; } } Ok(()) } -async fn write_lockb(splitted_lockb_2: &str, job_dir: &str) -> Result<()> { - write_file_binary( - job_dir, - "bun.lockb", - &base64::engine::general_purpose::STANDARD - .decode(splitted_lockb_2) - .map_err(|_| error::Error::InternalErr("Could not decode bun.lockb".to_string()))?, - ) - .await?; +async fn write_lock(splitted_lockb_2: &str, job_dir: &str, is_binary: bool) -> Result<()> { + if is_binary { + write_file_binary( + job_dir, + "bun.lockb", + &base64::engine::general_purpose::STANDARD + .decode(splitted_lockb_2) + .map_err(|_| error::Error::InternalErr(format!("Could not decode bun.lockb")))?, + ) + .await?; + } else { + write_file(job_dir, "bun.lock", splitted_lockb_2)?; + }; Ok(()) } @@ -900,26 +917,26 @@ pub async fn handle_bun_job( } else if let Some(codebase) = codebase.as_ref() { pull_codebase(&job.workspace_id, codebase, job_dir).await?; } else if let Some(reqs) = requirements_o.as_ref() { - let (pkg, lock, empty) = split_lockfile(reqs); + let (pkg, lock, empty, is_binary) = split_lockfile(reqs); if lock.is_none() && !annotation.npm { return Err(error::Error::ExecutionErr( - format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|") + format!("Invalid requirements, expected to find //bun.lock{} split pattern in reqs. Found: |{reqs}|", if is_binary {"b"} else {""}) )); } let _ = write_file(job_dir, "package.json", pkg)?; - let lockb = if annotation.npm { "" } else { lock.unwrap() }; + let lock = if annotation.npm { "" } else { lock.unwrap() }; if !empty { let mut skip_install = false; let mut create_buntar = false; let mut buntar_path = "".to_string(); if !annotation.npm { - let _ = write_lockb(lockb, job_dir).await?; + let _ = write_lock(lock, job_dir, is_binary).await?; let mut sha_path = sha2::Sha256::new(); - sha_path.update(lockb.as_bytes()); + sha_path.update(lock.as_bytes()); let buntar_name = base64::engine::general_purpose::URL_SAFE.encode(sha_path.finalize()); @@ -962,7 +979,7 @@ pub async fn handle_bun_job( Some(&vec![ "main.ts".to_string(), "package.json".to_string(), - "bun.lockb".to_string(), + if is_binary { "bun.lockb" } else { "bun.lock" }.to_string(), "shared".to_string(), "bunfig.toml".to_string(), ]), @@ -1592,25 +1609,29 @@ pub async fn start_worker( if let Some(codebase) = codebase.as_ref() { pull_codebase(w_id, codebase, job_dir).await?; } else if let Some(reqs) = requirements_o { - let (pkg, lock, empty) = split_lockfile(&reqs); + let (pkg, lock, empty, is_binary) = split_lockfile(&reqs); if lock.is_none() { return Err(error::Error::ExecutionErr( format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|") )); } let _ = write_file(job_dir, "package.json", pkg)?; - let lockb = lock.unwrap(); + let lock = lock.unwrap(); if !empty { - let _ = write_file_binary( - job_dir, - "bun.lockb", - &base64::engine::general_purpose::STANDARD - .decode(lockb) - .map_err(|_| { - error::Error::InternalErr("Could not decode bun.lockb".to_string()) - })?, - ) - .await?; + if is_binary { + let _ = write_file_binary( + job_dir, + "bun.lockb", + &base64::engine::general_purpose::STANDARD + .decode(lock) + .map_err(|_| { + error::Error::InternalErr("Could not decode bun.lockb".to_string()) + })?, + ) + .await?; + } else { + write_file(job_dir, "bun.lock", lock)?; + } install_bun_lockfile( &mut mem_peak, From 59bcce7a969ba57506b435d13f078c4e8c2f0465 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 22 Jan 2025 12:51:14 +0100 Subject: [PATCH 2/4] feat (frontend): improve capture UI (#5051) * feat (frontend): improve capture UI * show schema diff * add accept reject form * add diff for first node input * move update schema button * change modified properties display * propagate change through nested components * allow modification of nested object in schema * reject nested changes * clean logic * clean code * clean * nit * fix first strep input * fix argument preview * clean * restore indentation for nested objects * ignore undefined field in diff computation * fix run button disabled update * nit * fix update JSON * fix deletion of nested components * dark mode * clean * replace dropdown with sidebar * only check type for compatibility * add shadow * disbale dnd on edit * auto-scroll within schema * fix nested not dnd object diff viewer * fix captures drawer * open edit tab on add new arg * change button label when schema is the same * ajust padding * fix arg update * fix oneof display * propagate change event through nested schema * handle oneOf * fix preview arg sync * handle s3 object * fix arg sync * update schema input compatibility * clean compatible * accept empty array items * clean * increase of schema args gap * fix nested schema update * allow number and int compatibility * reset args when modifying schema * open fields on add with addPropertyV2 --------- Co-authored-by: Ruben Fiszel --- frontend/src/lib/components/ArgInput.svelte | 1280 +++++++++-------- .../lib/components/EditableSchemaForm.svelte | 93 +- .../src/lib/components/FirstStepInputs.svelte | 4 +- .../lib/components/FlowPreviewContent.svelte | 91 +- .../src/lib/components/HistoricInputs.svelte | 2 + .../lib/components/SavedInputsPicker.svelte | 6 + frontend/src/lib/components/SchemaForm.svelte | 75 +- .../src/lib/components/SimpleEditor.svelte | 5 +- .../components/flows/content/FlowInput.svelte | 349 +++-- .../flows/content/FlowInputEditor.svelte | 49 +- .../meltComponents/SideBarTab.svelte | 69 + .../lib/components/schema/AddProperty.svelte | 4 +- .../components/schema/AddPropertyV2.svelte | 14 +- .../schema/EditableSchemaDrawer.svelte | 15 +- .../schema/EditableSchemaWrapper.svelte | 5 + .../components/schema/SchemaFormDND.svelte | 36 +- .../src/lib/components/schema/schemaUtils.ts | 217 +++ .../components/triggers/CaptureTable.svelte | 12 +- 18 files changed, 1447 insertions(+), 879 deletions(-) create mode 100644 frontend/src/lib/components/meltComponents/SideBarTab.svelte create mode 100644 frontend/src/lib/components/schema/schemaUtils.ts diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 705da3e07c..c073eb2fcd 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -6,7 +6,7 @@ emptyString, getSchemaFromProperties } from '$lib/utils' - import { DollarSign, Pipette, Plus, X } from 'lucide-svelte' + import { DollarSign, Pipette, Plus, X, Check } from 'lucide-svelte' import { createEventDispatcher, onMount, tick } from 'svelte' import Multiselect from 'svelte-multiselect' import { fade } from 'svelte/transition' @@ -37,6 +37,7 @@ import { deepEqual } from 'fast-equals' import DynSelect from './DynSelect.svelte' import type { Script } from '$lib/gen' + import type { SchemaDiff } from '$lib/components/schema/schemaUtils' export let label: string = '' export let value: any @@ -96,6 +97,10 @@ | undefined = undefined export let otherArgs: Record = {} export let lightHeader = false + export let diffStatus: SchemaDiff | undefined = undefined + export let hideNested = false + export let nestedParent: { label: string; nestedParent: any | undefined } | undefined = undefined + export let nestedClasses = '' $: inputCat = computeInputCat(type, format, itemsType?.type, enum_, contentEncoding) @@ -347,464 +352,323 @@ /> -
-
- {#if displayHeader} - - {/if} +
+ {#if diffStatus && typeof diffStatus === 'object' && diffStatus.diff !== 'same'} +
+ + +
+ {/if} + {#if displayHeader} + + {/if} - {#if description} -
-
{description}
-
- {/if} + {#if description} +
+
{description}
+
+ {/if} -
- {#if inputCat == 'number'} - {#if extra['min'] != undefined && extra['max'] != undefined} - - {:else if extra['seconds'] !== undefined} - - {:else if extra?.currency} - - {:else} -
- { - ignoreValueUndefined = true - }} - class={valid - ? '' - : 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'} - placeholder={placeholder ?? defaultValue ?? ''} - bind:value - min={extra['min']} - max={extra['max']} - /> -
- {/if} - {:else if inputCat == 'boolean'} -
- { - e?.stopPropagation() - }} +
+ {#if inputCat == 'number'} + {#if extra['min'] != undefined && extra['max'] != undefined} + + {:else if extra['seconds'] !== undefined} + + {:else if extra?.currency} + + {:else} +
+ { + ignoreValueUndefined = true + }} class={valid ? '' : 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'} - bind:checked={value} + placeholder={placeholder ?? defaultValue ?? ''} + bind:value + min={extra['min']} + max={extra['max']} /> - {#if type == 'boolean' && value == undefined} -   Not set - {/if}
- {:else if inputCat == 'list' && !isListJson} -
-
- {#if Array.isArray(itemsType?.multiselect) && Array.isArray(value)} -
- { - dispatch('focus') - }} - /> -
- {:else if itemsType?.enum != undefined && Array.isArray(itemsType?.enum) && Array.isArray(value)} -
- { - dispatch('focus') - }} - /> -
- {:else} -
- {#key redraw} - {#if Array.isArray(value)} - {#each value ?? [] as v, i} - {#if i < itemsLimit} -
- {#if itemsType?.type == 'number'} - - {:else if itemsType?.type == 'string' && itemsType?.contentEncoding == 'base64'} - fileChanged(x, (val) => (value[i] = val))} - multiple={false} - /> - {:else if itemsType?.type == 'object' && itemsType?.resourceType === undefined && itemsType?.properties === undefined} - - {:else if Array.isArray(itemsType?.enum)} - { - dispatch('focus') - }} - on:blur={(e) => { - dispatch('blur') - }} - {defaultValue} - {valid} - {disabled} - {autofocus} - bind:value={v} - enum_={itemsType?.enum ?? []} - enumLabels={extra['enumLabels']} - /> - {:else if itemsType?.type == 'resource' && itemsType?.resourceType && resourceTypes?.includes(itemsType.resourceType)} - - {:else if itemsType?.type == 'resource'} - { - dispatch('focus') - }} - on:blur={(e) => { - dispatch('blur') - }} - code={JSON.stringify(v, null, 2)} - bind:value={v} - /> - {:else if itemsType?.type === 'object' && itemsType?.properties} -
- -
- {:else} - - {/if} - -
- {/if} - {/each} - {#if value.length > itemsLimit} - - {/if} - {/if} - {/key} -
-
- -
- {/if} -
-
- { - // Once the user has changed the input type, we should not change it back automatically - if (!hasIsListJsonChanged) { - hasIsListJsonChanged = true - } - - evalValueToRaw() - isListJson = !isListJson - }} - checked={isListJson} - textClass="text-secondary" - size="xs" - options={{ right: 'json' }} - /> -
-
- {:else if inputCat == 'dynselect'} - - {:else if inputCat == 'resource-object' && resourceTypes == undefined} - Loading resource types... - {:else if inputCat == 'resource-object' && (resourceTypes == undefined || (format.split('-').length > 1 && resourceTypes.includes(format.substring('resource-'.length))))} - { - defaultValue = null + {/if} + {:else if inputCat == 'boolean'} +
+ { + e?.stopPropagation() }} - {showSchemaExplorer} + {disabled} + class={valid + ? '' + : 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-30 bg-red-100'} + bind:checked={value} /> - {:else if inputCat == 'resource-object' && format.split('-').length > 1 && format - .replace('resource-', '') - .replace('_', '') - .toLowerCase() == 's3object'} -
- - {#if s3FileUploadRawMode} - { - dispatch('focus') - }} - on:blur={(e) => { - dispatch('blur') - }} - code={JSON.stringify(value ?? defaultValue ?? { s3: '' }, null, 2)} - bind:value - /> - - {:else} - { - value = { - s3: evt.detail?.path ?? '', - filename: evt.detail?.filename ?? '' - } - }} - on:deletion={(evt) => { - value = { - s3: '' - } - }} - defaultValue={defaultValue?.s3} - /> - {/if} -
- {:else if inputCat == 'object' || inputCat == 'resource-object' || isListJson} - {#if oneOf && oneOf.length >= 2} -
- {#if oneOf && oneOf.length >= 2} - { - value = { label: oneOfSelected } - redraw += 1 - }} - > - {#each oneOf as obj} - - {/each} - - {#if oneOfSelected} - {@const objIdx = oneOf.findIndex((o) => o.title === oneOfSelected)} - {@const obj = oneOf[objIdx]} - {#if obj && obj.properties && Object.keys(obj.properties).length > 0} - {#key redraw} -
- {#if orderEditable} - { - if (oneOf && oneOf[objIdx]) { - const keys = e.detail - oneOf[objIdx].order = keys - } - }} - on:change - /> - {:else} - - {/if} -
- {/key} - {:else if disabled} -