diff --git a/backend/.sqlx/query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json b/backend/.sqlx/query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json index aa608ecc57..053857a0a0 100644 --- a/backend/.sqlx/query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json +++ b/backend/.sqlx/query-0a686ca61444d7ad7484071727aa039a6ea6697e5a49a633b767c052aa3e0a18.json @@ -48,7 +48,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44.json b/backend/.sqlx/query-254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44.json index b2421d64f2..03bd3e00c1 100644 --- a/backend/.sqlx/query-254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44.json +++ b/backend/.sqlx/query-254776959f60455a00a2c29a1783ee6b27c836197df6454d3bb5c59441c34f44.json @@ -69,7 +69,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-2f42460fdd8aa125c8fd46b3cd02e47f57de0f073d3ce3bc7d21a7e404a83b5c.json b/backend/.sqlx/query-2f42460fdd8aa125c8fd46b3cd02e47f57de0f073d3ce3bc7d21a7e404a83b5c.json index c9110033a5..030a85c000 100644 --- a/backend/.sqlx/query-2f42460fdd8aa125c8fd46b3cd02e47f57de0f073d3ce3bc7d21a7e404a83b5c.json +++ b/backend/.sqlx/query-2f42460fdd8aa125c8fd46b3cd02e47f57de0f073d3ce3bc7d21a7e404a83b5c.json @@ -42,7 +42,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-4a393e61e33c1204c01044254692629e6ee2c053c7b1fa10f84462a327c739b3.json b/backend/.sqlx/query-4a393e61e33c1204c01044254692629e6ee2c053c7b1fa10f84462a327c739b3.json index a8c925c966..cac6b88592 100644 --- a/backend/.sqlx/query-4a393e61e33c1204c01044254692629e6ee2c053c7b1fa10f84462a327c739b3.json +++ b/backend/.sqlx/query-4a393e61e33c1204c01044254692629e6ee2c053c7b1fa10f84462a327c739b3.json @@ -28,7 +28,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json b/backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json index 89583ff5ef..4598ac08df 100644 --- a/backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json +++ b/backend/.sqlx/query-620ddf29c5e867079df4c2aa6e80bccb19beeb9ddfa308ca97f254cd5ba8157e.json @@ -62,7 +62,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json index 75b8108281..1fa370e682 100644 --- a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json +++ b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "?column?", + "name": "bool", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-6b313cc9a57ae3c943bda4a3213f7f6231a44b6ef5a52754074d136007f4f72a.json b/backend/.sqlx/query-6b313cc9a57ae3c943bda4a3213f7f6231a44b6ef5a52754074d136007f4f72a.json index 7d9d04bc37..72c175aff7 100644 --- a/backend/.sqlx/query-6b313cc9a57ae3c943bda4a3213f7f6231a44b6ef5a52754074d136007f4f72a.json +++ b/backend/.sqlx/query-6b313cc9a57ae3c943bda4a3213f7f6231a44b6ef5a52754074d136007f4f72a.json @@ -37,7 +37,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json b/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json index 983c1586f0..f296c4afc6 100644 --- a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json +++ b/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json @@ -42,7 +42,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-a28a83edc40e32815cb465338b53c8e892ac4fac6d78bc825cd0b0b1099f4e07.json b/backend/.sqlx/query-a28a83edc40e32815cb465338b53c8e892ac4fac6d78bc825cd0b0b1099f4e07.json new file mode 100644 index 0000000000..12581d6679 --- /dev/null +++ b/backend/.sqlx/query-a28a83edc40e32815cb465338b53c8e892ac4fac6d78bc825cd0b0b1099f4e07.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE instance_group SET name = $1 where name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a28a83edc40e32815cb465338b53c8e892ac4fac6d78bc825cd0b0b1099f4e07" +} diff --git a/backend/.sqlx/query-b69891c25dd029b1a54e97ace292433e1485324ff7dc802fe75d21c8c6db1d42.json b/backend/.sqlx/query-b69891c25dd029b1a54e97ace292433e1485324ff7dc802fe75d21c8c6db1d42.json index 86a1402ad0..f95d9d95b2 100644 --- a/backend/.sqlx/query-b69891c25dd029b1a54e97ace292433e1485324ff7dc802fe75d21c8c6db1d42.json +++ b/backend/.sqlx/query-b69891c25dd029b1a54e97ace292433e1485324ff7dc802fe75d21c8c6db1d42.json @@ -42,7 +42,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/.sqlx/query-ef132ac8d79579b08d7359789b6f22991f51e1c945efc2924df6253d62b83bba.json b/backend/.sqlx/query-ef132ac8d79579b08d7359789b6f22991f51e1c945efc2924df6253d62b83bba.json index 9ea1ab28f3..73af3f369c 100644 --- a/backend/.sqlx/query-ef132ac8d79579b08d7359789b6f22991f51e1c945efc2924df6253d62b83bba.json +++ b/backend/.sqlx/query-ef132ac8d79579b08d7359789b6f22991f51e1c945efc2924df6253d62b83bba.json @@ -42,7 +42,6 @@ "bash", "postgresql", "nativets", - "Nativets", "bun", "mysql", "bigquery", diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 08deab14c2..91e7b06ecc 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -1,6 +1,6 @@ openapi: 3.0.3 info: - version: 1.246.4 + version: 1.246.15 title: Windmill API contact: name: Windmill Team @@ -10503,6 +10503,8 @@ paths: properties: file_key: type: string + file_extension: + type: string part_content: type: array items: @@ -10528,8 +10530,7 @@ paths: s3_resource_path: type: string required: - - file_key - - part_content_base64 + - part_content - parts - is_final - cancel_upload @@ -10551,7 +10552,10 @@ paths: required: *ref_97 is_done: type: boolean + file_key: + type: string required: + - file_key - upload_id - parts - is_done diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index ef3d0330ff..fe79335c67 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -7070,6 +7070,8 @@ paths: properties: file_key: type: string + file_extension: + type: string part_content: type: array items: @@ -7087,8 +7089,7 @@ paths: s3_resource_path: type: string required: - - file_key - - part_content_base64 + - part_content - parts - is_final - cancel_upload @@ -7108,7 +7109,10 @@ paths: $ref: "#/components/schemas/UploadFilePart" is_done: type: boolean + file_key: + type: string required: + - file_key - upload_id - parts - is_done diff --git a/backend/windmill-api/src/job_helpers.rs b/backend/windmill-api/src/job_helpers.rs index 37aa41655a..de6b42618d 100644 --- a/backend/windmill-api/src/job_helpers.rs +++ b/backend/windmill-api/src/job_helpers.rs @@ -1,3 +1,4 @@ +use std::time::{SystemTime, UNIX_EPOCH}; use std::{cmp, time::Duration}; use crate::{db::DB, resources::get_resource_value_interpolated_internal, users::Tokened}; @@ -741,7 +742,8 @@ async fn move_s3_file( #[derive(Deserialize)] struct UploadFileQuery { - pub file_key: String, + pub file_key: Option, // if none, the file will be placed in windmill_uploads/ with a random name. + pub file_extension: Option, // preferred extension for the file in case a random name has to be generated pub part_content: Vec, pub upload_id: Option, // should be None for the first call to initiate the upload @@ -764,6 +766,7 @@ struct UploadFileResponse { pub upload_id: String, pub parts: Vec, // parts already uploaded, with their part_number and the tag associated pub is_done: bool, // whether the transfer is finished, either b/c it got cancelled or because the last chunk was uploaded + pub file_key: String, } async fn multipart_upload_s3_file( @@ -779,7 +782,23 @@ async fn multipart_upload_s3_file( query.parts.len(), query.is_final ); - let file_key = query.file_key.clone(); + let file_key = match query.file_key.clone() { + Some(fk) => fk, + None => { + // for now, we place all files into `windmill_uploads` folder with a random name + // TODO: make the folder configurable via the workspace settings + format!( + "windmill_uploads/upload_{}_{}.{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(), + rand::random::(), + query.file_extension.unwrap_or("file".to_string()) + ) + .to_string() + } + }; let s3_resource_opt = match query.s3_resource_path.clone() { Some(s3_resource_path) => { @@ -816,9 +835,10 @@ async fn multipart_upload_s3_file( error::Error::InternalErr(err.to_string()) })?; return Ok(Json(UploadFileResponse { - upload_id: upload_id, + upload_id, parts: vec![], // empty parts as the transfer has been cancelled is_done: true, + file_key, })); } @@ -885,7 +905,7 @@ async fn multipart_upload_s3_file( let _complete_multipart_upload_res = s3_client .complete_multipart_upload() .bucket(&s3_resource.bucket) - .key(&query.file_key) + .key(&file_key) .upload_id(&upload_id) .multipart_upload( CompletedMultipartUpload::builder() @@ -901,9 +921,10 @@ async fn multipart_upload_s3_file( } return Ok(Json(UploadFileResponse { - upload_id: upload_id, + upload_id, parts: new_parts, is_done: query.is_final, + file_key, })); } diff --git a/docker-compose.yml b/docker-compose.yml index 48322a2145..15956e630f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,8 @@ services: - db_data:/var/lib/postgresql/data expose: - 5432 + ports: + - 5432:5432 environment: POSTGRES_PASSWORD: changeme POSTGRES_DB: windmill diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 2a495f6449..c10a25589e 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -27,6 +27,7 @@ import DateTimeInput from './DateTimeInput.svelte' import S3FilePicker from './S3FilePicker.svelte' import CurrencyInput from './apps/components/inputs/currency/CurrencyInput.svelte' + import FileUpload from './common/fileUpload/FileUpload.svelte' export let label: string = '' export let value: any @@ -78,6 +79,7 @@ let error: string = '' let s3FilePicker: S3FilePicker + let s3FileUploadRawMode: false let el: HTMLTextAreaElement | undefined = undefined @@ -475,26 +477,49 @@ .replace('_', '') .toLowerCase() == 's3object'}
- { - dispatch('focus') - }} - code={JSON.stringify({ s3: '' }, null, 2)} - bind:value - /> - + options={{ left: 'Raw S3 object input' }} + /> + {#if s3FileUploadRawMode} + { + dispatch('focus') + }} + code={JSON.stringify({ s3: '' }, null, 2)} + bind:value + /> + + {:else} + { + value = { + s3: evt.detail?.path ?? '' + } + }} + on:deletion={(evt) => { + value = { + s3: '' + } + }} + /> + {/if}
{:else if inputCat == 'object' || inputCat == 'resource-object'} {#if properties && Object.keys(properties).length > 0} diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index d0300aec9d..27db7026a7 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -11,7 +11,7 @@ MoveRight } from 'lucide-svelte' import { workspaceStore } from '$lib/stores' - import { HelpersService, type UploadFilePart } from '$lib/gen' + import { HelpersService } from '$lib/gen' import { displayDate, displaySize, emptyString, sendUserToast } from '$lib/utils' import { Alert, Button, Drawer } from './common' import DrawerContent from './common/drawer/DrawerContent.svelte' @@ -30,11 +30,6 @@ let fileMoveInProgress = false let uploadModalOpen = false - let fileToUpload: File | undefined = undefined - let fileToUploadKey: string | undefined = undefined - let fileUploadProgress: number | undefined = undefined - let fileUploadCancelled: boolean = false - let fileUploadErrorMsg: string | undefined = undefined let workspaceSettingsInitialized = true @@ -251,81 +246,6 @@ await loadFileMetadataPlusPreviewAsync(selectedFileKey.s3) } - async function uploadFileToS3() { - fileUploadErrorMsg = undefined - if (fileToUpload === undefined || fileToUploadKey === undefined) { - return - } - if (allFilesByKey[fileToUploadKey] !== undefined) { - fileUploadErrorMsg = - 'A file with this name already exists in the S3 bucket. If you want to replace it, delete it first.' - return - } - - let upload_id: string | undefined = undefined - let parts: UploadFilePart[] = [] - - let reader = fileToUpload?.stream().getReader() - let { value: chunk, done: readerDone } = await reader.read() - if (chunk === undefined || readerDone) { - sendUserToast('Error reading file, no data read', true) - return - } - - fileUploadProgress = 0 - while (true) { - let { value: chunk_2, done: readerDone } = await reader.read() - if (!readerDone && chunk_2 !== undefined && chunk.length <= 5 * 1024 * 1024) { - // AWS enforces part to be bigger than 5MB, so we accumulate bytes until we reach that limit before triggering the request to the BE - chunk = new Uint8Array([...chunk, ...chunk_2]) - continue - } - fileUploadProgress += (chunk.length * 100) / fileToUpload.size - let response = await HelpersService.multipartFileUpload({ - workspace: $workspaceStore!, - requestBody: { - file_key: fileToUploadKey, - part_content: Array.from(chunk), - upload_id: upload_id, - parts: parts, - is_final: readerDone, - cancel_upload: fileUploadCancelled - } - }) - upload_id = response.upload_id - parts = response.parts - if (response.is_done) { - if (fileUploadCancelled) { - sendUserToast('File upload cancelled!') - } else { - sendUserToast('File upload finished!') - } - break - } - if (chunk_2 === undefined) { - sendUserToast( - 'File upload is not finished, yet there is no more data to stream. This is unexpected', - true - ) - return - } - chunk = chunk_2 - } - uploadModalOpen = false - - if (!fileUploadCancelled) { - selectedFileKey = { s3: fileToUploadKey } - await loadFiles() - await loadFileMetadataPlusPreviewAsync(selectedFileKey['s3']) - } - - fileToUpload = undefined - fileToUploadKey = undefined - fileUploadProgress = undefined - fileUploadCancelled = false - fileUploadErrorMsg = undefined - } - export async function open(preSelectedFileKey: { s3: string } | undefined = undefined) { if (preSelectedFileKey !== undefined) { initialFileKey = { ...preSelectedFileKey } @@ -701,24 +621,12 @@ { - if (fileUploadProgress !== undefined) { - fileUploadCancelled = true - fileUploadErrorMsg = 'Cancelling in progress, it might take a few seconds...' - } else { - fileUploadErrorMsg = undefined - uploadModalOpen = false + on:close={async (evt) => { + uploadModalOpen = false + if (evt.detail !== undefined && evt.detail !== null) { + selectedFileKey = { s3: evt.detail } + loadFiles() + loadFileMetadataPlusPreviewAsync(evt.detail) } }} - on:confirmed={() => { - uploadFileToS3() - }} - on:close={() => { - fileUploadCancelled = true - uploadModalOpen = false - }} - bind:progressPct={fileUploadProgress} - bind:errorMsg={fileUploadErrorMsg} /> diff --git a/frontend/src/lib/components/apps/components/inputs/AppS3FileInput.svelte b/frontend/src/lib/components/apps/components/inputs/AppS3FileInput.svelte index fca2432ed4..b74f6bd16e 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppS3FileInput.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppS3FileInput.svelte @@ -1,8 +1,5 @@ + +
+ {#if $fileUploads.length > 0 && !forceDisplayUploads} +
+
+ {#each $fileUploads as fileUpload} +
+
+
+ {fileUpload.name} + {`${Math.round((fileUpload.size / 1024 / 1024) * 100) / 100} MB`} +
+
+ {#if fileUpload.errorMessage} + + {:else if fileUpload.cancelled} + + {:else if fileUpload.progress === 100} + + {/if} + + {#if fileUpload.cancelled || fileUpload.errorMessage !== undefined} + + + {/if} + {#if fileUpload.progress < 100 && !fileUpload.cancelled && !fileUpload.errorMessage} + + {/if} + + {#if fileUpload.progress === 100 && !fileUpload.cancelled} + + {/if} +
+
+ + {#if fileUpload.errorMessage} + {fileUpload.errorMessage} + {:else if fileUpload.cancelled} + Upload cancelled + {/if} + + {#if !(fileUpload.cancelled || fileUpload.errorMessage !== undefined)} + + {fileUpload.progress === 100 ? 'Upload finished' : `Uploading`} to path: {fileUpload.path ?? + 'N/A'} + + {/if} +
+ {/each} +
+ {#if allowMultiple} +
+ {#if !$fileUploads.every((fileUpload) => fileUpload.progress === 100 || fileUpload.cancelled)} + + {/if} + +
+ {/if} +
+ {:else} + { + forceDisplayUploads = false + handleChange(detail) + }} + class={twMerge('w-full h-full', customClass, 'wm-file-input')} + style={customStyle} + > + {containerText} + + {/if} +
diff --git a/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte b/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte index b604817ac9..e3fe56fb4e 100644 --- a/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte +++ b/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte @@ -1,23 +1,32 @@ {#if open} @@ -49,8 +58,8 @@ {title} -