From a22dc985fa6f1b49e56fa0f3d4f5152a50e16a04 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 31 May 2024 11:57:41 +0200 Subject: [PATCH] feat: support multiple object storage + parquet_csv + polars -> datafusion" (#3853) * multiple storage * all * rm symlinks * all * all --- backend/ee-repo-ref.txt | 2 +- backend/substitute_ee_code.sh | 2 +- backend/windmill-api/openapi.yaml | 136 +++++++++++++- backend/windmill-api/src/workspaces.rs | 17 +- backend/windmill-common/src/s3_helpers.rs | 1 + backend/windmill-worker/src/common.rs | 105 +++++------ .../lib/components/AppConnectDrawer.svelte | 2 +- .../src/lib/components/AppConnectInner.svelte | 23 +-- .../src/lib/components/DisplayResult.svelte | 59 ++++-- ...r.svelte => ParqetCsvTableRenderer.svelte} | 41 ++++- .../src/lib/components/S3FilePicker.svelte | 28 ++- .../common/fileDownload/FileDownload.svelte | 6 +- .../propertyPicker/ObjectViewer.svelte | 8 +- frontend/src/lib/workspace_settings.ts | 101 ++++++++++ .../(logged)/workspace_settings/+page.svelte | 172 ++++++++++-------- .../routes/(root)/embed_connect/+page.svelte | 2 +- python-client/wmill/wmill/client.py | 4 +- python-client/wmill/wmill/s3_reader.py | 4 +- python-client/wmill/wmill/s3_types.py | 1 + typescript-client/client.ts | 3 + typescript-client/s3Types.ts | 24 +-- 21 files changed, 539 insertions(+), 202 deletions(-) rename frontend/src/lib/components/{ParqetTableRenderer.svelte => ParqetCsvTableRenderer.svelte} (73%) create mode 100644 frontend/src/lib/workspace_settings.ts diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 628f04e29e..14e50aef0a 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -5da40a29e6e4dfd4376fd917807e6936f244f6de \ No newline at end of file +d8aaa3f3a3fc16e41cb4a530611eba78e569923c \ No newline at end of file diff --git a/backend/substitute_ee_code.sh b/backend/substitute_ee_code.sh index d7f98bf0fe..c15d19a259 100755 --- a/backend/substitute_ee_code.sh +++ b/backend/substitute_ee_code.sh @@ -71,7 +71,7 @@ if [ "$REVERT" == "YES" ]; then ce_file="${ee_file/${EE_CODE_DIR}/.}" ce_file="${root_dirpath}/backend/${ce_file}" if [ "$REVERT_PREVIOUS" == "YES" ]; then - git checkout HEAD@{5} ${ce_file} || true + git checkout HEAD@{3} ${ce_file} || true else git restore --staged ${ce_file} || true git restore ${ce_file} || true diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index d41b3c93e1..660fca588d 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -7717,12 +7717,16 @@ paths: /w/{workspace}/job_helpers/test_connection: get: - summary: Test connection to the workspace datasets storage + summary: Test connection to the workspace object storage operationId: datasetStorageTestConnection tags: - helpers parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: storage + in: query + schema: + type: string responses: "200": description: Connection settings @@ -7732,7 +7736,7 @@ paths: /w/{workspace}/job_helpers/list_stored_files: get: - summary: List the file keys available in the workspace files storage (S3) + summary: List the file keys available in a workspace object storage operationId: listStoredFiles tags: - helpers @@ -7751,6 +7755,10 @@ paths: in: query schema: type: string + - name: storage + in: query + schema: + type: string responses: "200": description: List of file keys @@ -7783,6 +7791,10 @@ paths: required: true schema: type: string + - name: storage + in: query + schema: + type: string responses: "200": description: FileMetadata @@ -7828,6 +7840,10 @@ paths: in: query schema: type: integer + - name: storage + in: query + schema: + type: string responses: "200": description: FilePreview @@ -7869,7 +7885,10 @@ paths: in: query schema: type: string - + - name: storage + in: query + schema: + type: string responses: "200": description: Parquet Preview @@ -7877,6 +7896,50 @@ paths: application/json: schema: {} + /w/{workspace}/job_helpers/load_csv_preview/{path}: + get: + summary: Load a preview of a csv file + operationId: loadCsvPreview + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + - name: offset + in: query + schema: + type: number + - name: limit + in: query + schema: + type: number + - name: sort_col + in: query + schema: + type: string + - name: sort_desc + in: query + schema: + type: boolean + - name: search_col + in: query + schema: + type: string + - name: search_term + in: query + schema: + type: string + - name: storage + in: query + schema: + type: string + responses: + "200": + description: Csv Preview + content: + application/json: + schema: {} + /w/{workspace}/job_helpers/delete_s3_file: delete: summary: Permanently delete file from S3 @@ -7890,6 +7953,10 @@ paths: required: true schema: type: string + - name: storage + in: query + schema: + type: string responses: "200": description: Confirmation @@ -7915,6 +7982,10 @@ paths: required: true schema: type: string + - name: storage + in: query + schema: + type: string responses: "200": description: Confirmation @@ -7950,6 +8021,10 @@ paths: required: false schema: type: string + - name: storage + in: query + schema: + type: string requestBody: description: File content required: true @@ -7994,6 +8069,10 @@ paths: required: false schema: type: string + - name: storage + in: query + schema: + type: string responses: "200": description: Chunk of the downloaded file @@ -8003,6 +8082,37 @@ paths: type: string format: binary + /w/{workspace}/job_helpers/download_s3_parquet_file_as_csv: + get: + summary: Download file to S3 bucket + operationId: fileDownloadParquetAsCsv + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: file_key + in: query + required: true + schema: + type: string + - name: s3_resource_path + in: query + required: false + schema: + type: string + - name: resource_type + in: query + required: false + schema: + type: string + responses: + "200": + description: The downloaded file + content: + text/csv: + schema: + type: string + /w/{workspace}/job_metrics/get/{id}: post: summary: get job metrics @@ -10373,6 +10483,26 @@ components: type: string public_resource: type: boolean + secondary_storage: + type: object + additionalProperties: + type: object + properties: + type: + type: string + enum: + [ + "S3Storage", + "AzureBlobStorage", + "AzureWorkloadIdentity", + "S3AwsOidc", + ] + s3_resource_path: + type: string + azure_blob_resource_path: + type: string + public_resource: + type: boolean WindmillLargeFile: type: object diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 8efe435e60..65afd36e31 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -230,9 +230,17 @@ struct EditCopilotConfig { code_completion_enabled: bool, } -#[derive(Deserialize)] +#[derive(Deserialize, Serialize, Debug)] +struct LargeFileStorageWithSecondary { + #[serde(flatten)] + large_file_storage: LargeFileStorage, + + secondary_storage: HashMap, +} + +#[derive(Deserialize, Debug)] struct EditLargeFileStorageConfig { - large_file_storage: Option, + large_file_storage: Option, } #[derive(Deserialize)] @@ -952,8 +960,9 @@ async fn edit_large_file_storage_config( .await?; if let Some(lfs_config) = new_config.large_file_storage { - let serialized_lfs_config = serde_json::to_value::(lfs_config) - .map_err(|err| Error::InternalErr(err.to_string()))?; + let serialized_lfs_config = + serde_json::to_value::(lfs_config) + .map_err(|err| Error::InternalErr(err.to_string()))?; sqlx::query!( "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index fbf33af0e0..c9c2db30db 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -113,6 +113,7 @@ pub struct S3AwsOidcResource { #[derive(Deserialize, Clone)] pub struct S3Object { pub s3: String, + pub storage: Option, } #[cfg(feature = "parquet")] diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 47fd1618c0..3afd40ab6b 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1357,7 +1357,7 @@ pub async fn hash_args( k.hash(&mut dh); let arg_value = hm.get(k).unwrap(); #[cfg(feature = "parquet")] - let arg_additions = + let (_, arg_additions) = arg_value_hash_additions(_db, _client, _workspace_id, hm.get(k).unwrap()).await; arg_value.get().hash(&mut dh); #[cfg(feature = "parquet")] @@ -1376,21 +1376,33 @@ async fn get_workspace_s3_resource_path( db: &DB, client: &AuthedClient, workspace_id: &str, + storage: Option, ) -> windmill_common::error::Result> { use windmill_common::{ job_s3_helpers_ee::get_s3_resource_internal, s3_helpers::StorageResourceType, }; - let raw_lfs_opt = sqlx::query_scalar!( - "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", - workspace_id - ) - .fetch_one(db) - .await - .ok() - .flatten() - .map(|val| serde_json::from_value::(val).ok()) - .flatten(); + let raw_lfs_opt = if let Some(storage) = storage { + sqlx::query_scalar!( + "SELECT large_file_storage->'secondary_storage'->$2 FROM workspace_settings WHERE workspace_id = $1", + workspace_id, + storage + ) + .fetch_optional(db) + .await? + .flatten() + } else { + sqlx::query_scalar!( + "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", + workspace_id + ) + .fetch_optional(db) + .await? + .flatten() + }; + let raw_lfs_opt = raw_lfs_opt + .map(|val| serde_json::from_value::(val).ok()) + .flatten(); let (rt, path) = match raw_lfs_opt { Some(LargeFileStorage::S3Storage(s3_storage)) => { @@ -1440,47 +1452,26 @@ async fn arg_value_hash_additions( client: &AuthedClient, workspace_id: &str, raw_value: &Box, -) -> HashMap { +) -> (Option, HashMap) { let mut result: HashMap = HashMap::new(); - let parsed_s3_values: &mut Vec = &mut Vec::new(); - extract_all_s3_object_from_raw_value(raw_value, parsed_s3_values); - if parsed_s3_values.is_empty() { - // no s3 object, nothing to return - return result; - } - let s3_resource_opt = get_workspace_s3_resource_path(db, client, workspace_id).await; - if let Some(s3_resource) = s3_resource_opt.ok().flatten() { - for s3_object in parsed_s3_values { + let parsed_value = serde_json::from_str::(raw_value.get()); + + let mut storage = None; + if let Ok(s3_object) = parsed_value { + let s3_resource_opt = + get_workspace_s3_resource_path(db, client, workspace_id, s3_object.storage.clone()) + .await; + storage = s3_object.storage.clone(); + + if let Some(s3_resource) = s3_resource_opt.ok().flatten() { let etag = get_etag_or_empty(&s3_resource, s3_object.clone()).await; tracing::warn!("Enriching s3 arg value with etag: {:?}", etag); result.insert(s3_object.s3.clone(), etag.unwrap_or_default()); // TODO: maybe inject a random value to invalidate the cache? } } - return result; -} - -#[cfg(feature = "parquet")] -fn extract_all_s3_object_from_raw_value(raw_value: &Box, result: &mut Vec) { - let parsed_value = serde_json::from_str::(raw_value.get()); - if let Ok(parsed_value) = parsed_value { - result.push(parsed_value); - } else { - let parsed_value = serde_json::from_str::>>(raw_value.get()); - if let Ok(parsed_value) = parsed_value { - for (_, v) in parsed_value { - extract_all_s3_object_from_raw_value(&v, result); - } - } else { - let parsed_value = serde_json::from_str::>>(raw_value.get()); - if let Ok(parsed_value) = parsed_value { - for v in parsed_value { - extract_all_s3_object_from_raw_value(&v, result); - } - } - } - } + return (storage, result); } #[derive(Deserialize, Serialize)] @@ -1489,6 +1480,7 @@ struct CachedResource { #[serde(skip_serializing_if = "Option::is_none")] s3_etags: Option>, value: Box, + storage: Option, } pub async fn get_cached_resource_value_if_valid( @@ -1513,10 +1505,15 @@ pub async fn get_cached_resource_value_if_valid( let object_store_resource_opt: Option = if s3_etags.is_empty() { None } else { - get_workspace_s3_resource_path(_db, &client, _workspace_id) - .await - .ok() - .flatten() + get_workspace_s3_resource_path( + _db, + &client, + _workspace_id, + cached_resource.storage.clone(), + ) + .await + .ok() + .flatten() }; if !s3_etags.is_empty() && object_store_resource_opt.is_none() { @@ -1527,7 +1524,10 @@ pub async fn get_cached_resource_value_if_valid( if let Some(object_store_resource) = object_store_resource_opt.clone() { let etag = get_etag_or_empty( &object_store_resource, - S3Object { s3: s3_file_key.clone() }, + S3Object { + s3: s3_file_key.clone(), + storage: cached_resource.storage.clone(), + }, ) .await; if etag.is_none() || etag.clone().unwrap() != s3_file_etag { @@ -1552,7 +1552,8 @@ pub async fn save_in_cache( let expire = chrono::Utc::now().timestamp() + job.cache_ttl.unwrap() as i64; #[cfg(feature = "parquet")] - let s3_etags = arg_value_hash_additions(db, _client, job.workspace_id.as_str(), r).await; + let (storage, s3_etags) = + arg_value_hash_additions(db, _client, job.workspace_id.as_str(), r).await; #[cfg(feature = "parquet")] let s3_etags = if s3_etags.is_empty() { @@ -1562,9 +1563,9 @@ pub async fn save_in_cache( }; #[cfg(not(feature = "parquet"))] - let s3_etags = None; + let (storage, s3_etags) = (None, None); - let store_cache_resource = CachedResource { expire, s3_etags, value: r.clone() }; + let store_cache_resource = CachedResource { expire, s3_etags, value: r.clone(), storage }; let raw_json = sqlx::types::Json(store_cache_resource); if let Err(e) = sqlx::query!( diff --git a/frontend/src/lib/components/AppConnectDrawer.svelte b/frontend/src/lib/components/AppConnectDrawer.svelte index be1ae483d7..9cbd28edbd 100644 --- a/frontend/src/lib/components/AppConnectDrawer.svelte +++ b/frontend/src/lib/components/AppConnectDrawer.svelte @@ -11,7 +11,7 @@ let step = 1 let disabled = false let isGoogleSignin = false - let manual = false + let manual = true let appConnectInner: AppConnectInner | undefined = undefined diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index 117bee1724..9b6de1cfed 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -29,7 +29,7 @@ export let resourceType = '' export let isGoogleSignin = false export let disabled = false - export let manual = false + export let manual = true let isValid = true @@ -75,24 +75,18 @@ let pathError = '' export async function open(rt?: string) { + if (!rt) { + loadResourceTypes() + } step = 1 value = '' description = '' resourceType = rt ?? '' valueToken = undefined await loadConnects() - - const isConnect = connects?.includes(resourceType) - if (isConnect) { - manual = false + manual = !connects?.includes(resourceType) + if (rt) { next() - } else { - manual = true - if (rt) { - next() - } else { - loadResourceTypes() - } } } @@ -190,10 +184,11 @@ export async function next() { if (step == 1) { if (manual) { - await getResourceTypeInfo() + getResourceTypeInfo() args = {} } else { - await Promise.all([getScopesAndParams(), getResourceTypeInfo()]) + getResourceTypeInfo() + getScopesAndParams() } step += 1 } else if (step == 2 && !manual) { diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index ea0d7af912..d991f3a9e3 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -11,7 +11,8 @@ Table2, Braces, Highlighter, - InfoIcon + InfoIcon, + ArrowDownFromLine } from 'lucide-svelte' import Portal from 'svelte-portal' import ObjectViewer from './propertyPicker/ObjectViewer.svelte' @@ -22,13 +23,14 @@ import Toggle from './Toggle.svelte' import FileDownload from './common/fileDownload/FileDownload.svelte' - import ParqetTableRenderer from './ParqetTableRenderer.svelte' + import ParqetTableRenderer from './ParqetCsvTableRenderer.svelte' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import MapResult from './MapResult.svelte' import Popover from './Popover.svelte' import DownloadCsv from './table/DownloadCsv.svelte' import { convertJsonToCsv } from './table/tableUtils' + import Tooltip from './Tooltip.svelte' export let result: any export let requireHtmlApproval = false @@ -108,7 +110,7 @@ let largeObject: boolean | undefined = undefined function checkIfS3(result: any, keys: string[]) { - return keys.length === 1 && keys.includes('s3') && typeof result.s3 === 'string' + return keys.includes('s3') && typeof result.s3 === 'string' } let is_render_all = false @@ -335,6 +337,8 @@ } let globalForceJson: boolean = false + + let seeS3PreviewFileFromList = '' {#if is_render_all} @@ -554,8 +558,9 @@ class="flex" bind:checked={s3FileDisplayRawMode} size="xs" - options={{ right: 'Raw S3 object input' }} + options={{ right: 'Raw S3 object' }} /> + {#if s3FileDisplayRawMode} - {:else} + {:else if !result?.disable_download} {/if} - {#if typeof result?.s3 == 'string' && result?.s3?.endsWith('.parquet')} - + {#if typeof result?.s3 == 'string' && (result?.s3?.endsWith('.parquet') || result?.s3?.endsWith('.csv'))} + {/if} {:else if !forceJson && resultKind == 's3object-list'}
{#each result as s3object} {#if s3FileDisplayRawMode} @@ -609,8 +625,27 @@ >open preview - {:else} + {:else if !s3object?.disable_download} + {:else} +
{s3object?.s3} (download disabled)
+ {/if} + {#if s3object?.s3?.endsWith('.parquet') || s3object?.s3?.endsWith('.csv')} + {#if seeS3PreviewFileFromList == s3object?.s3} + {:else} + + {/if} {/if} {/each}
diff --git a/frontend/src/lib/components/ParqetTableRenderer.svelte b/frontend/src/lib/components/ParqetCsvTableRenderer.svelte similarity index 73% rename from frontend/src/lib/components/ParqetTableRenderer.svelte rename to frontend/src/lib/components/ParqetCsvTableRenderer.svelte index 778af77de7..164d1bcae0 100644 --- a/frontend/src/lib/components/ParqetTableRenderer.svelte +++ b/frontend/src/lib/components/ParqetCsvTableRenderer.svelte @@ -7,27 +7,38 @@ import DarkModeObserver from './DarkModeObserver.svelte' import { HelpersService } from '$lib/gen' import { enterpriseLicense, workspaceStore } from '$lib/stores' + import { Download } from 'lucide-svelte' // import 'ag-grid-community/dist/styles/ag-theme-alpine-dark.css' let selectedRowIndex = -1 export let s3resource: string + export let storage: string | undefined + export let workspaceId: string | undefined + export let disable_download: boolean = false let datasource: IDatasource = { rowCount: 0, getRows: async function (params) { try { const searchCol = params.filterModel ? Object.keys(params.filterModel)?.[0] : undefined - const res = (await HelpersService.loadParquetPreview({ - workspace: $workspaceStore!, + const requestBody = { + workspace: workspaceId ?? $workspaceStore!, path: s3resource, offset: params.startRow, limit: params.endRow - params.startRow, sortCol: params.sortModel?.[0]?.colId, sortDesc: params.sortModel?.[0]?.sort == 'desc', searchCol: searchCol, - searchTerm: searchCol ? params.filterModel?.[searchCol]?.filter : undefined - })) as any + searchTerm: searchCol ? params.filterModel?.[searchCol]?.filter : undefined, + storage: storage + } + const csv = s3resource.endsWith('.csv') + const res = ( + csv + ? await HelpersService.loadCsvPreview(requestBody) + : await HelpersService.loadParquetPreview(requestBody) + ) as any for (let i = 0; i < res.rows.length; i++) { res.rows[i]['__index'] = i + params.startRow if (!$enterpriseLicense) { @@ -71,7 +82,8 @@ const res = await HelpersService.loadParquetPreview({ workspace: $workspaceStore!, path: s3resource, - limit: 0 + limit: 0, + storage: storage }) createGrid( @@ -124,8 +136,23 @@ -
-
+
+ {#if !disable_download && !s3resource.endsWith('.csv')} +
CSV
 + {/if} + +
diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index 347df44194..8e334136e2 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -116,7 +116,8 @@ workspace: $workspaceStore!, maxKeys: maxKeys, // fixed pages of 1000 files for now marker: page == 0 ? undefined : listMarkers[page - 1], - prefix: filter.trim() != '' ? filter : undefined + prefix: filter.trim() != '' ? filter : undefined, + storage: storage }) if ( availableFiles.restricted_access === null || @@ -195,7 +196,8 @@ fileInfoLoading = true let fileMetadataRaw = await HelpersService.loadFileMetadata({ workspace: $workspaceStore!, - fileKey: fileKey + fileKey: fileKey, + storage: storage }) if (fileMetadataRaw !== undefined) { @@ -221,7 +223,8 @@ csvSeparator: csvSeparatorChar, csvHasHeader: csvHasHeader, readBytesFrom: 0, - readBytesLength: 128 * 1024 // For now static limit of 128Kb per file + readBytesLength: 128 * 1024, // For now static limit of 128Kb per file, + storage: storage }) let filePreviewContent = filePreviewRaw.content @@ -254,7 +257,8 @@ try { await HelpersService.deleteS3File({ workspace: $workspaceStore!, - fileKey: fileKey + fileKey: fileKey, + storage: storage }) } finally { fileDeletionInProgress = false @@ -279,7 +283,8 @@ await HelpersService.moveS3File({ workspace: $workspaceStore!, srcFileKey: srcFileKey, - destFileKey: destFileKey! + destFileKey: destFileKey!, + storage: storage }) } finally { fileMoveInProgress = false @@ -291,7 +296,11 @@ await loadFileMetadataPlusPreviewAsync(selectedFileKey.s3) } - export async function open(preSelectedFileKey: { s3: string } | undefined = undefined) { + let storage: string | undefined = undefined + export async function open( + preSelectedFileKey: { s3: string; storage: string | undefined } | undefined = undefined + ) { + storage = preSelectedFileKey?.storage if (preSelectedFileKey !== undefined) { initialFileKey = { ...preSelectedFileKey } selectedFileKey = { ...preSelectedFileKey } @@ -313,10 +322,13 @@ initialFileKeyInternalCopy = { ...initialFileKey } } try { - await HelpersService.datasetStorageTestConnection({ workspace: $workspaceStore! }) + await HelpersService.datasetStorageTestConnection({ + workspace: $workspaceStore!, + storage: storage + }) workspaceSettingsInitialized = true } catch (e) { - console.error('Workspace not connected to S3 bucket: ', e) + console.error('Workspace not connected to object storage: ', e) workspaceSettingsInitialized = false return } diff --git a/frontend/src/lib/components/common/fileDownload/FileDownload.svelte b/frontend/src/lib/components/common/fileDownload/FileDownload.svelte index 6847026604..6ef40e1d3c 100644 --- a/frontend/src/lib/components/common/fileDownload/FileDownload.svelte +++ b/frontend/src/lib/components/common/fileDownload/FileDownload.svelte @@ -10,9 +10,11 @@ border border-dashed border-gray-400 hover:border-blue-500 focus-within:border-blue-500 hover:bg-blue-50 dark:hover:bg-frost-900 focus-within:bg-blue-50 duration-200 rounded-lg p-1 gap-2" - href={`/api/w/${$workspaceStore}/job_helpers/download_s3_file?file_key=${s3object?.s3}`} + href={`/api/w/${$workspaceStore}/job_helpers/download_s3_file?file_key=${s3object?.s3}${ + s3object?.storage ? `&storage=${s3object.storage}` : '' + }`} download={s3object?.s3.split('/').pop() ?? 'unnamed_download.file'} > - s3://{s3object.s3} + s3://{s3object.s3} {s3object.storage ? ` (${s3object.storage})` : ''} diff --git a/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte b/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte index 76e3a06d6a..411e7112fd 100644 --- a/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte +++ b/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte @@ -109,7 +109,9 @@
{/if} +
+
+ {#each s3ResourceSettings.secondaryStorage ?? [] as secondaryStorage, idx} +
+ + + + + +
+ {/each} +
+ + + Secondary storage is a feature that allows you to read and write from storage that + isn't your main storage by specifying it in the s3 object as "secondary_storage" + with the name of it + +
+
+