From 9c2125bb89ca2a491b5dbe21e5868b03b2c5b366 Mon Sep 17 00:00:00 2001 From: Guillaume Bouvignies Date: Fri, 24 Nov 2023 20:18:02 +0100 Subject: [PATCH] fix: S3 bucket browser small improvements and fixes (#2700) * fix: S3 bucket browser small improvements and fixes * Bump file limits to 20 in CE and add a message in the FE --- backend/windmill-api/src/job_helpers.rs | 14 +++++--- .../src/lib/components/S3FilePicker.svelte | 34 +++++++++++++++---- .../(logged)/workspace_settings/+page.svelte | 6 ++++ 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/backend/windmill-api/src/job_helpers.rs b/backend/windmill-api/src/job_helpers.rs index de2c1f7932..0bec647e6f 100644 --- a/backend/windmill-api/src/job_helpers.rs +++ b/backend/windmill-api/src/job_helpers.rs @@ -142,7 +142,7 @@ async fn polars_connection_settings( let s3_resource = query.s3_resource; let response = PolarsConnectionSettingsResponse { - endpoint_url: s3_resource.endpoint, + endpoint_url: render_endpoint(&s3_resource), key: s3_resource.access_key, secret: s3_resource.secret_key, use_ssl: s3_resource.use_ssl, @@ -237,15 +237,21 @@ async fn list_stored_files( .collect::>(); #[cfg(not(feature = "enterprise"))] - if stored_datasets.len() > 10 { + if stored_datasets.len() > 20 { return Err(error::Error::ExecutionErr( - "The workspace s3 bucket contains more than 10 files. Consider upgrading to Windmill Enterprise Edition to continue to use this feature, " + "The workspace s3 bucket contains more than 20 files. Consider upgrading to Windmill Enterprise Edition to continue to use this feature." .to_string(), )); } let next_marker = if bucket_objects.is_truncated() { - bucket_objects.next_marker().map(|v| v.to_owned()) + if bucket_objects.next_marker().is_some() { + // some S3 providers returns the next marker for us. If that's the case just re-use it + bucket_objects.next_marker().map(|v| v.to_owned()) + } else { + // others, like AWS, doesn't and implicitly expect users to return the last key of the current page + stored_datasets.last().map(|v| v.s3.clone()) + } } else { None }; diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index 076ac0bc5b..d1c05d5d08 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -60,9 +60,10 @@ async function loadFiles() { let availableFiles = await HelpersService.listStoredFiles({ workspace: $workspaceStore!, - maxKeys: 500, // fixed pages of 500 files for now + maxKeys: 1000, // fixed pages of 1000 files for now marker: paginationMarker }) + for (let file_path of availableFiles.windmill_large_files) { let split_path = file_path.s3.split('/') let parent_path: string | undefined = undefined @@ -70,8 +71,11 @@ let nestingLevel = 0 for (let i = 0; i < split_path.length; i++) { parent_path = current_path - current_path = - current_path === undefined ? split_path[i] : current_path + '/' + split_path[i] + current_path = current_path === undefined ? split_path[i] : current_path + split_path[i] + + if (i < split_path.length - 1) { + current_path += '/' + } nestingLevel = i * 2 if (allFilesByKey[current_path] !== undefined) { @@ -81,17 +85,35 @@ type: i === split_path.length - 1 ? 'leaf' : 'folder', full_key: current_path, display_name: split_path[i], - collapsed: false, + collapsed: true, // folders collapsed by default parentPath: parent_path, nestingLevel: nestingLevel } - displayedFileKeys.push(current_path) + if (i == 0) { + displayedFileKeys.push(current_path) + } } } displayedFileKeys = displayedFileKeys.sort() if (availableFiles.next_marker !== undefined) { paginationMarker = availableFiles.next_marker } + + // before returning, un-collapse the folders containing the selected file (if any) + if (selectedFileKey !== undefined && !emptyString(selectedFileKey.s3)) { + let split_path = selectedFileKey.s3.split('/') + let current_path: string | undefined = undefined + for (let i = 0; i < split_path.length; i++) { + current_path = current_path === undefined ? split_path[i] : current_path + split_path[i] + if (i < split_path.length - 1) { + current_path += '/' + } + let indexOf = displayedFileKeys.indexOf(current_path) + if (indexOf >= 0) { + selectItem(indexOf, true) + } + } + } } async function loadFileMetadataPlusPreviewAsync(fileKey: string | undefined) { @@ -185,7 +207,7 @@ let elt_to_remove = 0 for (let i = index + 1; i < displayedFileKeys.length; i++) { let file_key = displayedFileKeys[i] - if (file_key.startsWith(item_key + '/')) { + if (file_key.startsWith(item_key)) { elt_to_remove += 1 } else { break diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 2a94429d81..a40a1c2d6b 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -583,6 +583,12 @@ {:else if tab == 'windmill_lfs'} + {#if !$enterpriseLicense} + + Windmill S3 bucket browser will not work for buckets containing more than 20 files. + Consider upgrading to Windmill EE to use this feature with large buckets. + + {/if}
{#key s3ResourceInitialPath}