fix: S3 bucket browser small improvements and fixes (#2700)

* fix: S3 bucket browser small improvements and fixes

* Bump file limits to 20 in CE and add a message in the FE

backend/windmill-api/src/job_helpers.rs

@@ -142,7 +142,7 @@ async fn polars_connection_settings(
     let s3_resource = query.s3_resource;
 
     let response = PolarsConnectionSettingsResponse {
-        endpoint_url: s3_resource.endpoint,
+        endpoint_url: render_endpoint(&s3_resource),
         key: s3_resource.access_key,
         secret: s3_resource.secret_key,
         use_ssl: s3_resource.use_ssl,
@@ -237,15 +237,21 @@ async fn list_stored_files(
         .collect::<Vec<WindmillLargeFile>>();
 
     #[cfg(not(feature = "enterprise"))]
-    if stored_datasets.len() > 10 {
+    if stored_datasets.len() > 20 {
         return Err(error::Error::ExecutionErr(
-            "The workspace s3 bucket contains more than 10 files. Consider upgrading to Windmill Enterprise Edition to continue to use this feature, "
+            "The workspace s3 bucket contains more than 20 files. Consider upgrading to Windmill Enterprise Edition to continue to use this feature."
                 .to_string(),
         ));
     }
 
     let next_marker = if bucket_objects.is_truncated() {
-        bucket_objects.next_marker().map(|v| v.to_owned())
+        if bucket_objects.next_marker().is_some() {
+            // some S3 providers returns the next marker for us. If that's the case just re-use it
+            bucket_objects.next_marker().map(|v| v.to_owned())
+        } else {
+            // others, like AWS, doesn't and implicitly expect users to return the last key of the current page
+            stored_datasets.last().map(|v| v.s3.clone())
+        }
     } else {
         None
     };

frontend/src/lib/components/S3FilePicker.svelte

@@ -60,9 +60,10 @@
 	async function loadFiles() {
 		let availableFiles = await HelpersService.listStoredFiles({
 			workspace: $workspaceStore!,
-			maxKeys: 500, // fixed pages of 500 files for now
+			maxKeys: 1000, // fixed pages of 1000 files for now
 			marker: paginationMarker
 		})
+
 		for (let file_path of availableFiles.windmill_large_files) {
 			let split_path = file_path.s3.split('/')
 			let parent_path: string | undefined = undefined
@@ -70,8 +71,11 @@
 			let nestingLevel = 0
 			for (let i = 0; i < split_path.length; i++) {
 				parent_path = current_path
-				current_path =
-					current_path === undefined ? split_path[i] : current_path + '/' + split_path[i]
+				current_path = current_path === undefined ? split_path[i] : current_path + split_path[i]
+
+				if (i < split_path.length - 1) {
+					current_path += '/'
+				}
 
 				nestingLevel = i * 2
 				if (allFilesByKey[current_path] !== undefined) {
@@ -81,17 +85,35 @@
 					type: i === split_path.length - 1 ? 'leaf' : 'folder',
 					full_key: current_path,
 					display_name: split_path[i],
-					collapsed: false,
+					collapsed: true, // folders collapsed by default
 					parentPath: parent_path,
 					nestingLevel: nestingLevel
 				}
-				displayedFileKeys.push(current_path)
+				if (i == 0) {
+					displayedFileKeys.push(current_path)
+				}
 			}
 		}
 		displayedFileKeys = displayedFileKeys.sort()
 		if (availableFiles.next_marker !== undefined) {
 			paginationMarker = availableFiles.next_marker
 		}
+
+		// before returning, un-collapse the folders containing the selected file (if any)
+		if (selectedFileKey !== undefined && !emptyString(selectedFileKey.s3)) {
+			let split_path = selectedFileKey.s3.split('/')
+			let current_path: string | undefined = undefined
+			for (let i = 0; i < split_path.length; i++) {
+				current_path = current_path === undefined ? split_path[i] : current_path + split_path[i]
+				if (i < split_path.length - 1) {
+					current_path += '/'
+				}
+				let indexOf = displayedFileKeys.indexOf(current_path)
+				if (indexOf >= 0) {
+					selectItem(indexOf, true)
+				}
+			}
+		}
 	}
 
 	async function loadFileMetadataPlusPreviewAsync(fileKey: string | undefined) {
@@ -185,7 +207,7 @@
 				let elt_to_remove = 0
 				for (let i = index + 1; i < displayedFileKeys.length; i++) {
 					let file_key = displayedFileKeys[i]
-					if (file_key.startsWith(item_key + '/')) {
+					if (file_key.startsWith(item_key)) {
 						elt_to_remove += 1
 					} else {
 						break

frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte

@@ -583,6 +583,12 @@
 			</div>
 		{:else if tab == 'windmill_lfs'}
 			<PageHeader title="Windmill Large File Storage" primary={false} />
+			{#if !$enterpriseLicense}
+				<Alert type="info" title="S3 storage it limited to 20 files in Windmill CE">
+					Windmill S3 bucket browser will not work for buckets containing more than 20 files.
+					Consider upgrading to Windmill EE to use this feature with large buckets.
+				</Alert>
+			{/if}
 			<div class="mt-5 flex gap-1">
 				{#key s3ResourceInitialPath}
 					<ResourcePicker