diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 25aefda398..acde7f20b5 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14675,6 +14675,7 @@ dependencies = [ "pin-project", "postgres-native-tls 0.5.3", "prometheus", + "quick-xml", "quick_cache", "rand 0.9.0", "regex", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 84d31776ef..02f80b95d2 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -467,6 +467,7 @@ php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb cron = "^0" mail-send = { version = "0.4.0", features = ["builder"], default-features=false } urlencoding = "^2" +quick-xml = { version = "^0.37", features = ["serialize"] } url = { version = "^2" , features = ["serde"]} async-oauth2 = "0.5.1" reqwest = { version = "^0.13", features = ["json", "stream", "gzip", "multipart", "query", "form"] } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index ecd7e0c3e5..c69a2444aa 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -a45bec03922d305aad5893ed354dc029c7f97bb4 +0373b4bfdaf8dd51533552e2e4de63ceb3c18b4d diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index eeca7213c3..4c410b7785 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -18,7 +18,7 @@ agent_worker_server = ["dep:windmill-worker", "dep:windmill-api-agent-workers"] enterprise_saml = ["dep:samael", "dep:libxml"] benchmark = [] embedding = ["windmill-api-embeddings/embedding"] -parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "windmill-api-workspaces/parquet", "dep:aws-sigv4", "dep:aws-sdk-config"] +parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "windmill-api-workspaces/parquet", "dep:aws-sigv4", "dep:aws-sdk-config", "dep:quick-xml"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker?/prometheus", "windmill-api-scripts/prometheus"] openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"] tantivy = ["dep:windmill-indexer"] @@ -128,6 +128,7 @@ hmac.workspace = true cookie.workspace = true sha2.workspace = true urlencoding.workspace = true +quick-xml = { workspace = true, optional = true } lazy_static.workspace = true prometheus = { workspace = true, optional = true } async_zip = { workspace = true, optional = true } diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 244070d6d1..fd40b6fa5e 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -31385,12 +31385,28 @@ "type": "string" } }, + { + "name": "search", + "in": "query", + "description": "Match keys by path prefix, case-sensitively, on the raw key rather than per path segment (so \"a/file1\" matches \"a/file1000\"). Pushed down to the storage provider as a seek; resume with the returned next_marker.", + "schema": { + "type": "string" + } + }, { "name": "storage", "in": "query", "schema": { "type": "string" } + }, + { + "name": "s3_resource_path", + "in": "query", + "description": "When set, list the files of this object storage resource instead of the workspace storage", + "schema": { + "type": "string" + } } ], "responses": { @@ -31424,6 +31440,100 @@ } } }, + "/w/{workspace}/job_helpers/list_stored_files_paged": { + "get": { + "summary": "List one page of a single folder level in a workspace object storage", + "operationId": "listStoredFilesPaged", + "tags": [ + "helpers" + ], + "parameters": [ + { + "$ref": "#/components/parameters/WorkspaceId" + }, + { + "name": "prefix", + "in": "query", + "description": "Folder to list; empty for the bucket root, otherwise must end with '/'", + "schema": { + "type": "string" + } + }, + { + "name": "max_keys", + "in": "query", + "description": "Maximum number of folders and files combined", + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 1000, + "default": 100 + } + }, + { + "name": "page_token", + "in": "query", + "description": "Opaque token from a previous response, to continue listing the same folder", + "schema": { + "type": "string" + } + }, + { + "name": "storage", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "s3_resource_path", + "in": "query", + "description": "When set, list the files of this object storage resource instead of the workspace storage", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "One page of folders and files at this level", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "folders": { + "type": "array", + "items": { + "$ref": "#/components/schemas/StorageFolder" + } + }, + "files": { + "type": "array", + "items": { + "$ref": "#/components/schemas/StorageFile" + } + }, + "next_page_token": { + "type": "string", + "description": "When set, more entries remain at this level" + }, + "restricted_access": { + "type": "boolean" + } + }, + "required": [ + "folders", + "files", + "restricted_access" + ] + } + } + } + } + } + } + }, "/w/{workspace}/job_helpers/load_file_metadata": { "get": { "summary": "Load metadata of the file", @@ -48450,6 +48560,44 @@ "required": [ "type" ] + }, + "StorageFolder": { + "type": "object", + "properties": { + "prefix": { + "type": "string", + "description": "Full key prefix of the folder, ending with '/'" + }, + "name": { + "type": "string", + "description": "Last path segment, without the trailing '/'" + } + }, + "required": [ + "prefix", + "name" + ] + }, + "StorageFile": { + "type": "object", + "properties": { + "key": { + "type": "string" + }, + "name": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "last_modified": { + "type": "string" + } + }, + "required": [ + "key", + "name" + ] } } } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 4c6214025a..d480bfdd23 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -21606,6 +21606,11 @@ paths: in: query schema: type: string + - name: search + in: query + description: Match keys by path prefix, case-sensitively, on the raw key rather than per path segment (so "a/file1" matches "a/file1000"). Pushed down to the storage provider as a seek; resume with the returned next_marker. + schema: + type: string - name: storage in: query schema: @@ -21634,6 +21639,67 @@ paths: required: - windmill_large_files + /w/{workspace}/job_helpers/list_stored_files_paged: + get: + summary: List one page of a single folder level in a workspace object storage + operationId: listStoredFilesPaged + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: prefix + in: query + description: Folder to list; empty for the bucket root, otherwise must end with '/' + schema: + type: string + - name: max_keys + in: query + description: Maximum number of folders and files combined + schema: + type: integer + minimum: 1 + maximum: 1000 + default: 100 + - name: page_token + in: query + description: Opaque token from a previous response, to continue listing the same folder + schema: + type: string + - name: storage + in: query + schema: + type: string + - name: s3_resource_path + in: query + description: When set, list the files of this object storage resource instead of the workspace storage + schema: + type: string + responses: + "200": + description: One page of folders and files at this level + content: + application/json: + schema: + type: object + properties: + folders: + type: array + items: + $ref: "#/components/schemas/StorageFolder" + files: + type: array + items: + $ref: "#/components/schemas/StorageFile" + next_page_token: + type: string + description: When set, more entries remain at this level + restricted_access: + type: boolean + required: + - folders + - files + - restricted_access + /w/{workspace}/job_helpers/load_file_metadata: get: summary: Load metadata of the file @@ -31994,6 +32060,34 @@ components: required: - s3 + StorageFolder: + type: object + properties: + prefix: + type: string + description: Full key prefix of the folder, ending with '/' + name: + type: string + description: Last path segment, without the trailing '/' + required: + - prefix + - name + + StorageFile: + type: object + properties: + key: + type: string + name: + type: string + size: + type: integer + last_modified: + type: string + required: + - key + - name + WindmillFileMetadata: type: object properties: diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index ee8906975d..3e63831828 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -106,6 +106,10 @@ mod runnables; #[cfg(all(feature = "private", feature = "parquet"))] pub mod s3_proxy_ee; mod s3_proxy_oss; +#[cfg(all(feature = "private", feature = "parquet"))] +pub mod storage_list_ee; +#[cfg(feature = "parquet")] +mod storage_list_oss; mod workspace_dependencies; mod approvals; diff --git a/backend/windmill-api/src/storage_list_oss.rs b/backend/windmill-api/src/storage_list_oss.rs new file mode 100644 index 0000000000..326071d41f --- /dev/null +++ b/backend/windmill-api/src/storage_list_oss.rs @@ -0,0 +1,6 @@ +// OSS stub for paged object storage listing +// The actual implementation is in storage_list_ee.rs (Enterprise Edition) + +#[cfg(all(feature = "private", feature = "parquet"))] +#[allow(unused)] +pub use crate::storage_list_ee::*; diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index 9121f6183d..2482cf14dd 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -355,10 +355,41 @@ pub async fn attempt_fetch_bytes( return Ok(bytes); } +/// Whether an S3 resource carries static credentials. When it does not, the +/// ambient AWS chain (env, profile, ECS/EC2 instance role) is used instead. +/// Shared so callers that sign requests by hand resolve credentials on exactly the +/// same condition as `build_s3_client`. +#[cfg(feature = "parquet")] +pub fn s3_resource_has_static_credentials(s3_resource: &S3Resource) -> bool { + s3_resource.access_key.as_ref().is_some_and(|x| x != "") + || s3_resource.secret_key.as_ref().is_some_and(|x| x != "") +} + +/// Ambient AWS credentials from the shared, cached provider backing +/// `build_s3_client`. Callers that sign their own requests must go through this +/// rather than resolving the default chain themselves: the cache is what keeps a +/// burst of requests from hitting the instance metadata service once each. +/// +/// These are the **instance's own** credentials, not any caller's, and they are +/// returned in the clear. A caller therefore MUST: +/// - authorize the request target itself — reaching this function implies no +/// permission check, and the credentials typically outrank the requesting user; +/// - use them only to sign a request it has already authorized, never surface them +/// in a response, log, or error message, and never hand them to a caller-supplied +/// endpoint. +/// +/// Prefer `build_s3_client`, which confines them to the object-store client; reach +/// for this only where a request must be signed by hand. +#[cfg(feature = "parquet")] +pub async fn ambient_aws_credentials( + region: &str, +) -> anyhow::Result { + ambient_aws_credentials_provider(region).await.get().await +} + #[cfg(feature = "parquet")] pub async fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result> { - let static_creds = s3_resource_ref.access_key.as_ref().is_some_and(|x| x != "") - || s3_resource_ref.secret_key.as_ref().is_some_and(|x| x != ""); + let static_creds = s3_resource_has_static_credentials(s3_resource_ref); let credentials_provider = if !static_creds { Some(ambient_aws_credentials_provider(&s3_resource_ref.region).await) diff --git a/frontend/src/lib/components/GitRepoViewer.svelte b/frontend/src/lib/components/GitRepoViewer.svelte index 69fd651c3b..8383748e5f 100644 --- a/frontend/src/lib/components/GitRepoViewer.svelte +++ b/frontend/src/lib/components/GitRepoViewer.svelte @@ -319,6 +319,7 @@ readOnlyMode hideS3SpecificDetails rootPath={`gitrepos/${ws}/${gitRepoResourcePath}/${commitHash}/`} + lazyFolders={false} listStoredFilesRequest={HelpersService.listGitRepoFiles} loadFilePreviewRequest={HelpersService.loadGitRepoFilePreview} testConnectionRequest={(async (_d) => { diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index 9f702a53c3..5006e2df0b 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -18,6 +18,11 @@ selectedFileKey?: { s3: string; storage?: string } | undefined folderOnly?: boolean regexFilter?: RegExp | undefined + /** + * Expand one folder level at a time. Turn off to keep listing every key up + * front, which is what lets `regexFilter` prune folders with no match. + */ + lazyFolders?: boolean /** Workspace to browse S3 storage in — the acting workspace of the editor that * opened the picker, else the nav workspace. */ workspace?: string | undefined @@ -33,6 +38,7 @@ selectedFileKey = $bindable(undefined), folderOnly = false, regexFilter = undefined, + lazyFolders = true, workspace = undefined, onClose, onSelectAndClose @@ -125,6 +131,7 @@ {s3ResourcePath} {folderOnly} {regexFilter} + {lazyFolders} workspace={effectiveWorkspace} /> {#snippet actions()} diff --git a/frontend/src/lib/components/S3FilePickerInner.svelte b/frontend/src/lib/components/S3FilePickerInner.svelte index 8c6ee879fa..5737744034 100644 --- a/frontend/src/lib/components/S3FilePickerInner.svelte +++ b/frontend/src/lib/components/S3FilePickerInner.svelte @@ -3,6 +3,7 @@ File as FileIcon, FolderClosed, FolderOpen, + ChevronDown, RotateCw, Loader2, Download, @@ -18,6 +19,8 @@ type DeleteS3FileData, type DeleteS3FileResponse, type ListStoredFilesData, + type ListStoredFilesPagedData, + type ListStoredFilesPagedResponse, type ListStoredFilesResponse, type LoadFileMetadataData, type LoadFileMetadataResponse, @@ -86,7 +89,16 @@ > allowDelete?: boolean replaceUnauthorizedWarning?: Snippet + /** + * Expand one folder level at a time instead of listing every key up front. + * Callers that override `listStoredFilesRequest` with a listing that has no + * paged counterpart (e.g. git repo files) must turn this off. + */ + lazyFolders?: boolean listStoredFilesRequest?: (d: ListStoredFilesData) => CancelablePromise + listStoredFilesPagedRequest?: ( + d: ListStoredFilesPagedData + ) => CancelablePromise loadFilePreviewRequest?: (d: LoadFilePreviewData) => CancelablePromise loadFileMetadataRequest?: ( d: LoadFileMetadataData @@ -115,7 +127,9 @@ allFilesByKey = $bindable({}), allowDelete = false, replaceUnauthorizedWarning, + lazyFolders = true, listStoredFilesRequest = HelpersService.listStoredFiles, + listStoredFilesPagedRequest = HelpersService.listStoredFilesPaged, loadFilePreviewRequest = HelpersService.loadFilePreview, loadFileMetadataRequest = HelpersService.loadFileMetadata, deleteS3FileRequest = HelpersService.deleteS3File, @@ -163,13 +177,26 @@ } | undefined = $state(undefined) + /** Identifies the metadata request that currently owns the preview pane. */ + let metadataRequestId = 0 + + /** + * Flat pagination cursor: `listMarkers[n]` is where page `n + 1` resumes, so `page` + * may only advance over a page that actually loaded. Running ahead of `listMarkers` + * sends no marker and silently replays the first page, and never recovers, because + * the `listMarkers.length == page` guard stops recording from then on. + */ let listMarkers: string[] let page = $state(0) const maxKeys = 1000 + /** Entries fetched per folder level before a "Load more" row appears. */ + const pageSize = 500 let count = $state(0) let displayedCount = $state(0) + /** Flat (non-lazy) listing: whether the last page came back full. */ + let flatHasMore = $state(false) let filter = $state('') @@ -187,17 +214,356 @@ } } + /** + * Marks the synthetic "Load more" row belonging to a folder. Appended to the + * folder's own prefix so the plain lexicographic sort that orders the tree also + * places the row last among that folder's children. + */ + const LOAD_MORE_SUFFIX = '￿' + + type FolderState = { nextPageToken?: string; loading: boolean; loaded: boolean } + let folderState: Record = $state({}) + + /** Per-level listing only makes sense while browsing; searching stays flat. */ + let lazyMode = $derived(lazyFolders && filter.trim() === '') + + function nestingLevelOf(key: string): number { + const slashes = (key.match(/\//g) ?? []).length + return (key.endsWith('/') ? slashes - 1 : slashes) * 2 + } + + function parentPathOf(key: string): string | undefined { + const body = key.endsWith('/') ? key.slice(0, -1) : key + const idx = body.lastIndexOf('/') + return idx === -1 ? undefined : body.slice(0, idx + 1) + } + + /** The parentPath value carried by entries sitting at the browsing root. */ + let rootParentPath = $derived(rootPath === '' ? undefined : rootPath) + + function addEntry(key: string, type: 'folder' | 'leaf', displayName: string) { + if (allFilesByKey[key] !== undefined) return + allFilesByKey[key] = { + type, + full_key: key, + display_name: displayName, + collapsed: true, + parentPath: parentPathOf(key), + nestingLevel: nestingLevelOf(key), + count: 0 + } + } + + /** + * A level shows only when every folder above it is expanded. The browsing root is + * spelled `rootPath` by `folderState` but `undefined` by an entry's `parentPath` + * (there is no parent entry), so both spellings must resolve here — otherwise the + * root's own "Load more" row is filtered out and the top level is stuck on one page. + */ + function isLevelVisible(prefix: string | undefined): boolean { + if (prefix === rootParentPath || prefix === rootPath) return true + if (prefix === undefined) return false + const info = allFilesByKey[prefix] + if (info === undefined || info.collapsed) return false + return isLevelVisible(info.parentPath) + } + + /** + * Every key whose ancestors are all expanded, in depth-first order — full keys + * sort that way because a child always starts with its parent's + * delimiter-terminated prefix. + */ + function computeVisibleKeys(): string[] { + const visible: string[] = [] + for (const key in allFilesByKey) { + if (!key.startsWith(rootPath)) continue + if (isLevelVisible(allFilesByKey[key].parentPath)) visible.push(key) + } + for (const prefix in folderState) { + if (folderState[prefix].nextPageToken && isLevelVisible(prefix)) { + visible.push(prefix + LOAD_MORE_SUFFIX) + } + } + return visible.sort() + } + + function refreshDisplayed() { + const visible = computeVisibleKeys() + displayedFileKeys = visible + displayedCount = visible.filter((k) => !k.endsWith(LOAD_MORE_SUFFIX)).length + } + + /** + * In-flight request per level. Callers that await `loadFolderPage` need the data + * to be there when it resolves; returning early on a concurrent load would hand + * them a resolved promise and no entries. + */ + let inFlightFolderLoads: Record> = {} + + /** + * Bumped when a single level is invalidated on its own (a delete refetches it from + * page one). Requests are keyed by prefix alone, so without this a pending "Load + * more" for that level would be joined by the refetch, which would then return + * believing page one had been fetched — leaving the level showing only its later + * pages until a full reload. + */ + let folderEpoch: Record = {} + + /** + * Bumped whenever the listing is thrown away (storage switch, filter change, + * reload). A request started before the bump belongs to the previous listing, so + * its response must not repopulate the cleared state — otherwise switching + * storage mid-load leaves the previous bucket's entries on screen. + */ + let listingGeneration = 0 + + async function loadFolderPage(prefix: string, append: boolean = false): Promise { + const generation = listingGeneration + const epoch = folderEpoch[prefix] ?? 0 + const pending = inFlightFolderLoads[prefix] + if (pending) { + await pending + // Only reuse the joined result if it belongs to the same listing *and* the + // same epoch of this level. + if (generation === listingGeneration && epoch === (folderEpoch[prefix] ?? 0) && !append) + return + } + const run = loadFolderPageInner(prefix, append) + inFlightFolderLoads[prefix] = run.catch(() => {}) + try { + await run + } finally { + delete inFlightFolderLoads[prefix] + } + } + + async function loadFolderPageInner(prefix: string, append: boolean) { + const generation = listingGeneration + const epoch = folderEpoch[prefix] ?? 0 + /** Whether this response still belongs to the listing and level that asked for it. */ + const stillCurrent = () => + generation === listingGeneration && epoch === (folderEpoch[prefix] ?? 0) + const current = folderState[prefix] ?? { loading: false, loaded: false } + if (append && current.nextPageToken === undefined) return + folderState[prefix] = { ...current, loading: true } + try { + const page = await listStoredFilesPagedRequest({ + workspace: ws!, + prefix, + maxKeys: pageSize, + pageToken: append ? current.nextPageToken : undefined, + storage, + s3ResourcePath + }) + // This listing, or just this level, has been thrown away since the request + // went out. Writing the entries back would resurrect the previous storage's + // contents, or reinstate a file that was just deleted. + if (!stillCurrent()) return + // Absent counts as restricted, matching `loadFlatFiles`: the two paths must + // not disagree on which way an omitted value falls. + if ( + page.restricted_access === null || + page.restricted_access === undefined || + page.restricted_access === true + ) { + fileListUnavailable = true + folderState[prefix] = { loading: false, loaded: true } + return + } + fileListUnavailable = false + for (const folder of page.folders ?? []) { + addEntry(folder.prefix, 'folder', folder.name) + } + for (const file of page.files ?? []) { + if (regexFilter && !regexFilter.test(file.key)) continue + addEntry(file.key, 'leaf', file.name) + } + folderState[prefix] = { + loading: false, + loaded: true, + // An exhausted level serializes as JSON `null`, which is not `undefined` + // — leaving it as-is makes "no more pages" look like another page and + // sends the level round again with no token. + nextPageToken: page.next_page_token ?? undefined + } + } catch (e) { + if (stillCurrent()) { + folderState[prefix] = { ...current, loading: false } + } + throw e + } finally { + // A response that lands after the user typed a filter must not rebuild the + // list: flat mode owns `displayedFileKeys` then, and rebuilding it under + // the lazy visibility rules would prune most of the search results. + if (lazyMode && stillCurrent()) { + refreshDisplayed() + } + } + } + + /** Surface listing failures instead of leaving a folder looking empty. */ + function reportFolderError(prefix: string, e: unknown) { + console.error('Error listing folder', prefix, e) + sendUserToast(`Could not list ${prefix || 'the bucket root'}`, true) + } + + function expandFolder(prefix: string) { + loadFolderPage(prefix).catch((e) => reportFolderError(prefix, e)) + } + + function loadMore(prefix: string) { + loadFolderPage(prefix, true).catch((e) => reportFolderError(prefix, e)) + } + + /** + * Bound on the pages fetched while hunting for one entry, so a preselected key + * that no longer exists cannot walk an entire bucket. + */ + const MAX_PAGES_WHILE_REVEALING = 20 + + /** + * Page through `parent` until `child` shows up. A single page is not enough: the + * target can sort past the first page of its own folder, and giving up there + * leaves the selection looking absent. + */ + async function revealChild(parent: string, child: string, generation: number) { + for (let fetched = 0; fetched < MAX_PAGES_WHILE_REVEALING; fetched++) { + if (allFilesByKey[child] !== undefined) return + const state = folderState[parent] + if (state === undefined || !state.loaded) { + await loadFolderPage(parent) + } else if (state.nextPageToken !== undefined) { + await loadFolderPage(parent, true) + } else { + return + } + if (generation !== listingGeneration) return + } + } + + /** + * Reveal a key by loading each folder above it in turn — with per-level listing + * an ancestor's children are not known until that level is fetched. + * + * `generation` is the listing this walk belongs to. A reveal is a chain of round + * trips, so the check has to be repeated after every one of them: a filter typed + * mid-walk switches the picker to the search, and the loads that follow would date + * themselves to that listing and graft browse results onto it. + */ + async function expandToKey(key: string, generation: number) { + if (!key.startsWith(rootPath)) return + const rest = key.slice(rootPath.length).split('/') + let prefix = rootPath + for (let i = 0; i < rest.length - 1; i++) { + const child = prefix + rest[i] + '/' + await revealChild(prefix, child, generation) + if (generation !== listingGeneration) return + prefix = child + const info = allFilesByKey[prefix] + // The folder is genuinely absent; deeper levels cannot exist either. + if (info === undefined) break + info.collapsed = false + } + if (allFilesByKey[key] === undefined) { + await revealChild(prefix, key, generation) + if (generation !== listingGeneration) return + } + refreshDisplayed() + } + let lastKeyFolders: string[] = $state([]) - async function loadFiles() { + + /** Reports whether the listing completed, so a caller that moved the cursor can undo it. */ + async function loadFiles(): Promise { + if (lazyMode) { + // Typing in the filter switches the picker to the flat listing while this is in + // flight, and `loadFolderPage` resolves rather than throwing once superseded. + // What follows belongs to the listing that started it: expanding a preselected + // file would graft browse results into the search, and clearing the flags would + // retire the search's spinner. + const generation = listingGeneration + fileListLoading = true + try { + await loadFolderPage(rootPath) + if ( + generation === listingGeneration && + selectedFileKey !== undefined && + !emptyString(selectedFileKey.s3) + ) { + await expandToKey(selectedFileKey.s3, generation) + } + } catch (e) { + // `reloadContent` is called un-awaited from `open()`, so without this a + // failing root listing surfaces as an unhandled rejection and an empty + // tree indistinguishable from an empty bucket. + reportFolderError(rootPath, e) + return false + } finally { + if (generation === listingGeneration) { + fileListLoading = false + fileInfoLoading = false + } + } + return true + } + // Same contract as the lazy branch above: every caller here is un-awaited, so an + // uncaught rejection would leave the drawer on a spinner with nothing said. The + // generation guard keeps a superseded listing from clearing the spinner of the one + // that replaced it — `loadFlatFiles` returns early rather than throwing in that case. + const generation = listingGeneration + fileListLoading = true + try { + await loadFlatFiles() + } catch (e) { + console.error('Error listing files', e) + sendUserToast('Could not list the files', true) + // Nothing will load a preview now, so the right-hand pane has to stop waiting + // too. On the other exits it is owned by whoever is still fetching metadata. + if (generation === listingGeneration) { + fileInfoLoading = false + } + return false + } finally { + if (generation === listingGeneration) { + fileListLoading = false + } + } + return true + } + + /** Flat "Load more" / "Keep looking": a page that fails has to give the cursor back. */ + async function loadNextFlatPage() { + // The page number alone does not identify our advance: a filter or storage change + // resets the cursor, and the replacement listing can reach the same number before + // this request fails. Rolling that one back would strand *its* cursor instead. + const generation = listingGeneration + page += 1 + const requested = page + const ok = await loadFiles() + // Only undo our own advance: another click may have moved it on meanwhile. + if (!ok && generation === listingGeneration && page === requested) { + page = requested - 1 + } + } + + async function loadFlatFiles() { + // Debounced searches overlap: an older response must not add its keys to a newer + // search's results or overwrite its pagination state. + const generation = listingGeneration fileListLoading = true let availableFiles = await listStoredFilesRequest({ workspace: ws!, maxKeys: maxKeys, // fixed pages of 1000 files for now marker: page == 0 ? undefined : listMarkers[page - 1], - prefix: rootPath ?? (filter.trim() != '' ? filter : undefined), + // `prefix` is evaluated per path *segment* by the storage layer, so sending the + // query there matched only whole folder names. `search` matches the raw key + // prefix instead, which is what the box means. + prefix: rootPath !== '' ? rootPath : undefined, + search: filter.trim() !== '' ? filter.trim() : undefined, storage: storage, s3ResourcePath }) + if (generation !== listingGeneration) return if ( availableFiles.restricted_access === null || availableFiles.restricted_access === undefined || @@ -212,7 +578,12 @@ if (regexFilter && !regexFilter.test(file_path.s3)) { continue } - displayedCount += 1 + // Only count keys not already in the tree: a page can legitimately repeat + // entries, and counting them again makes the total climb while the list + // stays put. + if (allFilesByKey[file_path.s3] === undefined) { + displayedCount += 1 + } let split_path = file_path.s3.split('/') let parent_path: string | undefined = undefined let current_path: string | undefined = undefined @@ -249,9 +620,19 @@ } } } + // A short page means the listing is exhausted. Deriving "there is more" from + // `count % maxKeys` instead would keep offering another page whenever the total + // happens to be an exact multiple of the page size. + // Searching returns an explicit cursor because it skips over keys that did not + // match, so the last *returned* key is not where the next page resumes. + const serverMarker = availableFiles.next_marker ?? undefined + flatHasMore = + serverMarker !== undefined || + (filter.trim() === '' && availableFiles.windmill_large_files.length === maxKeys) if (listMarkers.length == page) { count += availableFiles.windmill_large_files.length const nextMarker = + serverMarker ?? availableFiles.windmill_large_files?.[availableFiles.windmill_large_files.length - 1]?.s3 if (nextMarker) listMarkers.push(nextMarker) } @@ -277,7 +658,12 @@ } } } - displayedFileKeys = [...new Set(displayedFileKeys)].sort() + // The loop above only lists entries at the browsing root, so a later page's + // keys land in `allFilesByKey` without ever being displayed — the folder they + // belong to is already expanded and nothing re-scans it. Recomputing from the + // expansion state picks them up without needing a collapse/expand round trip. + // `displayedCount` is left alone: in this mode it counts files loaded, not rows shown. + displayedFileKeys = computeVisibleKeys() fileListLoading = false fileInfoLoading = false } @@ -287,14 +673,36 @@ fileInfoLoading = false return } + // The pane belongs to the newest request, not to a key: switching storage reloads + // the same key, so comparing keys would let an older request speak for a newer one. + const requestId = ++metadataRequestId fileInfoLoading = true - let fileMetadataRaw = await loadFileMetadataRequest({ - workspace: ws!, - fileKey: fileKey, - storage: storage, - s3ResourcePath - }) + let fileMetadataRaw: LoadFileMetadataResponse + try { + fileMetadataRaw = await loadFileMetadataRequest({ + workspace: ws!, + fileKey: fileKey, + storage: storage, + s3ResourcePath + }) + } catch (e) { + // Every caller invokes this un-awaited, so a key that no longer exists would + // otherwise leave the preview pane on "Loading..." forever. + console.error('Error loading metadata for', fileKey, e) + // Unless a later request has taken the pane over: it will report its own + // outcome, including the loading flag, and blanking here would undo it. + if (requestId !== metadataRequestId) { + return + } + fileMetadata = undefined + filePreview = undefined + fileInfoLoading = false + return + } + if (requestId !== metadataRequestId) { + return + } if (fileMetadataRaw !== undefined) { fileMetadata = { fileKey: fileKey, @@ -305,10 +713,15 @@ } } // async call - loadFilePreview(fileKey, fileMetadataRaw.size_in_bytes, fileMetadataRaw.mime_type) + loadFilePreview(fileKey, requestId, fileMetadataRaw.size_in_bytes, fileMetadataRaw.mime_type) } - async function loadFilePreview(fileKey: string, fileSizeInBytes?: number, fileMimeType?: string) { + async function loadFilePreview( + fileKey: string, + requestId: number, + fileSizeInBytes?: number, + fileMimeType?: string + ) { let filePreviewRaw = await loadFilePreviewRequest({ workspace: ws!, fileKey: fileKey, @@ -333,6 +746,9 @@ '\n\n ... FILE CONTENT TRUNCATED ...\n\n' } + if (requestId !== metadataRequestId) { + return + } if (filePreviewRaw !== undefined) { filePreview = { fileKey: fileKey, @@ -371,11 +787,45 @@ } sendUserToast(`${fileKey} deleted from S3 bucket`) selectedFileKey = { s3: '', storage } + // The preview pane and its toolbar render from these rather than from the + // selection, so the deleted file stays previewed — and offers to download, move + // and delete itself — until they are cleared too. + fileMetadata = undefined + filePreview = undefined + // A metadata load still in flight belongs to the file just deleted; retire it, or + // its response repopulates the pane it was cleared from. Retiring it also means + // nobody is left to report its outcome, so the pane's loading flag is ours. + metadataRequestId += 1 + fileInfoLoading = false + if (lazyMode) { + // Only the level the file lived in changed; refetch it from its first page + // and keep the rest of the expanded tree. Its already-fetched entries have + // to go with the cursor, or the reset cursor would hand out a "Load more" + // for pages that are still displayed. + const parent = parentPathOf(fileKey) ?? rootPath + for (const key of Object.keys(allFilesByKey)) { + if (allFilesByKey[key].parentPath === (parent === '' ? undefined : parent)) { + delete allFilesByKey[key] + } + } + delete folderState[parent] + // Invalidate this level so a pending "Load more" for it is not joined below. + folderEpoch[parent] = (folderEpoch[parent] ?? 0) + 1 + delete inFlightFolderLoads[parent] + await loadFolderPage(parent).catch((e) => reportFolderError(parent, e)) + return + } const currentPage = page - await clearAndLoadFiles() - for (let i = 0; i < currentPage; i++) { - page = i + 1 - await loadFiles() + // Every page here has to land, starting with the fresh first one. + if (await clearAndLoadFiles()) { + for (let i = 0; i < currentPage; i++) { + page = i + 1 + if (!(await loadFiles())) { + // Stop at the last page that actually loaded. + page = i + break + } + } } const fileKeyFolders = fileKey.split('/').slice(0, -1) let current_path: string | undefined = undefined @@ -399,11 +849,20 @@ displayedFileKeys = [...new Set(displayedFileKeys)].sort() } - async function clearAndLoadFiles({ keepFilter }: { keepFilter?: boolean } = {}) { + /** Reports whether the fresh listing loaded, so a caller replaying pages can stop. */ + async function clearAndLoadFiles({ + keepFilter + }: { keepFilter?: boolean } = {}): Promise { + // Anything already in flight belongs to the listing being discarded. + listingGeneration += 1 + inFlightFolderLoads = {} + folderEpoch = {} displayedFileKeys = [] allFilesByKey = {} + folderState = {} count = 0 displayedCount = 0 + flatHasMore = false page = 0 listMarkers = [] fileMetadata = undefined @@ -411,7 +870,7 @@ if (!keepFilter) { filter = '' } - await loadFiles() + return await loadFiles() } async function moveS3File(srcFileKey: string | undefined, destFileKey: string | undefined) { @@ -483,9 +942,18 @@ } await clearAndLoadFiles() if (selectedFileKey !== undefined) { - if (allFilesByKey[selectedFileKey.s3] === undefined) { + const entry = allFilesByKey[selectedFileKey.s3] + if (entry !== undefined) { + if (entry.type !== 'folder') { + loadFileMetadataPlusPreviewAsync(selectedFileKey.s3) + } + } else if (!lazyMode) { + // Flat mode has listed everything it is going to, so a missing key really + // is missing. Per-level listing has not: only the levels on the way to + // the key were fetched, and `selectedFileKey` is bound out to the caller + // — blanking it there would silently clear the configured object. selectedFileKey = { s3: '', storage } - } else if (allFilesByKey[selectedFileKey.s3].type !== 'folder') { + } else { loadFileMetadataPlusPreviewAsync(selectedFileKey.s3) } } @@ -506,6 +974,27 @@ function selectItem(index: number, toggleCollapsed: boolean = true) { let item_key = displayedFileKeys[index] let item = allFilesByKey[item_key] + if (item === undefined) return + if (lazyMode) { + if (item.type === 'folder') { + if (folderOnly) { + selectedFileKey = { s3: item_key, storage } + } + if (toggleCollapsed) { + item.collapsed = !item.collapsed + } + if (!item.collapsed && !folderState[item_key]?.loaded) { + // Children are unknown until this level is fetched. + expandFolder(item_key) + } else { + refreshDisplayed() + } + } else { + selectedFileKey = { s3: item_key, storage } + loadFileMetadataPlusPreviewAsync(selectedFileKey.s3) + } + return + } if (item.type === 'folder') { if (folderOnly) { selectedFileKey = { @@ -612,7 +1101,12 @@
{#if !rootPath}
- +
{/if} {#if displayedFileKeys.length === 0} @@ -624,8 +1118,22 @@
{:else}
- No files in the workspace S3 bucket at that prefix + {#if filter.trim() !== ''} + No files starting with "{filter.trim()}" + {:else} + No files in the workspace S3 bucket + {/if}
+ {#if flatHasMore} + +
+ +
+ {/if} {/if} {:else}
@@ -638,7 +1146,12 @@ {#snippet header()}{/snippet} {#snippet footer()}{/snippet} {#snippet item({ index, style })} - {@const file_info = allFilesByKey[displayedFileKeys[index]]} + + {@const item_key = displayedFileKeys[index] ?? ''} + {@const is_load_more = item_key.endsWith(LOAD_MORE_SUFFIX)} + {@const load_more_prefix = is_load_more ? item_key.slice(0, -1) : ''} + {@const file_info = allFilesByKey[item_key]}
- {#if file_info} + {#if is_load_more} + + {@const loadMoreNesting = + nestingLevelOf(load_more_prefix + 'x') - 2 * rootPathNestingLevel} + {@const loadingMore = folderState[load_more_prefix]?.loading === true} + + +
!loadingMore && loadMore(load_more_prefix)} + class={twMerge( + 'flex flex-row h-full text-xs items-center justify-start text-secondary', + loadingMore ? 'cursor-default' : 'cursor-pointer' + )} + > +
+ +
+ {#if loadingMore} + + {:else} + + {/if} +
+
Load more
+
+
+ {:else if file_info} {@const nestingLevel = file_info.nestingLevel - 2 * rootPathNestingLevel} @@ -665,14 +1209,23 @@ style={`margin-left: ${(2 + nestingLevel) * 0.25}rem;`} > {#if file_info.type === 'folder'} - {#if file_info.collapsed}{:else} + {:else if file_info.collapsed}{:else}{/if}
- {file_info.display_name} ({file_info.count}{count % 1000 === 0 && - lastKeyFolders[file_info.nestingLevel / 2] === file_info.display_name - ? '+' - : ''} item{file_info.count === 1 ? '' : 's'}) + + {#if file_info.display_name === ''} + (empty name) + {:else}{file_info.display_name}{/if} + {#if !lazyMode} + ({file_info.count}{count % 1000 === 0 && + lastKeyFolders[file_info.nestingLevel / 2] === file_info.display_name + ? '+' + : ''} item{file_info.count === 1 ? '' : 's'}) + {/if}
{:else} @@ -688,27 +1241,24 @@
{#if fileListLoading === true}
Loading content
+ {:else if lazyMode} + +
{displayedCount} item{displayedCount === 1 ? '' : 's'} shown
{:else}
- {displayedCount}{count % maxKeys === 0 ? '+' : ''} + {displayedCount}{flatHasMore ? '+' : ''} {displayedCount !== count ? 'filtered ' : ''}items (including inside folders)
- {#if count % maxKeys === 0} - {/if} diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecEditor.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecEditor.svelte index 6a6d820582..f921f4aeca 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecEditor.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecEditor.svelte @@ -359,6 +359,7 @@ }} readOnlyMode={false} regexFilter={/\.(png|jpg|jpeg|svg|webp)$/i} + lazyFolders={false} /> {:else if componentInput?.type === 'user'} Field's value is set by the user