diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index b2d35dc125..012f205d9c 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -599eae0034b34c5e0d3d42df37cf18bbe01f9b49 +06610ecc950ed64a2709d7f1aa8c7b49a95fdbae diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 43b1f776c2..bab02327ec 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -25310,9 +25310,12 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewS3ResourcePath" + - $ref: "#/components/parameters/TablePreviewFilters" responses: "200": - description: Parquet Preview + description: > + Parquet Preview: `columns`, `column_types` (Arrow type names) and `rows`. content: application/json: {} @@ -25337,6 +25340,13 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewS3ResourcePath" + - name: csv_separator + in: query + schema: + type: string + - $ref: "#/components/parameters/TablePreviewCsvHasHeader" + - $ref: "#/components/parameters/TablePreviewFilters" responses: "200": description: Table count @@ -25385,13 +25395,20 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewS3ResourcePath" - name: csv_separator in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewCsvHasHeader" + - $ref: "#/components/parameters/TablePreviewFilters" responses: "200": - description: Csv Preview + description: > + Csv Preview: `columns`, `column_types` (Arrow type names), `rows`, and the + `csv_separator` the file was read with, guessed from its first bytes when + none is given. + A `.jsonl` or `.ndjson` file is read as newline-delimited JSON. content: application/json: {} @@ -27751,6 +27768,28 @@ components: required: true schema: type: string + TablePreviewS3ResourcePath: + name: s3_resource_path + description: Read the file from this object storage resource instead of the workspace storage. + in: query + schema: + type: string + TablePreviewCsvHasHeader: + name: csv_has_header + description: Whether the first CSV line names the columns. Defaults to true. + in: query + schema: + type: boolean + TablePreviewFilters: + name: filters + description: > + JSON object, column name to filter. A number column takes an optional comparison + operator and a number (`>=10`), a boolean column a boolean, and any other column a + case-insensitive substring, or `=value` for an exact match. A `search_term` without + a `search_col` matches any column, case-insensitively. + in: query + schema: + type: string HubPublishFolder: name: folder in: query diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 54095f0dc6..430aa801a0 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -5848,6 +5848,10 @@ impl AppLoadCountQuery { search_col: self.search_col, search_term: self.search_term, storage: self.storage, + s3_resource_path: None, + csv_separator: None, + csv_has_header: None, + filters: None, }, ) } @@ -5881,6 +5885,9 @@ impl AppLoadPreviewQuery { search_term: self.search_term, storage: self.storage, csv_separator: self.csv_separator, + s3_resource_path: None, + csv_has_header: None, + filters: None, }, ) } @@ -5968,7 +5975,9 @@ async fn app_load_table_count( let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = - crate::job_helpers_oss::load_table_count_internal(job_authed, &db, &w_id, file_key, inner) + crate::job_helpers_oss::load_table_count_internal( + job_authed, &db, None, &w_id, file_key, inner, + ) .await?; Ok(Json(resp).into_response()) } @@ -5987,7 +5996,7 @@ async fn app_load_parquet_preview( let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = crate::job_helpers_oss::load_preview_internal( - job_authed, &db, &w_id, file_key, inner, true, + job_authed, &db, None, &w_id, file_key, inner, true, ) .await?; Ok(Json(resp).into_response()) @@ -6007,7 +6016,7 @@ async fn app_load_csv_preview( let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = crate::job_helpers_oss::load_preview_internal( - job_authed, &db, &w_id, file_key, inner, false, + job_authed, &db, None, &w_id, file_key, inner, false, ) .await?; Ok(Json(resp).into_response()) diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index eb8986a986..0d6f20cdfa 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -263,6 +263,10 @@ mod app_s3_display_stubs { pub search_col: Option, pub search_term: Option, pub storage: Option, + pub s3_resource_path: Option, + pub csv_separator: Option, + pub csv_has_header: Option, + pub filters: Option, } #[derive(Serialize)] @@ -278,7 +282,10 @@ mod app_s3_display_stubs { pub search_col: Option, pub search_term: Option, pub storage: Option, + pub s3_resource_path: Option, pub csv_separator: Option, + pub csv_has_header: Option, + pub filters: Option, } pub async fn load_file_metadata_internal( @@ -308,6 +315,7 @@ mod app_s3_display_stubs { pub async fn load_table_count_internal( _authed: OptJobAuthed, _db: &DB, + _user_db: Option, _w_id: &str, _file_key: String, _query: LoadCountQuery, @@ -320,6 +328,7 @@ mod app_s3_display_stubs { pub async fn load_preview_internal( _authed: OptJobAuthed, _db: &DB, + _user_db: Option, _w_id: &str, _file_key: String, _query: LoadPreviewQuery, diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index ce1af47118..a5e4849c57 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -105,6 +105,7 @@ size="1200px" > { s3FilePickerInner?.exit?.() diff --git a/frontend/src/lib/components/S3FilePickerInner.svelte b/frontend/src/lib/components/S3FilePickerInner.svelte index e3ac7322a8..5f8dcfc3d7 100644 --- a/frontend/src/lib/components/S3FilePickerInner.svelte +++ b/frontend/src/lib/components/S3FilePickerInner.svelte @@ -8,8 +8,36 @@ Loader2, Download, Trash, - MoveRight + MoveRight, + Ellipsis, + FileText, + FileCode, + FileJson, + FileSpreadsheet, + FileImage, + FileArchive, + FilePlay, + FileAudio, + Table2 } from 'lucide-svelte' + import { Pane, Splitpanes } from 'svelte-splitpanes' + import TextInput from './text_input/TextInput.svelte' + import Select from './select/Select.svelte' + import Toggle from './Toggle.svelte' + import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' + import DBTable from './DBTable.svelte' + import type { IDbTableOps } from './dbOps' + import { resource } from 'runed' + import { staticDbTableOps } from './staticDbTableOps' + import { + fetchJsonTableRows, + isJsonFile, + JSON_TABLE_MAX_BYTES, + s3TableOps, + serverTableFormat, + type S3TableSource + } from './s3TableOps' import { CancelablePromise, HelpersService, @@ -34,14 +62,13 @@ emptyString, parseS3Object, sendUserToast, - type S3Object + type S3Object, + type IconType } from '$lib/utils' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' - import { Alert, Button } from './common' - import Section from './Section.svelte' + import { Alert, Button, CopyButton } from './common' import { createEventDispatcher, untrack, type Snippet } from 'svelte' import VirtualList from '@tutorlatin/svelte-tiny-virtual-list' - import TableSimple from './TableSimple.svelte' import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' import FileUploadModal from './common/fileUpload/FileUploadModal.svelte' import S3FilePreview from './S3FilePreview.svelte' @@ -143,8 +170,14 @@ let rootPath = $state(initialRootPath) let rootPathNestingLevel = $derived(1 * (rootPath.split('/').length - 1)) - let csvSeparatorChar: string = $state(',') + let csvSeparatorChar: string | undefined = $state(undefined) let csvHasHeader: boolean = $state(true) + const CSV_SEPARATORS = [ + { value: ',', label: ',' }, + { value: ';', label: ';' }, + { value: '\t', label: 'Tab' }, + { value: '|', label: '|' } + ] let dispatch = createEventDispatcher<{ close: { s3: string; storage: string | undefined } | undefined @@ -178,6 +211,68 @@ } | undefined = $state(undefined) + let viewMode: 'table' | 'text' = $state('table') + /** The selected file, when it can be shown as a table. Its own requests can't read + * files behind a custom listing (git repos), so those keep the text preview. */ + let tableSource = $derived.by(() => { + if (hideS3SpecificDetails || !fileMetadata || !ws) return undefined + const fileKey = fileMetadata.fileKey + const inBrowser = isJsonFile(fileKey) && (fileMetadata.size ?? Infinity) <= JSON_TABLE_MAX_BYTES + if (!inBrowser && !serverTableFormat(fileKey)) return undefined + return { + workspace: ws, + fileKey, + storage, + s3ResourcePath, + csvSeparator: csvSeparatorChar, + csvHasHeader, + inBrowser + } + }) + let tableKey = $derived(tableSource && JSON.stringify(tableSource)) + let tableOps = resource( + () => (viewMode === 'table' ? tableSource : undefined), + async (source, _previous, { signal }) => { + if (!source) return undefined + const key = JSON.stringify(source) + // A response for a file or options since left must not replace the current one; + // the resource drops a fetch that throws an AbortError. + const superseded = () => new DOMException('Superseded', 'AbortError') + try { + const table = source.inBrowser ? await jsonTable(source) : await s3TableOps(source) + if (signal.aborted) throw superseded() + return { key, ops: table?.ops, csvSeparator: table?.csvSeparator } + } catch (e) { + throw signal.aborted ? superseded() : e + } + } + ) + async function jsonTable( + source: S3TableSource + ): Promise<{ ops: IDbTableOps; csvSeparator?: string } | undefined> { + const rows = await fetchJsonTableRows(source) + return rows && { ops: staticDbTableOps(rows, 'duckdb', source.fileKey.split('/').pop() ?? '') } + } + /** The loaded table, once it belongs to the selected file and options. */ + let currentTable = $derived(tableOps.current?.key === tableKey ? tableOps.current : undefined) + /** A JSON file that turns out not to be an array has no table to show. */ + let showTable = $derived( + viewMode === 'table' && !!tableSource && !(currentTable && !currentTable.ops) + ) + // Each file starts on the separator the server guesses from its first bytes. + $effect(() => { + fileMetadata?.fileKey + untrack(() => { + csvSeparatorChar = undefined + detectedCsvSeparator = undefined + }) + }) + /** Kept while the file stays selected, so the text view still shows what was guessed. */ + let detectedCsvSeparator: string | undefined = $state(undefined) + $effect(() => { + if (currentTable?.csvSeparator) detectedCsvSeparator = currentTable.csvSeparator + }) + /** Identifies the metadata request that currently owns the preview pane. */ let metadataRequestId = 0 @@ -228,6 +323,25 @@ /** Per-level listing only makes sense while browsing; searching stays flat. */ let lazyMode = $derived(lazyFolders && filter.trim() === '') + /** `nestingLevel` counts two per folder depth. */ + function treeRowIndent(nestingLevel: number): string { + return `padding-left: ${12 + (nestingLevel / 2) * 16}px;` + } + + const FILE_ICONS: [RegExp, IconType][] = [ + [/\.(csv|tsv|xlsx?|parquet|ods)$/i, FileSpreadsheet], + [/\.(json|jsonl|ndjson)$/i, FileJson], + [/\.(png|jpe?g|gif|webp|svg|bmp|ico|tiff?)$/i, FileImage], + [/\.(zip|tar|gz|tgz|bz2|xz|7z|rar|zst)$/i, FileArchive], + [/\.(mp4|mov|webm|mkv|avi)$/i, FilePlay], + [/\.(mp3|wav|ogg|flac|m4a)$/i, FileAudio], + [/\.(py|ts|js|go|rs|sh|sql|java|rb|php|ya?ml|toml|html|css|xml)$/i, FileCode], + [/\.(txt|md|log|pdf|docx?|rtf)$/i, FileText] + ] + function fileIconFor(name: string): IconType { + return FILE_ICONS.find(([re]) => re.test(name))?.[1] ?? FileIcon + } + function nestingLevelOf(key: string): number { const slashes = (key.match(/\//g) ?? []).length return (key.endsWith('/') ? slashes - 1 : slashes) * 2 @@ -1047,40 +1161,43 @@ {#if workspaceSettingsInitialized === false} - {#if fromWorkspaceSettings} - -
-

Double check the S3 resource fields and try again.

-
- - {:else if s3ResourcePath} - -
-

- Double check the resource fields and that its object storage is reachable, then try again. -

-
- - {:else} - -
-

- The workspace needs to be connected to an S3 storage to use this feature. You can configure it here. -

-
- - {/if} +
+ {#if fromWorkspaceSettings} + +
+

Double check the S3 resource fields and try again.

+
+ + {:else if s3ResourcePath} + +
+

+ Double check the resource fields and that its object storage is reachable, then try + again. +

+
+ + {:else} + +
+

+ The workspace needs to be connected to an S3 storage to use this feature. You can configure it here. +

+
+ + {/if} +
{:else} {#if fileListUnavailable == true} {#if replaceUnauthorizedWarning} {@render replaceUnauthorizedWarning()} {:else} -
+

You don't have access to the S3 bucket resource and your administrator has restricted @@ -1097,28 +1214,25 @@

{/if} {/if} -
+ {#if !fileListUnavailable} -
+ {#if !rootPath} -
- +
{/if} {#if displayedFileKeys.length === 0} {#if fileListLoading} -
-
- Loading content -
+
+ + Loading...
{:else} -
+
{#if filter.trim() !== ''} No files starting with "{filter.trim()}" {:else} @@ -1130,19 +1244,19 @@ can remove every match. Without this the only control that could resume the listing would be hidden behind the empty state. -->
-
{/if} {/if} {:else} -
+
{#snippet header()}{/snippet} {#snippet footer()}{/snippet} @@ -1154,199 +1268,249 @@ {@const load_more_prefix = is_load_more ? item_key.slice(0, -1) : ''} {@const file_info = allFilesByKey[item_key]} -
+
{#if is_load_more} {@const loadMoreNesting = nestingLevelOf(load_more_prefix + 'x') - 2 * rootPathNestingLevel} {@const loadingMore = folderState[load_more_prefix]?.loading === true} - - -
!loadingMore && loadMore(load_more_prefix)} - class={twMerge( - 'flex flex-row h-full text-xs items-center justify-start text-secondary', - loadingMore ? 'cursor-default' : 'cursor-pointer' - )} + disabled={loadingMore} + class="w-full text-xs font-normal flex gap-2 items-center h-8 pr-1 text-secondary hover:bg-surface-hover cursor-pointer disabled:cursor-default disabled:hover:bg-transparent" + style={treeRowIndent(loadMoreNesting)} > -
- -
- {#if loadingMore} - - {:else} - - {/if} -
-
Load more
-
-
+ + {#if loadingMore} + + {:else} + + {/if} + Load more + {:else if file_info} {@const nestingLevel = file_info.nestingLevel - 2 * rootPathNestingLevel} - - -
selectItem(index)} + title={file_info.full_key.slice(rootPath.length)} class={twMerge( - 'flex flex-row h-full font-semibold text-xs items-center justify-start', - selectedFileKey !== undefined && selectedFileKey.s3 === file_info.full_key - ? 'bg-surface-hover' - : '' + 'w-full text-xs font-normal text-primary flex gap-2 items-center h-8 pr-1 cursor-pointer', + isSelected ? 'bg-surface-secondary' : 'hover:bg-surface-hover' )} + style={treeRowIndent(nestingLevel)} > -
- {#if file_info.type === 'folder'} - {#if folderState[file_info.full_key]?.loading} - - {:else if file_info.collapsed}{:else}{/if} -
- - {#if file_info.display_name === ''} - (empty name) - {:else}{file_info.display_name}{/if} - {#if !lazyMode} - ({file_info.count}{count % 1000 === 0 && - lastKeyFolders[file_info.nestingLevel / 2] === file_info.display_name - ? '+' - : ''} item{file_info.count === 1 ? '' : 's'}) - {/if} -
+ {#if isFolder} + {#if folderState[file_info.full_key]?.loading} + {:else} - -
- {file_info.display_name} -
+ {/if} -
-
+ {:else} + + {/if} + + + + {#if file_info.display_name === ''} + (empty name) + {:else}{file_info.display_name}{/if} + + + {#if isFolder && !lazyMode} + + {file_info.count}{count % 1000 === 0 && + lastKeyFolders[file_info.nestingLevel / 2] === file_info.display_name + ? '+' + : ''} + + {/if} + {/if}
{/snippet}
{#if fileListLoading === true} -
- Loading content +
+ Loading...
{:else if lazyMode}
{displayedCount} item{displayedCount === 1 ? '' : 's'} shown
{:else} -
+
{displayedCount}{flatHasMore ? '+' : ''} {displayedCount !== count ? 'filtered ' : ''}items (including inside folders)
{#if flatHasMore} - {/if} {/if}
{/if} -
+ {/if} -
- {#if fileMetadata === undefined} -
- {#if fileInfoLoading} -
- {:else if fileListUnavailable} -
- {:else} -
- {/if} -
- {:else} -
-
-

- {((p) => (p.startsWith(rootPath) ? p.slice(rootPath.length) : p))( - fileMetadata.fileKey - )} -

- {#if filePreview !== undefined && (!hideS3SpecificDetails || !readOnlyMode || allowDelete)} -
- {#if !hideS3SpecificDetails} - {@const downloadApiPath = `/w/${ws}/job_helpers/download_s3_file?file_key=${encodeURIComponent(fileMetadata?.fileKey ?? '')}${storage ? `&storage=${storage}` : ''}${s3ResourcePath ? `&s3_resource_path=${encodeURIComponent(s3ResourcePath)}` : ''}`} - {@const downloadName = - fileMetadata?.fileKey.split('/').pop() ?? 'unnamed_download.file'} - {#if shouldDownloadViaClient()} -
+ {:else if !csvOptions} +
+ {#if filePreview.contentType === 'Csv'} + Previewing a {filePreview.contentType?.toLowerCase()} file. Separator character: +
+ +
+ Header row: +
+ +
+ {:else} + Previewing a {filePreview.contentType?.toLowerCase()} file. + {/if} +
+ {/if} 
{#if !emptyString(filePreview.contentPreview)}{filePreview.contentPreview}{:else if filePreview.contentType !== undefined}Preview impossible.{/if}
diff --git a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts index 21371aa08b..ae3a82e97c 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts @@ -41,6 +41,8 @@ export type ColumnDef = { children?: ColumnDef[] // DBExplorer ignored?: boolean + /** False when the column cannot be filtered on, e.g. a nested value. */ + filterable?: boolean hideInsert?: boolean editable?: boolean overrideDefaultValue?: boolean diff --git a/frontend/src/lib/components/dbTableFilters.ts b/frontend/src/lib/components/dbTableFilters.ts index 9d9c83e6b0..c3266b3ac7 100644 --- a/frontend/src/lib/components/dbTableFilters.ts +++ b/frontend/src/lib/components/dbTableFilters.ts @@ -52,7 +52,7 @@ export function buildDbTableFilterSchema(columns: ColumnDef[]): DbTableFilterSch const columnOfKey: Record = {} const keyOfColumn: Record = {} for (const col of columns) { - if (!col.field) continue + if (!col.field || col.filterable === false) continue let base = col.field.replace(/\W+/g, '_') || 'column' if (base === FREE_TEXT_KEY) base = `${base}col` let key = base diff --git a/frontend/src/lib/components/s3TableOps.test.ts b/frontend/src/lib/components/s3TableOps.test.ts new file mode 100644 index 0000000000..f32ce47c56 --- /dev/null +++ b/frontend/src/lib/components/s3TableOps.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest' +import { datatypeOf, jsonTableRows, serverTableFormat } from './s3TableOps' +import { dbColumnKind } from './dbTableFilters' + +describe('serverTableFormat', () => { + it('pages delimited text and newline-delimited JSON through the csv reader', () => { + expect(serverTableFormat('a/b.parquet')).toBe('parquet') + for (const key of ['a.csv', 'a.TSV', 'a.jsonl', 'a.ndjson']) { + expect(serverTableFormat(key)).toBe('csv') + } + }) + + it('leaves plain JSON to the browser', () => { + expect(serverTableFormat('a.json')).toBeUndefined() + expect(serverTableFormat('a.txt')).toBeUndefined() + }) +}) + +describe('datatypeOf', () => { + it('maps Arrow types to the kinds the grid filters by', () => { + expect(dbColumnKind(datatypeOf('Int64'))).toBe('number') + expect(dbColumnKind(datatypeOf('UInt8'))).toBe('number') + expect(dbColumnKind(datatypeOf('Float64'))).toBe('number') + expect(dbColumnKind(datatypeOf('Decimal128(21, 1)'))).toBe('number') + expect(dbColumnKind(datatypeOf('Boolean'))).toBe('boolean') + expect(dbColumnKind(datatypeOf('Utf8View'))).toBe('text') + }) + + it('names a nested type without its field list', () => { + expect(datatypeOf('List(Field { name: "item", data_type: Utf8 })')).toBe('list') + expect(datatypeOf('Timestamp(Microsecond, None)')).toBe('timestamp') + }) +}) + +describe('jsonTableRows', () => { + it('keeps objects and wraps other elements in a value column', () => { + expect(jsonTableRows([{ a: 1 }, 2, null])).toEqual([{ a: 1 }, { value: 2 }, { value: null }]) + }) + + it('has no table for an empty array or a non-array', () => { + expect(jsonTableRows([])).toBeUndefined() + expect(jsonTableRows({ a: 1 })).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/s3TableOps.ts b/frontend/src/lib/components/s3TableOps.ts new file mode 100644 index 0000000000..f5d7bd758b --- /dev/null +++ b/frontend/src/lib/components/s3TableOps.ts @@ -0,0 +1,170 @@ +import { HelpersService, OpenAPI } from '$lib/gen' +import { getHeaders } from '$lib/gen/core/request' +import { ColumnIdentity, type ColumnDef } from './apps/components/display/dbtable/utils' +import type { IDbTableOps } from './dbOps' + +/** Files the server can page through as a table, without reading them whole. */ +export function serverTableFormat(fileKey: string): 'parquet' | 'csv' | undefined { + const key = fileKey.toLowerCase() + if (key.endsWith('.parquet')) return 'parquet' + if (/\.(csv|tsv|jsonl|ndjson)$/.test(key)) return 'csv' + return undefined +} + +export type S3TableSource = { + workspace: string + fileKey: string + storage?: string + s3ResourcePath?: string + csvSeparator?: string + csvHasHeader?: boolean +} + +type Chunk = { + columns: string[] + column_types?: string[] + rows: Record[] + csv_separator?: string +} + +/** An Arrow type as the SQL type name the grid filters and formats by. */ +export function datatypeOf(arrowType: string | undefined): string { + const t = arrowType ?? '' + if (/^U?Int\d+$/.test(t)) return 'bigint' + if (/^Float\d+$/.test(t)) return 'double' + if (/^Decimal/.test(t)) return 'decimal' + if (t === 'Boolean') return 'boolean' + if (/^(Large)?Utf8/.test(t)) return 'text' + // A nested type spells out its fields, e.g. `List(Field { name: "item", … })`. + return t.split('(')[0].toLowerCase() || 'text' +} + +/** Read-only table ops over an object storage file, paged, searched, filtered and sorted + * by the server, with the CSV separator it read the file with (guessed when none is given). + */ +export async function s3TableOps( + source: S3TableSource +): Promise<{ ops: IDbTableOps; csvSeparator?: string }> { + const format = serverTableFormat(source.fileKey) + if (!format) throw new Error(`${source.fileKey} cannot be read as a table`) + const common = { + workspace: source.workspace, + path: source.fileKey, + storage: source.storage, + s3ResourcePath: source.s3ResourcePath + } + const csvOptions: { csvSeparator?: string; csvHasHeader?: boolean } = + format === 'csv' ? { csvSeparator: source.csvSeparator, csvHasHeader: source.csvHasHeader } : {} + const filtersParam = (columnFilters: Record | undefined) => + columnFilters && Object.keys(columnFilters).length ? JSON.stringify(columnFilters) : undefined + const searchParam = (quicksearch: string) => quicksearch.trim() || undefined + + async function loadChunk(params: { + offset: number + limit: number + quicksearch?: string + columnFilters?: Record + orderBy?: string + isDesc?: boolean + }): Promise { + const data = { + ...common, + ...csvOptions, + offset: params.offset, + limit: params.limit, + searchTerm: searchParam(params.quicksearch ?? ''), + filters: filtersParam(params.columnFilters), + sortCol: params.orderBy, + sortDesc: params.isDesc + } + return (await (format === 'parquet' + ? HelpersService.loadParquetPreview(data) + : HelpersService.loadCsvPreview(data))) as Chunk + } + + const head = await loadChunk({ offset: 0, limit: 0 }) + // Later reads reuse the separator the server guessed, rather than each guessing again. + if (format === 'csv') csvOptions.csvSeparator ??= head.csv_separator + const colDefs: ColumnDef[] = head.columns.map((field, i) => ({ + field, + datatype: datatypeOf(head.column_types?.[i]), + // The server cannot read a struct, map or union as text to filter it. + filterable: !/^(Struct|Map|Union)/.test(head.column_types?.[i] ?? ""), + defaultvalue: '', + isprimarykey: false, + isidentity: ColumnIdentity.No, + isnullable: 'YES', + isenum: false + })) + + const ops: IDbTableOps = { + dbType: 'duckdb', + tableKey: source.fileKey.split('/').pop() ?? source.fileKey, + colDefs, + getCount: async ({ quicksearch, columnFilters }) => { + const res = await HelpersService.loadTableRowCount({ + ...common, + ...csvOptions, + searchTerm: searchParam(quicksearch), + filters: filtersParam(columnFilters) + }) + return res.count ?? 0 + }, + getRows: async ({ + offset, + limit, + quicksearch, + columnFilters, + order_by, + is_desc, + explicitSort + }) => + ( + await loadChunk({ + offset, + limit, + quicksearch, + columnFilters, + // Unsorted, a file keeps its own row order. + orderBy: explicitSort ? order_by : undefined, + isDesc: is_desc + }) + ).rows + } + return { ops, csvSeparator: head.csv_separator } +} + +/** Plain JSON cannot be paged by the server, so only a file this small is read whole into + * the browser to be shown as a table. */ +export const JSON_TABLE_MAX_BYTES = 10 * 1024 * 1024 + +export function isJsonFile(fileKey: string): boolean { + return fileKey.toLowerCase().endsWith('.json') +} + +/** Rows of a JSON array, an element that is not an object becoming a one-column row; + * undefined for anything else. */ +export function jsonTableRows(value: unknown): Record[] | undefined { + if (!Array.isArray(value) || value.length === 0) return undefined + return value.map((v) => + v !== null && typeof v === 'object' && !Array.isArray(v) + ? (v as Record) + : { value: v } + ) +} + +export async function fetchJsonTableRows( + source: S3TableSource +): Promise[] | undefined> { + const query = new URLSearchParams({ file_key: source.fileKey }) + if (source.storage) query.set('storage', source.storage) + if (source.s3ResourcePath) query.set('s3_resource_path', source.s3ResourcePath) + const apiPath = `/w/${source.workspace}/job_helpers/download_s3_file?${query}` + const headers = await getHeaders(OpenAPI, { method: 'GET', url: apiPath }) + const response = await fetch(`${OpenAPI.BASE}${apiPath}`, { + headers, + credentials: OpenAPI.CREDENTIALS + }) + if (!response.ok) throw new Error(`${response.status} ${response.statusText}`) + return jsonTableRows(await response.json()) +}