improve parquet renderer

backend/Cargo.toml

@@ -213,7 +213,8 @@ candle-nn = "0.3.0"
 tiberius = { version = "0.12.2", default-features = false, features = ["rustls", "tds73", "chrono"] }
 aws-sdk-s3 = "1.5.0"
 aws-config = "1.0.3"
-polars = { version = "0.35.2", features = ["lazy", "parquet", "aws", "csv", "dtype-full", "serde"] }
-polars-io = { version = "0.35.2", features = ["csv"] }
+
+polars = { version = "0.35.4", features = ["lazy", "parquet", "aws", "csv", "dtype-full", "serde", "strings", "extract_groups"] }
+polars-io = { version = "0.35.4", features = ["csv"] }
 object_store = { version = "0.8.0", features = ["aws"] }
 openidconnect = { version = "3.4.0" }

backend/windmill-api/openapi.yaml

@@ -7083,7 +7083,11 @@ paths:
           in: query
           schema:
             type: boolean
-        - name: search
+        - name: search_col
+          in: query
+          schema:
+            type: string
+        - name: search_term
           in: query
           schema:
             type: string

backend/windmill-api/src/job_helpers.rs

@@ -17,7 +17,6 @@ use axum::{
 use hyper::http;
 use itertools::Itertools;
 use object_store::ClientConfigKey;
-use polars::chunked_array::ops::SortOptions;
 use polars::{
     io::{
         cloud::{AmazonS3ConfigKey, CloudOptions},
@@ -546,7 +545,8 @@ struct LoadParquetQuery {
     offset: Option<i64>,
     sort_col: Option<String>,
     sort_desc: Option<bool>,
-    search: Option<String>,
+    search_col: Option<String>,
+    search_term: Option<String>,
 }
 
 async fn load_parquet_preview(
@@ -570,7 +570,9 @@ async fn load_parquet_preview(
         query
             .sort_col
             .map(|v| (v.to_string(), query.sort_desc.unwrap_or(false))),
-        query.search,
+        query
+            .search_col
+            .map(|v| (v.to_string(), query.search_term.unwrap_or_default())),
     )
     .await
     .map(Json);
@@ -1353,12 +1355,12 @@ async fn read_s3_parquet_chunk(
     limit: Option<u32>,
     offset: Option<i64>,
     sort: Option<(String, bool)>,
-    _search: Option<String>,
+    search: Option<(String, String)>,
 ) -> error::Result<Box<RawValue>> {
     let s3_cloud_config = build_polars_s3_config(s3_resource_ref);
 
     let args: ScanArgsParquet = ScanArgsParquet {
-        n_rows: Some(1),
+        n_rows: None,
         cache: false,
         parallel: polars::io::parquet::ParallelStrategy::Auto,
         rechunk: false,
@@ -1397,12 +1399,12 @@ async fn read_s3_parquet_chunk(
                 } else {
                     df
                 };
-
-                // let df = if let Some(search) = search {
-                //     df.filter(col("*").str().contains(&search))
-                // } else {
-                //     df
-                // };
+                use polars::prelude::*;
+                let df = if let Some(search) = search {
+                    df.filter(col(&search.0).str().contains_literal(lit(search.1.clone())))
+                } else {
+                    df
+                };
                 let df = df
                     .collect()
                     .map_err(|err| error::Error::InternalErr(err.to_string()))?;

frontend/src/lib/components/DisplayResult.svelte

@@ -351,7 +351,12 @@
 				>
 			</div>
 		{:else if !forceJson && resultKind == 's3object'}
-			<div class="absolute top-1 h-full w-full">
+			<div
+				class="absolute top-1 h-full w-full {typeof result?.s3 == 'string' &&
+				result?.s3?.endsWith('.parquet')
+					? 'h-min-[600px]'
+					: ''}"
+			>
 				<div class="flex flex-col gap-2">
 					<Toggle
 						class="flex"
@@ -370,6 +375,13 @@
 						</button>
 					{:else}
 						<FileDownload s3object={result} />
+						<button
+							class="text-secondary underline text-2xs whitespace-nowrap"
+							on:click={() => {
+								s3FileViewer?.open?.(result)
+							}}
+							><span class="flex items-center gap-1"><PanelRightOpen size={12} />open preview</span>
+						</button>
 					{/if}
 				</div>
 				{#if typeof result?.s3 == 'string' && result?.s3?.endsWith('.parquet')}

frontend/src/lib/components/ParqetTableRenderer.svelte

@@ -17,11 +17,17 @@
 		rowCount: 0,
 		getRows: async function (params) {
 			try {
+				const searchCol = params.filterModel ? Object.keys(params.filterModel)?.[0] : undefined
+				console.log(params.filterModel)
 				const res = await HelpersService.loadParquetPreview({
 					workspace: $workspaceStore!,
 					path: s3resource,
-					// offset: params.startRow,
-					limit: params.endRow - params.startRow
+					offset: params.startRow,
+					limit: params.endRow - params.startRow,
+					sortCol: params.sortModel?.[0]?.colId,
+					sortDesc: params.sortModel?.[0]?.sort == 'desc',
+					searchCol: searchCol,
+					searchTerm: searchCol ? params.filterModel?.[searchCol]?.filter : undefined
 				})
 
 				const data: any[] = []
@@ -35,7 +41,6 @@
 					})
 				})
 
-				console.log(res)
 				params.successCallback(data)
 			} catch (e) {
 				console.error(e)
@@ -63,8 +68,6 @@
 
 	$: eGui && mountGrid()
 
-	let firstRow = 0
-	let lastRow = 0
 	async function mountGrid() {
 		if (eGui) {
 			const res = await HelpersService.loadParquetPreview({
@@ -78,11 +81,16 @@
 				{
 					rowModelType: 'infinite',
 					datasource,
+					// @ts-ignore
 					columnDefs: res.columns.map((c) => {
 						return {
 							field: c.name,
 							sortable: true,
-							filter: true
+							filter: true,
+							filterParams: {
+								filterOptions: ['contains'],
+								maxNumConditions: 1
+							}
 						}
 					}),
 					pagination: false,
@@ -93,10 +101,6 @@
 					rowSelection: 'multiple',
 					rowMultiSelectWithClick: true,
 					suppressRowDeselection: true,
-					onViewportChanged: (e) => {
-						firstRow = e.firstRow
-						lastRow = e.lastRow
-					},
 
 					onSelectionChanged: (e) => {
 						onSelectionChanged(e.api)

frontend/src/lib/components/runs/JobPreview.svelte

@@ -36,7 +36,7 @@
 
 <TestJobLoader bind:job={currentJob} bind:watchJob on:done={onDone} />
 
-<div class="p-4 flex flex-col gap-2 items-start">
+<div class="p-4 flex flex-col gap-2 items-start h-full">
 	{#if job}
 		<div class="flex gap-2">
 			{#if job?.['priority']}
@@ -77,7 +77,7 @@
 			</div>
 		{/if}
 
-		<div class=" w-full rounded-md overflow-auto">
+		<div class=" w-full rounded-md overflow-auto h-full">
 			{#if job?.type === Job.type.COMPLETED_JOB}
 				<Tabs bind:selected={viewTab}>
 					<Tab size="xs" value="result">Result</Tab>
@@ -96,7 +96,7 @@
 							</div>
 						</div>
 					{:else}
-						<div class="flex flex-row border rounded-md p-2 mt-2 max-h-1/2 overflow-auto">
+						<div class="flex flex-col border rounded-md p-2 mt-2 h-full overflow-auto">
 							{#if viewTab == 'logs'}
 								<div class="w-full">
 									<LogViewer

frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte

@@ -519,7 +519,7 @@
 
 				<Skeleton loading={!job} layout={[[5]]} />
 				{#if job}
-					<div class="flex flex-row border rounded-md p-2 mt-2 overflow-auto">
+					<div class="flex flex-row border rounded-md p-2 mt-2 overflow-auto min-h-[600px]">
 						{#if viewTab == 'logs'}
 							<div class="w-full">
 								<LogViewer