diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 67ac2c2bb8..b66ab03660 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -213,7 +213,8 @@ candle-nn = "0.3.0" tiberius = { version = "0.12.2", default-features = false, features = ["rustls", "tds73", "chrono"] } aws-sdk-s3 = "1.5.0" aws-config = "1.0.3" -polars = { version = "0.35.2", features = ["lazy", "parquet", "aws", "csv", "dtype-full", "serde"] } -polars-io = { version = "0.35.2", features = ["csv"] } + +polars = { version = "0.35.4", features = ["lazy", "parquet", "aws", "csv", "dtype-full", "serde", "strings", "extract_groups"] } +polars-io = { version = "0.35.4", features = ["csv"] } object_store = { version = "0.8.0", features = ["aws"] } openidconnect = { version = "3.4.0" } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 9394a3816f..e6baffbd32 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -7083,7 +7083,11 @@ paths: in: query schema: type: boolean - - name: search + - name: search_col + in: query + schema: + type: string + - name: search_term in: query schema: type: string diff --git a/backend/windmill-api/src/job_helpers.rs b/backend/windmill-api/src/job_helpers.rs index 0f6c3025b6..c31f8f4e68 100644 --- a/backend/windmill-api/src/job_helpers.rs +++ b/backend/windmill-api/src/job_helpers.rs @@ -17,7 +17,6 @@ use axum::{ use hyper::http; use itertools::Itertools; use object_store::ClientConfigKey; -use polars::chunked_array::ops::SortOptions; use polars::{ io::{ cloud::{AmazonS3ConfigKey, CloudOptions}, @@ -546,7 +545,8 @@ struct LoadParquetQuery { offset: Option, sort_col: Option, sort_desc: Option, - search: Option, + search_col: Option, + search_term: Option, } async fn load_parquet_preview( @@ -570,7 +570,9 @@ async fn load_parquet_preview( query .sort_col .map(|v| (v.to_string(), query.sort_desc.unwrap_or(false))), - query.search, + query + .search_col + .map(|v| (v.to_string(), query.search_term.unwrap_or_default())), ) .await .map(Json); @@ -1353,12 +1355,12 @@ async fn read_s3_parquet_chunk( limit: Option, offset: Option, sort: Option<(String, bool)>, - _search: Option, + search: Option<(String, String)>, ) -> error::Result> { let s3_cloud_config = build_polars_s3_config(s3_resource_ref); let args: ScanArgsParquet = ScanArgsParquet { - n_rows: Some(1), + n_rows: None, cache: false, parallel: polars::io::parquet::ParallelStrategy::Auto, rechunk: false, @@ -1397,12 +1399,12 @@ async fn read_s3_parquet_chunk( } else { df }; - - // let df = if let Some(search) = search { - // df.filter(col("*").str().contains(&search)) - // } else { - // df - // }; + use polars::prelude::*; + let df = if let Some(search) = search { + df.filter(col(&search.0).str().contains_literal(lit(search.1.clone()))) + } else { + df + }; let df = df .collect() .map_err(|err| error::Error::InternalErr(err.to_string()))?; diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index 71e1f96524..7c0e2a63e1 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -351,7 +351,12 @@ > {:else if !forceJson && resultKind == 's3object'} -
+
{:else} + {/if}
{#if typeof result?.s3 == 'string' && result?.s3?.endsWith('.parquet')} diff --git a/frontend/src/lib/components/ParqetTableRenderer.svelte b/frontend/src/lib/components/ParqetTableRenderer.svelte index 1a08cb2cf1..acd0839100 100644 --- a/frontend/src/lib/components/ParqetTableRenderer.svelte +++ b/frontend/src/lib/components/ParqetTableRenderer.svelte @@ -17,11 +17,17 @@ rowCount: 0, getRows: async function (params) { try { + const searchCol = params.filterModel ? Object.keys(params.filterModel)?.[0] : undefined + console.log(params.filterModel) const res = await HelpersService.loadParquetPreview({ workspace: $workspaceStore!, path: s3resource, - // offset: params.startRow, - limit: params.endRow - params.startRow + offset: params.startRow, + limit: params.endRow - params.startRow, + sortCol: params.sortModel?.[0]?.colId, + sortDesc: params.sortModel?.[0]?.sort == 'desc', + searchCol: searchCol, + searchTerm: searchCol ? params.filterModel?.[searchCol]?.filter : undefined }) const data: any[] = [] @@ -35,7 +41,6 @@ }) }) - console.log(res) params.successCallback(data) } catch (e) { console.error(e) @@ -63,8 +68,6 @@ $: eGui && mountGrid() - let firstRow = 0 - let lastRow = 0 async function mountGrid() { if (eGui) { const res = await HelpersService.loadParquetPreview({ @@ -78,11 +81,16 @@ { rowModelType: 'infinite', datasource, + // @ts-ignore columnDefs: res.columns.map((c) => { return { field: c.name, sortable: true, - filter: true + filter: true, + filterParams: { + filterOptions: ['contains'], + maxNumConditions: 1 + } } }), pagination: false, @@ -93,10 +101,6 @@ rowSelection: 'multiple', rowMultiSelectWithClick: true, suppressRowDeselection: true, - onViewportChanged: (e) => { - firstRow = e.firstRow - lastRow = e.lastRow - }, onSelectionChanged: (e) => { onSelectionChanged(e.api) diff --git a/frontend/src/lib/components/runs/JobPreview.svelte b/frontend/src/lib/components/runs/JobPreview.svelte index de2451dd01..967edee92b 100644 --- a/frontend/src/lib/components/runs/JobPreview.svelte +++ b/frontend/src/lib/components/runs/JobPreview.svelte @@ -36,7 +36,7 @@ -
+
{#if job}
{#if job?.['priority']} @@ -77,7 +77,7 @@
{/if} -
+
{#if job?.type === Job.type.COMPLETED_JOB} Result @@ -96,7 +96,7 @@
{:else} -
+
{#if viewTab == 'logs'}
{#if job} -
+
{#if viewTab == 'logs'}