diff --git a/backend/Cargo.lock b/backend/Cargo.lock
index 4ad66cf524..26c9d12cc2 100644
--- a/backend/Cargo.lock
+++ b/backend/Cargo.lock
@@ -3762,6 +3762,12 @@ version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
+[[package]]
+name = "itoap"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9028f49264629065d057f340a86acb84867925865f73bbf8d47b4d149a7e88b8"
+
[[package]]
name = "jobserver"
version = "0.1.27"
@@ -5145,6 +5151,7 @@ checksum = "dd503430a6d9779b07915d858865fe998317ef3cfef8973881f578ac5d4baae7"
dependencies = [
"ahash 0.8.6",
"arrow-format",
+ "atoi 2.0.0",
"atoi_simd",
"bytemuck",
"chrono",
@@ -5184,6 +5191,7 @@ dependencies = [
"either",
"hashbrown 0.14.2",
"indexmap 2.1.0",
+ "itoap",
"num-traits",
"once_cell",
"polars-arrow",
diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml
index 661bead073..0009af0768 100644
--- a/backend/windmill-api/Cargo.toml
+++ b/backend/windmill-api/Cargo.toml
@@ -84,6 +84,6 @@ candle-transformers.workspace = true
candle-nn.workspace = true
aws-sdk-s3 = "0.36.0"
aws-config = "0.57.2"
-polars = { version = "0.35.2", features = ["lazy", "parquet", "aws", "csv"] }
+polars = { version = "0.35.2", features = ["lazy", "parquet", "aws", "csv", "dtype-full"] }
polars-io = { version = "0.35.2", features = ["csv"] }
object_store = { version = "0.8.0", features = ["aws"] }
diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml
index 72f92dda43..79b00433a1 100644
--- a/backend/windmill-api/openapi-deref.yaml
+++ b/backend/windmill-api/openapi-deref.yaml
@@ -1,6 +1,6 @@
openapi: 3.0.3
info:
- version: 1.211.0
+ version: 1.213.0
title: Windmill API
contact:
name: Windmill Team
@@ -3301,32 +3301,32 @@ paths:
id:
type: string
value:
- oneOf: &ref_180
+ oneOf: &ref_182
- type: object
- properties: &ref_164
+ properties: &ref_166
input_transforms:
type: object
additionalProperties:
oneOf: &ref_24
- type: object
- properties: &ref_160
+ properties: &ref_162
value: {}
type:
type: string
enum:
- javascript
- required: &ref_161
+ required: &ref_163
- expr
- type
- type: object
- properties: &ref_162
+ properties: &ref_164
expr:
type: string
type:
type: string
enum:
- javascript
- required: &ref_163
+ required: &ref_165
- expr
- type
discriminator: &ref_25
@@ -3366,13 +3366,13 @@ paths:
type: number
concurrency_time_window_s:
type: number
- required: &ref_165
+ required: &ref_167
- type
- content
- language
- input_transforms
- type: object
- properties: &ref_166
+ properties: &ref_168
input_transforms:
type: object
additionalProperties:
@@ -3386,12 +3386,12 @@ paths:
type: string
enum:
- script
- required: &ref_167
+ required: &ref_169
- type
- path
- input_transforms
- type: object
- properties: &ref_168
+ properties: &ref_170
input_transforms:
type: object
additionalProperties:
@@ -3403,12 +3403,12 @@ paths:
type: string
enum:
- flow
- required: &ref_169
+ required: &ref_171
- type
- path
- input_transforms
- type: object
- properties: &ref_170
+ properties: &ref_172
modules:
type: array
items:
@@ -3430,13 +3430,13 @@ paths:
type: boolean
parallelism:
type: integer
- required: &ref_171
+ required: &ref_173
- modules
- iterator
- skip_failures
- type
- type: object
- properties: &ref_172
+ properties: &ref_174
branches:
type: array
items:
@@ -3467,12 +3467,12 @@ paths:
type: string
enum:
- branchone
- required: &ref_173
+ required: &ref_175
- branches
- default
- type
- type: object
- properties: &ref_174
+ properties: &ref_176
branches:
type: array
items:
@@ -3497,28 +3497,28 @@ paths:
- branchall
parallel:
type: boolean
- required: &ref_175
- - branches
- - type
- - type: object
- properties: &ref_176
- type:
- type: string
- enum:
- - identity
- flow:
- type: boolean
required: &ref_177
+ - branches
- type
- type: object
properties: &ref_178
type:
type: string
enum:
- - graphql
+ - identity
+ flow:
+ type: boolean
required: &ref_179
- type
- discriminator: &ref_181
+ - type: object
+ properties: &ref_180
+ type:
+ type: string
+ enum:
+ - graphql
+ required: &ref_181
+ - type
+ discriminator: &ref_183
propertyName: type
mapping:
rawscript: '#/components/schemas/RawScript'
@@ -3574,7 +3574,7 @@ paths:
type: number
retry:
type: object
- properties: &ref_182
+ properties: &ref_184
constant:
type: object
properties:
@@ -9487,6 +9487,57 @@ paths:
properties:
connection_settings_str:
type: string
+ /w/{workspace}/job_helpers/polars_connection_settings:
+ post:
+ summary: >-
+ Converts an S3 resource to the set of arguments necessary to connect
+ Polars to an S3 bucket
+ operationId: polarsConnectionSettings
+ tags:
+ - helpers
+ parameters:
+ - name: workspace
+ in: path
+ required: true
+ schema: *ref_0
+ requestBody:
+ description: S3 resource to connect to
+ required: true
+ content:
+ application/json:
+ schema:
+ s3_resource:
+ $ref: '#/components/schemas/S3Resource'
+ responses:
+ '200':
+ description: Connection settings
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ endpoint_url:
+ type: string
+ key:
+ type: string
+ secret:
+ type: string
+ use_ssl:
+ type: boolean
+ cache_regions:
+ type: boolean
+ client_kwargs:
+ type: object
+ properties: &ref_155
+ region_name:
+ type: string
+ required: &ref_156
+ - region_name
+ required:
+ - endpoint_url
+ - use_ssl
+ - cache_regions
+ - client_kwargs
/w/{workspace}/job_helpers/test_connection:
get:
summary: Test connection to the workspace datasets storage
@@ -9538,10 +9589,10 @@ paths:
type: array
items:
type: object
- properties: &ref_155
+ properties: &ref_157
s3:
type: string
- required: &ref_156
+ required: &ref_158
- s3
required:
- windmill_large_files
@@ -9568,7 +9619,7 @@ paths:
application/json:
schema:
type: object
- properties: &ref_157
+ properties: &ref_159
mime_type:
type: string
size_in_bytes:
@@ -9624,7 +9675,7 @@ paths:
application/json:
schema:
type: object
- properties: &ref_158
+ properties: &ref_160
msg:
type: string
content:
@@ -9636,7 +9687,7 @@ paths:
- Csv
- Parquet
- Unknown
- required: &ref_159
+ required: &ref_161
- content_type
components:
securitySchemes:
@@ -10236,20 +10287,24 @@ components:
HubScriptKind:
name: kind
schema: *ref_29
+ PolarsClientKwargs:
+ type: object
+ properties: *ref_155
+ required: *ref_156
LargeFileStorage:
type: object
properties: *ref_15
WindmillLargeFile:
type: object
- properties: *ref_155
- required: *ref_156
+ properties: *ref_157
+ required: *ref_158
WindmillFileMetadata:
type: object
- properties: *ref_157
+ properties: *ref_159
WindmillFilePreview:
type: object
- properties: *ref_158
- required: *ref_159
+ properties: *ref_160
+ required: *ref_161
S3Resource:
type: object
properties:
@@ -10274,58 +10329,58 @@ components:
- useSSL
- pathStyle
StaticTransform:
- type: object
- properties: *ref_160
- required: *ref_161
- JavascriptTransform:
type: object
properties: *ref_162
required: *ref_163
+ JavascriptTransform:
+ type: object
+ properties: *ref_164
+ required: *ref_165
InputTransform:
oneOf: *ref_24
discriminator: *ref_25
RawScript:
- type: object
- properties: *ref_164
- required: *ref_165
- PathScript:
type: object
properties: *ref_166
required: *ref_167
- PathFlow:
+ PathScript:
type: object
properties: *ref_168
required: *ref_169
+ PathFlow:
+ type: object
+ properties: *ref_170
+ required: *ref_171
FlowModule:
type: object
properties: *ref_26
required: *ref_27
ForloopFlow:
- type: object
- properties: *ref_170
- required: *ref_171
- BranchOne:
type: object
properties: *ref_172
required: *ref_173
- BranchAll:
+ BranchOne:
type: object
properties: *ref_174
required: *ref_175
- Identity:
+ BranchAll:
type: object
properties: *ref_176
required: *ref_177
- Graphql:
+ Identity:
type: object
properties: *ref_178
required: *ref_179
+ Graphql:
+ type: object
+ properties: *ref_180
+ required: *ref_181
FlowModuleValue:
- oneOf: *ref_180
- discriminator: *ref_181
+ oneOf: *ref_182
+ discriminator: *ref_183
Retry:
type: object
- properties: *ref_182
+ properties: *ref_184
FlowValue:
type: object
properties: *ref_48
diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml
index ce8b95d7ac..595328c768 100644
--- a/backend/windmill-api/openapi.yaml
+++ b/backend/windmill-api/openapi.yaml
@@ -6320,6 +6320,49 @@ paths:
properties:
connection_settings_str:
type: string
+ /w/{workspace}/job_helpers/polars_connection_settings:
+ post:
+ summary:
+ Converts an S3 resource to the set of arguments necessary to connect
+ Polars to an S3 bucket
+ operationId: polarsConnectionSettings
+ tags:
+ - helpers
+ parameters:
+ - $ref: "#/components/parameters/WorkspaceId"
+ requestBody:
+ description: S3 resource to connect to
+ required: true
+ content:
+ application/json:
+ schema:
+ s3_resource:
+ $ref: "#/components/schemas/S3Resource"
+ responses:
+ "200":
+ description: Connection settings
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ endpoint_url:
+ type: string
+ key:
+ type: string
+ secret:
+ type: string
+ use_ssl:
+ type: boolean
+ cache_regions:
+ type: boolean
+ client_kwargs:
+ $ref: "#/components/schemas/PolarsClientKwargs"
+ required:
+ - endpoint_url
+ - use_ssl
+ - cache_regions
+ - client_kwargs
/w/{workspace}/job_helpers/test_connection:
get:
@@ -8412,6 +8455,14 @@ components:
type: string
enum: [script, failure, trigger, approval]
+ PolarsClientKwargs:
+ type: object
+ properties:
+ region_name:
+ type: string
+ required:
+ - region_name
+
LargeFileStorage:
type: object
properties:
diff --git a/backend/windmill-api/src/job_helpers.rs b/backend/windmill-api/src/job_helpers.rs
index 2328d5d171..de2c1f7932 100644
--- a/backend/windmill-api/src/job_helpers.rs
+++ b/backend/windmill-api/src/job_helpers.rs
@@ -398,17 +398,6 @@ async fn load_file_preview(
let content_type: WindmillContentType;
let content_preview = match s3_object_mime_type.as_deref() {
- Some("application/json") | Some("application/x-yaml") => {
- content_type = WindmillContentType::RawText;
- read_s3_text_object_head(
- &s3_client,
- &s3_bucket,
- &file_key,
- query.read_bytes_from,
- file_chunk_length,
- )
- .await
- }
Some("text/csv") => {
content_type = WindmillContentType::Csv;
read_s3_csv_object_head(
@@ -420,19 +409,11 @@ async fn load_file_preview(
)
.await
}
- Some("application/octet-stream") => {
- if file_key.to_lowercase().ends_with(".parquet") {
- content_type = WindmillContentType::Parquet;
- read_s3_parquet_object_head(&s3_resource, &file_key).await
- } else {
- content_type = WindmillContentType::Unknown;
- Err(error::Error::ExecutionErr(
- "Preview is not available for content of type application/octet-stream"
- .to_string(),
- ))
- }
- }
- Some(mt) if mt.starts_with("text/") => {
+ Some(mt)
+ if mt.starts_with("text/")
+ || mt == "application/json"
+ || mt == "application/x-yaml" =>
+ {
content_type = WindmillContentType::RawText;
read_s3_text_object_head(
&s3_client,
@@ -443,11 +424,32 @@ async fn load_file_preview(
)
.await
}
- _ => {
- content_type = WindmillContentType::Unknown;
- Err(error::Error::ExecutionErr(
- "Preview is not available for content of type application/octet-stream".to_string(),
- ))
+ mt_opt => {
+ // sometimes S3 doesn't infer the content type on upload. Guess it from the file extension
+ if file_key.to_lowercase().ends_with(".parquet") {
+ content_type = WindmillContentType::Parquet;
+ read_s3_parquet_object_head(&s3_resource, &file_key).await
+ } else if file_key.to_lowercase().ends_with(".csv") {
+ content_type = WindmillContentType::Csv;
+ read_s3_csv_object_head(
+ &s3_client,
+ &s3_bucket,
+ &file_key,
+ file_chunk_length,
+ query.csv_separator,
+ )
+ .await
+ } else {
+ content_type = WindmillContentType::Unknown;
+ let msg = match mt_opt {
+ Some(mt) => {
+ format!("Preview is not available for content of type '{}'", mt).to_string()
+ }
+ None => "Preview is not available. Content type is unknown or not supported"
+ .to_string(),
+ };
+ Err(error::Error::ExecutionErr(msg))
+ }
}
};
diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte
index de6e6e82a8..076ac0bc5b 100644
--- a/frontend/src/lib/components/S3FilePicker.svelte
+++ b/frontend/src/lib/components/S3FilePicker.svelte
@@ -18,7 +18,6 @@
let initialFileKeyInternalCopy: { s3: string }
export let selectedFileKey: { s3: string } | undefined = undefined
- let csvSeparatorDefaultChar: string = ','
let csvSeparatorChar: string = ','
let dispatch = createEventDispatcher()
@@ -175,8 +174,6 @@
}
function selectItem(index: number, toggleCollapsed: boolean = true) {
- csvSeparatorChar = csvSeparatorDefaultChar
-
let item_key = displayedFileKeys[index]
let item = allFilesByKey[item_key]
if (item.type === 'folder') {
@@ -339,6 +336,7 @@
>
+
{:else}