From 0882279055beeef87f4a9bf6ec096d6eb933dd9e Mon Sep 17 00:00:00 2001 From: Guillaume Bouvignies Date: Wed, 22 Nov 2023 12:29:50 +0100 Subject: [PATCH] fix: Various fixes and improvements for Windmill S3 capabilities (#2674) --- backend/Cargo.lock | 8 + backend/windmill-api/Cargo.toml | 2 +- backend/windmill-api/openapi-deref.yaml | 175 ++++++++++++------ backend/windmill-api/openapi.yaml | 51 +++++ backend/windmill-api/src/job_helpers.rs | 60 +++--- .../src/lib/components/S3FilePicker.svelte | 4 +- 6 files changed, 207 insertions(+), 93 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 4ad66cf524..26c9d12cc2 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -3762,6 +3762,12 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "itoap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9028f49264629065d057f340a86acb84867925865f73bbf8d47b4d149a7e88b8" + [[package]] name = "jobserver" version = "0.1.27" @@ -5145,6 +5151,7 @@ checksum = "dd503430a6d9779b07915d858865fe998317ef3cfef8973881f578ac5d4baae7" dependencies = [ "ahash 0.8.6", "arrow-format", + "atoi 2.0.0", "atoi_simd", "bytemuck", "chrono", @@ -5184,6 +5191,7 @@ dependencies = [ "either", "hashbrown 0.14.2", "indexmap 2.1.0", + "itoap", "num-traits", "once_cell", "polars-arrow", diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 661bead073..0009af0768 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -84,6 +84,6 @@ candle-transformers.workspace = true candle-nn.workspace = true aws-sdk-s3 = "0.36.0" aws-config = "0.57.2" -polars = { version = "0.35.2", features = ["lazy", "parquet", "aws", "csv"] } +polars = { version = "0.35.2", features = ["lazy", "parquet", "aws", "csv", "dtype-full"] } polars-io = { version = "0.35.2", features = ["csv"] } object_store = { version = "0.8.0", features = ["aws"] } diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 72f92dda43..79b00433a1 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -1,6 +1,6 @@ openapi: 3.0.3 info: - version: 1.211.0 + version: 1.213.0 title: Windmill API contact: name: Windmill Team @@ -3301,32 +3301,32 @@ paths: id: type: string value: - oneOf: &ref_180 + oneOf: &ref_182 - type: object - properties: &ref_164 + properties: &ref_166 input_transforms: type: object additionalProperties: oneOf: &ref_24 - type: object - properties: &ref_160 + properties: &ref_162 value: {} type: type: string enum: - javascript - required: &ref_161 + required: &ref_163 - expr - type - type: object - properties: &ref_162 + properties: &ref_164 expr: type: string type: type: string enum: - javascript - required: &ref_163 + required: &ref_165 - expr - type discriminator: &ref_25 @@ -3366,13 +3366,13 @@ paths: type: number concurrency_time_window_s: type: number - required: &ref_165 + required: &ref_167 - type - content - language - input_transforms - type: object - properties: &ref_166 + properties: &ref_168 input_transforms: type: object additionalProperties: @@ -3386,12 +3386,12 @@ paths: type: string enum: - script - required: &ref_167 + required: &ref_169 - type - path - input_transforms - type: object - properties: &ref_168 + properties: &ref_170 input_transforms: type: object additionalProperties: @@ -3403,12 +3403,12 @@ paths: type: string enum: - flow - required: &ref_169 + required: &ref_171 - type - path - input_transforms - type: object - properties: &ref_170 + properties: &ref_172 modules: type: array items: @@ -3430,13 +3430,13 @@ paths: type: boolean parallelism: type: integer - required: &ref_171 + required: &ref_173 - modules - iterator - skip_failures - type - type: object - properties: &ref_172 + properties: &ref_174 branches: type: array items: @@ -3467,12 +3467,12 @@ paths: type: string enum: - branchone - required: &ref_173 + required: &ref_175 - branches - default - type - type: object - properties: &ref_174 + properties: &ref_176 branches: type: array items: @@ -3497,28 +3497,28 @@ paths: - branchall parallel: type: boolean - required: &ref_175 - - branches - - type - - type: object - properties: &ref_176 - type: - type: string - enum: - - identity - flow: - type: boolean required: &ref_177 + - branches - type - type: object properties: &ref_178 type: type: string enum: - - graphql + - identity + flow: + type: boolean required: &ref_179 - type - discriminator: &ref_181 + - type: object + properties: &ref_180 + type: + type: string + enum: + - graphql + required: &ref_181 + - type + discriminator: &ref_183 propertyName: type mapping: rawscript: '#/components/schemas/RawScript' @@ -3574,7 +3574,7 @@ paths: type: number retry: type: object - properties: &ref_182 + properties: &ref_184 constant: type: object properties: @@ -9487,6 +9487,57 @@ paths: properties: connection_settings_str: type: string + /w/{workspace}/job_helpers/polars_connection_settings: + post: + summary: >- + Converts an S3 resource to the set of arguments necessary to connect + Polars to an S3 bucket + operationId: polarsConnectionSettings + tags: + - helpers + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: S3 resource to connect to + required: true + content: + application/json: + schema: + s3_resource: + $ref: '#/components/schemas/S3Resource' + responses: + '200': + description: Connection settings + content: + application/json: + schema: + type: object + properties: + endpoint_url: + type: string + key: + type: string + secret: + type: string + use_ssl: + type: boolean + cache_regions: + type: boolean + client_kwargs: + type: object + properties: &ref_155 + region_name: + type: string + required: &ref_156 + - region_name + required: + - endpoint_url + - use_ssl + - cache_regions + - client_kwargs /w/{workspace}/job_helpers/test_connection: get: summary: Test connection to the workspace datasets storage @@ -9538,10 +9589,10 @@ paths: type: array items: type: object - properties: &ref_155 + properties: &ref_157 s3: type: string - required: &ref_156 + required: &ref_158 - s3 required: - windmill_large_files @@ -9568,7 +9619,7 @@ paths: application/json: schema: type: object - properties: &ref_157 + properties: &ref_159 mime_type: type: string size_in_bytes: @@ -9624,7 +9675,7 @@ paths: application/json: schema: type: object - properties: &ref_158 + properties: &ref_160 msg: type: string content: @@ -9636,7 +9687,7 @@ paths: - Csv - Parquet - Unknown - required: &ref_159 + required: &ref_161 - content_type components: securitySchemes: @@ -10236,20 +10287,24 @@ components: HubScriptKind: name: kind schema: *ref_29 + PolarsClientKwargs: + type: object + properties: *ref_155 + required: *ref_156 LargeFileStorage: type: object properties: *ref_15 WindmillLargeFile: type: object - properties: *ref_155 - required: *ref_156 + properties: *ref_157 + required: *ref_158 WindmillFileMetadata: type: object - properties: *ref_157 + properties: *ref_159 WindmillFilePreview: type: object - properties: *ref_158 - required: *ref_159 + properties: *ref_160 + required: *ref_161 S3Resource: type: object properties: @@ -10274,58 +10329,58 @@ components: - useSSL - pathStyle StaticTransform: - type: object - properties: *ref_160 - required: *ref_161 - JavascriptTransform: type: object properties: *ref_162 required: *ref_163 + JavascriptTransform: + type: object + properties: *ref_164 + required: *ref_165 InputTransform: oneOf: *ref_24 discriminator: *ref_25 RawScript: - type: object - properties: *ref_164 - required: *ref_165 - PathScript: type: object properties: *ref_166 required: *ref_167 - PathFlow: + PathScript: type: object properties: *ref_168 required: *ref_169 + PathFlow: + type: object + properties: *ref_170 + required: *ref_171 FlowModule: type: object properties: *ref_26 required: *ref_27 ForloopFlow: - type: object - properties: *ref_170 - required: *ref_171 - BranchOne: type: object properties: *ref_172 required: *ref_173 - BranchAll: + BranchOne: type: object properties: *ref_174 required: *ref_175 - Identity: + BranchAll: type: object properties: *ref_176 required: *ref_177 - Graphql: + Identity: type: object properties: *ref_178 required: *ref_179 + Graphql: + type: object + properties: *ref_180 + required: *ref_181 FlowModuleValue: - oneOf: *ref_180 - discriminator: *ref_181 + oneOf: *ref_182 + discriminator: *ref_183 Retry: type: object - properties: *ref_182 + properties: *ref_184 FlowValue: type: object properties: *ref_48 diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index ce8b95d7ac..595328c768 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -6320,6 +6320,49 @@ paths: properties: connection_settings_str: type: string + /w/{workspace}/job_helpers/polars_connection_settings: + post: + summary: + Converts an S3 resource to the set of arguments necessary to connect + Polars to an S3 bucket + operationId: polarsConnectionSettings + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: S3 resource to connect to + required: true + content: + application/json: + schema: + s3_resource: + $ref: "#/components/schemas/S3Resource" + responses: + "200": + description: Connection settings + content: + application/json: + schema: + type: object + properties: + endpoint_url: + type: string + key: + type: string + secret: + type: string + use_ssl: + type: boolean + cache_regions: + type: boolean + client_kwargs: + $ref: "#/components/schemas/PolarsClientKwargs" + required: + - endpoint_url + - use_ssl + - cache_regions + - client_kwargs /w/{workspace}/job_helpers/test_connection: get: @@ -8412,6 +8455,14 @@ components: type: string enum: [script, failure, trigger, approval] + PolarsClientKwargs: + type: object + properties: + region_name: + type: string + required: + - region_name + LargeFileStorage: type: object properties: diff --git a/backend/windmill-api/src/job_helpers.rs b/backend/windmill-api/src/job_helpers.rs index 2328d5d171..de2c1f7932 100644 --- a/backend/windmill-api/src/job_helpers.rs +++ b/backend/windmill-api/src/job_helpers.rs @@ -398,17 +398,6 @@ async fn load_file_preview( let content_type: WindmillContentType; let content_preview = match s3_object_mime_type.as_deref() { - Some("application/json") | Some("application/x-yaml") => { - content_type = WindmillContentType::RawText; - read_s3_text_object_head( - &s3_client, - &s3_bucket, - &file_key, - query.read_bytes_from, - file_chunk_length, - ) - .await - } Some("text/csv") => { content_type = WindmillContentType::Csv; read_s3_csv_object_head( @@ -420,19 +409,11 @@ async fn load_file_preview( ) .await } - Some("application/octet-stream") => { - if file_key.to_lowercase().ends_with(".parquet") { - content_type = WindmillContentType::Parquet; - read_s3_parquet_object_head(&s3_resource, &file_key).await - } else { - content_type = WindmillContentType::Unknown; - Err(error::Error::ExecutionErr( - "Preview is not available for content of type application/octet-stream" - .to_string(), - )) - } - } - Some(mt) if mt.starts_with("text/") => { + Some(mt) + if mt.starts_with("text/") + || mt == "application/json" + || mt == "application/x-yaml" => + { content_type = WindmillContentType::RawText; read_s3_text_object_head( &s3_client, @@ -443,11 +424,32 @@ async fn load_file_preview( ) .await } - _ => { - content_type = WindmillContentType::Unknown; - Err(error::Error::ExecutionErr( - "Preview is not available for content of type application/octet-stream".to_string(), - )) + mt_opt => { + // sometimes S3 doesn't infer the content type on upload. Guess it from the file extension + if file_key.to_lowercase().ends_with(".parquet") { + content_type = WindmillContentType::Parquet; + read_s3_parquet_object_head(&s3_resource, &file_key).await + } else if file_key.to_lowercase().ends_with(".csv") { + content_type = WindmillContentType::Csv; + read_s3_csv_object_head( + &s3_client, + &s3_bucket, + &file_key, + file_chunk_length, + query.csv_separator, + ) + .await + } else { + content_type = WindmillContentType::Unknown; + let msg = match mt_opt { + Some(mt) => { + format!("Preview is not available for content of type '{}'", mt).to_string() + } + None => "Preview is not available. Content type is unknown or not supported" + .to_string(), + }; + Err(error::Error::ExecutionErr(msg)) + } } }; diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index de6e6e82a8..076ac0bc5b 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -18,7 +18,6 @@ let initialFileKeyInternalCopy: { s3: string } export let selectedFileKey: { s3: string } | undefined = undefined - let csvSeparatorDefaultChar: string = ',' let csvSeparatorChar: string = ',' let dispatch = createEventDispatcher() @@ -175,8 +174,6 @@ } function selectItem(index: number, toggleCollapsed: boolean = true) { - csvSeparatorChar = csvSeparatorDefaultChar - let item_key = displayedFileKeys[index] let item = allFilesByKey[item_key] if (item.type === 'folder') { @@ -339,6 +336,7 @@ > + {:else}