mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-20 16:02:19 +00:00
fix: Various fixes and improvements for Windmill S3 capabilities (#2674)
This commit is contained in:
Guillaume Bouvignies
committed by
GitHub
GitHub
parent
5f31f9b317
commit
0882279055
Generated
+8
@@ -3762,6 +3762,12 @@ version = "1.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
|
||||
|
||||
[[package]]
|
||||
name = "itoap"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9028f49264629065d057f340a86acb84867925865f73bbf8d47b4d149a7e88b8"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.27"
|
||||
@@ -5145,6 +5151,7 @@ checksum = "dd503430a6d9779b07915d858865fe998317ef3cfef8973881f578ac5d4baae7"
|
||||
dependencies = [
|
||||
"ahash 0.8.6",
|
||||
"arrow-format",
|
||||
"atoi 2.0.0",
|
||||
"atoi_simd",
|
||||
"bytemuck",
|
||||
"chrono",
|
||||
@@ -5184,6 +5191,7 @@ dependencies = [
|
||||
"either",
|
||||
"hashbrown 0.14.2",
|
||||
"indexmap 2.1.0",
|
||||
"itoap",
|
||||
"num-traits",
|
||||
"once_cell",
|
||||
"polars-arrow",
|
||||
|
||||
@@ -84,6 +84,6 @@ candle-transformers.workspace = true
|
||||
candle-nn.workspace = true
|
||||
aws-sdk-s3 = "0.36.0"
|
||||
aws-config = "0.57.2"
|
||||
polars = { version = "0.35.2", features = ["lazy", "parquet", "aws", "csv"] }
|
||||
polars = { version = "0.35.2", features = ["lazy", "parquet", "aws", "csv", "dtype-full"] }
|
||||
polars-io = { version = "0.35.2", features = ["csv"] }
|
||||
object_store = { version = "0.8.0", features = ["aws"] }
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
version: 1.211.0
|
||||
version: 1.213.0
|
||||
title: Windmill API
|
||||
contact:
|
||||
name: Windmill Team
|
||||
@@ -3301,32 +3301,32 @@ paths:
|
||||
id:
|
||||
type: string
|
||||
value:
|
||||
oneOf: &ref_180
|
||||
oneOf: &ref_182
|
||||
- type: object
|
||||
properties: &ref_164
|
||||
properties: &ref_166
|
||||
input_transforms:
|
||||
type: object
|
||||
additionalProperties:
|
||||
oneOf: &ref_24
|
||||
- type: object
|
||||
properties: &ref_160
|
||||
properties: &ref_162
|
||||
value: {}
|
||||
type:
|
||||
type: string
|
||||
enum:
|
||||
- javascript
|
||||
required: &ref_161
|
||||
required: &ref_163
|
||||
- expr
|
||||
- type
|
||||
- type: object
|
||||
properties: &ref_162
|
||||
properties: &ref_164
|
||||
expr:
|
||||
type: string
|
||||
type:
|
||||
type: string
|
||||
enum:
|
||||
- javascript
|
||||
required: &ref_163
|
||||
required: &ref_165
|
||||
- expr
|
||||
- type
|
||||
discriminator: &ref_25
|
||||
@@ -3366,13 +3366,13 @@ paths:
|
||||
type: number
|
||||
concurrency_time_window_s:
|
||||
type: number
|
||||
required: &ref_165
|
||||
required: &ref_167
|
||||
- type
|
||||
- content
|
||||
- language
|
||||
- input_transforms
|
||||
- type: object
|
||||
properties: &ref_166
|
||||
properties: &ref_168
|
||||
input_transforms:
|
||||
type: object
|
||||
additionalProperties:
|
||||
@@ -3386,12 +3386,12 @@ paths:
|
||||
type: string
|
||||
enum:
|
||||
- script
|
||||
required: &ref_167
|
||||
required: &ref_169
|
||||
- type
|
||||
- path
|
||||
- input_transforms
|
||||
- type: object
|
||||
properties: &ref_168
|
||||
properties: &ref_170
|
||||
input_transforms:
|
||||
type: object
|
||||
additionalProperties:
|
||||
@@ -3403,12 +3403,12 @@ paths:
|
||||
type: string
|
||||
enum:
|
||||
- flow
|
||||
required: &ref_169
|
||||
required: &ref_171
|
||||
- type
|
||||
- path
|
||||
- input_transforms
|
||||
- type: object
|
||||
properties: &ref_170
|
||||
properties: &ref_172
|
||||
modules:
|
||||
type: array
|
||||
items:
|
||||
@@ -3430,13 +3430,13 @@ paths:
|
||||
type: boolean
|
||||
parallelism:
|
||||
type: integer
|
||||
required: &ref_171
|
||||
required: &ref_173
|
||||
- modules
|
||||
- iterator
|
||||
- skip_failures
|
||||
- type
|
||||
- type: object
|
||||
properties: &ref_172
|
||||
properties: &ref_174
|
||||
branches:
|
||||
type: array
|
||||
items:
|
||||
@@ -3467,12 +3467,12 @@ paths:
|
||||
type: string
|
||||
enum:
|
||||
- branchone
|
||||
required: &ref_173
|
||||
required: &ref_175
|
||||
- branches
|
||||
- default
|
||||
- type
|
||||
- type: object
|
||||
properties: &ref_174
|
||||
properties: &ref_176
|
||||
branches:
|
||||
type: array
|
||||
items:
|
||||
@@ -3497,28 +3497,28 @@ paths:
|
||||
- branchall
|
||||
parallel:
|
||||
type: boolean
|
||||
required: &ref_175
|
||||
- branches
|
||||
- type
|
||||
- type: object
|
||||
properties: &ref_176
|
||||
type:
|
||||
type: string
|
||||
enum:
|
||||
- identity
|
||||
flow:
|
||||
type: boolean
|
||||
required: &ref_177
|
||||
- branches
|
||||
- type
|
||||
- type: object
|
||||
properties: &ref_178
|
||||
type:
|
||||
type: string
|
||||
enum:
|
||||
- graphql
|
||||
- identity
|
||||
flow:
|
||||
type: boolean
|
||||
required: &ref_179
|
||||
- type
|
||||
discriminator: &ref_181
|
||||
- type: object
|
||||
properties: &ref_180
|
||||
type:
|
||||
type: string
|
||||
enum:
|
||||
- graphql
|
||||
required: &ref_181
|
||||
- type
|
||||
discriminator: &ref_183
|
||||
propertyName: type
|
||||
mapping:
|
||||
rawscript: '#/components/schemas/RawScript'
|
||||
@@ -3574,7 +3574,7 @@ paths:
|
||||
type: number
|
||||
retry:
|
||||
type: object
|
||||
properties: &ref_182
|
||||
properties: &ref_184
|
||||
constant:
|
||||
type: object
|
||||
properties:
|
||||
@@ -9487,6 +9487,57 @@ paths:
|
||||
properties:
|
||||
connection_settings_str:
|
||||
type: string
|
||||
/w/{workspace}/job_helpers/polars_connection_settings:
|
||||
post:
|
||||
summary: >-
|
||||
Converts an S3 resource to the set of arguments necessary to connect
|
||||
Polars to an S3 bucket
|
||||
operationId: polarsConnectionSettings
|
||||
tags:
|
||||
- helpers
|
||||
parameters:
|
||||
- name: workspace
|
||||
in: path
|
||||
required: true
|
||||
schema: *ref_0
|
||||
requestBody:
|
||||
description: S3 resource to connect to
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
s3_resource:
|
||||
$ref: '#/components/schemas/S3Resource'
|
||||
responses:
|
||||
'200':
|
||||
description: Connection settings
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
endpoint_url:
|
||||
type: string
|
||||
key:
|
||||
type: string
|
||||
secret:
|
||||
type: string
|
||||
use_ssl:
|
||||
type: boolean
|
||||
cache_regions:
|
||||
type: boolean
|
||||
client_kwargs:
|
||||
type: object
|
||||
properties: &ref_155
|
||||
region_name:
|
||||
type: string
|
||||
required: &ref_156
|
||||
- region_name
|
||||
required:
|
||||
- endpoint_url
|
||||
- use_ssl
|
||||
- cache_regions
|
||||
- client_kwargs
|
||||
/w/{workspace}/job_helpers/test_connection:
|
||||
get:
|
||||
summary: Test connection to the workspace datasets storage
|
||||
@@ -9538,10 +9589,10 @@ paths:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
properties: &ref_155
|
||||
properties: &ref_157
|
||||
s3:
|
||||
type: string
|
||||
required: &ref_156
|
||||
required: &ref_158
|
||||
- s3
|
||||
required:
|
||||
- windmill_large_files
|
||||
@@ -9568,7 +9619,7 @@ paths:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties: &ref_157
|
||||
properties: &ref_159
|
||||
mime_type:
|
||||
type: string
|
||||
size_in_bytes:
|
||||
@@ -9624,7 +9675,7 @@ paths:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties: &ref_158
|
||||
properties: &ref_160
|
||||
msg:
|
||||
type: string
|
||||
content:
|
||||
@@ -9636,7 +9687,7 @@ paths:
|
||||
- Csv
|
||||
- Parquet
|
||||
- Unknown
|
||||
required: &ref_159
|
||||
required: &ref_161
|
||||
- content_type
|
||||
components:
|
||||
securitySchemes:
|
||||
@@ -10236,20 +10287,24 @@ components:
|
||||
HubScriptKind:
|
||||
name: kind
|
||||
schema: *ref_29
|
||||
PolarsClientKwargs:
|
||||
type: object
|
||||
properties: *ref_155
|
||||
required: *ref_156
|
||||
LargeFileStorage:
|
||||
type: object
|
||||
properties: *ref_15
|
||||
WindmillLargeFile:
|
||||
type: object
|
||||
properties: *ref_155
|
||||
required: *ref_156
|
||||
properties: *ref_157
|
||||
required: *ref_158
|
||||
WindmillFileMetadata:
|
||||
type: object
|
||||
properties: *ref_157
|
||||
properties: *ref_159
|
||||
WindmillFilePreview:
|
||||
type: object
|
||||
properties: *ref_158
|
||||
required: *ref_159
|
||||
properties: *ref_160
|
||||
required: *ref_161
|
||||
S3Resource:
|
||||
type: object
|
||||
properties:
|
||||
@@ -10274,58 +10329,58 @@ components:
|
||||
- useSSL
|
||||
- pathStyle
|
||||
StaticTransform:
|
||||
type: object
|
||||
properties: *ref_160
|
||||
required: *ref_161
|
||||
JavascriptTransform:
|
||||
type: object
|
||||
properties: *ref_162
|
||||
required: *ref_163
|
||||
JavascriptTransform:
|
||||
type: object
|
||||
properties: *ref_164
|
||||
required: *ref_165
|
||||
InputTransform:
|
||||
oneOf: *ref_24
|
||||
discriminator: *ref_25
|
||||
RawScript:
|
||||
type: object
|
||||
properties: *ref_164
|
||||
required: *ref_165
|
||||
PathScript:
|
||||
type: object
|
||||
properties: *ref_166
|
||||
required: *ref_167
|
||||
PathFlow:
|
||||
PathScript:
|
||||
type: object
|
||||
properties: *ref_168
|
||||
required: *ref_169
|
||||
PathFlow:
|
||||
type: object
|
||||
properties: *ref_170
|
||||
required: *ref_171
|
||||
FlowModule:
|
||||
type: object
|
||||
properties: *ref_26
|
||||
required: *ref_27
|
||||
ForloopFlow:
|
||||
type: object
|
||||
properties: *ref_170
|
||||
required: *ref_171
|
||||
BranchOne:
|
||||
type: object
|
||||
properties: *ref_172
|
||||
required: *ref_173
|
||||
BranchAll:
|
||||
BranchOne:
|
||||
type: object
|
||||
properties: *ref_174
|
||||
required: *ref_175
|
||||
Identity:
|
||||
BranchAll:
|
||||
type: object
|
||||
properties: *ref_176
|
||||
required: *ref_177
|
||||
Graphql:
|
||||
Identity:
|
||||
type: object
|
||||
properties: *ref_178
|
||||
required: *ref_179
|
||||
Graphql:
|
||||
type: object
|
||||
properties: *ref_180
|
||||
required: *ref_181
|
||||
FlowModuleValue:
|
||||
oneOf: *ref_180
|
||||
discriminator: *ref_181
|
||||
oneOf: *ref_182
|
||||
discriminator: *ref_183
|
||||
Retry:
|
||||
type: object
|
||||
properties: *ref_182
|
||||
properties: *ref_184
|
||||
FlowValue:
|
||||
type: object
|
||||
properties: *ref_48
|
||||
|
||||
@@ -6320,6 +6320,49 @@ paths:
|
||||
properties:
|
||||
connection_settings_str:
|
||||
type: string
|
||||
/w/{workspace}/job_helpers/polars_connection_settings:
|
||||
post:
|
||||
summary:
|
||||
Converts an S3 resource to the set of arguments necessary to connect
|
||||
Polars to an S3 bucket
|
||||
operationId: polarsConnectionSettings
|
||||
tags:
|
||||
- helpers
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
requestBody:
|
||||
description: S3 resource to connect to
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
s3_resource:
|
||||
$ref: "#/components/schemas/S3Resource"
|
||||
responses:
|
||||
"200":
|
||||
description: Connection settings
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
endpoint_url:
|
||||
type: string
|
||||
key:
|
||||
type: string
|
||||
secret:
|
||||
type: string
|
||||
use_ssl:
|
||||
type: boolean
|
||||
cache_regions:
|
||||
type: boolean
|
||||
client_kwargs:
|
||||
$ref: "#/components/schemas/PolarsClientKwargs"
|
||||
required:
|
||||
- endpoint_url
|
||||
- use_ssl
|
||||
- cache_regions
|
||||
- client_kwargs
|
||||
|
||||
/w/{workspace}/job_helpers/test_connection:
|
||||
get:
|
||||
@@ -8412,6 +8455,14 @@ components:
|
||||
type: string
|
||||
enum: [script, failure, trigger, approval]
|
||||
|
||||
PolarsClientKwargs:
|
||||
type: object
|
||||
properties:
|
||||
region_name:
|
||||
type: string
|
||||
required:
|
||||
- region_name
|
||||
|
||||
LargeFileStorage:
|
||||
type: object
|
||||
properties:
|
||||
|
||||
@@ -398,17 +398,6 @@ async fn load_file_preview(
|
||||
|
||||
let content_type: WindmillContentType;
|
||||
let content_preview = match s3_object_mime_type.as_deref() {
|
||||
Some("application/json") | Some("application/x-yaml") => {
|
||||
content_type = WindmillContentType::RawText;
|
||||
read_s3_text_object_head(
|
||||
&s3_client,
|
||||
&s3_bucket,
|
||||
&file_key,
|
||||
query.read_bytes_from,
|
||||
file_chunk_length,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Some("text/csv") => {
|
||||
content_type = WindmillContentType::Csv;
|
||||
read_s3_csv_object_head(
|
||||
@@ -420,19 +409,11 @@ async fn load_file_preview(
|
||||
)
|
||||
.await
|
||||
}
|
||||
Some("application/octet-stream") => {
|
||||
if file_key.to_lowercase().ends_with(".parquet") {
|
||||
content_type = WindmillContentType::Parquet;
|
||||
read_s3_parquet_object_head(&s3_resource, &file_key).await
|
||||
} else {
|
||||
content_type = WindmillContentType::Unknown;
|
||||
Err(error::Error::ExecutionErr(
|
||||
"Preview is not available for content of type application/octet-stream"
|
||||
.to_string(),
|
||||
))
|
||||
}
|
||||
}
|
||||
Some(mt) if mt.starts_with("text/") => {
|
||||
Some(mt)
|
||||
if mt.starts_with("text/")
|
||||
|| mt == "application/json"
|
||||
|| mt == "application/x-yaml" =>
|
||||
{
|
||||
content_type = WindmillContentType::RawText;
|
||||
read_s3_text_object_head(
|
||||
&s3_client,
|
||||
@@ -443,11 +424,32 @@ async fn load_file_preview(
|
||||
)
|
||||
.await
|
||||
}
|
||||
_ => {
|
||||
content_type = WindmillContentType::Unknown;
|
||||
Err(error::Error::ExecutionErr(
|
||||
"Preview is not available for content of type application/octet-stream".to_string(),
|
||||
))
|
||||
mt_opt => {
|
||||
// sometimes S3 doesn't infer the content type on upload. Guess it from the file extension
|
||||
if file_key.to_lowercase().ends_with(".parquet") {
|
||||
content_type = WindmillContentType::Parquet;
|
||||
read_s3_parquet_object_head(&s3_resource, &file_key).await
|
||||
} else if file_key.to_lowercase().ends_with(".csv") {
|
||||
content_type = WindmillContentType::Csv;
|
||||
read_s3_csv_object_head(
|
||||
&s3_client,
|
||||
&s3_bucket,
|
||||
&file_key,
|
||||
file_chunk_length,
|
||||
query.csv_separator,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
content_type = WindmillContentType::Unknown;
|
||||
let msg = match mt_opt {
|
||||
Some(mt) => {
|
||||
format!("Preview is not available for content of type '{}'", mt).to_string()
|
||||
}
|
||||
None => "Preview is not available. Content type is unknown or not supported"
|
||||
.to_string(),
|
||||
};
|
||||
Err(error::Error::ExecutionErr(msg))
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
let initialFileKeyInternalCopy: { s3: string }
|
||||
export let selectedFileKey: { s3: string } | undefined = undefined
|
||||
|
||||
let csvSeparatorDefaultChar: string = ','
|
||||
let csvSeparatorChar: string = ','
|
||||
|
||||
let dispatch = createEventDispatcher()
|
||||
@@ -175,8 +174,6 @@
|
||||
}
|
||||
|
||||
function selectItem(index: number, toggleCollapsed: boolean = true) {
|
||||
csvSeparatorChar = csvSeparatorDefaultChar
|
||||
|
||||
let item_key = displayedFileKeys[index]
|
||||
let item = allFilesByKey[item_key]
|
||||
if (item.type === 'folder') {
|
||||
@@ -339,6 +336,7 @@
|
||||
>
|
||||
<option value=",">,</option>
|
||||
<option value=";">;</option>
|
||||
<option value="|">|</option>
|
||||
</select>
|
||||
</div>
|
||||
{:else}
|
||||
|
||||
Reference in New Issue
Block a user