From f8c7a8edf87226e020cec5e602e2dcb31f87d95e Mon Sep 17 00:00:00 2001 From: Guillaume Bouvignies Date: Thu, 14 Dec 2023 15:45:37 +0100 Subject: [PATCH] feat: Add download button to S3 file picker (#2857) --- backend/windmill-api/openapi-deref.yaml | 60 +++++++++- backend/windmill-api/openapi.yaml | 111 +++++++++++------- backend/windmill-api/src/job_helpers.rs | 76 +++++++++++- .../src/lib/components/S3FilePicker.svelte | 36 +++++- python-client/tests/wmill_client_test.py | 13 +- python-client/wmill/wmill/client.py | 30 +++++ typescript-client/client.ts | 22 +++- 7 files changed, 291 insertions(+), 57 deletions(-) diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 0f2b8fff13..d7dfe23799 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -1,6 +1,6 @@ openapi: 3.0.3 info: - version: 1.225.0 + version: 1.226.1 title: Windmill API contact: name: Windmill Team @@ -2238,6 +2238,13 @@ paths: in: query schema: type: boolean + - name: include_encrypted + description: > + ask to include the encrypted value if secret and decrypt secret is + not true (default: false) + in: query + schema: + type: boolean responses: '200': description: variable @@ -9945,7 +9952,54 @@ paths: - aws_allow_http required: - s3fs_args - - cloud_options + - polars_cloud_options + /w/{workspace}/job_helpers/v2/boto3_connection_settings: + post: + summary: >- + Converts an S3 resource to the set of arguments necessary to connect to + an S3 bucket using boto3 in Python + operationId: boto3ConnectionSettingsV2 + tags: + - helpers + parameters: + - name: workspace + in: path + required: true + schema: *ref_0 + requestBody: + description: >- + S3 resource path to use to generate the connection settings. If empty, + the S3 resource defined in the workspace settings will be used + required: true + content: + application/json: + schema: + type: object + properties: + s3_resource_path: + type: string + responses: + '200': + description: Connection settings + content: + application/json: + schema: + type: object + properties: + endpoint_url: + type: string + region_name: + type: string + use_ssl: + type: boolean + aws_access_key_id: + type: string + aws_secret_access_key: + type: string + required: + - endpoint_url + - region_name + - use_ssl /w/{workspace}/job_helpers/test_connection: get: summary: Test connection to the workspace datasets storage @@ -10099,6 +10153,8 @@ paths: - Csv - Parquet - Unknown + download_url: + type: string required: &ref_170 - content_type components: diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 19d6714002..0ff1a67a69 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -2231,8 +2231,7 @@ paths: /w/{workspace}/resources/get_value_interpolated/{path}: get: - summary: - get resource interpolated (variables and resources are fully unrolled) + summary: get resource interpolated (variables and resources are fully unrolled) operationId: getResourceValueInterpolated tags: - resource @@ -2920,8 +2919,7 @@ paths: schema: type: string - name: first_parent_hash - description: - mask to filter scripts whom first direct parent has exact hash + description: mask to filter scripts whom first direct parent has exact hash in: query schema: type: string @@ -3116,8 +3114,7 @@ paths: /workers/custom_tags: get: - summary: - get all instance custom tags (tags are used to dispatch jobs to + summary: get all instance custom tags (tags are used to dispatch jobs to different worker groups) operationId: getCustomTags tags: @@ -3168,8 +3165,7 @@ paths: /w/{workspace}/scripts/delete/h/{hash}: post: - summary: - delete script by hash (erase content but keep hash, require admin) + summary: delete script by hash (erase content but keep hash, require admin) operationId: deleteScriptByHash tags: - script @@ -3408,16 +3404,14 @@ paths: type: string format: date-time - name: scheduled_in_secs - description: - schedule the script to execute in the number of seconds starting now + description: schedule the script to execute in the number of seconds starting now in: query schema: type: integer - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/NewJobId" - name: invisible_to_owner - description: - make the run invisible to the the script owner (default false) + description: make the run invisible to the the script owner (default false) in: query schema: type: boolean @@ -4448,8 +4442,7 @@ paths: type: string format: date-time - name: scheduled_in_secs - description: - schedule the script to execute in the number of seconds starting now + description: schedule the script to execute in the number of seconds starting now in: query schema: type: integer @@ -4457,8 +4450,7 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner - description: - make the run invisible to the the flow owner (default false) + description: make the run invisible to the the flow owner (default false) in: query schema: type: boolean @@ -4510,8 +4502,7 @@ paths: type: string format: date-time - name: scheduled_in_secs - description: - schedule the script to execute in the number of seconds starting now + description: schedule the script to execute in the number of seconds starting now in: query schema: type: integer @@ -4519,8 +4510,7 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner - description: - make the run invisible to the the flow owner (default false) + description: make the run invisible to the the flow owner (default false) in: query schema: type: boolean @@ -4558,8 +4548,7 @@ paths: type: string format: date-time - name: scheduled_in_secs - description: - schedule the script to execute in the number of seconds starting now + description: schedule the script to execute in the number of seconds starting now in: query schema: type: integer @@ -4567,8 +4556,7 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner - description: - make the run invisible to the the script owner (default false) + description: make the run invisible to the the script owner (default false) in: query schema: type: boolean @@ -4599,8 +4587,7 @@ paths: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner - description: - make the run invisible to the the script owner (default false) + description: make the run invisible to the the script owner (default false) in: query schema: type: boolean @@ -4633,8 +4620,7 @@ paths: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner - description: - make the run invisible to the the script owner (default false) + description: make the run invisible to the the script owner (default false) in: query schema: type: boolean @@ -4849,8 +4835,7 @@ paths: - job responses: "200": - description: - the timestamp of the db that can be used to compute the drift + description: the timestamp of the db that can be used to compute the drift content: application/json: schema: @@ -5114,8 +5099,7 @@ paths: /w/{workspace}/jobs/resume_urls/{id}/{resume_id}: get: - summary: - get resume urls given a job_id, resume_id and a nonce to resume a flow + summary: get resume urls given a job_id, resume_id and a nonce to resume a flow operationId: getResumeUrls tags: - job @@ -5666,8 +5650,7 @@ paths: - $ref: "#/components/parameters/WorkspaceId" - name: only_member_of in: query - description: - only list the groups the user is member of (default false) + description: only list the groups the user is member of (default false) schema: type: boolean responses: @@ -5855,8 +5838,7 @@ paths: - $ref: "#/components/parameters/WorkspaceId" - name: only_member_of in: query - description: - only list the folders the user is member of (default false) + description: only list the folders the user is member of (default false) schema: type: boolean responses: @@ -6681,7 +6663,51 @@ paths: - aws_allow_http required: - s3fs_args - - cloud_options + - polars_cloud_options + /w/{workspace}/job_helpers/v2/boto3_connection_settings: + post: + summary: + Converts an S3 resource to the set of arguments necessary to connect + to an S3 bucket using boto3 in Python + operationId: boto3ConnectionSettingsV2 + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: + S3 resource path to use to generate the connection settings. If empty, + the S3 resource defined in the workspace settings will be used + required: true + content: + application/json: + schema: + type: object + properties: + s3_resource_path: + type: string + responses: + "200": + description: Connection settings + content: + application/json: + schema: + type: object + properties: + endpoint_url: + type: string + region_name: + type: string + use_ssl: + type: boolean + aws_access_key_id: + type: string + aws_secret_access_key: + type: string + required: + - endpoint_url + - region_name + - use_ssl /w/{workspace}/job_helpers/test_connection: get: @@ -6896,8 +6922,7 @@ components: type: integer PerPage: name: per_page - description: - number of items to return for a given page (default 30, max 100) + description: number of items to return for a given page (default 30, max 100) in: query schema: type: integer @@ -7031,8 +7056,7 @@ components: type: boolean ArgsFilter: name: args - description: - filter on jobs containing those args as a json subset (@> in postgres) + description: filter on jobs containing those args as a json subset (@> in postgres) in: query schema: type: string @@ -7044,8 +7068,7 @@ components: type: string ResultFilter: name: result - description: - filter on jobs containing those result as a json subset (@> in postgres) + description: filter on jobs containing those result as a json subset (@> in postgres) in: query schema: type: string @@ -8878,6 +8901,8 @@ components: content_type: type: string enum: ["RawText", "Csv", "Parquet", "Unknown"] + download_url: + type: string required: - content_type diff --git a/backend/windmill-api/src/job_helpers.rs b/backend/windmill-api/src/job_helpers.rs index b8fa0e2f40..9d74f65ccc 100644 --- a/backend/windmill-api/src/job_helpers.rs +++ b/backend/windmill-api/src/job_helpers.rs @@ -1,10 +1,13 @@ -use std::cmp; +use std::{cmp, time::Duration}; use crate::{ db::DB, resources::get_resource_value_interpolated_internal, users::Tokened, workspaces::LargeFileStorage, }; -use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region}; +use aws_sdk_s3::{ + config::{BehaviorVersion, Credentials, Region}, + presigning::PresigningConfig, +}; use axum::{ extract::{Path, Query}, routing::{get, post}, @@ -53,8 +56,8 @@ pub fn workspaced_service() -> Router { post(polars_connection_settings_v2).layer(cors.clone()), ) .route( - "/blah", - post(polars_connection_settings_v2).layer(cors.clone()), + "/v2/boto3_connection_settings", + post(boto3_connection_settings_v2).layer(cors.clone()), ) .route("/test_connection", get(test_connection).layer(cors.clone())) .route( @@ -268,6 +271,55 @@ async fn polars_connection_settings_v2( return Ok(Json(response)); } +#[derive(Deserialize)] +struct Boto3ConnectionSettingsQueryV2 { + s3_resource_path: Option, +} + +#[derive(Serialize)] +struct Boto3ConnectionSettingsResponse { + region_name: String, + use_ssl: bool, + endpoint_url: String, + aws_access_key_id: Option, + aws_secret_access_key: Option, +} + +async fn boto3_connection_settings_v2( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Tokened { token }: Tokened, + Path(w_id): Path, + Json(query): Json, +) -> error::JsonResult { + let s3_resource_opt = match query.s3_resource_path { + Some(s3_resource_path) => { + get_s3_resource( + &authed, + &user_db, + &db, + &token, + &w_id, + s3_resource_path.as_str(), + ) + .await? + } + None => get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?, + }; + let s3_resource = s3_resource_opt.ok_or(error::Error::NotFound( + "No datasets storage resource defined at the workspace level".to_string(), + ))?; + let response = Boto3ConnectionSettingsResponse { + endpoint_url: render_endpoint(&s3_resource), + region_name: s3_resource.region, + use_ssl: s3_resource.use_ssl, + aws_access_key_id: s3_resource.access_key, + aws_secret_access_key: s3_resource.secret_key, + }; + return Ok(Json(response)); +} + #[derive(Serialize, Deserialize, Clone)] struct WindmillLargeFile { s3: String, @@ -416,6 +468,7 @@ struct LoadFilePreviewResponse { pub content: Option, pub content_type: WindmillContentType, pub msg: Option, + pub download_url: Option, } #[derive(Serialize)] @@ -510,6 +563,19 @@ async fn load_file_preview( ) }; + // URL expires 30 minutes after its generation + let presigned_config = PresigningConfig::expires_in(Duration::from_secs(60 * 30)) + .map_err(|err| error::Error::InternalErr(err.to_string()))?; + let download_url = s3_client + .get_object() + .bucket(&s3_bucket) + .key(&file_key) + .presigned(presigned_config) + .await + .map_err(|err| error::Error::InternalErr(err.to_string()))? + .uri() + .to_string(); + let file_chunk_length = if s3_object_content_length.is_some() { cmp::min( query.read_bytes_length, @@ -599,12 +665,14 @@ async fn load_file_preview( content_type: content_type, content: Some(content), msg: None, + download_url: Some(download_url), }, Err(err) => LoadFilePreviewResponse { content_type: content_type, content: None, msg: Some(err.to_string()), + download_url: Some(download_url), }, }; diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index 025d70eee0..d7568bfd08 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -1,5 +1,5 @@