starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-11 16:09:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add download button to S3 file picker (#2857)

Browse Source
This commit is contained in:
Guillaume Bouvignies
2023-12-14 15:45:37 +01:00
committed by GitHub
parent 70c2669b1e
commit f8c7a8edf8
7 changed files with 291 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/windmill-api/openapi-deref.yaml
+58 -2
View File
@@ -1,6 +1,6 @@
openapi: 3.0.3
info:
version: 1.225.0
version: 1.226.1
title: Windmill API
contact:
name: Windmill Team
@@ -2238,6 +2238,13 @@ paths:
in: query
schema:
type: boolean
- name: include_encrypted
description: >
ask to include the encrypted value if secret and decrypt secret is
not true (default: false)
in: query
schema:
type: boolean
responses:
'200':
description: variable
@@ -9945,7 +9952,54 @@ paths:
- aws_allow_http
required:
- s3fs_args
- cloud_options
- polars_cloud_options
/w/{workspace}/job_helpers/v2/boto3_connection_settings:
post:
summary: >-
Converts an S3 resource to the set of arguments necessary to connect to
an S3 bucket using boto3 in Python
operationId: boto3ConnectionSettingsV2
tags:
- helpers
parameters:
- name: workspace
in: path
required: true
schema: *ref_0
requestBody:
description: >-
S3 resource path to use to generate the connection settings. If empty,
the S3 resource defined in the workspace settings will be used
required: true
content:
application/json:
schema:
type: object
properties:
s3_resource_path:
type: string
responses:
'200':
description: Connection settings
content:
application/json:
schema:
type: object
properties:
endpoint_url:
type: string
region_name:
type: string
use_ssl:
type: boolean
aws_access_key_id:
type: string
aws_secret_access_key:
type: string
required:
- endpoint_url
- region_name
- use_ssl
/w/{workspace}/job_helpers/test_connection:
get:
summary: Test connection to the workspace datasets storage
@@ -10099,6 +10153,8 @@ paths:
- Csv
- Parquet
- Unknown
download_url:
type: string
required: &ref_170
- content_type
components:
backend/windmill-api/openapi.yaml
+68 -43
View File
@@ -2231,8 +2231,7 @@ paths:
/w/{workspace}/resources/get_value_interpolated/{path}:
get:
summary:
get resource interpolated (variables and resources are fully unrolled)
summary: get resource interpolated (variables and resources are fully unrolled)
operationId: getResourceValueInterpolated
tags:
- resource
@@ -2920,8 +2919,7 @@ paths:
schema:
type: string
- name: first_parent_hash
description:
mask to filter scripts whom first direct parent has exact hash
description: mask to filter scripts whom first direct parent has exact hash
in: query
schema:
type: string
@@ -3116,8 +3114,7 @@ paths:
/workers/custom_tags:
get:
summary:
get all instance custom tags (tags are used to dispatch jobs to
summary: get all instance custom tags (tags are used to dispatch jobs to
different worker groups)
operationId: getCustomTags
tags:
@@ -3168,8 +3165,7 @@ paths:
/w/{workspace}/scripts/delete/h/{hash}:
post:
summary:
delete script by hash (erase content but keep hash, require admin)
summary: delete script by hash (erase content but keep hash, require admin)
operationId: deleteScriptByHash
tags:
- script
@@ -3408,16 +3404,14 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
- $ref: "#/components/parameters/ParentJob"
- $ref: "#/components/parameters/NewJobId"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4448,8 +4442,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4457,8 +4450,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the flow owner (default false)
description: make the run invisible to the the flow owner (default false)
in: query
schema:
type: boolean
@@ -4510,8 +4502,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4519,8 +4510,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the flow owner (default false)
description: make the run invisible to the the flow owner (default false)
in: query
schema:
type: boolean
@@ -4558,8 +4548,7 @@ paths:
type: string
format: date-time
- name: scheduled_in_secs
description:
schedule the script to execute in the number of seconds starting now
description: schedule the script to execute in the number of seconds starting now
in: query
schema:
type: integer
@@ -4567,8 +4556,7 @@ paths:
- $ref: "#/components/parameters/NewJobId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4599,8 +4587,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4633,8 +4620,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/IncludeHeader"
- name: invisible_to_owner
description:
make the run invisible to the the script owner (default false)
description: make the run invisible to the the script owner (default false)
in: query
schema:
type: boolean
@@ -4849,8 +4835,7 @@ paths:
- job
responses:
"200":
description:
the timestamp of the db that can be used to compute the drift
description: the timestamp of the db that can be used to compute the drift
content:
application/json:
schema:
@@ -5114,8 +5099,7 @@ paths:
/w/{workspace}/jobs/resume_urls/{id}/{resume_id}:
get:
summary:
get resume urls given a job_id, resume_id and a nonce to resume a flow
summary: get resume urls given a job_id, resume_id and a nonce to resume a flow
operationId: getResumeUrls
tags:
- job
@@ -5666,8 +5650,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- name: only_member_of
in: query
description:
only list the groups the user is member of (default false)
description: only list the groups the user is member of (default false)
schema:
type: boolean
responses:
@@ -5855,8 +5838,7 @@ paths:
- $ref: "#/components/parameters/WorkspaceId"
- name: only_member_of
in: query
description:
only list the folders the user is member of (default false)
description: only list the folders the user is member of (default false)
schema:
type: boolean
responses:
@@ -6681,7 +6663,51 @@ paths:
- aws_allow_http
required:
- s3fs_args
- cloud_options
- polars_cloud_options
/w/{workspace}/job_helpers/v2/boto3_connection_settings:
post:
summary:
Converts an S3 resource to the set of arguments necessary to connect
to an S3 bucket using boto3 in Python
operationId: boto3ConnectionSettingsV2
tags:
- helpers
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description:
S3 resource path to use to generate the connection settings. If empty,
the S3 resource defined in the workspace settings will be used
required: true
content:
application/json:
schema:
type: object
properties:
s3_resource_path:
type: string
responses:
"200":
description: Connection settings
content:
application/json:
schema:
type: object
properties:
endpoint_url:
type: string
region_name:
type: string
use_ssl:
type: boolean
aws_access_key_id:
type: string
aws_secret_access_key:
type: string
required:
- endpoint_url
- region_name
- use_ssl
/w/{workspace}/job_helpers/test_connection:
get:
@@ -6896,8 +6922,7 @@ components:
type: integer
PerPage:
name: per_page
description:
number of items to return for a given page (default 30, max 100)
description: number of items to return for a given page (default 30, max 100)
in: query
schema:
type: integer
@@ -7031,8 +7056,7 @@ components:
type: boolean
ArgsFilter:
name: args
description:
filter on jobs containing those args as a json subset (@> in postgres)
description: filter on jobs containing those args as a json subset (@> in postgres)
in: query
schema:
type: string
@@ -7044,8 +7068,7 @@ components:
type: string
ResultFilter:
name: result
description:
filter on jobs containing those result as a json subset (@> in postgres)
description: filter on jobs containing those result as a json subset (@> in postgres)
in: query
schema:
type: string
@@ -8878,6 +8901,8 @@ components:
content_type:
type: string
enum: ["RawText", "Csv", "Parquet", "Unknown"]
download_url:
type: string
required:
- content_type
backend/windmill-api/src/job_helpers.rs
+72 -4
View File
@@ -1,10 +1,13 @@
use std::cmp;
use std::{cmp, time::Duration};
use crate::{
db::DB, resources::get_resource_value_interpolated_internal, users::Tokened,
workspaces::LargeFileStorage,
};
use aws_sdk_s3::config::{BehaviorVersion, Credentials, Region};
use aws_sdk_s3::{
config::{BehaviorVersion, Credentials, Region},
presigning::PresigningConfig,
};
use axum::{
extract::{Path, Query},
routing::{get, post},
@@ -53,8 +56,8 @@ pub fn workspaced_service() -> Router {
post(polars_connection_settings_v2).layer(cors.clone()),
)
.route(
"/blah",
post(polars_connection_settings_v2).layer(cors.clone()),
"/v2/boto3_connection_settings",
post(boto3_connection_settings_v2).layer(cors.clone()),
)
.route("/test_connection", get(test_connection).layer(cors.clone()))
.route(
@@ -268,6 +271,55 @@ async fn polars_connection_settings_v2(
return Ok(Json(response));
}
#[derive(Deserialize)]
struct Boto3ConnectionSettingsQueryV2 {
s3_resource_path: Option<String>,
}
#[derive(Serialize)]
struct Boto3ConnectionSettingsResponse {
region_name: String,
use_ssl: bool,
endpoint_url: String,
aws_access_key_id: Option<String>,
aws_secret_access_key: Option<String>,
}
async fn boto3_connection_settings_v2(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Tokened { token }: Tokened,
Path(w_id): Path<String>,
Json(query): Json<Boto3ConnectionSettingsQueryV2>,
) -> error::JsonResult<Boto3ConnectionSettingsResponse> {
let s3_resource_opt = match query.s3_resource_path {
Some(s3_resource_path) => {
get_s3_resource(
&authed,
&user_db,
&db,
&token,
&w_id,
s3_resource_path.as_str(),
)
.await?
}
None => get_workspace_s3_resource(&authed, &user_db, &db, &token, &w_id).await?,
};
let s3_resource = s3_resource_opt.ok_or(error::Error::NotFound(
"No datasets storage resource defined at the workspace level".to_string(),
))?;
let response = Boto3ConnectionSettingsResponse {
endpoint_url: render_endpoint(&s3_resource),
region_name: s3_resource.region,
use_ssl: s3_resource.use_ssl,
aws_access_key_id: s3_resource.access_key,
aws_secret_access_key: s3_resource.secret_key,
};
return Ok(Json(response));
}
#[derive(Serialize, Deserialize, Clone)]
struct WindmillLargeFile {
s3: String,
@@ -416,6 +468,7 @@ struct LoadFilePreviewResponse {
pub content: Option<String>,
pub content_type: WindmillContentType,
pub msg: Option<String>,
pub download_url: Option<String>,
}
#[derive(Serialize)]
@@ -510,6 +563,19 @@ async fn load_file_preview(
)
};
// URL expires 30 minutes after its generation
let presigned_config = PresigningConfig::expires_in(Duration::from_secs(60 * 30))
.map_err(|err| error::Error::InternalErr(err.to_string()))?;
let download_url = s3_client
.get_object()
.bucket(&s3_bucket)
.key(&file_key)
.presigned(presigned_config)
.await
.map_err(|err| error::Error::InternalErr(err.to_string()))?
.uri()
.to_string();
let file_chunk_length = if s3_object_content_length.is_some() {
cmp::min(
query.read_bytes_length,
@@ -599,12 +665,14 @@ async fn load_file_preview(
content_type: content_type,
content: Some(content),
msg: None,
download_url: Some(download_url),
},
Err(err) => LoadFilePreviewResponse {
content_type: content_type,
content: None,
msg: Some(err.to_string()),
download_url: Some(download_url),
},
};
frontend/src/lib/components/S3FilePicker.svelte
+30 -6
View File
@@ -1,5 +1,5 @@
<script lang="ts">
import { File, FolderClosed, FolderOpen, RotateCw, Loader2 } from 'lucide-svelte'
import { File, FolderClosed, FolderOpen, RotateCw, Loader2, Download } from 'lucide-svelte'
import { workspaceStore } from '$lib/stores'
import { HelpersService } from '$lib/gen'
import { displayDate, displaySize, emptyString } from '$lib/utils'
@@ -24,6 +24,8 @@
let dispatch = createEventDispatcher()
let drawer: Drawer
let fileListLoading: boolean = true
let allFilesByKey: Record<
string,
{
@@ -55,10 +57,12 @@
fileKey: string
contentPreview: string | undefined
contentType: string | undefined
downloadUrl: string | undefined
}
| undefined = undefined
async function loadFiles() {
fileListLoading = true
let availableFiles = await HelpersService.listStoredFiles({
workspace: $workspaceStore!,
maxKeys: 1000, // fixed pages of 1000 files for now
@@ -113,6 +117,7 @@
}
}
}
fileListLoading = false
}
async function loadFileMetadataPlusPreviewAsync(fileKey: string | undefined) {
@@ -165,7 +170,8 @@
filePreview = {
fileKey: fileKey,
contentPreview: filePreviewContent,
contentType: filePreviewRaw.content_type
contentType: filePreviewRaw.content_type,
downloadUrl: filePreviewRaw.download_url
}
}
filePreviewLoading = false
@@ -294,7 +300,7 @@
{:else}
<div class="flex flex-row border rounded-md h-full" bind:clientHeight={listDivHeight}>
<div class="min-w-[30%] border-r">
{#if displayedFileKeys.length === 0}
{#if fileListLoading === false && displayedFileKeys.length === 0}
<div class="p-4 text-tertiary text-xs text-center italic">
No files in the workspace S3 bucket
</div>
@@ -344,6 +350,11 @@
>More files in bucket. Click here to load more...
</button>
{/if}
{#if fileListLoading === true}
<div class="flex text-secondary mt-1 text-xs justify-center items-center w-full">
<Loader2 size={12} class="animate-spin mr-1" /> Loading content
</div>
{/if}
</div>
</VirtualList>
{/if}
@@ -355,7 +366,20 @@
</div>
{:else}
<div class="p-4 gap-2">
<Section label={fileMetadata.fileKey} />
<Section label={fileMetadata.fileKey}>
<div slot="action">
{#if filePreview !== undefined}
<Button
title="Download file from S3"
variant="border"
color="light"
href={filePreview.downloadUrl}
startIcon={{ icon: Download }}
iconOnly={true}
/>
{/if}
</div>
</Section>
<TableSimple
headers={['Last modified', 'Size', 'Type']}
data={[fileMetadata]}
@@ -368,7 +392,7 @@
{#if fileMetadata !== undefined && filePreview !== undefined}
<div class="flex h-6 items-center text-tertiary mb-4">
{#if filePreview.contentType === 'Unknown'}
Type of file not supported for preview
Type of file not supported for preview.
{:else if filePreview.contentType === 'Csv'}
Previewing a {filePreview.contentType?.toLowerCase()} file. Separator character:
<div class="inline-flex w-12 ml-2 mr-2">
@@ -407,7 +431,7 @@
/>
</div>
{:else}
Previewing a {filePreview.contentType?.toLowerCase()} file
Previewing a {filePreview.contentType?.toLowerCase()} file.
{/if}
</div>
<pre class="grow whitespace-no-wrap break-words"
python-client/tests/wmill_client_test.py
+12 -1
View File
@@ -4,7 +4,7 @@ import os
class TestStringMethods(unittest.TestCase):
_token = "wx9B4WcQhlGSrmhbHY8HqADxx6f4oy"
_token = "<WM_TOKEN>"
_workspace = "storage"
_host = "http://localhost:8000"
_resource_path = "u/admin/docker_minio"
@@ -64,6 +64,17 @@ SET s3_secret_access_key='80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4';
}
self.assertEqual(settings, expected_settings)
def test_boto3_connection_settings(self):
settings = wmill.boto3_connection_settings(self._resource_path)
expected_settings = {
"endpoint_url": "http://localhost:9000",
"region_name": "fr-paris",
"use_ssl": False,
"aws_access_key_id": "IeuKPSYLKTO2h9CWfCVR",
"aws_secret_access_key": "80yMndIMcyXwEujxVNINQbf0tBlIzRaLPyM2m1n4",
}
self.assertEqual(settings, expected_settings)
if __name__ == "__main__":
unittest.main()
python-client/wmill/wmill/client.py
+30
View File
@@ -352,6 +352,25 @@ class Windmill:
return None
raise Exception("Could not generate Polars S3 connection settings from the provided resource") from e
def get_boto3_connection_settings(
self,
s3_resource_path: str = "",
none_if_undefined: bool = False,
) -> Any:
"""
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
initiate an S3 connection using boto3
"""
try:
return self.post(
f"/w/{self.workspace}/job_helpers/v2/boto3_connection_settings",
json={} if s3_resource_path == "" else {"s3_resource_path": s3_resource_path},
).json()
except JSONDecodeError as e:
if none_if_undefined:
return None
raise Exception("Could not generate Polars S3 connection settings from the provided resource") from e
def whoami(self) -> dict:
return self.get("/users/whoami").json()
@@ -479,6 +498,7 @@ def run_script_async(
scheduled_in_secs=scheduled_in_secs,
)
@init_global_client
def run_flow_async(
path: str,
@@ -491,6 +511,7 @@ def run_flow_async(
scheduled_in_secs=scheduled_in_secs,
)
@init_global_client
def run_script_sync(
hash: str,
@@ -572,6 +593,15 @@ def polars_connection_settings(s3_resource_path: str = "", none_if_undefined: bo
return _client.get_polars_connection_settings(s3_resource_path, none_if_undefined)
@init_global_client
def boto3_connection_settings(s3_resource_path: str = "", none_if_undefined: bool = False) -> Any:
"""
Convenient helpers that takes an S3 resource as input and returns the settings necessary to
initiate an S3 connection using boto3
"""
return _client.get_boto3_connection_settings(s3_resource_path, none_if_undefined)
@init_global_client
def whoami() -> dict:
"""
typescript-client/client.ts
+21 -1
View File
@@ -1,4 +1,4 @@
import { ResourceService, VariableService, JobService } from "./index";
import { ResourceService, VariableService, JobService, HelpersService } from "./index";
import { OpenAPI } from "./index";
export {
@@ -247,6 +247,26 @@ export async function databaseUrlFromResource(path: string): Promise<string> {
return `postgresql://${resource.user}:${resource.password}@${resource.host}:${resource.port}/${resource.dbname}?sslmode=${resource.sslmode}`;
}
export async function polarsConnectionSettings(s3_resource_path: string | undefined): Promise<any> {
const workspace = getWorkspace();
return await HelpersService.polarsConnectionSettingsV2({
workspace: workspace,
requestBody: {
s3_resource_path: s3_resource_path
}
});
}
export async function duckdbConnectionSettings(s3_resource_path: string | undefined): Promise<any> {
const workspace = getWorkspace();
return await HelpersService.duckdbConnectionSettingsV2({
workspace: workspace,
requestBody: {
s3_resource_path: s3_resource_path
}
});
}
/**
* Get URLs needed for resuming a flow after this step
* @param approver approver name