diff --git a/backend/.sqlx/query-43aa468aac174529a74e6108af55a383f8a20e98b8c502929f4dc5041a55e72f.json b/backend/.sqlx/query-43aa468aac174529a74e6108af55a383f8a20e98b8c502929f4dc5041a55e72f.json deleted file mode 100644 index e33b0dc0b8..0000000000 --- a/backend/.sqlx/query-43aa468aac174529a74e6108af55a383f8a20e98b8c502929f4dc5041a55e72f.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS (\n SELECT 1 FROM v2_as_completed_job \n WHERE workspace_id = $2 \n AND (job_kind = 'appscript' OR job_kind = 'preview')\n AND created_by = 'anonymous' \n AND started_at > now() - interval '3 hours'\n AND script_path LIKE $3 || '/%' \n AND result @> ('{\"s3\":\"' || $1 || '\"}')::jsonb \n )", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "43aa468aac174529a74e6108af55a383f8a20e98b8c502929f4dc5041a55e72f" -} diff --git a/backend/.sqlx/query-ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b.json b/backend/.sqlx/query-ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b.json new file mode 100644 index 0000000000..d87e680abe --- /dev/null +++ b/backend/.sqlx/query-ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (\n SELECT 1 FROM v2_as_completed_job\n WHERE workspace_id = $2\n AND (job_kind = 'appscript' OR job_kind = 'preview')\n AND created_by = 'anonymous'\n AND started_at > now() - interval '3 hours'\n AND script_path LIKE $3 || '/%'\n AND result @> ('{\"s3\":\"' || $1 || '\"}')::jsonb\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ec9980f80a8bfa4b09225035e8a1f78d7d61fbf83dda9b39bb22e0f9584d221b" +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 81b1b3759b..f1c3e0a195 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -6307,6 +6307,38 @@ paths: schema: type: boolean + /w/{workspace}/apps/sign_s3_objects: + post: + summary: sign s3 objects, to be used by anonymous users in public apps + operationId: signS3Objects + tags: + - app + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: s3 objects to sign + required: true + content: + application/json: + schema: + type: object + properties: + s3_objects: + type: array + items: + $ref: "#/components/schemas/S3Object" + required: + - s3_objects + responses: + "200": + description: signed s3 objects + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/S3Object" + /w/{workspace}/apps_u/execute_component/{path}: post: summary: executeComponent @@ -16455,3 +16487,17 @@ components: required: - account_id - installation_id + + S3Object: + type: object + properties: + s3: + type: string + filename: + type: string + storage: + type: string + presigned: + type: string + required: + - s3 \ No newline at end of file diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 78159218c7..b3ffdb896b 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -20,8 +20,7 @@ use crate::{ use crate::{ job_helpers_ee::{ download_s3_file_internal, get_random_file_name, get_s3_resource, - get_workspace_s3_resource, load_image_preview_internal, upload_file_from_req, - DownloadFileQuery, LoadImagePreviewQuery, + get_workspace_s3_resource, upload_file_from_req, DownloadFileQuery, }, users::fetch_api_authed_from_permissioned_as, }; @@ -51,7 +50,6 @@ use sqlx::{types::Uuid, FromRow}; use std::str; use windmill_audit::audit_ee::audit_log; use windmill_audit::ActionKind; -use windmill_common::variables::encrypt; use windmill_common::{ apps::{AppScriptId, ListAppQuery}, cache::{self, future::FutureCachedExt}, @@ -63,16 +61,24 @@ use windmill_common::{ http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath, }, - variables::{build_crypt, build_crypt_with_key_suffix}, + variables::{build_crypt, build_crypt_with_key_suffix, encrypt}, worker::{to_raw_value, CLOUD_HOSTED}, HUB_BASE_URL, }; -#[cfg(feature = "parquet")] -use windmill_common::{jwt, s3_helpers::build_object_store_client}; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; use windmill_queue::{push, PushArgs, PushArgsOwned, PushIsolationLevel}; +#[cfg(feature = "parquet")] +use hmac::Mac; +#[cfg(feature = "parquet")] +use windmill_common::{ + jwt, + oauth2::HmacSha256, + s3_helpers::{build_object_store_client, S3Object}, + variables::get_workspace_key, +}; + pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_apps)) @@ -94,6 +100,7 @@ pub fn workspaced_service() -> Router { get(list_paths_from_workspace_runnable), ) .route("/custom_path_exists/*custom_path", get(custom_path_exists)) + .route("/sign_s3_objects", post(sign_s3_objects)) } pub fn unauthed_service() -> Router { @@ -102,10 +109,6 @@ pub fn unauthed_service() -> Router { .route("/upload_s3_file/*path", post(upload_s3_file_from_app)) .route("/delete_s3_file", delete(delete_s3_file_from_app)) .route("/download_s3_file/*path", get(download_s3_file_from_app)) - .route( - "/load_image_preview/*path", - get(load_s3_file_image_preview_from_app), - ) .route("/public_app/:secret", get(get_public_app_by_secret)) .route("/public_resource/*path", get(get_public_resource)) } @@ -232,7 +235,8 @@ pub struct S3Input { #[derive(Serialize, Deserialize, Debug, Clone)] pub struct S3Key { s3_path: String, - resource: String, + #[serde(skip_serializing_if = "Option::is_none")] + storage: Option, } #[derive(Serialize, Deserialize, Debug, Clone, Default)] @@ -1554,15 +1558,97 @@ struct UploadFileToS3Query { #[cfg(feature = "parquet")] #[derive(Serialize, Deserialize)] -struct DeleteTokenClaims { +struct S3DeleteTokenClaims { file_key: String, on_behalf_of_email: String, permissioned_as: String, username: String, s3_resource_path: Option, + workspace: String, pub exp: usize, } +#[cfg(feature = "parquet")] +#[derive(Deserialize)] +struct S3TokenRequestBody { + s3_objects: Vec, +} +#[cfg(feature = "parquet")] +async fn sign_s3_objects( + Extension(db): Extension, + Path(w_id): Path, + Json(body): Json, +) -> Result>> { + let workspace_key = get_workspace_key(&w_id, &db).await?; + + let futures = body.s3_objects.into_iter().map(|s3_object| async { + let exp = (chrono::Utc::now() + chrono::Duration::hours(12)).timestamp(); + let mut message = format!("file_key={}&exp={}", s3_object.s3.clone(), exp); + if let Some(ref storage) = s3_object.storage { + message = format!("{}&storage={}", message, storage); + } + + let mut max = HmacSha256::new_from_slice(workspace_key.as_bytes()) + .map_err(|err| Error::internal_err(format!("Failed to create hmac: {}", err)))?; + max.update(message.as_bytes()); + let result = max.finalize(); + let signature = hex::encode(result.into_bytes()); + + let presigned = format!("exp={}&sig={}", exp, signature); + + Ok::<_, Error>(S3Object { presigned: Some(presigned), ..s3_object }) + }); + + let signed_s3_objects = futures::future::try_join_all(futures).await?; + + Ok(Json(signed_s3_objects)) +} + +#[cfg(feature = "parquet")] +async fn validate_s3_signature(file_query: &AppS3FileQuery, w_id: &str, db: &DB) -> Result<()> { + let workspace_key = get_workspace_key(w_id, &db).await?; + + let Some(exp) = file_query + .exp + .as_ref() + .map(|e| e.parse::().unwrap_or_default()) + else { + return Err(Error::BadRequest("Missing exp".to_string())); + }; + + let Some(ref sig) = file_query.sig else { + return Err(Error::BadRequest("Missing signature".to_string())); + }; + + let mut message = format!("file_key={}&exp={}", file_query.s3, exp); + + if let Some(ref storage) = file_query.storage { + message = format!("{}&storage={}", message, storage); + } + + let mut mac = HmacSha256::new_from_slice(workspace_key.as_bytes()) + .map_err(|err| Error::internal_err(format!("Failed to create hmac: {}", err)))?; + + mac.update(message.as_bytes()); + + let sig_bytes = hex::decode(sig)?; + mac.verify_slice(&sig_bytes) + .map_err(|err| Error::BadRequest(format!("Invalid signature: {}", err)))?; + + if exp < chrono::Utc::now().timestamp() { + return Err(Error::BadRequest("Signature expired".to_string())); + } + + Ok(()) +} + +#[cfg(not(feature = "parquet"))] +async fn sign_s3_objects() -> Result<()> { + return Err(Error::BadRequest( + "This endpoint requires the parquet feature to be enabled".to_string(), + )); +} + #[cfg(feature = "parquet")] #[derive(Serialize)] struct AppUploadFileResponse { @@ -1817,13 +1903,14 @@ async fn upload_s3_file_from_app( upload_file_from_req(s3_client, &file_key, request, options).await?; - let delete_token = jwt::encode_with_internal_secret(DeleteTokenClaims { + let delete_token = jwt::encode_with_internal_secret(S3DeleteTokenClaims { file_key: file_key.clone(), on_behalf_of_email, permissioned_as, username, s3_resource_path: query.s3_resource_path, - exp: (chrono::Utc::now() + chrono::Duration::seconds(3600 * 24)).timestamp() as usize, + workspace: w_id.clone(), + exp: (chrono::Utc::now() + chrono::Duration::hours(12)).timestamp() as usize, }) .await?; @@ -1843,14 +1930,19 @@ async fn delete_s3_file_from_app( Path(w_id): Path, Query(query): Query, ) -> Result<()> { - let DeleteTokenClaims { + let S3DeleteTokenClaims { file_key, on_behalf_of_email, permissioned_as, username, s3_resource_path, + workspace, .. - } = jwt::decode_with_internal_secret::(&query.delete_token).await?; + } = jwt::decode_with_internal_secret::(&query.delete_token).await?; + + if workspace != w_id { + return Err(Error::BadRequest("Invalid workspace".to_string())); + } let on_behalf_authed = fetch_api_authed_from_permissioned_as( permissioned_as, @@ -1958,7 +2050,7 @@ async fn get_on_behalf_authed_from_app( async fn check_if_allowed_to_access_s3_file_from_app( db: &DB, opt_authed: &Option, - file_key: &str, + file_query: &AppS3FileQuery, w_id: &str, path: &str, policy: &Policy, @@ -1966,40 +2058,59 @@ async fn check_if_allowed_to_access_s3_file_from_app( // if anonymous, check that the file was the result of an app script ran by an anonymous user in the last 3 hours // otherwise, if logged in, allow any file (TODO: change that when we implement better s3 policy) - let allowed = opt_authed.is_some() - || sqlx::query_scalar!( - r#"SELECT EXISTS ( - SELECT 1 FROM v2_as_completed_job - WHERE workspace_id = $2 - AND (job_kind = 'appscript' OR job_kind = 'preview') - AND created_by = 'anonymous' - AND started_at > now() - interval '3 hours' - AND script_path LIKE $3 || '/%' - AND result @> ('{"s3":"' || $1 || '"}')::jsonb - )"#, - file_key, - w_id, - path, - ) - .fetch_one(db) - .await? - .unwrap_or(false) - - // check if the file is allowed by the allowed_s3_keys policy - || policy.allowed_s3_keys.as_ref().unwrap().iter().any(|key| key.s3_path == file_key); - - if !allowed { - Err(Error::BadRequest("File restricted".to_string())) - } else { + if file_query.sig.is_some() { + validate_s3_signature(file_query, w_id, &db).await + } else if opt_authed.is_some() { Ok(()) + } else { + let allowed = policy + .allowed_s3_keys + .as_ref() + .unwrap() + .iter() + .any(|key| key.s3_path == file_query.s3 && key.storage == file_query.storage) + || { + sqlx::query_scalar!( + r#"SELECT EXISTS ( + SELECT 1 FROM v2_as_completed_job + WHERE workspace_id = $2 + AND (job_kind = 'appscript' OR job_kind = 'preview') + AND created_by = 'anonymous' + AND started_at > now() - interval '3 hours' + AND script_path LIKE $3 || '/%' + AND result @> ('{"s3":"' || $1 || '"}')::jsonb + )"#, + file_query.s3, + w_id, + path, + ) + .fetch_one(db) + .await? + .unwrap_or(false) + }; + + if !allowed { + Err(Error::BadRequest("File restricted".to_string())) + } else { + Ok(()) + } } } #[cfg(feature = "parquet")] -#[derive(Deserialize)] -pub struct DownloadFileQueryWithForceViewerAllowedS3Keys { +#[derive(Deserialize, Debug)] +struct AppS3FileQuery { + s3: String, + storage: Option, + sig: Option, + exp: Option, +} + +#[cfg(feature = "parquet")] +#[derive(Deserialize, Debug)] +struct AppS3FileQueryWithForceViewerAllowedS3Keys { #[serde(flatten)] - pub file_query: DownloadFileQuery, + pub file_query: AppS3FileQuery, pub force_viewer_allowed_s3_keys: Option, } @@ -2008,7 +2119,7 @@ async fn download_s3_file_from_app( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, - Query(query): Query, + Query(query): Query, ) -> Result { let path = path.to_path(); @@ -2027,46 +2138,26 @@ async fn download_s3_file_from_app( check_if_allowed_to_access_s3_file_from_app( &db, &opt_authed, - &query.file_query.file_key, + &query.file_query, &w_id, &path, &policy, ) .await?; - download_s3_file_internal(on_behalf_authed, &db, None, "", &w_id, query.file_query).await -} - -#[cfg(not(feature = "parquet"))] -async fn load_s3_file_image_preview_from_app() -> Result<()> { - return Err(Error::BadRequest( - "This endpoint requires the parquet feature to be enabled".to_string(), - )); -} - -#[cfg(feature = "parquet")] -async fn load_s3_file_image_preview_from_app( - OptAuthed(opt_authed): OptAuthed, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Query(query): Query, -) -> Result { - let path = path.to_path(); - - let (on_behalf_authed, policy) = - get_on_behalf_authed_from_app(&db, &path, &w_id, &opt_authed, None).await?; - - check_if_allowed_to_access_s3_file_from_app( + download_s3_file_internal( + on_behalf_authed, &db, - &opt_authed, - &query.file_key, + None, + "", &w_id, - &path, - &policy, + DownloadFileQuery { + file_key: query.file_query.s3, + s3_resource_path: None, + storage: query.file_query.storage, + }, ) - .await?; - - load_image_preview_internal(on_behalf_authed, &db, "", &w_id, query).await + .await } fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> { diff --git a/backend/windmill-api/src/job_helpers_ee.rs b/backend/windmill-api/src/job_helpers_ee.rs index b35f40d661..4e14d16c5f 100644 --- a/backend/windmill-api/src/job_helpers_ee.rs +++ b/backend/windmill-api/src/job_helpers_ee.rs @@ -30,12 +30,20 @@ pub struct UploadFileResponse { #[derive(Deserialize)] pub struct LoadImagePreviewQuery { + #[allow(dead_code)] pub file_key: String, + #[allow(dead_code)] + pub storage: Option, } #[derive(Deserialize)] pub struct DownloadFileQuery { + #[allow(dead_code)] pub file_key: String, + #[allow(dead_code)] + pub storage: Option, + #[allow(dead_code)] + pub s3_resource_path: Option, } pub fn workspaced_service() -> Router { @@ -111,16 +119,3 @@ pub async fn download_s3_file_internal( "Not implemented in Windmill's Open Source repository".to_string(), )) } - -#[cfg(feature = "parquet")] -pub async fn load_image_preview_internal( - _authed: ApiAuthed, - _db: &DB, - _token: &str, - _w_id: &str, - _query: LoadImagePreviewQuery, -) -> error::Result { - Err(error::Error::internal_err( - "Not implemented in Windmill's Open Source repository".to_string(), - )) -} diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 2c5a76a2e1..e3dff5350c 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -111,13 +111,15 @@ pub struct S3AwsOidcResource { pub audience: Option, } -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, Default)] pub struct S3Object { pub s3: String, #[serde(skip_serializing_if = "Option::is_none")] pub storage: Option, #[serde(skip_serializing_if = "Option::is_none")] pub filename: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub presigned: Option, } #[cfg(feature = "parquet")] diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index baaaeadc72..2699b89d02 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -903,7 +903,7 @@ pub async fn get_cached_resource_value_if_valid( S3Object { s3: s3_file_key.clone(), storage: resource.storage.clone(), - filename: None, + ..Default::default() }, ) .await; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 19a7ba80c2..c7127b4173 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -79,7 +79,8 @@ "y-monaco": "^0.1.4", "y-websocket": "^1.5.4", "yaml": "^2.3.4", - "yjs": "^13.6.7" + "yjs": "^13.6.7", + "zod": "^3.24.2" }, "devDependencies": { "@floating-ui/core": "^1.3.1", @@ -11085,9 +11086,9 @@ }, "node_modules/zod": { "version": "3.24.2", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.2.tgz", + "integrity": "sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ==", "license": "MIT", - "optional": true, - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/frontend/package.json b/frontend/package.json index f5e7c1cbd0..d2cb878ce5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -152,7 +152,8 @@ "y-monaco": "^0.1.4", "y-websocket": "^1.5.4", "yaml": "^2.3.4", - "yjs": "^13.6.7" + "yjs": "^13.6.7", + "zod": "^3.24.2" }, "peerDependencies": { "svelte": "^4.0.0" diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index b737ed21cd..f7d100c1a4 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -767,18 +767,6 @@ }} /> {/await} - {:else} {/if} + {:else if inputCat == 'object' || inputCat == 'resource-object' || isListJson} {#if oneOf && oneOf.length >= 2} diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index 1d0008139f..aa39489723 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -774,24 +774,30 @@ preview rendered {:else if result?.s3?.endsWith('.pdf')}
{/if} diff --git a/frontend/src/lib/components/S3ObjectPicker.svelte b/frontend/src/lib/components/S3ObjectPicker.svelte index e19af40079..f877681615 100644 --- a/frontend/src/lib/components/S3ObjectPicker.svelte +++ b/frontend/src/lib/components/S3ObjectPicker.svelte @@ -66,18 +66,6 @@ bind:value /> {/await} - {:else} {/if} + diff --git a/frontend/src/lib/components/apps/components/display/AppImage.svelte b/frontend/src/lib/components/apps/components/display/AppImage.svelte index a7be34ca3a..1ac67dd8e6 100644 --- a/frontend/src/lib/components/apps/components/display/AppImage.svelte +++ b/frontend/src/lib/components/apps/components/display/AppImage.svelte @@ -11,7 +11,7 @@ import ResolveStyle from '../helpers/ResolveStyle.svelte' import { defaultIfEmptyString } from '$lib/utils' import { userStore } from '$lib/stores' - import { computeS3ImageViewerPolicy } from '../../editor/appUtilsS3' + import { computeS3ImageViewerPolicy, isPartialS3Object } from '../../editor/appUtilsS3' export let id: string export let configuration: RichConfigurations @@ -22,7 +22,7 @@ if (!isEditor) { return undefined } - const policy = computeS3ImageViewerPolicy(configuration, $app) + const policy = computeS3ImageViewerPolicy(configuration) return policy } @@ -46,25 +46,36 @@ let imageUrl: string | undefined = undefined - async function getS3Image(source: string | undefined) { + async function getS3Image(source: string | undefined, storage?: string, presigned?: string) { if (!source) return '' const appPathOrUser = defaultIfEmptyString( $appPath, `u/${$userStore?.username ?? 'unknown'}/newapp` ) const params = new URLSearchParams() - params.append('file_key', source) + params.append('s3', source) + if (storage) { + params.append('storage', storage) + } const forceViewerPolicies = computeForceViewerPolicies() if (forceViewerPolicies) { params.append('force_viewer_allowed_s3_keys', JSON.stringify([forceViewerPolicies])) } - return `/api/w/${workspace}/apps_u/download_s3_file/${appPathOrUser}?${params.toString()}` + return `/api/w/${workspace}/apps_u/download_s3_file/${appPathOrUser}?${params.toString()}${presigned ? `&${presigned}` : ''}` } async function loadImage() { - if ( + if (isPartialS3Object(resolvedConfig.source)) { + imageUrl = await getS3Image( + resolvedConfig.source.s3, + resolvedConfig.source.storage, + resolvedConfig.source.presigned + ) + } else if (resolvedConfig.source && typeof resolvedConfig.source !== 'string') { + throw new Error('Invalid image object' + typeof resolvedConfig.source) + } else if ( resolvedConfig.sourceKind === 's3 (workspace storage)' || resolvedConfig.source?.startsWith('s3://') ) { diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte index ae669ba17e..46fde6f024 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte @@ -352,10 +352,10 @@ .filter((x) => (x.data as AppComponent).type === 'imagecomponent') .map((x) => { const c = x.data as AppComponent - const config = c.configuration as any - return computeS3ImageViewerPolicy(config, $app) + const config = c.configuration + return computeS3ImageViewerPolicy(config) }) - .filter(Boolean) as { s3_path?: string | undefined; resource?: string | undefined }[] + .filter(Boolean) as { s3_path: string; storage?: string | undefined }[] policy.allowed_s3_keys = s3FileKeys } diff --git a/frontend/src/lib/components/apps/editor/appUtilsS3.ts b/frontend/src/lib/components/apps/editor/appUtilsS3.ts index 5938db0435..4bef5ef6b7 100644 --- a/frontend/src/lib/components/apps/editor/appUtilsS3.ts +++ b/frontend/src/lib/components/apps/editor/appUtilsS3.ts @@ -1,7 +1,7 @@ import type { AppInput, EvalInputV2 } from '../inputType' -import type { App } from '../types' +import type { App, RichConfigurations } from '../types' import { collectOneOfFields } from './appUtils' - +import { z } from 'zod' function filenameExprToRegex(template: string) { const filenameEscaped = template.replaceAll('${file.name}', '') // replace filename with placeholder const escapedTemplate = filenameEscaped @@ -51,12 +51,12 @@ export function computeS3FileInputPolicy(s3Config: any, app: App) { ? resourceInput.value ? [removeResourcePrefix(resourceInput.value)] : [] - : collectOneOfFields( + : (collectOneOfFields( { s3_resource: resourceInput }, app - ).s3_resource?.map((s) => removeResourcePrefix(s)) ?? [] + ).s3_resource?.map((s) => removeResourcePrefix(s)) ?? []) : [] const allow_user_resources = @@ -81,12 +81,33 @@ export function computeS3FileInputPolicy(s3Config: any, app: App) { } } -export function computeS3ImageViewerPolicy(config: any, app: App) { - if ( - config.sourceKind.value === 's3 (workspace storage)' || - config.source.value.startsWith('s3://') +const partialS3ObjectSchema = z.object({ + s3: z.string(), + storage: z.string().optional(), + presigned: z.string().optional() +}) + +export function isPartialS3Object(input: unknown): input is z.infer { + return partialS3ObjectSchema.safeParse(input).success +} + +export function computeS3ImageViewerPolicy(config: RichConfigurations) { + if (config.source.type === 'uploadS3' && isPartialS3Object(config.source.value)) { + return { + s3_path: config.source.value.s3, + storage: config.source.value.storage + } + } else if ( + config.source.type === 'static' && + typeof config.source.value === 'string' && + ((config.sourceKind.type === 'static' && + config.sourceKind.value === 's3 (workspace storage)') || + config.source.value.startsWith('s3://')) ) { - return { s3_path: config.source.value.replace('s3://', ''), resource: 'default' } + return { + s3_path: config.source.value.replace('s3://', ''), + storage: undefined + } } else { return undefined } diff --git a/frontend/src/lib/components/apps/editor/component/components.ts b/frontend/src/lib/components/apps/editor/component/components.ts index 2344b57947..52cb6b9ad5 100644 --- a/frontend/src/lib/components/apps/editor/component/components.ts +++ b/frontend/src/lib/components/apps/editor/component/components.ts @@ -484,6 +484,12 @@ export const selectOptions = { animationTimingFunctionOptions: ['linear', 'ease', 'ease-in', 'ease-out', 'ease-in-out'], prose: ['sm', 'Default', 'lg', 'xl', '2xl'], imageSourceKind: [ + 'url', + 'png encoded as base64', + 'jpeg encoded as base64', + 'svg encoded as base64' + ], + imageSourceKindWithS3: [ 'url', 's3 (workspace storage)', 'png encoded as base64', @@ -2903,7 +2909,11 @@ See date-fns format for more information. By default, it is 'dd.MM.yyyy HH:mm' type: 'static', fieldType: 'array', subFieldType: 'object', - value: [{ header: 'First', foo: 1 }, { header: 'Second', foo: 2 }, { header: 'Third', foo: 3 }] as object[] + value: [ + { header: 'First', foo: 1 }, + { header: 'Second', foo: 2 }, + { header: 'Third', foo: 3 } + ] as object[] }, numberOfSubgrids: 1 } @@ -3089,8 +3099,8 @@ See date-fns format for more information. By default, it is 'dd.MM.yyyy HH:mm' sourceKind: { fieldType: 'select', type: 'static', - selectOptions: selectOptions.imageSourceKind, - value: 'url' as (typeof selectOptions.imageSourceKind)[number] + selectOptions: selectOptions.imageSourceKindWithS3, + value: 'url' as (typeof selectOptions.imageSourceKindWithS3)[number] }, imageFit: { fieldType: 'select', diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecEditor.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecEditor.svelte index bbc342e5e2..b39d3354dd 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecEditor.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecEditor.svelte @@ -64,15 +64,16 @@ let s3FilePicker: S3FilePicker | undefined let s3PickerSelection: { s3: string; storage?: string } | undefined = undefined let s3FolderPrefix: string = '' + let s3FileUploadRawMode = componentInput?.type == 'uploadS3' && !!componentInput.value?.s3 function updateSelectedS3File() { if (s3PickerSelection) { - // @ts-ignore - componentInput = { - ...componentInput, - type: 'static', - value: `s3://${s3PickerSelection.s3}` + if (componentInput.type === 'uploadS3') { + componentInput.value = { + ...s3PickerSelection + } } + s3FileUploadRawMode = true } } @@ -162,6 +163,11 @@ (componentInput['expr'] == '' || componentInput['expr'] == undefined) ) { componentInput['expr'] = JSON.stringify(componentInput['value']) + } else if (fileUploadS3 && fieldType === 'text' && e.detail != 'uploadS3') { + componentInput['value'] = '' + } else if (e.detail == 'uploadS3') { + s3FileUploadRawMode = false + componentInput['value'] = { s3: '' } } if (shouldFormatExpression) { @@ -238,34 +244,52 @@ {:else if componentInput?.type === 'upload'} {:else if componentInput?.type === 'uploadS3'} -
- + + {#if s3FileUploadRawMode} + {#await import('$lib/components/JsonEditor.svelte')} + + {:then Module} + + {/await} + {:else} + + + {/if} +
- - | undefined export let fieldType: InputType | undefined = undefined @@ -34,6 +36,9 @@ const { onchange } = getContext('AppViewerContext') + let s3FileUploadRawMode = false + let s3FilePicker: S3FilePicker | undefined = undefined + $: componentInput && onchange?.() @@ -42,7 +47,8 @@ {#if fieldType === 'number' || fieldType === 'integer'} {:else if fieldType === 'textarea'} - + {:else if fieldType === 'date'} {:else if fieldType === 'time'} @@ -135,7 +141,72 @@ {:else if fieldType === 'color'} {:else if fieldType === 'object' || fieldType == 'labeledselect'} - {#if format?.startsWith('resource-') && (componentInput.value == undefined || typeof componentInput.value == 'string')} + {#if format && format.split('-').length > 1 && format + .replace('resource-', '') + .replace('_', '') + .toLowerCase() == 's3object'} +
+ + {#if s3FileUploadRawMode} + {#await import('$lib/components/JsonEditor.svelte')} + + {:then Module} + + {/await} + {:else} + { + if (componentInput) { + componentInput.value = { + s3: evt.detail?.path ?? '', + filename: evt.detail?.filename ?? '' + } + s3FileUploadRawMode = true + } + }} + on:deletion={(evt) => { + if (componentInput) { + componentInput.value = { + s3: '' + } + } + }} + /> + {/if} + +
+ { + if (componentInput?.value?.s3) { + s3FileUploadRawMode = true + } + }} + bind:selectedFileKey={componentInput.value} + /> + {:else if format?.startsWith('resource-') && (componentInput.value == undefined || typeof componentInput.value == 'string')} { diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/UploadInputEditor.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/UploadInputEditor.svelte index 4758d1f357..85d975278b 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/UploadInputEditor.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/UploadInputEditor.svelte @@ -15,6 +15,7 @@ export let s3: boolean | undefined = false export let prefix: string | undefined = undefined export let workspace: string | undefined = undefined + export let s3FileUploadRawMode: boolean = false let fileUploads: Writable = writable([]) @@ -38,11 +39,13 @@ return `${cleanPrefix}${file.name}` }} on:addition={({ detail }) => { + // @ts-ignore componentInput = { ...componentInput, - type: 'static', - value: `s3://${detail.path}` + type: 'uploadS3', + value: { s3: detail.path } } + s3FileUploadRawMode = true }} /> {:else} diff --git a/frontend/src/lib/components/apps/inputType.ts b/frontend/src/lib/components/apps/inputType.ts index 4bc2fc8532..e348300628 100644 --- a/frontend/src/lib/components/apps/inputType.ts +++ b/frontend/src/lib/components/apps/inputType.ts @@ -78,7 +78,7 @@ export type UploadInput = { export type UploadS3Input = { type: 'uploadS3' - value: string + value: any } export type FileUploadData = { diff --git a/frontend/src/lib/components/common/fileDownload/FileDownload.svelte b/frontend/src/lib/components/common/fileDownload/FileDownload.svelte index 519acbb1d9..ec94243179 100644 --- a/frontend/src/lib/components/common/fileDownload/FileDownload.svelte +++ b/frontend/src/lib/components/common/fileDownload/FileDownload.svelte @@ -15,9 +15,9 @@ focus-within:border-blue-500 hover:bg-blue-50 dark:hover:bg-frost-900 focus-with duration-200 rounded-lg p-1 gap-2" href={`${base}/api/w/${workspaceId ?? $workspaceStore}${ appPath ? `/apps_u/download_s3_file/${appPath}` : '/job_helpers/download_s3_file' - }?file_key=${encodeURIComponent(s3object?.s3 ?? '')}${ + }?${appPath ? 's3' : 'file_key'}=${encodeURIComponent(s3object?.s3 ?? '')}${ s3object?.storage ? `&storage=${s3object.storage}` : '' - }`} + }${appPath && s3object?.presigned ? `&${s3object.presigned}` : ''}`} download={s3object?.s3.split('/').pop() ?? 'unnamed_download.file'} > diff --git a/frontend/src/lib/components/common/fileUpload/FileUpload.svelte b/frontend/src/lib/components/common/fileUpload/FileUpload.svelte index 1d2555f577..9e5180b76c 100644 --- a/frontend/src/lib/components/common/fileUpload/FileUpload.svelte +++ b/frontend/src/lib/components/common/fileUpload/FileUpload.svelte @@ -19,8 +19,8 @@ export let containerText: string = folderOnly ? 'Drag and drop a folder here or click to browse' : allowMultiple - ? 'Drag and drop files here or click to browse' - : 'Drag and drop a file here or click to browse' + ? 'Drag and drop files here or click to browse' + : 'Drag and drop a file here or click to browse' export let customResourcePath: string | undefined = undefined export let customResourceType: 's3' | 'azure_blob' | undefined = undefined // when customResourcePath is provided, this should be provided as well. Will default to S3 if not export let customClass: string = '' @@ -70,7 +70,10 @@ | undefined) | undefined = undefined - const dispatch = createEventDispatcher() + const dispatch = createEventDispatcher<{ + addition: { path?: string; filename?: string } + deletion: { path: string } + }>() type FileUploadData = { name: string @@ -146,9 +149,9 @@ } else { path = typeof pathTransformer == 'function' - ? (await pathTransformer?.({ + ? ((await pathTransformer?.({ file: fileToUpload - })) ?? fileToUploadKey + })) ?? fileToUploadKey) : fileToUploadKey } const uploadData: FileUploadData = { @@ -269,10 +272,10 @@ appPath ? `/api/w/${ workspace ?? $workspaceStore - }/apps_u/upload_s3_file/${appPath}?${params.toString()}` + }/apps_u/upload_s3_file/${appPath}?${params.toString()}` : `/api/w/${ workspace ?? $workspaceStore - }/job_helpers/upload_s3_file?${params.toString()}`, + }/job_helpers/upload_s3_file?${params.toString()}`, true ) xhr?.setRequestHeader('Content-Type', 'application/octet-stream') @@ -469,10 +472,10 @@ color={fileUpload.errorMessage ? '#ef4444' : fileUpload.cancelled - ? '#eab308' - : fileUpload.progress === 100 - ? '#22c55e' - : '#3b82f6'} + ? '#eab308' + : fileUpload.progress === 100 + ? '#22c55e' + : '#3b82f6'} ended={fileUpload.cancelled || fileUpload.errorMessage !== undefined} > {#if fileUpload.errorMessage} diff --git a/frontend/src/lib/components/multiselect/MultiSelect.svelte b/frontend/src/lib/components/multiselect/MultiSelect.svelte index 5bf1a96ed3..1e4f00c66d 100644 --- a/frontend/src/lib/components/multiselect/MultiSelect.svelte +++ b/frontend/src/lib/components/multiselect/MultiSelect.svelte @@ -572,7 +572,7 @@ {/if} {/if} - {#if (searchText && noMatchingOptionsMsg) || options?.length > 0} + {#if allowUserOptions || (searchText && noMatchingOptionsMsg) || options?.length > 0}
0) { + $: if (portalRef && outerDiv && (allowUserOptions || items?.length > 0)) { tick().then(() => { moveOptionsToPortal() }) @@ -53,6 +53,7 @@ {#if !value || Array.isArray(value)}
{/if} @@ -385,20 +385,6 @@ code={JSON.stringify(static_asset_config ?? { s3: '' }, null, 2)} /> {/if} - {#if can_write} - - {/if} {:else} {#key is_static_website} {/key} {/if} + {#if can_write} + + {/if}
{:else} @@ -612,10 +612,10 @@ href={itemKind === 'flow' ? `/flows/add?${SECRET_KEY_PATH}=${encodeURIComponent(variable_path)}&hub=${ HubFlow.SIGNATURE_TEMPLATE - }` + }` : `/scripts/add?${SECRET_KEY_PATH}=${encodeURIComponent( variable_path - )}&hub=hub%2F${HUB_SCRIPT_ID}`} + )}&hub=hub%2F${HUB_SCRIPT_ID}`} target="_blank">Create from template diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 614d74faaf..d4ed2d7dd8 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -2,7 +2,6 @@ import { WorkspaceService, type AIConfig, type AIProvider } from '$lib/gen' import { setCopilotInfo, workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' - import MultiSelect from 'svelte-multiselect' import { AI_DEFAULT_MODELS } from '../copilot/lib' import TestAiKey from '../copilot/TestAIKey.svelte' import Description from '../Description.svelte' @@ -11,6 +10,7 @@ import Toggle from '../Toggle.svelte' import ArgEnum from '../ArgEnum.svelte' import Button from '../common/button/Button.svelte' + import MultiSelectWrapper from '../multiselect/MultiSelectWrapper.svelte' const aiProviderLabels: [AIProvider, string][] = [ ['openai', 'OpenAI'], @@ -122,6 +122,22 @@ aiProviders = Object.fromEntries( Object.entries(aiProviders).filter(([key]) => key !== provider) ) + if (defaultModel) { + const currentDefaultModel = Object.values(aiProviders).find( + (p) => defaultModel && p.models.includes(defaultModel) + ) + if (!currentDefaultModel) { + defaultModel = undefined + } + } + if (codeCompletionModel) { + const currentCodeCompletionModel = Object.values(aiProviders).find( + (p) => codeCompletionModel && p.models.includes(codeCompletionModel) + ) + if (!currentCodeCompletionModel) { + codeCompletionModel = undefined + } + } } }} /> @@ -139,8 +155,8 @@ bind:value={aiProviders[provider].resource_path} on:change={() => { if ( - aiProviders[provider].resource_path && - aiProviders[provider].models.length === 0 && + aiProviders[provider]?.resource_path && + aiProviders[provider]?.models.length === 0 && AI_DEFAULT_MODELS[provider].length > 0 ) { aiProviders[provider].models = AI_DEFAULT_MODELS[provider].slice(0, 1) @@ -158,11 +174,11 @@ @@ -177,16 +193,18 @@

Settings

diff --git a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte new file mode 100644 index 0000000000..51caaa1a42 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte @@ -0,0 +1,252 @@ + + + + + + +
+
+
Workspace Object Storage (S3/Azure Blob)
+ + Connect your Windmill workspace to your S3 bucket or your Azure Blob storage to enable users + to read and write from S3 without having to have access to the credentials. + +
+
+{#if !$enterpriseLicense} + + Windmill S3 bucket browser will not work for buckets containing more than 20 files and uploads + are limited to files {'<'} 50MB. Consider upgrading to Windmill EE to use this feature with large + buckets. + +{:else} + + This setting is only for storage of large files allowing to upload files directly to object + storage using S3Object and use the wmill sdk to read and write large files backed by an object + storage. Large-scale log management and distributed dependency caching is under Instance object storage, set by the superadmins in the instance settings UI. + +{/if} +{#if s3ResourceSettings} +
+
+ + + + S3 + Azure Blob + AWS OIDC + Azure Workload Identity + +
+
+ + + + +
+
+ {#if s3ResourceSettings.resourceType == 's3'} +
+ + + + {#if s3ResourceSettings.publicResource === true} +
+ + + S3 resource public access is ON, which means that the entire content of the S3 bucket will + be accessible to all the users of this workspace regardless of whether they have access + the resource or not. Similarly, certain Windmill SDK endpoints can be used in scripts to + access the resource details, including public and private keys. + + {/if} +
+ {:else} +
+ + + + {#if s3ResourceSettings.publicResource === true} +
+ + object public access is ON, which means that the entire content of the object store will + be accessible to all the users of this workspace regardless of whether they have access + the resource or not. + + {/if} +
+ {/if} +
+
+ {#each s3ResourceSettings.secondaryStorage ?? [] as _, idx} +
+ s3ResourceSettings.secondaryStorage?.[idx]?.[0] || '', + (v) => { + if (s3ResourceSettings.secondaryStorage?.[idx]) { + s3ResourceSettings.secondaryStorage[idx][0] = v + } + } + } + placeholder="Storage name" + /> + + + + s3ResourceSettings.secondaryStorage?.[idx]?.[1].resourcePath || '', + (v) => { + if (s3ResourceSettings.secondaryStorage?.[idx]) { + s3ResourceSettings.secondaryStorage[idx][1].resourcePath = v + } + } + } + /> + + +
+ {/each} +
+ + + Secondary storage is a feature that allows you to read and write from storage that isn't + your main storage by specifying it in the s3 object as "secondary_storage" with the name + of it + +
+
+
+
+ +
+{/if} diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 2809592688..a672288ff2 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -10,7 +10,6 @@ import PageHeader from '$lib/components/PageHeader.svelte' import ResourcePicker from '$lib/components/ResourcePicker.svelte' import ScriptPicker from '$lib/components/ScriptPicker.svelte' - import S3FilePicker from '$lib/components/S3FilePicker.svelte' import Tooltip from '$lib/components/Tooltip.svelte' import WorkspaceUserSettings from '$lib/components/settings/WorkspaceUserSettings.svelte' @@ -46,7 +45,6 @@ import PremiumInfo from '$lib/components/settings/PremiumInfo.svelte' import Toggle from '$lib/components/Toggle.svelte' - import Portal from '$lib/components/Portal.svelte' import { fade } from 'svelte/transition' import ChangeWorkspaceName from '$lib/components/settings/ChangeWorkspaceName.svelte' @@ -54,14 +52,14 @@ import ChangeWorkspaceColor from '$lib/components/settings/ChangeWorkspaceColor.svelte' import { convertBackendSettingsToFrontendSettings, - convertFrontendToBackendSetting, type S3ResourceSettings } from '$lib/workspace_settings' import { base } from '$lib/base' import { hubPaths } from '$lib/hub' import Description from '$lib/components/Description.svelte' import ConnectionSection from '$lib/components/ConnectionSection.svelte' - import AiSettings from '$lib/components/workspaceSettings/AISettings.svelte' + import AISettings from '$lib/components/workspaceSettings/AISettings.svelte' + import StorageSettings from '$lib/components/workspaceSettings/StorageSettings.svelte' type GitSyncTypeMap = { scripts: boolean @@ -89,8 +87,6 @@ | 'user' | 'group' - let s3FileViewer: S3FilePicker - let slackInitialPath: string let slackScriptPath: string let teamsInitialPath: string @@ -212,18 +208,6 @@ } } - async function editWindmillLFSSettings(): Promise { - const large_file_storage = convertFrontendToBackendSetting(s3ResourceSettings) - await WorkspaceService.editLargeFileStorageConfig({ - workspace: $workspaceStore!, - requestBody: { - large_file_storage: large_file_storage - } - }) - console.log('Large file storage settings changed', large_file_storage) - sendUserToast(`Large file storage settings changed`) - } - async function editWindmillGitSyncSettings(): Promise { let alreadySeenResource: string[] = [] let repositories = gitSyncSettings.repositories.map((elmt) => { @@ -629,10 +613,6 @@ $: updateFromSearchTab($page.url.searchParams.get('tab')) - - - - {#if $userStore?.is_admin || $superadmin}
{:else if tab == 'ai'} - {:else if tab == 'windmill_lfs'} -
-
-
Workspace Object Storage (S3/Azure Blob)
- - Connect your Windmill workspace to your S3 bucket or your Azure Blob storage to enable - users to read and write from S3 without having to have access to the credentials. - -
-
- {#if !$enterpriseLicense} - - Windmill S3 bucket browser will not work for buckets containing more than 20 files and - uploads are limited to files {'<'} 50MB. Consider upgrading to Windmill EE to use this feature - with large buckets. - - {:else} - - This setting is only for storage of large files allowing to upload files directly to - object storage using S3Object and use the wmill sdk to read and write large files backed - by an object storage. Large-scale log management and distributed dependency caching is - under Instance object storage, set by the superadmins in the instance settings UI. - - {/if} - {#if s3ResourceSettings} -
-
- - S3 - Azure Blob - AWS OIDC - Azure Workload Identity - -
-
- - -
-
- {#if s3ResourceSettings.resourceType == 's3'} -
- - {#if s3ResourceSettings.publicResource === true} -
- - - S3 resource public access is ON, which means that the entire content of the S3 - bucket will be accessible to all the users of this workspace regardless of whether - they have access the resource or not. Similarly, certain Windmill SDK endpoints can - be used in scripts to access the resource details, including public and private - keys. - - {/if} -
- {:else} -
- - {#if s3ResourceSettings.publicResource === true} -
- - object public access is ON, which means that the entire content of the object store - will be accessible to all the users of this workspace regardless of whether they - have access the resource or not. - - {/if} -
- {/if} -
-
- {#each s3ResourceSettings.secondaryStorage ?? [] as secondaryStorage, idx} -
- - - - - -
- {/each} -
- - - Secondary storage is a feature that allows you to read and write from storage that - isn't your main storage by specifying it in the s3 object as "secondary_storage" - with the name of it - -
-
-
-
- -
- {/if} + {:else if tab == 'git_sync'}
diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index e1f16765ff..bca007b438 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -586,6 +586,12 @@ class Windmill: raise Exception("Could not write file to S3") from e return S3Object(s3=response["file_key"]) + def sign_s3_objects(self, s3_objects: list[S3Object]) -> list[S3Object]: + return self.post(f"/w/{self.workspace}/apps/sign_s3_objects", json={"s3_objects": s3_objects}).json() + + def sign_s3_object(self, s3_object: S3Object) -> S3Object: + return self.post(f"/w/{self.workspace}/apps/sign_s3_objects", json={"s3_objects": [s3_object]}).json()[0] + def __boto3_connection_settings(self, s3_resource) -> Boto3ConnectionSettings: endpoint_url_prefix = "https://" if s3_resource["useSSL"] else "http://" return Boto3ConnectionSettings( @@ -974,6 +980,24 @@ def write_s3_file( return _client.write_s3_file(s3object, file_content, s3_resource_path if s3_resource_path != "" else None, content_type, content_disposition) +@init_global_client +def sign_s3_objects(s3_objects: list[S3Object]) -> list[S3Object]: + """ + Sign S3 objects to be used by anonymous users in public apps + Returns a list of signed s3 tokens + """ + return _client.sign_s3_objects(s3_objects) + + +@init_global_client +def sign_s3_object(s3_object: S3Object) -> S3Object: + """ + Sign S3 object to be used by anonymous users in public apps + Returns a signed s3 object + """ + return _client.sign_s3_object(s3_object) + + @init_global_client def whoami() -> dict: """ diff --git a/python-client/wmill/wmill/s3_types.py b/python-client/wmill/wmill/s3_types.py index f633532591..b1736db9e9 100644 --- a/python-client/wmill/wmill/s3_types.py +++ b/python-client/wmill/wmill/s3_types.py @@ -1,6 +1,7 @@ class S3Object(dict): s3: str storage: str | None + presigned: str | None def __getattr__(self, attr): return self[attr] diff --git a/typescript-client/build.jsr.sh b/typescript-client/build.jsr.sh index 4d2859f204..06dae6605b 100755 --- a/typescript-client/build.jsr.sh +++ b/typescript-client/build.jsr.sh @@ -14,5 +14,5 @@ cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/build.sh b/typescript-client/build.sh index 73e2486d6b..5f9cdbc4ec 100755 --- a/typescript-client/build.sh +++ b/typescript-client/build.sh @@ -39,4 +39,4 @@ cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 37fd4c9df6..258ea76215 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -3,10 +3,11 @@ import { VariableService, JobService, HelpersService, + AppService, MetricsService, OidcService, UserService, - TeamsService + TeamsService, } from "./index"; import { OpenAPI } from "./index"; // import type { DenoS3LightClientSettings } from "./index"; @@ -770,6 +771,33 @@ export async function writeS3File( }; } +/** + * Sign S3 objects to be used by anonymous users in public apps + * @param s3objects s3 objects to sign + * @returns signed s3 objects + */ +export async function signS3Objects( + s3objects: S3Object[] +): Promise { + const signedKeys = await AppService.signS3Objects({ + workspace: getWorkspace(), + requestBody: { + s3_objects: s3objects, + }, + }); + return signedKeys; +} + +/** + * Sign S3 object to be used by anonymous users in public apps + * @param s3object s3 object to sign + * @returns signed s3 object + */ +export async function signS3Object(s3object: S3Object): Promise { + const [signedObject] = await signS3Objects([s3object]); + return signedObject; +} + /** * Get URLs needed for resuming a flow after this step * @param approver approver name diff --git a/typescript-client/s3Types.ts b/typescript-client/s3Types.ts index 16641477d4..a46248d778 100644 --- a/typescript-client/s3Types.ts +++ b/typescript-client/s3Types.ts @@ -1,6 +1,7 @@ export type S3Object = { s3: string; storage?: string; + presigned?: string; }; export type DenoS3LightClientSettings = {