From 167100510032ab53cd609fb2c7629e67faceb093 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Tue, 12 Nov 2024 20:07:59 +0100 Subject: [PATCH] feat: s3 input available for public apps (#4685) --- backend/ee-repo-ref.txt | 2 +- backend/windmill-api/openapi.yaml | 4 + backend/windmill-api/src/apps.rs | 350 ++++++++++++++++-- backend/windmill-api/src/job_helpers_ee.rs | 38 ++ backend/windmill-api/src/users.rs | 12 +- .../lib/components/LightweightArgInput.svelte | 15 + .../components/LightweightSchemaForm.svelte | 15 + .../components/buttons/AppSchemaForm.svelte | 26 +- .../helpers/RunnableComponent.svelte | 12 + .../components/inputs/AppS3FileInput.svelte | 14 + .../apps/editor/AppEditorHeader.svelte | 45 ++- .../lib/components/apps/editor/appUtilsS3.ts | 82 ++++ .../settingsPanel/InputsSpecEditor.svelte | 2 +- .../common/fileUpload/FileUpload.svelte | 44 ++- 14 files changed, 614 insertions(+), 47 deletions(-) create mode 100644 frontend/src/lib/components/apps/editor/appUtilsS3.ts diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 6a33fdaadd..7a1f8f391e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -6534b0f31fb4a75dd706fca2ce91e37e77e4ad02 \ No newline at end of file +8f45974252a7ce6fcf8f49482751ffa75b81bed7 \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index e394d48eaf..154200bdcf 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -12186,6 +12186,10 @@ components: type: object additionalProperties: type: object + s3_inputs: + type: array + items: + type: object execution_mode: type: string enum: [viewer, publisher, anonymous] diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index cd65136a2a..4ec07393dc 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -7,6 +7,12 @@ use std::collections::HashMap; * Please see the included NOTICE for copyright information and * LICENSE-AGPL for a copy of the license. */ + +#[cfg(feature = "parquet")] +use crate::{job_helpers_ee::{ + get_random_file_name, get_s3_resource, get_workspace_s3_resource, upload_file_internal, + UploadFileResponse, +}, users::fetch_api_authed_from_permissioned_as}; use crate::{ db::{ApiAuthed, DB}, resources::get_resource_value_interpolated_internal, @@ -23,7 +29,13 @@ use axum::{ Router, }; use hyper::StatusCode; +#[cfg(feature = "parquet")] +use itertools::Itertools; use magic_crypt::MagicCryptTrait; +#[cfg(feature = "parquet")] +use object_store::{Attribute, Attributes}; +#[cfg(feature = "parquet")] +use regex::Regex; use serde::{Deserialize, Serialize}; use serde_json::{json, value::RawValue}; use sha2::{Digest, Sha256}; @@ -32,6 +44,8 @@ use sqlx::{types::Uuid, FromRow}; use std::str; use windmill_audit::audit_ee::audit_log; use windmill_audit::ActionKind; +#[cfg(feature = "parquet")] +use windmill_common::s3_helpers::build_object_store_client; use windmill_common::{ apps::ListAppQuery, db::UserDB, @@ -69,6 +83,7 @@ pub fn workspaced_service() -> Router { pub fn unauthed_service() -> Router { Router::new() .route("/execute_component/*path", post(execute_component)) + .route("/upload_s3_file/*path", post(upload_s3_file_from_app)) .route("/public_app/:secret", get(get_public_app_by_secret)) .route("/public_resource/*path", get(get_public_resource)) } @@ -179,6 +194,14 @@ pub struct PolicyTriggerableInputs { allow_user_resources: AllowUserResources, } +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct S3Input { + allowed_resources: Vec, + allow_user_resources: bool, + allow_workspace_resource: bool, + file_key_regex: String, +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Policy { pub on_behalf_of: Option, @@ -192,6 +215,7 @@ pub struct Policy { #[serde(skip_serializing_if = "Option::is_none")] pub triggerables_v2: Option>, pub execution_mode: ExecutionMode, + pub s3_inputs: Option>, } #[derive(Deserialize)] @@ -432,9 +456,7 @@ async fn get_latest_version( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, - ) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; let row = sqlx::query!( "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg @@ -457,7 +479,6 @@ async fn get_latest_version( } else { return Ok(Json(None)); } - } async fn update_app_history( @@ -1067,6 +1088,49 @@ fn digest(code: &str) -> String { format!("rawscript/{:x}", result) } +async fn get_on_behalf_details_from_policy_and_authed( + policy: &Policy, + opt_authed: &Option, +) -> Result<(String, String, String)> { + let (username, permissioned_as, email) = match policy.execution_mode { + ExecutionMode::Anonymous => { + let username = opt_authed + .as_ref() + .map(|a| a.username.clone()) + .unwrap_or_else(|| "anonymous".to_string()); + let (permissioned_as, email) = get_on_behalf_of(&policy)?; + (username, permissioned_as, email) + } + ExecutionMode::Publisher => { + let username = opt_authed + .as_ref() + .map(|a| a.username.clone()) + .ok_or_else(|| { + Error::BadRequest( + "publisher execution mode requires authentication".to_string(), + ) + })?; + let (permissioned_as, email) = get_on_behalf_of(&policy)?; + (username, permissioned_as, email) + } + ExecutionMode::Viewer => { + let (username, email) = opt_authed + .as_ref() + .map(|a| (a.username.clone(), a.email.clone())) + .ok_or_else(|| { + Error::BadRequest("Required to be authed in viewer mode".to_string()) + })?; + ( + username.clone(), + username_to_permissioned_as(&username), + email, + ) + } + }; + + Ok((username, permissioned_as, email)) +} + async fn execute_component( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, @@ -1129,6 +1193,7 @@ async fn execute_component( triggerables_v2: Some(hm), on_behalf_of: None, on_behalf_of_email: None, + s3_inputs: None, } } _ => { @@ -1146,41 +1211,8 @@ async fn execute_component( } }; - let (username, permissioned_as, email) = match policy.execution_mode { - ExecutionMode::Anonymous => { - let username = opt_authed - .as_ref() - .map(|a| a.username.clone()) - .unwrap_or_else(|| "anonymous".to_string()); - let (permissioned_as, email) = get_on_behalf_of(&policy)?; - (username, permissioned_as, email) - } - ExecutionMode::Publisher => { - let username = opt_authed - .as_ref() - .map(|a| a.username.clone()) - .ok_or_else(|| { - Error::BadRequest( - "publisher execution mode requires authentication".to_string(), - ) - })?; - let (permissioned_as, email) = get_on_behalf_of(&policy)?; - (username, permissioned_as, email) - } - ExecutionMode::Viewer => { - let (username, email) = opt_authed - .as_ref() - .map(|a| (a.username.clone(), a.email.clone())) - .ok_or_else(|| { - Error::BadRequest("Required to be authed in viewer mode".to_string()) - })?; - ( - username.clone(), - username_to_permissioned_as(&username), - email, - ) - } - }; + let (username, permissioned_as, email) = + get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; let (job_payload, (args, job_id), tag) = match payload { ExecuteApp { args, component, raw_code: Some(raw_code), path: None, .. } => { @@ -1249,6 +1281,248 @@ async fn execute_component( Ok(uuid.to_string()) } +#[cfg(not(feature = "parquet"))] +async fn upload_s3_file_from_app() -> Result<()> { + return Err(Error::BadRequest( + "This endpoint requires the parquet feature to be enabled".to_string(), + )); +} + + +#[cfg(feature = "parquet")] +#[derive(Debug, Deserialize, Clone)] +struct UploadFileToS3Query { + file_key: Option, + file_extension: Option, + s3_resource_path: Option, + content_type: Option, + content_disposition: Option, + force_viewer_file_key_regex: Option, + force_viewer_allow_user_resources: Option, + force_viewer_allow_workspace_resource: Option, + force_viewer_allowed_resources: Option, +} + +#[cfg(feature = "parquet")] +async fn upload_s3_file_from_app( + OptAuthed(opt_authed): OptAuthed, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, + request: axum::extract::Request, +) -> JsonResult { + let policy = if let Some(file_key_regex) = query.force_viewer_file_key_regex { + Some(Policy { + execution_mode: ExecutionMode::Viewer, + triggerables: None, + triggerables_v2: None, + on_behalf_of: None, + on_behalf_of_email: None, + s3_inputs: Some(vec![S3Input { + file_key_regex: file_key_regex, + allow_user_resources: query.force_viewer_allow_user_resources.unwrap_or(false), + allow_workspace_resource: query + .force_viewer_allow_workspace_resource + .unwrap_or(false), + allowed_resources: query + .force_viewer_allowed_resources + .map(|s| s.split(',').map(|s| s.to_string()).collect()) + .unwrap_or_default(), + }]), + }) + } else { + let policy_o = sqlx::query_scalar!( + "SELECT policy from app WHERE path = $1 AND workspace_id = $2", + &path.0, + &w_id + ) + .fetch_optional(&db) + .await?; + + policy_o + .map(|p| serde_json::from_value::(p).map_err(to_anyhow)) + .transpose()? + }; + + let user_db = UserDB::new(db.clone()); + + let (s3_resource_opt, file_key) = if policy.as_ref().is_some_and(|p| p.s3_inputs.is_some()) { + let policy = policy.unwrap(); + let s3_inputs = policy.s3_inputs.as_ref().unwrap(); + + let (username, permissioned_as, email) = + get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; + + let on_behalf_authed = + fetch_api_authed_from_permissioned_as(permissioned_as, email, &w_id, &db, username) + .await?; + + if let Some(file_key) = query.file_key { + // file key is provided => requires workspace, user or list policy and must match the regex + let matching_s3_inputs = if let Some(ref s3_resource_path) = query.s3_resource_path { + s3_inputs + .iter() + .filter(|s3_input| { + s3_input.allowed_resources.contains(s3_resource_path) + || s3_input.allow_user_resources + }) + .sorted_by_key(|i| i.allow_user_resources) // consider user resources last + .collect::>() + } else { + s3_inputs + .iter() + .filter(|s3_input| s3_input.allow_workspace_resource) + .collect::>() + }; + + let matched_input = matching_s3_inputs.iter().find(|s3_input| { + match Regex::new(&s3_input.file_key_regex) { + Ok(re) => re.is_match(&file_key), + Err(e) => { + tracing::error!("Error compiling regex: {}", e); + false + } + } + }); + + if let Some(matched_input) = matched_input { + if let Some(ref s3_resource_path) = query.s3_resource_path { + if matched_input.allow_user_resources { + if let Some(authed) = opt_authed { + ( + Some( + get_s3_resource( + &authed, + &db, + Some(user_db), + "", + &w_id, + s3_resource_path, + None, + None, + ) + .await?, + ), + file_key, + ) + } else { + return Err(Error::BadRequest( + "User resources are not allowed without being logged in" + .to_string(), + )); + } + } else { + ( + Some( + get_s3_resource( + &on_behalf_authed, + &db, + Some(user_db), + "", + &w_id, + s3_resource_path, + None, + None, + ) + .await?, + ), + file_key, + ) + } + } else { + let (_, s3_resource_opt) = + get_workspace_s3_resource(&on_behalf_authed, &db, None, "", &w_id, None) + .await?; + (s3_resource_opt, file_key) + } + } else { + return Err(Error::BadRequest( + "No matching s3 resource found for the given file key".to_string(), + )); + } + } else { + // no file key => requires unnamed upload policy => allow workspace resource and file_key_regex is empty + let has_unnamed_policy = s3_inputs.iter().any(|s3_input| { + s3_input.allow_workspace_resource && s3_input.file_key_regex.is_empty() + }); + + if !has_unnamed_policy { + return Err(Error::BadRequest( + "no policy found for unnamed s3 file uplooad".to_string(), + )); + } + + // for now, we place all files into `windmill_uploads` folder with a random name + // TODO: make the folder configurable via the workspace settings + let file_key = get_random_file_name(query.file_extension); + + let (_, s3_resource_opt) = + get_workspace_s3_resource(&on_behalf_authed, &db, None, "", &w_id, None).await?; + + (s3_resource_opt, file_key) + } + } else { + // backward compatibility (no policy) + // if no policy but logged in, use the user's auth to get the s3 resource + if let Some(authed) = opt_authed { + let file_key = query + .file_key + .unwrap_or_else(|| get_random_file_name(query.file_extension)); + + if let Some(ref s3_resource_path) = query.s3_resource_path { + ( + Some( + get_s3_resource( + &authed, + &db, + Some(user_db), + "", + &w_id, + s3_resource_path, + None, + None, + ) + .await?, + ), + file_key, + ) + } else { + let (_, s3_resource) = + get_workspace_s3_resource(&authed, &db, None, "", &w_id, None).await?; + + (s3_resource, file_key) + } + } else { + return Err(Error::BadRequest("Missing s3 policy".to_string())); + } + }; + + let s3_resource = s3_resource_opt.ok_or(Error::InternalErr( + "No files storage resource defined at the workspace level".to_string(), + ))?; + let s3_client = build_object_store_client(&s3_resource).await?; + + let options = Attributes::from_iter(vec![ + ( + Attribute::ContentType, + query.content_type.unwrap_or_else(|| { + mime_guess::from_path(&file_key) + .first_or_octet_stream() + .to_string() + }), + ), + ( + Attribute::ContentDisposition, + query.content_disposition.unwrap_or("inline".to_string()), + ), + ]) + .into(); + + upload_file_internal(s3_client, &file_key, request, options).await?; + + return Ok(Json(UploadFileResponse { file_key })); +} + fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> { let permissioned_as = policy .on_behalf_of diff --git a/backend/windmill-api/src/job_helpers_ee.rs b/backend/windmill-api/src/job_helpers_ee.rs index b64ccde92d..3033a2ec5a 100644 --- a/backend/windmill-api/src/job_helpers_ee.rs +++ b/backend/windmill-api/src/job_helpers_ee.rs @@ -1,9 +1,21 @@ use axum::Router; +use serde::Serialize; +use uuid::Uuid; +use windmill_common::s3_helpers::StorageResourceType; #[cfg(feature = "parquet")] use crate::db::{ApiAuthed, DB}; #[cfg(feature = "parquet")] +use object_store::{ObjectStore, PutMultipartOpts}; +#[cfg(feature = "parquet")] +use std::sync::Arc; +use windmill_common::error; +#[cfg(feature = "parquet")] use windmill_common::{db::UserDB, s3_helpers::ObjectStoreResource}; +#[derive(Serialize)] +pub struct UploadFileResponse { + pub file_key: String, +} pub fn workspaced_service() -> Router { Router::new() @@ -21,3 +33,29 @@ pub async fn get_workspace_s3_resource<'c>( // implementation is not open source Ok((None, None)) } + +pub fn get_random_file_name(_file_extension: Option) -> String { + todo!() +} + +pub async fn get_s3_resource<'c>( + _authed: &ApiAuthed, + _db: &DB, + _user_db: Option, + _token: &str, + _w_id: &str, + _resource_path: &str, + _resource_type: Option, + _job_id: Option, +) -> error::Result { + todo!() +} + +pub async fn upload_file_internal( + _s3_client: Arc, + _file_key: &str, + _request: axum::extract::Request, + _options: PutMultipartOpts, +) -> error::Result<()> { + todo!() +} diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 005dfad342..380084d524 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -746,10 +746,20 @@ pub async fn fetch_api_authed( username_override: String, ) -> error::Result { let permissioned_as = username_to_permissioned_as(username.as_str()); + fetch_api_authed_from_permissioned_as(permissioned_as, email, w_id, db, username_override).await +} + +pub async fn fetch_api_authed_from_permissioned_as( + permissioned_as: String, + email: String, + w_id: &str, + db: &DB, + username_override: String, +) -> error::Result { let authed = fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?; Ok(ApiAuthed { - username: username, + username: authed.username, email: email, is_admin: authed.is_admin, is_operator: authed.is_operator, diff --git a/frontend/src/lib/components/LightweightArgInput.svelte b/frontend/src/lib/components/LightweightArgInput.svelte index bc15a41926..f9c4a04ed0 100644 --- a/frontend/src/lib/components/LightweightArgInput.svelte +++ b/frontend/src/lib/components/LightweightArgInput.svelte @@ -61,6 +61,18 @@ export let render = true export let title: string | undefined = undefined export let placeholder: string | undefined = undefined + export let appPath: string | undefined = undefined + export let computeS3ForceViewerPolicies: + | (() => + | { + allowed_resources: string[] + allow_user_resources: boolean + allow_workspace_resource: boolean + file_key_regex: string + } + | undefined) + | undefined = undefined + export let workspace: string | undefined = undefined let oneOfSelected: string | undefined = undefined async function updateOneOfSelected(oneOf: SchemaProperty[] | undefined) { @@ -428,6 +440,9 @@ .toLowerCase() == 's3object'}
{ diff --git a/frontend/src/lib/components/LightweightSchemaForm.svelte b/frontend/src/lib/components/LightweightSchemaForm.svelte index d4da597066..55b1373b4d 100644 --- a/frontend/src/lib/components/LightweightSchemaForm.svelte +++ b/frontend/src/lib/components/LightweightSchemaForm.svelte @@ -16,6 +16,18 @@ export let defaultValues: Record = {} export let dynamicEnums: Record = {} export let disabled: boolean = false + export let appPath: string | undefined = undefined + export let computeS3ForceViewerPolicies: + | (() => + | { + allowed_resources: string[] + allow_user_resources: boolean + allow_workspace_resource: boolean + file_key_regex: string + } + | undefined) + | undefined = undefined + export let workspace: string | undefined = undefined let inputCheck: { [id: string]: boolean } = {} let errors: { [id: string]: string } = {} @@ -104,6 +116,9 @@ {displayType} {css} disabled={disabled || schema.properties[argName].disabled} + {appPath} + {computeS3ForceViewerPolicies} + {workspace} /> {/if} {/each} diff --git a/frontend/src/lib/components/apps/components/buttons/AppSchemaForm.svelte b/frontend/src/lib/components/apps/components/buttons/AppSchemaForm.svelte index fcbd4e789a..a77d708e13 100644 --- a/frontend/src/lib/components/apps/components/buttons/AppSchemaForm.svelte +++ b/frontend/src/lib/components/apps/components/buttons/AppSchemaForm.svelte @@ -18,6 +18,9 @@ import ResolveConfig from '../helpers/ResolveConfig.svelte' import ResolveStyle from '../helpers/ResolveStyle.svelte' import { deepEqual } from 'fast-equals' + import { computeWorkspaceS3FileInputPolicy } from '../../editor/appUtilsS3' + import { defaultIfEmptyString } from '$lib/utils' + import { userStore } from '$lib/stores' export let id: string export let componentInput: AppInput | undefined @@ -26,8 +29,16 @@ export let configuration: RichConfigurations export let customCss: ComponentCustomCSS<'schemaformcomponent'> | undefined = undefined - const { worldStore, connectingInput, app, selectedComponent, componentControl } = - getContext('AppViewerContext') + const { + worldStore, + connectingInput, + app, + selectedComponent, + componentControl, + appPath, + isEditor, + workspace + } = getContext('AppViewerContext') const iterContext = getContext('ListWrapperContext') const listInputs: ListInputs | undefined = getContext('ListInputs') @@ -100,6 +111,14 @@ previousDefault = structuredClone(resolvedConfig.defaultValues) args = previousDefault ?? {} } + + function computeS3ForceViewerPolicies() { + if (!isEditor) { + return undefined + } + const policy = computeWorkspaceS3FileInputPolicy() + return policy + } {#each Object.keys(components['schemaformcomponent'].initialData.configuration) as key (key)} @@ -140,6 +159,9 @@ bind:this={schemaForm} displayType={Boolean(resolvedConfig.displayType)} largeGap={Boolean(resolvedConfig.largeGap)} + appPath={defaultIfEmptyString(appPath, `u/${$userStore?.username ?? 'unknown'}/newapp`)} + {computeS3ForceViewerPolicies} + {workspace} {css} />
diff --git a/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte b/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte index 66fd746285..286c5ac573 100644 --- a/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte +++ b/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte @@ -26,6 +26,7 @@ import { get } from 'svelte/store' import RefreshButton from '$lib/components/apps/components/helpers/RefreshButton.svelte' import { ctxRegex } from '../../utils' + import { computeWorkspaceS3FileInputPolicy } from '../../editor/appUtilsS3' // Component props export let id: string @@ -671,6 +672,14 @@ return undefined } } + + function computeS3ForceViewerPolicies() { + if (!isEditor) { + return undefined + } + const policy = computeWorkspaceS3FileInputPolicy() + return policy + } {#each Object.entries(fields ?? {}) as [key, v] (key)} @@ -754,6 +763,9 @@
('AppViewerContext') + function computeForceViewerPolicies() { + if (!isEditor) { + return undefined + } + const policy = computeS3FileInputPolicy((configuration as any)?.type?.configuration?.s3, $app) + return policy + } @@ -139,5 +151,7 @@ outputs.result.set(value) }} {forceDisplayUploads} + appPath={defaultIfEmptyString(appPath, `u/${$userStore?.username ?? 'unknown'}/newapp`)} + {computeForceViewerPolicies} /> {/if} diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte index af2f4563df..662631f752 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte @@ -87,6 +87,7 @@ import ToggleEnable from '$lib/components/common/toggleButton-v2/ToggleEnable.svelte' import HideButton from './settingsPanel/HideButton.svelte' import DeployOverrideConfirmationModal from '$lib/components/common/confirmationModal/DeployOverrideConfirmationModal.svelte' + import { computeS3FileInputPolicy, computeWorkspaceS3FileInputPolicy } from './appUtilsS3' async function hash(message) { try { @@ -195,8 +196,12 @@ } async function computeTriggerables() { + const items = allItems($app.grid, $app.subgrids) + + console.log('items', items) + const allTriggers: ([string, TriggerableV2] | undefined)[] = (await Promise.all( - allItems($app.grid, $app.subgrids) + items .flatMap((x) => { let c = x.data as AppComponent let r: { input: AppInput | undefined; id: string }[] = [ @@ -296,6 +301,44 @@ allTriggers.filter(Boolean) as [string, TriggerableV2][] ) policy.triggerables_v2 = ntriggerables + + const s3_inputs = items + .filter((x) => (x.data as AppComponent).type === 's3fileinputcomponent') + .map((x) => { + const c = x.data as AppComponent + const config = c.configuration as any + return computeS3FileInputPolicy(config?.type?.configuration?.s3, $app) + }) + .filter(Boolean) as { + allowed_resources: string[] + allow_user_resources: boolean + file_key_regex: string + }[] + + if ( + items.findIndex((x) => { + const c = x.data as AppComponent + if (c.type === 'schemaformcomponent') { + return ( + Object.values((c.componentInput as any)?.value?.properties ?? {}).findIndex( + (p: any) => p?.type === 'object' && p?.format === 'resource-s3_object' + ) !== -1 + ) + } else if (c.type === 'formbuttoncomponent' || c.type === 'formcomponent') { + return ( + Object.values((c.componentInput as any)?.fields ?? {}).findIndex( + (p: any) => p?.fieldType === 'object' && p?.format === 'resource-s3_object' + ) !== -1 + ) + } else { + return false + } + }) !== -1 + ) { + s3_inputs.push(computeWorkspaceS3FileInputPolicy()) + } + + policy.s3_inputs = s3_inputs } async function processRunnable( diff --git a/frontend/src/lib/components/apps/editor/appUtilsS3.ts b/frontend/src/lib/components/apps/editor/appUtilsS3.ts new file mode 100644 index 0000000000..2256331cc4 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/appUtilsS3.ts @@ -0,0 +1,82 @@ +import type { AppInput, EvalInputV2 } from '../inputType' +import type { App } from '../types' +import { collectOneOfFields } from './appUtils' + +function filenameExprToRegex(template: string) { + const filenameEscaped = template.replaceAll('${file.name}', '') // replace filename with placeholder + const escapedTemplate = filenameEscaped + .slice(1, -1) // remove quotes + .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') // escape regex special characters + const regexPattern = escapedTemplate.replaceAll('', '[^/]+') // replace filename placeholder with regex pattern + return `^${regexPattern}$` +} + +function staticToRegex(str: string) { + return `^${str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$` +} + +function checkIfExprIsString(input: string) { + return /^(['"`])[^'"`]*\1$/g.test(input) +} + +function checkIfEvalIsStringWithFilename(input: EvalInputV2) { + if (input.connections.length > 0) { + return false + } else { + return checkIfExprIsString(input.expr.replaceAll('${file.name}', '')) + } +} + +function removeResourcePrefix(resource: string) { + return resource.replace(/^\$res:/, '') +} + +export function computeWorkspaceS3FileInputPolicy() { + return { + allow_workspace_resource: true, + allowed_resources: [], + allow_user_resources: false, + file_key_regex: '' + } +} + +export function computeS3FileInputPolicy(s3Config: any, app: App) { + const resourceInput = s3Config?.resource as AppInput | undefined + const pathTemplateInput = s3Config?.pathTemplate as AppInput | undefined + + const allow_workspace_resource = + !resourceInput || (resourceInput.type === 'static' && !resourceInput.value) + const allowed_resources: string[] = resourceInput + ? resourceInput.type === 'static' + ? resourceInput.value + ? [removeResourcePrefix(resourceInput.value)] + : [] + : collectOneOfFields( + { + s3_resource: resourceInput + }, + app + ).s3_resource?.map((s) => removeResourcePrefix(s)) ?? [] + : [] + + const allow_user_resources = + (resourceInput?.type === 'evalv2' && resourceInput?.allowUserResources) ?? false + let file_key_regex = '^.*$' + if (pathTemplateInput) { + if (pathTemplateInput.type === 'static') { + file_key_regex = staticToRegex(pathTemplateInput.value) + } else if ( + pathTemplateInput.type === 'evalv2' && + checkIfEvalIsStringWithFilename(pathTemplateInput) + ) { + file_key_regex = filenameExprToRegex(pathTemplateInput.expr) + } + } + + return { + allow_workspace_resource, + allowed_resources, + allow_user_resources, + file_key_regex + } +} diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecEditor.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecEditor.svelte index 14ecb4855c..02cfce4ea4 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecEditor.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/InputsSpecEditor.svelte @@ -193,7 +193,7 @@ {:else if componentInput?.type === 'user'} Field's value is set by the user {/if} - {#if (componentInput?.type === 'evalv2' || componentInput?.type === 'connected' || componentInput?.type === 'user') && fieldType == 'object' && format?.startsWith('resource-')} + {#if (componentInput?.type === 'evalv2' || componentInput?.type === 'connected' || componentInput?.type === 'user') && ((fieldType == 'object' && format?.startsWith('resource-') && format !== 'resource-s3_object') || fieldType == 'resource')}
= writable([]) + export let appPath: string | undefined = undefined + export let computeForceViewerPolicies: + | (() => + | { + allowed_resources: string[] + allow_user_resources: boolean + allow_workspace_resource: boolean + file_key_regex: string + } + | undefined) + | undefined = undefined const dispatch = createEventDispatcher() @@ -115,6 +126,26 @@ params.append('content_type', fileToUpload.type) } + if (computeForceViewerPolicies !== undefined) { + const forceViewerPolicies = computeForceViewerPolicies() + + if (forceViewerPolicies) { + params.append( + 'force_viewer_allowed_resources', + forceViewerPolicies.allowed_resources.join(',') + ) + params.append( + 'force_viewer_allow_user_resources', + JSON.stringify(forceViewerPolicies.allow_user_resources) + ) + params.append( + 'force_viewer_allow_workspace_resource', + JSON.stringify(forceViewerPolicies.allow_workspace_resource) + ) + params.append('force_viewer_file_key_regex', forceViewerPolicies.file_key_regex) + } + } + // let response = await fetch( // `/api/w/${$workspaceStore}/job_helpers/multipart_upload_s3_file?${params.toString()}`, // { @@ -158,9 +189,16 @@ } } }) + xhr?.open( 'POST', - `/api/w/${workspace ?? $workspaceStore}/job_helpers/upload_s3_file?${params.toString()}`, + appPath + ? `/api/w/${ + workspace ?? $workspaceStore + }/apps_u/upload_s3_file/${appPath}?${params.toString()}` + : `/api/w/${ + workspace ?? $workspaceStore + }/job_helpers/upload_s3_file?${params.toString()}`, true ) xhr?.setRequestHeader('Content-Type', 'application/octet-stream') @@ -309,7 +347,7 @@ {/if} - {#if fileUpload.progress === 100 && !fileUpload.cancelled} + {#if fileUpload.progress === 100 && !fileUpload.cancelled && $userStore}