This commit is contained in:
HugoCasa
2025-04-11 15:00:16 +02:00
parent 5409b280c3
commit 62b0925db9
19 changed files with 503 additions and 370 deletions
+2
View File
@@ -6327,6 +6327,8 @@ paths:
type: array
items:
$ref: "#/components/schemas/S3Object"
required:
- s3_objects
responses:
"200":
description: signed s3 tokens
+60 -97
View File
@@ -20,8 +20,7 @@ use crate::{
use crate::{
job_helpers_ee::{
download_s3_file_internal, get_random_file_name, get_s3_resource,
get_workspace_s3_resource, load_image_preview_internal, upload_file_from_req,
DownloadFileQuery, LoadImagePreviewQuery,
get_workspace_s3_resource, upload_file_from_req, DownloadFileQuery,
},
users::fetch_api_authed_from_permissioned_as,
};
@@ -110,10 +109,6 @@ pub fn unauthed_service() -> Router {
.route("/upload_s3_file/*path", post(upload_s3_file_from_app))
.route("/delete_s3_file", delete(delete_s3_file_from_app))
.route("/download_s3_file/*path", get(download_s3_file_from_app))
.route(
"/load_image_preview/*path",
get(load_s3_file_image_preview_from_app),
)
.route("/public_app/:secret", get(get_public_app_by_secret))
.route("/public_resource/*path", get(get_public_resource))
}
@@ -1610,27 +1605,24 @@ async fn sign_s3_objects(
}
#[cfg(feature = "parquet")]
async fn validate_s3_signature(s3_object: &S3Object, w_id: &str, db: &DB) -> Result<()> {
use url::form_urlencoded;
async fn validate_s3_signature(file_query: &AppS3FileQuery, w_id: &str, db: &DB) -> Result<()> {
let workspace_key = get_workspace_key(w_id, &db).await?;
let params: HashMap<_, _> =
form_urlencoded::parse(s3_object.presigned.as_ref().unwrap().as_bytes())
.into_owned()
.collect();
let Some(exp) = file_query
.exp
.as_ref()
.map(|e| e.parse::<i64>().unwrap_or_default())
else {
return Err(Error::BadRequest("Missing exp".to_string()));
};
let exp = params
.get("exp")
.map(|s| s.parse::<i64>().unwrap_or_default())
.ok_or_else(|| Error::NotAuthorized("Missing exp".to_string()))?;
let Some(ref sig) = file_query.sig else {
return Err(Error::BadRequest("Missing signature".to_string()));
};
let signature = params
.get("sig")
.ok_or_else(|| Error::NotAuthorized("Missing signature".to_string()))?;
let mut message = format!("file_key={}&exp={}", file_query.s3, exp);
let mut message = format!("file_key={}&exp={}", s3_object.s3.clone(), exp);
if let Some(ref storage) = s3_object.storage {
if let Some(ref storage) = file_query.storage {
message = format!("{}&storage={}", message, storage);
}
@@ -1639,12 +1631,12 @@ async fn validate_s3_signature(s3_object: &S3Object, w_id: &str, db: &DB) -> Res
mac.update(message.as_bytes());
let signature_bytes = hex::decode(signature)?;
mac.verify_slice(&signature_bytes)
.map_err(|err| Error::NotAuthorized(format!("Invalid signature: {}", err)))?;
let sig_bytes = hex::decode(sig)?;
mac.verify_slice(&sig_bytes)
.map_err(|err| Error::BadRequest(format!("Invalid signature: {}", err)))?;
if exp < chrono::Utc::now().timestamp() {
return Err(Error::NotAuthorized("Signature expired".to_string()));
return Err(Error::BadRequest("Signature expired".to_string()));
}
Ok(())
@@ -2058,7 +2050,7 @@ async fn get_on_behalf_authed_from_app(
async fn check_if_allowed_to_access_s3_file_from_app(
db: &DB,
opt_authed: &Option<ApiAuthed>,
file_query: &mut S3Object,
file_query: &AppS3FileQuery,
w_id: &str,
path: &str,
policy: &Policy,
@@ -2066,49 +2058,59 @@ async fn check_if_allowed_to_access_s3_file_from_app(
// if anonymous, check that the file was the result of an app script ran by an anonymous user in the last 3 hours
// otherwise, if logged in, allow any file (TODO: change that when we implement better s3 policy)
if file_query.presigned.is_some() {
validate_s3_signature(file_query, w_id, &db).await?;
}
let allowed = opt_authed.is_some()
|| policy
if file_query.sig.is_some() {
validate_s3_signature(file_query, w_id, &db).await
} else if opt_authed.is_some() {
Ok(())
} else {
let allowed = policy
.allowed_s3_keys
.as_ref()
.unwrap()
.iter()
.any(|key| key.s3_path == file_query.s3 && key.storage == file_query.storage)
|| {
sqlx::query_scalar!(
r#"SELECT EXISTS (
SELECT 1 FROM v2_as_completed_job
WHERE workspace_id = $2
|| {
sqlx::query_scalar!(
r#"SELECT EXISTS (
SELECT 1 FROM v2_as_completed_job
WHERE workspace_id = $2
AND (job_kind = 'appscript' OR job_kind = 'preview')
AND created_by = 'anonymous'
AND created_by = 'anonymous'
AND started_at > now() - interval '3 hours'
AND script_path LIKE $3 || '/%'
AND result @> ('{"s3":"' || $1 || '"}')::jsonb
AND script_path LIKE $3 || '/%'
AND result @> ('{"s3":"' || $1 || '"}')::jsonb
)"#,
file_query.s3,
w_id,
path,
)
.fetch_one(db)
.await?
.unwrap_or(false)
};
file_query.s3,
w_id,
path,
)
.fetch_one(db)
.await?
.unwrap_or(false)
};
if !allowed {
Err(Error::BadRequest("File restricted".to_string()))
} else {
Ok(())
if !allowed {
Err(Error::BadRequest("File restricted".to_string()))
} else {
Ok(())
}
}
}
#[cfg(feature = "parquet")]
#[derive(Deserialize)]
pub struct S3ObjectWithForceViewerAllowedS3Keys {
#[derive(Deserialize, Debug)]
struct AppS3FileQuery {
s3: String,
storage: Option<String>,
sig: Option<String>,
exp: Option<String>,
}
#[cfg(feature = "parquet")]
#[derive(Deserialize, Debug)]
struct AppS3FileQueryWithForceViewerAllowedS3Keys {
#[serde(flatten)]
pub file_query: S3Object,
pub file_query: AppS3FileQuery,
pub force_viewer_allowed_s3_keys: Option<String>,
}
@@ -2117,7 +2119,7 @@ async fn download_s3_file_from_app(
OptAuthed(opt_authed): OptAuthed,
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
Query(mut query): Query<S3ObjectWithForceViewerAllowedS3Keys>,
Query(query): Query<AppS3FileQueryWithForceViewerAllowedS3Keys>,
) -> Result<Response> {
let path = path.to_path();
@@ -2136,7 +2138,7 @@ async fn download_s3_file_from_app(
check_if_allowed_to_access_s3_file_from_app(
&db,
&opt_authed,
&mut query.file_query,
&query.file_query,
&w_id,
&path,
&policy,
@@ -2158,45 +2160,6 @@ async fn download_s3_file_from_app(
.await
}
#[cfg(not(feature = "parquet"))]
async fn load_s3_file_image_preview_from_app() -> Result<()> {
return Err(Error::BadRequest(
"This endpoint requires the parquet feature to be enabled".to_string(),
));
}
#[cfg(feature = "parquet")]
async fn load_s3_file_image_preview_from_app(
OptAuthed(opt_authed): OptAuthed,
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
Query(mut query): Query<S3Object>,
) -> Result<Response> {
let path = path.to_path();
let (on_behalf_authed, policy) =
get_on_behalf_authed_from_app(&db, &path, &w_id, &opt_authed, None).await?;
check_if_allowed_to_access_s3_file_from_app(
&db,
&opt_authed,
&mut query,
&w_id,
&path,
&policy,
)
.await?;
load_image_preview_internal(
on_behalf_authed,
&db,
"",
&w_id,
LoadImagePreviewQuery { file_key: query.s3, storage: query.storage },
)
.await
}
fn get_on_behalf_of(policy: &Policy) -> Result<(String, String)> {
let permissioned_as = policy
.on_behalf_of
@@ -772,24 +772,30 @@
<img
alt="preview rendered"
class="w-auto h-full"
src={`/api/w/${workspaceId}/${
src="{`/api/w/${workspaceId}/${
appPath
? 'apps_u/load_image_preview/' + appPath
? 'apps_u/download_s3_file/' + appPath
: 'job_helpers/load_image_preview'
}?${appPath ? 's3' : 'file_key'}=${encodeURIComponent(result.s3)}` +
(result.storage ? `&storage=${result.storage}` : '')}
(result.storage ? `&storage=${result.storage}` : '')}{appPath &&
result.presigned
? `&${result.presigned}`
: ''}"
/>
</div>
{:else if result?.s3?.endsWith('.pdf')}
<div class="h-96 mt-2 border">
<PdfViewer
allowFullscreen
source={`/api/w/${workspaceId}/${
source="{`/api/w/${workspaceId}/${
appPath
? 'apps_u/load_image_preview/' + appPath
? 'apps_u/download_s3_file/' + appPath
: 'job_helpers/load_image_preview'
}?${appPath ? 's3' : 'file_key'}=${encodeURIComponent(result.s3)}` +
(result.storage ? `&storage=${result.storage}` : '')}
(result.storage ? `&storage=${result.storage}` : '')}{appPath &&
result.presigned
? `&${result.presigned}`
: ''}"
/>
</div>
{/if}
@@ -46,7 +46,7 @@
let imageUrl: string | undefined = undefined
async function getS3Image(source: string | undefined, storage?: string) {
async function getS3Image(source: string | undefined, storage?: string, presigned?: string) {
if (!source) return ''
const appPathOrUser = defaultIfEmptyString(
$appPath,
@@ -63,14 +63,18 @@
params.append('force_viewer_allowed_s3_keys', JSON.stringify([forceViewerPolicies]))
}
return `/api/w/${workspace}/apps_u/download_s3_file/${appPathOrUser}?${params.toString()}`
return `/api/w/${workspace}/apps_u/download_s3_file/${appPathOrUser}?${params.toString()}${presigned ? `&${presigned}` : ''}`
}
async function loadImage() {
if (isPartialS3Object(resolvedConfig.source)) {
imageUrl = await getS3Image(resolvedConfig.source.s3, resolvedConfig.source.storage)
imageUrl = await getS3Image(
resolvedConfig.source.s3,
resolvedConfig.source.storage,
resolvedConfig.source.presigned
)
} else if (resolvedConfig.source && typeof resolvedConfig.source !== 'string') {
throw new Error('Invalid s3 object' + typeof resolvedConfig.source)
throw new Error('Invalid image object' + typeof resolvedConfig.source)
} else if (
resolvedConfig.sourceKind === 's3 (workspace storage)' ||
resolvedConfig.source?.startsWith('s3://')
@@ -83,7 +83,8 @@ export function computeS3FileInputPolicy(s3Config: any, app: App) {
const partialS3ObjectSchema = z.object({
s3: z.string(),
storage: z.string().optional()
storage: z.string().optional(),
presigned: z.string().optional()
})
export function isPartialS3Object(input: unknown): input is z.infer<typeof partialS3ObjectSchema> {
@@ -91,24 +92,21 @@ export function isPartialS3Object(input: unknown): input is z.infer<typeof parti
}
export function computeS3ImageViewerPolicy(config: RichConfigurations) {
if (config.source.type === 'static') {
if (isPartialS3Object(config.source.value)) {
return {
s3_path: config.source.value.s3,
storage: config.source.value.storage
}
} else if (
typeof config.source.value === 'string' &&
((config.sourceKind.type === 'static' &&
config.sourceKind.value === 's3 (workspace storage)') ||
config.source.value.startsWith('s3://'))
) {
return {
s3_path: config.source.value.replace('s3://', ''),
storage: undefined
}
} else {
return undefined
if (config.source.type === 'uploadS3' && isPartialS3Object(config.source.value)) {
return {
s3_path: config.source.value.s3,
storage: config.source.value.storage
}
} else if (
config.source.type === 'static' &&
typeof config.source.value === 'string' &&
((config.sourceKind.type === 'static' &&
config.sourceKind.value === 's3 (workspace storage)') ||
config.source.value.startsWith('s3://'))
) {
return {
s3_path: config.source.value.replace('s3://', ''),
storage: undefined
}
} else {
return undefined
@@ -66,7 +66,6 @@
let s3FolderPrefix: string = ''
let s3FileUploadRawMode = componentInput?.type == 'uploadS3' && !!componentInput.value?.s3
function updateSelectedS3File() {
if (s3PickerSelection) {
if (componentInput.type === 'uploadS3') {
@@ -120,7 +119,6 @@
onConnect: applyConnection
}
}
</script>
{#if !(resourceOnly && (fieldType !== 'object' || !format?.startsWith('resource-')))}
@@ -258,9 +256,6 @@
<Loader2 class="animate-spin" />
{:then Module}
<Module.default
on:focus={(e) => {
dispatch('focus')
}}
code={JSON.stringify(componentInput.value ?? { s3: '' }, null, 2)}
bind:value={componentInput.value}
/>
@@ -10,7 +10,7 @@
import Toggle from '$lib/components/Toggle.svelte'
import autosize from '$lib/autosize'
import Button from '$lib/components/common/button/Button.svelte'
import { Settings } from 'lucide-svelte'
import { Loader2, Pipette, Settings } from 'lucide-svelte'
import AgGridWizard from '$lib/components/wizards/AgGridWizard.svelte'
import TableColumnWizard from '$lib/components/wizards/TableColumnWizard.svelte'
import PlotlyWizard from '$lib/components/wizards/PlotlyWizard.svelte'
@@ -23,6 +23,8 @@
import EditableSchemaDrawer from '$lib/components/schema/EditableSchemaDrawer.svelte'
import AppPicker from '$lib/components/wizards/AppPicker.svelte'
import JsonEditor from '$lib/components/JsonEditor.svelte'
import S3FilePicker from '$lib/components/S3FilePicker.svelte'
import FileUpload from '$lib/components/common/fileUpload/FileUpload.svelte'
export let componentInput: StaticInput<any> | undefined
export let fieldType: InputType | undefined = undefined
@@ -34,6 +36,9 @@
const { onchange } = getContext<AppViewerContext>('AppViewerContext')
let s3FileUploadRawMode = false
let s3FilePicker: S3FilePicker | undefined = undefined
$: componentInput && onchange?.()
</script>
@@ -136,7 +141,72 @@
{:else if fieldType === 'color'}
<ColorInput bind:value={componentInput.value} />
{:else if fieldType === 'object' || fieldType == 'labeledselect'}
{#if format?.startsWith('resource-') && (componentInput.value == undefined || typeof componentInput.value == 'string')}
{#if format && format.split('-').length > 1 && format
.replace('resource-', '')
.replace('_', '')
.toLowerCase() == 's3object'}
<div class="flex flex-col w-full gap-1">
<Toggle
class="flex justify-end"
bind:checked={s3FileUploadRawMode}
size="xs"
options={{ left: 'Raw S3 object input' }}
/>
{#if s3FileUploadRawMode}
{#await import('$lib/components/JsonEditor.svelte')}
<Loader2 class="animate-spin" />
{:then Module}
<Module.default
code={JSON.stringify(componentInput.value ?? { s3: '' }, null, 2)}
bind:value={componentInput.value}
/>
{/await}
{:else}
<FileUpload
allowMultiple={false}
randomFileKey={true}
on:addition={(evt) => {
if (componentInput) {
componentInput.value = {
s3: evt.detail?.path ?? '',
filename: evt.detail?.filename ?? ''
}
s3FileUploadRawMode = true
}
}}
on:deletion={(evt) => {
if (componentInput) {
componentInput.value = {
s3: ''
}
}
}}
/>
{/if}
<Button
variant="border"
color="light"
size="xs"
btnClasses="mt-1"
on:click={() => {
s3FilePicker?.open?.()
}}
startIcon={{ icon: Pipette }}
>
Choose an object from the catalog
</Button>
</div>
<S3FilePicker
bind:this={s3FilePicker}
readOnlyMode={false}
on:close={(e) => {
if (componentInput?.value?.s3) {
s3FileUploadRawMode = true
}
}}
bind:selectedFileKey={componentInput.value}
/>
{:else if format?.startsWith('resource-') && (componentInput.value == undefined || typeof componentInput.value == 'string')}
<ResourcePicker
initialValue={componentInput.value?.split('$res:')?.[1] || ''}
on:change={(e) => {
@@ -17,7 +17,7 @@ duration-200 rounded-lg p-1 gap-2"
appPath ? `/apps_u/download_s3_file/${appPath}` : '/job_helpers/download_s3_file'
}?${appPath ? 's3' : 'file_key'}=${encodeURIComponent(s3object?.s3 ?? '')}${
s3object?.storage ? `&storage=${s3object.storage}` : ''
}`}
}${appPath && s3object?.presigned ? `&${s3object.presigned}` : ''}`}
download={s3object?.s3.split('/').pop() ?? 'unnamed_download.file'}
>
<Download />
@@ -572,7 +572,7 @@
{/if}
{/if}
<!-- only render options dropdown if options or searchText is not empty (needed to avoid briefly flashing empty dropdown) -->
{#if (searchText && noMatchingOptionsMsg) || options?.length > 0}
{#if allowUserOptions || (searchText && noMatchingOptionsMsg) || options?.length > 0}
<div class="options bg-surface shadow-md rounded-component">
<VirtualList
width="100%"
@@ -15,6 +15,7 @@
export let placeholder: string | undefined = undefined
export let target: string | HTMLElement | undefined = undefined
export let topPlacement = false
export let allowUserOptions: boolean | 'append' | undefined = undefined
const [floatingRef, floatingContent] = createFloatingActions({
strategy: 'absolute',
placement: topPlacement ? 'top-start' : 'bottom-start',
@@ -27,14 +28,13 @@
function moveOptionsToPortal() {
// Find ul element with class 'options' within the outerDiv
const ul = outerDiv?.querySelector('.options')
if (ul) {
// Move the ul element to the portal
portalRef?.appendChild(ul)
}
}
$: if (portalRef && outerDiv && items?.length > 0) {
$: if (portalRef && outerDiv && (allowUserOptions || items?.length > 0)) {
tick().then(() => {
moveOptionsToPortal()
})
@@ -53,6 +53,7 @@
{#if !value || Array.isArray(value)}
<div class="border rounded-md border-gray-300 shadow-sm dark:border-gray-600 !w-full">
<MultiSelect
{allowUserOptions}
outerDivClass={`!text-xs`}
ulSelectedClass="overflow-auto"
bind:outerDiv
@@ -2,7 +2,6 @@
import { WorkspaceService, type AIConfig, type AIProvider } from '$lib/gen'
import { setCopilotInfo, workspaceStore } from '$lib/stores'
import { sendUserToast } from '$lib/toast'
import MultiSelect from 'svelte-multiselect'
import { AI_DEFAULT_MODELS } from '../copilot/lib'
import TestAiKey from '../copilot/TestAIKey.svelte'
import Description from '../Description.svelte'
@@ -11,6 +10,7 @@
import Toggle from '../Toggle.svelte'
import ArgEnum from '../ArgEnum.svelte'
import Button from '../common/button/Button.svelte'
import MultiSelectWrapper from '../multiselect/MultiSelectWrapper.svelte'
const aiProviderLabels: [AIProvider, string][] = [
['openai', 'OpenAI'],
@@ -122,6 +122,22 @@
aiProviders = Object.fromEntries(
Object.entries(aiProviders).filter(([key]) => key !== provider)
)
if (defaultModel) {
const currentDefaultModel = Object.values(aiProviders).find(
(p) => defaultModel && p.models.includes(defaultModel)
)
if (!currentDefaultModel) {
defaultModel = undefined
}
}
if (codeCompletionModel) {
const currentCodeCompletionModel = Object.values(aiProviders).find(
(p) => codeCompletionModel && p.models.includes(codeCompletionModel)
)
if (!currentCodeCompletionModel) {
codeCompletionModel = undefined
}
}
}
}}
/>
@@ -139,8 +155,8 @@
bind:value={aiProviders[provider].resource_path}
on:change={() => {
if (
aiProviders[provider].resource_path &&
aiProviders[provider].models.length === 0 &&
aiProviders[provider]?.resource_path &&
aiProviders[provider]?.models.length === 0 &&
AI_DEFAULT_MODELS[provider].length > 0
) {
aiProviders[provider].models = AI_DEFAULT_MODELS[provider].slice(0, 1)
@@ -158,11 +174,11 @@
<Label label="Enabled models">
<!-- this can be removed once the parent component moves to runes -->
<!-- svelte-ignore binding_property_non_reactive -->
<MultiSelect
options={AI_DEFAULT_MODELS[provider]}
ulOptionsClass={'!bg-surface-secondary'}
<MultiSelectWrapper
items={AI_DEFAULT_MODELS[provider]}
bind:value={aiProviders[provider].models}
placeholder="Select models"
allowUserOptions="append"
bind:selected={aiProviders[provider].models}
/>
</Label>
</div>
@@ -177,16 +193,18 @@
<p class="font-semibold">Settings</p>
<div class="flex flex-col gap-4">
<Label label="Default chat model">
<ArgEnum
enum_={availableAIModels}
bind:value={defaultModel}
disabled={false}
autofocus={false}
defaultValue={undefined}
valid={true}
create={false}
required={false}
/>
{#key Object.keys(aiProviders).length}
<ArgEnum
enum_={availableAIModels}
bind:value={defaultModel}
disabled={false}
autofocus={false}
defaultValue={undefined}
valid={true}
create={false}
required={false}
/>
{/key}
</Label>
<div class="flex flex-col gap-2">
@@ -0,0 +1,252 @@
<script lang="ts">
import { enterpriseLicense, workspaceStore } from '$lib/stores'
import { emptyString, sendUserToast } from '$lib/utils'
import { Plus, X } from 'lucide-svelte'
import Alert from '../common/alert/Alert.svelte'
import Button from '../common/button/Button.svelte'
import Tab from '../common/tabs/Tab.svelte'
import Tabs from '../common/tabs/Tabs.svelte'
import Description from '../Description.svelte'
import ResourcePicker from '../ResourcePicker.svelte'
import Toggle from '../Toggle.svelte'
import Tooltip from '../Tooltip.svelte'
import { convertFrontendToBackendSetting, type S3ResourceSettings } from '$lib/workspace_settings'
import { WorkspaceService } from '$lib/gen'
import S3FilePicker from '../S3FilePicker.svelte'
import Portal from '../Portal.svelte'
import { fade } from 'svelte/transition'
let { s3ResourceSettings = $bindable() }: { s3ResourceSettings: S3ResourceSettings } = $props()
let s3FileViewer: S3FilePicker | undefined = $state()
async function editWindmillLFSSettings(): Promise<void> {
const large_file_storage = convertFrontendToBackendSetting(s3ResourceSettings)
await WorkspaceService.editLargeFileStorageConfig({
workspace: $workspaceStore!,
requestBody: {
large_file_storage: large_file_storage
}
})
console.log('Large file storage settings changed', large_file_storage)
sendUserToast(`Large file storage settings changed`)
}
</script>
<Portal name="workspace-settings">
<S3FilePicker bind:this={s3FileViewer} readOnlyMode={false} fromWorkspaceSettings={true} />
</Portal>
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">
<div class="text-primary text-lg font-semibold">Workspace Object Storage (S3/Azure Blob)</div>
<Description
link="https://www.windmill.dev/docs/core_concepts/object_storage_in_windmill#workspace-object-storage"
>
Connect your Windmill workspace to your S3 bucket or your Azure Blob storage to enable users
to read and write from S3 without having to have access to the credentials.
</Description>
</div>
</div>
{#if !$enterpriseLicense}
<Alert type="info" title="S3 storage is limited to 20 files in Windmill CE">
Windmill S3 bucket browser will not work for buckets containing more than 20 files and uploads
are limited to files {'<'} 50MB. Consider upgrading to Windmill EE to use this feature with large
buckets.
</Alert>
{:else}
<Alert type="info" title="Logs storage is set at the instance level">
This setting is only for storage of large files allowing to upload files directly to object
storage using S3Object and use the wmill sdk to read and write large files backed by an object
storage. Large-scale log management and distributed dependency caching is under <a
href="https://www.windmill.dev/docs/core_concepts/object_storage_in_windmill#instance-object-storage"
class="text-blue-500">Instance object storage</a
>, set by the superadmins in the instance settings UI.
</Alert>
{/if}
{#if s3ResourceSettings}
<div class="mt-5">
<div class="w-full">
<!-- this can be removed once parent moves to runes -->
<!-- svelte-ignore binding_property_non_reactive -->
<Tabs bind:selected={s3ResourceSettings.resourceType}>
<Tab exact size="xs" value="s3">S3</Tab>
<Tab size="xs" value="azure_blob">Azure Blob</Tab>
<Tab exact size="xs" value="s3_aws_oidc">AWS OIDC</Tab>
<Tab size="xs" value="azure_workload_identity">Azure Workload Identity</Tab>
</Tabs>
</div>
<div class="w-full flex gap-1 mt-4">
<!-- this can be removed once parent moves to runes -->
<!-- svelte-ignore binding_property_non_reactive -->
<ResourcePicker
resourceType={s3ResourceSettings.resourceType}
bind:value={s3ResourceSettings.resourcePath}
/>
<Button
size="sm"
variant="contained"
color="dark"
disabled={emptyString(s3ResourceSettings.resourcePath)}
on:click={async () => {
if ($workspaceStore) {
s3FileViewer?.open?.(undefined)
}
}}>Browse content (save first)</Button
>
</div>
</div>
{#if s3ResourceSettings.resourceType == 's3'}
<div class="flex flex-col mt-5 mb-1 gap-1">
<!-- this can be removed once parent moves to runes -->
<!-- svelte-ignore binding_property_non_reactive -->
<Toggle
disabled={emptyString(s3ResourceSettings.resourcePath)}
bind:checked={s3ResourceSettings.publicResource}
options={{
right: 'S3 resource details and content can be accessed by all users of this workspace',
rightTooltip:
'If set, all users of this workspace will have access the to entire content of the S3 bucket, as well as the resource details and the "open preview" button. This effectively by-pass the permissions set on the resource and makes it public to everyone.'
}}
/>
{#if s3ResourceSettings.publicResource === true}
<div class="pt-2"></div>
<Alert type="warning" title="S3 bucket content and resource details are shared">
S3 resource public access is ON, which means that the entire content of the S3 bucket will
be accessible to all the users of this workspace regardless of whether they have access
the resource or not. Similarly, certain Windmill SDK endpoints can be used in scripts to
access the resource details, including public and private keys.
</Alert>
{/if}
</div>
{:else}
<div class="flex flex-col mt-5 mb-1 gap-1">
<!-- this can be removed once parent moves to runes -->
<!-- svelte-ignore binding_property_non_reactive -->
<Toggle
disabled={emptyString(s3ResourceSettings.resourcePath)}
bind:checked={s3ResourceSettings.publicResource}
options={{
right: 'object storage content can be accessed by all users of this workspace',
rightTooltip:
'If set, all users of this workspace will have access the to entire content of the object storage.'
}}
/>
{#if s3ResourceSettings.publicResource === true}
<div class="pt-2"></div>
<Alert type="warning" title="object content">
object public access is ON, which means that the entire content of the object store will
be accessible to all the users of this workspace regardless of whether they have access
the resource or not.
</Alert>
{/if}
</div>
{/if}
<div class="mt-6">
<div class="flex mt-2 flex-col gap-y-4 max-w-3xl">
{#each s3ResourceSettings.secondaryStorage ?? [] as _, idx}
<div class="flex gap-1 items-center">
<input
class="max-w-[200px]"
type="text"
bind:value={
() => s3ResourceSettings.secondaryStorage?.[idx]?.[0] || '',
(v) => {
if (s3ResourceSettings.secondaryStorage?.[idx]) {
s3ResourceSettings.secondaryStorage[idx][0] = v
}
}
}
placeholder="Storage name"
/>
<select
class="max-w-[125px]"
bind:value={
() => s3ResourceSettings.secondaryStorage?.[idx]?.[1].resourceType || 's3',
(v) => {
if (s3ResourceSettings.secondaryStorage?.[idx]) {
s3ResourceSettings.secondaryStorage[idx][1].resourceType = v
}
}
}
>
<option value="s3">S3</option>
<option value="azure_blob">Azure Blob</option>
<option value="s3_aws_oidc">AWS OIDC</option>
<option value="azure_workload_identity">Azure Workload Identity</option>
</select>
<!-- this can be removed once parent moves to runes -->
<!-- svelte-ignore binding_property_non_reactive -->
<ResourcePicker
resourceType={s3ResourceSettings.secondaryStorage?.[idx]?.[1].resourceType || 's3'}
bind:value={
() => s3ResourceSettings.secondaryStorage?.[idx]?.[1].resourcePath || '',
(v) => {
if (s3ResourceSettings.secondaryStorage?.[idx]) {
s3ResourceSettings.secondaryStorage[idx][1].resourcePath = v
}
}
}
/>
<Button
size="sm"
variant="contained"
color="dark"
disabled={emptyString(s3ResourceSettings.secondaryStorage?.[idx]?.[1].resourcePath)}
on:click={async () => {
if ($workspaceStore) {
s3FileViewer?.open?.({
s3: '',
storage: s3ResourceSettings.secondaryStorage?.[idx]?.[0] || ''
})
}
}}>Browse content (save first)</Button
>
<button
transition:fade|local={{ duration: 100 }}
class="rounded-full p-1 bg-surface-secondary duration-200 hover:bg-surface-hover ml-2"
aria-label="Clear"
onclick={() => {
if (s3ResourceSettings.secondaryStorage) {
s3ResourceSettings.secondaryStorage.splice(idx, 1)
s3ResourceSettings.secondaryStorage = [...s3ResourceSettings.secondaryStorage]
}
}}
>
<X size={14} />
</button>
</div>
{/each}
<div class="flex gap-1">
<Button
size="xs"
variant="border"
on:click={() => {
if (s3ResourceSettings.secondaryStorage === undefined) {
s3ResourceSettings.secondaryStorage = []
}
s3ResourceSettings.secondaryStorage.push([
`storage_${s3ResourceSettings.secondaryStorage.length + 1}`,
{ resourcePath: '', resourceType: 's3', publicResource: false }
])
s3ResourceSettings.secondaryStorage = s3ResourceSettings.secondaryStorage
}}><Plus size={14} />Add secondary storage</Button
>
<Tooltip>
Secondary storage is a feature that allows you to read and write from storage that isn't
your main storage by specifying it in the s3 object as "secondary_storage" with the name
of it
</Tooltip>
</div>
</div>
</div>
<div class="flex mt-5 mb-5 gap-1">
<Button
on:click={() => {
editWindmillLFSSettings()
console.log('Saving S3 settings', s3ResourceSettings)
}}>Save storage settings</Button
>
</div>
{/if}
@@ -10,7 +10,6 @@
import PageHeader from '$lib/components/PageHeader.svelte'
import ResourcePicker from '$lib/components/ResourcePicker.svelte'
import ScriptPicker from '$lib/components/ScriptPicker.svelte'
import S3FilePicker from '$lib/components/S3FilePicker.svelte'
import Tooltip from '$lib/components/Tooltip.svelte'
import WorkspaceUserSettings from '$lib/components/settings/WorkspaceUserSettings.svelte'
@@ -46,7 +45,6 @@
import PremiumInfo from '$lib/components/settings/PremiumInfo.svelte'
import Toggle from '$lib/components/Toggle.svelte'
import Portal from '$lib/components/Portal.svelte'
import { fade } from 'svelte/transition'
import ChangeWorkspaceName from '$lib/components/settings/ChangeWorkspaceName.svelte'
@@ -54,14 +52,14 @@
import ChangeWorkspaceColor from '$lib/components/settings/ChangeWorkspaceColor.svelte'
import {
convertBackendSettingsToFrontendSettings,
convertFrontendToBackendSetting,
type S3ResourceSettings
} from '$lib/workspace_settings'
import { base } from '$lib/base'
import { hubPaths } from '$lib/hub'
import Description from '$lib/components/Description.svelte'
import ConnectionSection from '$lib/components/ConnectionSection.svelte'
import AiSettings from '$lib/components/workspaceSettings/AISettings.svelte'
import AISettings from '$lib/components/workspaceSettings/AISettings.svelte'
import StorageSettings from '$lib/components/workspaceSettings/StorageSettings.svelte'
type GitSyncTypeMap = {
scripts: boolean
@@ -89,8 +87,6 @@
| 'user'
| 'group'
let s3FileViewer: S3FilePicker
let slackInitialPath: string
let slackScriptPath: string
let teamsInitialPath: string
@@ -212,18 +208,6 @@
}
}
async function editWindmillLFSSettings(): Promise<void> {
const large_file_storage = convertFrontendToBackendSetting(s3ResourceSettings)
await WorkspaceService.editLargeFileStorageConfig({
workspace: $workspaceStore!,
requestBody: {
large_file_storage: large_file_storage
}
})
console.log('Large file storage settings changed', large_file_storage)
sendUserToast(`Large file storage settings changed`)
}
async function editWindmillGitSyncSettings(): Promise<void> {
let alreadySeenResource: string[] = []
let repositories = gitSyncSettings.repositories.map((elmt) => {
@@ -629,10 +613,6 @@
$: updateFromSearchTab($page.url.searchParams.get('tab'))
</script>
<Portal name="workspace-settings">
<S3FilePicker bind:this={s3FileViewer} readOnlyMode={false} fromWorkspaceSettings={true} />
</Portal>
<CenteredPage>
{#if $userStore?.is_admin || $superadmin}
<PageHeader title="Workspace settings: {$workspaceStore}"
@@ -1009,196 +989,14 @@
</div>
</div>
{:else if tab == 'ai'}
<AiSettings
{aiProviders}
{codeCompletionModel}
{defaultModel}
{usingOpenaiClientCredentialsOauth}
<AISettings
bind:aiProviders
bind:codeCompletionModel
bind:defaultModel
bind:usingOpenaiClientCredentialsOauth
/>
{:else if tab == 'windmill_lfs'}
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">
<div class="text-primary text-lg font-semibold"
>Workspace Object Storage (S3/Azure Blob)</div
>
<Description
link="https://www.windmill.dev/docs/core_concepts/object_storage_in_windmill#workspace-object-storage"
>
Connect your Windmill workspace to your S3 bucket or your Azure Blob storage to enable
users to read and write from S3 without having to have access to the credentials.
</Description>
</div>
</div>
{#if !$enterpriseLicense}
<Alert type="info" title="S3 storage is limited to 20 files in Windmill CE">
Windmill S3 bucket browser will not work for buckets containing more than 20 files and
uploads are limited to files {'<'} 50MB. Consider upgrading to Windmill EE to use this feature
with large buckets.
</Alert>
{:else}
<Alert type="info" title="Logs storage is set at the instance level">
This setting is only for storage of large files allowing to upload files directly to
object storage using S3Object and use the wmill sdk to read and write large files backed
by an object storage. Large-scale log management and distributed dependency caching is
under <a
href="https://www.windmill.dev/docs/core_concepts/object_storage_in_windmill#instance-object-storage"
class="text-blue-500">Instance object storage</a
>, set by the superadmins in the instance settings UI.
</Alert>
{/if}
{#if s3ResourceSettings}
<div class="mt-5">
<div class="w-full">
<Tabs bind:selected={s3ResourceSettings.resourceType}>
<Tab exact size="xs" value="s3">S3</Tab>
<Tab size="xs" value="azure_blob">Azure Blob</Tab>
<Tab exact size="xs" value="s3_aws_oidc">AWS OIDC</Tab>
<Tab size="xs" value="azure_workload_identity">Azure Workload Identity</Tab>
</Tabs>
</div>
<div class="w-full flex gap-1 mt-4">
<ResourcePicker
resourceType={s3ResourceSettings.resourceType}
bind:value={s3ResourceSettings.resourcePath}
/>
<Button
size="sm"
variant="contained"
color="dark"
disabled={emptyString(s3ResourceSettings.resourcePath)}
on:click={async () => {
if ($workspaceStore) {
s3FileViewer?.open?.(undefined)
}
}}>Browse content (save first)</Button
>
</div>
</div>
{#if s3ResourceSettings.resourceType == 's3'}
<div class="flex flex-col mt-5 mb-1 gap-1">
<Toggle
disabled={emptyString(s3ResourceSettings.resourcePath)}
bind:checked={s3ResourceSettings.publicResource}
options={{
right:
'S3 resource details and content can be accessed by all users of this workspace',
rightTooltip:
'If set, all users of this workspace will have access the to entire content of the S3 bucket, as well as the resource details and the "open preview" button. This effectively by-pass the permissions set on the resource and makes it public to everyone.'
}}
/>
{#if s3ResourceSettings.publicResource === true}
<div class="pt-2"></div>
<Alert type="warning" title="S3 bucket content and resource details are shared">
S3 resource public access is ON, which means that the entire content of the S3
bucket will be accessible to all the users of this workspace regardless of whether
they have access the resource or not. Similarly, certain Windmill SDK endpoints can
be used in scripts to access the resource details, including public and private
keys.
</Alert>
{/if}
</div>
{:else}
<div class="flex flex-col mt-5 mb-1 gap-1">
<Toggle
disabled={emptyString(s3ResourceSettings.resourcePath)}
bind:checked={s3ResourceSettings.publicResource}
options={{
right: 'object storage content can be accessed by all users of this workspace',
rightTooltip:
'If set, all users of this workspace will have access the to entire content of the object storage.'
}}
/>
{#if s3ResourceSettings.publicResource === true}
<div class="pt-2"></div>
<Alert type="warning" title="object content">
object public access is ON, which means that the entire content of the object store
will be accessible to all the users of this workspace regardless of whether they
have access the resource or not.
</Alert>
{/if}
</div>
{/if}
<div class="mt-6">
<div class="flex mt-2 flex-col gap-y-4 max-w-3xl">
{#each s3ResourceSettings.secondaryStorage ?? [] as secondaryStorage, idx}
<div class="flex gap-1 items-center">
<input
class="max-w-[200px]"
type="text"
bind:value={secondaryStorage[0]}
placeholder="Storage name"
/>
<select class="max-w-[125px]" bind:value={secondaryStorage[1].resourceType}>
<option value="s3">S3</option>
<option value="azure_blob">Azure Blob</option>
<option value="s3_aws_oidc">AWS OIDC</option>
<option value="azure_workload_identity">Azure Workload Identity</option>
</select>
<ResourcePicker
resourceType={secondaryStorage[1].resourceType}
bind:value={secondaryStorage[1].resourcePath}
/>
<Button
size="sm"
variant="contained"
color="dark"
disabled={emptyString(secondaryStorage[1].resourcePath)}
on:click={async () => {
if ($workspaceStore) {
s3FileViewer?.open?.({ s3: '', storage: secondaryStorage[0] })
}
}}>Browse content (save first)</Button
>
<button
transition:fade|local={{ duration: 100 }}
class="rounded-full p-1 bg-surface-secondary duration-200 hover:bg-surface-hover ml-2"
aria-label="Clear"
on:click={() => {
if (s3ResourceSettings.secondaryStorage) {
s3ResourceSettings.secondaryStorage.splice(idx, 1)
s3ResourceSettings.secondaryStorage = [...s3ResourceSettings.secondaryStorage]
}
}}
>
<X size={14} />
</button>
</div>
{/each}
<div class="flex gap-1">
<Button
size="xs"
variant="border"
on:click={() => {
if (s3ResourceSettings.secondaryStorage === undefined) {
s3ResourceSettings.secondaryStorage = []
}
s3ResourceSettings.secondaryStorage.push([
`storage_${s3ResourceSettings.secondaryStorage.length + 1}`,
{ resourcePath: '', resourceType: 's3', publicResource: false }
])
s3ResourceSettings.secondaryStorage = s3ResourceSettings.secondaryStorage
}}><Plus size={14} />Add secondary storage</Button
>
<Tooltip>
Secondary storage is a feature that allows you to read and write from storage that
isn't your main storage by specifying it in the s3 object as "secondary_storage"
with the name of it
</Tooltip>
</div>
</div>
</div>
<div class="flex mt-5 mb-5 gap-1">
<Button
color="blue"
disabled={emptyString(s3ResourceSettings.resourcePath)}
on:click={() => {
editWindmillLFSSettings()
console.log('Saving S3 settings', s3ResourceSettings)
}}>Save storage settings</Button
>
</div>
{/if}
<StorageSettings bind:s3ResourceSettings />
{:else if tab == 'git_sync'}
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">
+14 -2
View File
@@ -586,9 +586,12 @@ class Windmill:
raise Exception("Could not write file to S3") from e
return S3Object(s3=response["file_key"])
def sign_s3_objects(self, s3_objects: list[S3Object]) -> list[str]:
def sign_s3_objects(self, s3_objects: list[S3Object]) -> list[S3Object]:
return self.post(f"/w/{self.workspace}/apps/sign_s3_objects", json={"s3_objects": s3_objects}).json()
def sign_s3_object(self, s3_object: S3Object) -> S3Object:
return self.post(f"/w/{self.workspace}/apps/sign_s3_objects", json={"s3_objects": [s3_object]}).json()[0]
def __boto3_connection_settings(self, s3_resource) -> Boto3ConnectionSettings:
endpoint_url_prefix = "https://" if s3_resource["useSSL"] else "http://"
return Boto3ConnectionSettings(
@@ -978,7 +981,7 @@ def write_s3_file(
@init_global_client
def sign_s3_objects(s3_objects: list[S3Object]) -> list[str]:
def sign_s3_objects(s3_objects: list[S3Object]) -> list[S3Object]:
"""
Sign S3 objects to be used by anonymous users in public apps
Returns a list of signed s3 tokens
@@ -986,6 +989,15 @@ def sign_s3_objects(s3_objects: list[S3Object]) -> list[str]:
return _client.sign_s3_objects(s3_objects)
@init_global_client
def sign_s3_object(s3_object: S3Object) -> S3Object:
"""
Sign S3 object to be used by anonymous users in public apps
Returns a signed s3 object
"""
return _client.sign_s3_object(s3_object)
@init_global_client
def whoami() -> dict:
"""
+1
View File
@@ -1,6 +1,7 @@
class S3Object(dict):
s3: str
storage: str | None
presigned: str | None
def __getattr__(self, attr):
return self[attr]
+1 -1
View File
@@ -14,5 +14,5 @@ cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/"
echo "" >> "${script_dirpath}/src/index.ts"
echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts"
echo "" >> "${script_dirpath}/src/index.ts"
echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts"
echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts"
+1 -1
View File
@@ -39,4 +39,4 @@ cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/"
echo "" >> "${script_dirpath}/src/index.ts"
echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts"
echo "" >> "${script_dirpath}/src/index.ts"
echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts"
echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts"
+14 -2
View File
@@ -774,9 +774,11 @@ export async function writeS3File(
/**
* Sign S3 objects to be used by anonymous users in public apps
* @param s3objects s3 objects to sign
* @returns signed s3 tokens
* @returns signed s3 objects
*/
export async function signS3Objects(s3objects: S3Object[]): Promise<string[]> {
export async function signS3Objects(
s3objects: S3Object[]
): Promise<S3Object[]> {
const signedKeys = await AppService.signS3Objects({
workspace: getWorkspace(),
requestBody: {
@@ -786,6 +788,16 @@ export async function signS3Objects(s3objects: S3Object[]): Promise<string[]> {
return signedKeys;
}
/**
* Sign S3 object to be used by anonymous users in public apps
* @param s3object s3 object to sign
* @returns signed s3 object
*/
export async function signS3Object(s3object: S3Object): Promise<S3Object> {
const [signedObject] = await signS3Objects([s3object]);
return signedObject;
}
/**
* Get URLs needed for resuming a flow after this step
* @param approver approver name
+1
View File
@@ -1,6 +1,7 @@
export type S3Object = {
s3: string;
storage?: string;
presigned?: string;
};
export type DenoS3LightClientSettings = {