feat(frontend): S3 resource schema (#3083)

* feat(frontend): make s3 a top level selected type + fix wand position + fix for loop double overflow

* feat(frontend): add search

* feat(frontend): fix default field

* feat(frontend): fix default build

* feat(frontend): support for s3 object

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>

frontend/src/lib/components/ObjectResourceInput.svelte

@@ -1,5 +1,6 @@
 <script lang="ts">
 	import ResourcePicker from './ResourcePicker.svelte'
+	import S3ObjectPicker from './S3ObjectPicker.svelte'
 
 	export let format: string
 	export let value: any
@@ -34,14 +35,18 @@
 </script>
 
 <div class="flex flex-row w-full flex-wrap gap-x-2 gap-y-0.5">
-	<ResourcePicker
-		{disablePortal}
-		on:change={(e) => {
-			path = e.detail
-			resourceToValue()
-		}}
-		bind:value={path}
-		resourceType={format.split('-').length > 1 ? format.substring('resource-'.length) : undefined}
-		{showSchemaExplorer}
-	/>
+	{#if format === 'resource-s3_object'}
+		<S3ObjectPicker bind:value />
+	{:else}
+		<ResourcePicker
+			{disablePortal}
+			on:change={(e) => {
+				path = e.detail
+				resourceToValue()
+			}}
+			bind:value={path}
+			resourceType={format.split('-').length > 1 ? format.substring('resource-'.length) : undefined}
+			{showSchemaExplorer}
+		/>
+	{/if}
 </div>

frontend/src/lib/components/ResourceTypePicker.svelte

@@ -4,7 +4,7 @@
 	import { ResourceService } from '$lib/gen'
 	import { workspaceStore } from '$lib/stores'
 	import IconedResourceType from './IconedResourceType.svelte'
-	import { Button } from './common'
+	import { Button, ClearableInput } from './common'
 
 	let resources: string[] = []
 
@@ -27,10 +27,16 @@
 	$: if ($workspaceStore) {
 		loadResources()
 	}
+	let search: string = ''
+
+	$: filteredResources = resources.filter((r) => r.toLowerCase().includes(search.toLowerCase()))
 </script>
 
+<div class="mb-2">
+	<ClearableInput bind:value={search} placeholder="Search resource..." />
+</div>
 <div class="grid sm:grid-cols-2 md:grid-cols-3 gap-x-2 gap-y-1 items-center mb-2">
-	{#if nonePickable}
+	{#if nonePickable && search === ''}
 		{@const isPicked = value === undefined}
 		<Button
 			size="sm"
@@ -43,7 +49,7 @@
 			None
 		</Button>
 	{/if}
-	{#each resources as r}
+	{#each filteredResources as r}
 		{@const isPicked = value === r}
 		<Button
 			size="sm"
@@ -56,4 +62,8 @@
 			<IconedResourceType name={r} after={true} width="20px" height="20px" />
 		</Button>
 	{/each}
+
+	{#if filteredResources.length === 0 && search !== ''}
+		<div class="text-tertiary text-sm">No resources found</div>
+	{/if}
 </div>

frontend/src/lib/components/S3ObjectPicker.svelte

@@ -0,0 +1,94 @@
+<script lang="ts">
+	import { Pipette } from 'lucide-svelte'
+	import { createEventDispatcher } from 'svelte'
+	// @ts-ignore
+	import SimpleEditor from './SimpleEditor.svelte'
+
+	import JsonEditor from './apps/editor/settingsPanel/inputEditor/JsonEditor.svelte'
+	import { Button } from './common'
+
+	import Toggle from './Toggle.svelte'
+
+	import S3FilePicker from './S3FilePicker.svelte'
+	import FileUpload from './common/fileUpload/FileUpload.svelte'
+
+	export let value: any
+	export let editor: SimpleEditor | undefined = undefined
+
+	const dispatch = createEventDispatcher()
+
+	let s3FilePicker: S3FilePicker
+	let s3FileUploadRawMode: false
+	let el: HTMLTextAreaElement | undefined = undefined
+	let rawValue: string | undefined = undefined
+
+	function evalValueToRaw() {
+		rawValue = JSON.stringify(value, null, 2)
+	}
+
+	evalValueToRaw()
+
+	export function focus() {
+		el?.focus()
+		if (el) {
+			el.style.height = '5px'
+			el.style.height = el.scrollHeight + 50 + 'px'
+		}
+	}
+</script>
+
+<S3FilePicker
+	bind:this={s3FilePicker}
+	bind:selectedFileKey={value}
+	on:close={() => {
+		rawValue = JSON.stringify(value, null, 2)
+		editor?.setCode(rawValue)
+	}}
+	readOnlyMode={false}
+/>
+
+<div class="flex flex-col w-full gap-1">
+	<Toggle
+		class="flex justify-end"
+		bind:checked={s3FileUploadRawMode}
+		size="xs"
+		options={{ left: 'Raw S3 object input' }}
+	/>
+	{#if s3FileUploadRawMode}
+		<JsonEditor
+			bind:editor
+			on:focus={(e) => {
+				dispatch('focus')
+			}}
+			code={JSON.stringify({ s3: '' }, null, 2)}
+			bind:value
+		/>
+		<Button
+			variant="border"
+			color="light"
+			size="xs"
+			btnClasses="mt-1"
+			on:click={() => {
+				s3FilePicker?.open?.(value)
+			}}
+			startIcon={{ icon: Pipette }}
+		>
+			Choose an object from the catalog
+		</Button>
+	{:else}
+		<FileUpload
+			allowMultiple={false}
+			randomFileKey={true}
+			on:addition={(evt) => {
+				value = {
+					s3: evt.detail?.path ?? ''
+				}
+			}}
+			on:deletion={(evt) => {
+				value = {
+					s3: ''
+				}
+			}}
+		/>
+	{/if}
+</div>

frontend/src/lib/components/SchemaModal.svelte

@@ -5,7 +5,26 @@
 	import Tabs from './common/tabs/Tabs.svelte'
 	import SchemaEditor from './SchemaEditor.svelte'
 
-	export const ARG_TYPES = ['integer', 'number', 'string', 'boolean', 'object', 'array'] as const
+	export const ARG_TYPES = [
+		'integer',
+		'number',
+		'string',
+		'boolean',
+		'object',
+		'array',
+		'S3'
+	] as const
+
+	const argTypesLabels = {
+		integer: 'Integer',
+		number: 'Number',
+		string: 'String',
+		boolean: 'Boolean',
+		object: 'Object',
+		array: 'Array',
+		S3: 'S3 Object'
+	} as const
+
 	export type ArgType = (typeof ARG_TYPES)[number]
 
 	export function schemaToModal(
@@ -123,6 +142,14 @@
 
 	let title = ''
 	$: title = editing ? `Edit ${oldArgName} argument` : 'Add an argument'
+
+	function getResourceTypesFromFormat(format: string | undefined): string[] {
+		if (format?.startsWith('resource-')) {
+			return [format.split('-')[1]]
+		}
+
+		return []
+	}
 </script>
 
 <Drawer bind:this={drawer} placement="right">
@@ -162,15 +189,25 @@
 				<div class="mb-1 font-semibold text-secondary">Type<Required required={true} /></div>
 				<div class="grid sm:grid-cols-3 md:grid-cols-4 gap-x-2 gap-y-1 items-center mb-2 w-full">
 					{#each ARG_TYPES as argType}
-						{@const isSelected = argType == property.selectedType}
+						{@const isSelected =
+							(argType === 'S3' && property.format === 'resource-s3_object') ||
+							(argType === property.selectedType && property.format !== 'resource-s3_object')}
 						<Button
 							size="sm"
 							variant="border"
 							color={isSelected ? 'blue' : 'light'}
 							btnClasses={isSelected ? '!border-2' : 'm-[1px]'}
 							on:click={() => {
-								property.selectedType = argType
-								property.format = undefined
+								const isS3 = argType == 'S3'
+
+								if (isS3) {
+									property.selectedType = 'object'
+									property.format = 'resource-s3_object'
+								} else {
+									property.format = undefined
+									property.selectedType = argType
+								}
+
 								property.contentEncoding = undefined
 								property.enum_ = undefined
 								property.pattern = undefined
@@ -189,7 +226,7 @@
 							}}
 							id={`schema-modal-type-${argType}`}
 						>
-							{argType}
+							{argTypesLabels[argType]}
 						</Button>
 					{/each}
 					<Button
@@ -208,7 +245,7 @@
 			<div>
 				<div class="flex flex-row gap-x-4 items-center">
 					<ArgInput
-						resourceTypes={[]}
+						resourceTypes={getResourceTypesFromFormat(property.format)}
 						label="Default"
 						bind:value={property.default}
 						type={property.selectedType}
@@ -254,12 +291,12 @@
 						bind:currency={property.currency}
 						bind:currencyLocale={property.currencyLocale}
 					/>
-				{:else if property.selectedType == 'object'}
+				{:else if property.selectedType == 'object' && property.format !== 'resource-s3'}
 					<Tabs selected="custom-object">
 						<Tab value="custom-object">Custom Object</Tab>
 						<Tab value="resource">Resource</Tab>
 						<svelte:fragment slot="content">
-							<div class="overflow-auto pt-2">
+							<div class="pt-2">
 								<TabContent value="custom-object">
 									<SchemaEditor bind:schema={property.schema} />
 								</TabContent>

frontend/src/lib/components/StringTypeNarrowing.svelte

@@ -3,6 +3,7 @@
 	import ResourceTypePicker from './ResourceTypePicker.svelte'
 	import Toggle from './Toggle.svelte'
 	import { Button } from './common'
+	import Alert from './common/alert/Alert.svelte'
 	import RegexGen from './copilot/RegexGen.svelte'
 
 	export let pattern: string | undefined
@@ -200,6 +201,16 @@
 			<input type="number" bind:value={minRows} />
 		</label>
 	{/if}
+{:else if kind === 'base64'}
+	<Alert
+		type="warning"
+		title="S3 Object recommended"
+		collapsible
+		size="xs"
+		documentationLink="Pattern (Regex) https://www.windmill.dev/docs/core_concepts/persistent_storage#large-data-files-s3-r2-minio"
+	>
+		For large files, we recommend using the S3 Object type instead of the base64 string type.
+	</Alert>
 {/if}
 {#if (kind == 'none' || kind == 'pattern' || kind == 'format') && !noExtra}
 	<div class="mt-1" />

frontend/src/lib/components/copilot/StepGen.svelte

@@ -127,7 +127,7 @@
 			{#if funcDesc.length === 0}
 				<Wand2
 					size={14}
-					class="absolute left-[65px] qhd:left-[75px] top-[18px] qhd:top-[20px] fill-current opacity-70"
+					class="absolute right-4 top-1/2 -translate-y-1/2 fill-current opacity-70"
 				/>
 			{/if}
 		</div>

frontend/src/lib/components/flows/content/FlowLoop.svelte

@@ -68,18 +68,19 @@
 		<div slot="header" class="grow">
 			<input bind:value={mod.summary} placeholder={'Summary'} />
 		</div>
-		<Alert
-			type="info"
-			title="For loops"
-			tooltip="For loops"
-			documentationLink="https://www.windmill.dev/docs/flows/flow_loops"
-			class="m-2"
-		>
-			Add steps inside the loop and specify an iterator expression that defines the sequence over
-			which your subsequent steps will iterate.
-		</Alert>
+
 		<Splitpanes horizontal class="!max-h-[calc(100%-48px)]">
 			<Pane size={60} minSize={20} class="p-4">
+				<Alert
+					type="info"
+					title="For loops"
+					tooltip="For loops"
+					documentationLink="https://www.windmill.dev/docs/flows/flow_loops"
+					class="mb-4"
+				>
+					Add steps inside the loop and specify an iterator expression that defines the sequence
+					over which your subsequent steps will iterate.
+				</Alert>
 				{#if mod.value.type === 'forloopflow'}
 					<div class="flex flex-row gap-8 mt-2 mb-6">
 						<div>

frontend/src/lib/components/preview/FlowPreviewStatus.svelte

@@ -8,8 +8,8 @@
 <div class="grid grid-cols-2 gap-4 mb-1 text-tertiary dark:text-gray-400">
 	<JobStatus {job} />
 	{#if job}
-		<div
-			><div class="text-primary whitespace-nowrap truncate text-sm">
+		<div>
+			<div class="text-primary whitespace-nowrap truncate text-sm">
 				<span class="font-semibold mr-1">Flow's job Id:</span>
 				<a rel="noreferrer" target="_blank" href="/run/{job?.id}?workspace={job?.workspace_id}">
 					{job?.id}