Better flow builder (#6596)

* fly animation

* output picker fly transition

* svelte 5 migrate

* createCache

* cache integrations

* fly transition for triggers add btn

* better hub scripts fetching (code style + caching + fewer states)

* loadItemsCached in WorkspaceScriptPickerQuick

* usePromise fixes

* createCache initial keys, no flicker at all

* Prettier template editor

* fix null access

* fix bad z-index issue

* Revert "Prettier template editor"

This reverts commit ff2a952656.

* type error

* ee repo ref

* Update ee-repo-ref.txt

* Fix formatting of ee-repo-ref.txt

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>

backend/ee-repo-ref.txt

@@ -1 +1 @@
-daa1c26c268c778e55756f02a459b6c7628c9267
+daa1c26c268c778e55756f02a459b6c7628c9267

frontend/src/lib/components/DropdownV2.svelte

@@ -18,6 +18,7 @@
 	import { twMerge } from 'tailwind-merge'
 	import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte'
 	import { untrack } from 'svelte'
+	import { fly } from 'svelte/transition'
 
 	interface Props {
 		aiId?: string | undefined
@@ -33,6 +34,7 @@
 		customWidth?: number | undefined
 		customMenu?: boolean
 		class?: string | undefined
+		enableFlyTransition?: boolean
 		buttonReplacement?: import('svelte').Snippet
 		menu?: import('svelte').Snippet
 	}
@@ -51,6 +53,7 @@
 		customWidth = undefined,
 		customMenu = false,
 		class: classNames = undefined,
+		enableFlyTransition = false,
 		buttonReplacement,
 		menu
 	}: Props = $props()
@@ -148,7 +151,12 @@
 </button>
 
 {#if open && !hidePopup}
-	<div use:melt={$menuEl} data-menu class="z-[6000] transition-all duration-100">
+	<div
+		use:melt={$menuEl}
+		data-menu
+		class="z-[6000] transition-all duration-100"
+		transition:fly={{ duration: enableFlyTransition ? 100 : 0, y: -16 }}
+	>
 		{#if customMenu}
 			{@render menu?.()}
 		{:else}

frontend/src/lib/components/EditableSchemaForm.svelte

@@ -186,7 +186,7 @@
 		let index = 0
 		let hasChanged = false
 		for (let k of properties) {
-			if (schema.properties[k].type === 'object' && schema.properties[k].properties) {
+			if (schema.properties[k]?.type === 'object' && schema.properties[k].properties) {
 				hasChanged = hasChanged || alignOrderWithProperties(schema.properties[k])
 			}
 			if (!norder.includes(k)) {
@@ -220,7 +220,7 @@
 
 	function computeSelected(property: any) {
 		if (!opened) return ''
-		if (property.type !== 'object') return property.type
+		if (property?.type !== 'object') return property?.type
 		if (property.format === 'resource-s3_object') return 'S3'
 		if (property.format?.startsWith('dynselect-')) return 'dynselect'
 		if (property.format?.startsWith('dynmultiselect-')) return 'dynmultiselect'
@@ -344,7 +344,7 @@
 		Object.entries(schema?.properties ?? {})
 			.filter(([_, property]) => {
 				const props = property as any
-				return props.type === 'object' && DynamicInput.isDynInputFormat(props.format)
+				return props?.type === 'object' && DynamicInput.isDynInputFormat(props.format)
 			})
 			.map(([fieldName, _]) => fieldName.replace(/\s+/g, '_'))
 	)
@@ -652,7 +652,7 @@
 													{#if typeof args == 'object' && schema?.properties[argName]}
 														<PropertyEditor
 															bind:description={schema.properties[argName].description}
-															type={schema.properties[argName].type}
+															type={schema.properties[argName]?.type}
 															bind:oneOf={schema.properties[argName].oneOf}
 															bind:pattern={schema.properties[argName].pattern}
 															bind:enum_={schema.properties[argName].enum}
@@ -795,7 +795,7 @@
 																	{itemPicker}
 																	bind:nullable={schema.properties[argName].nullable}
 																	bind:disabled={schema.properties[argName].disabled}
-																	type={schema.properties[argName].type}
+																	type={schema.properties[argName]?.type}
 																	bind:oneOf={schema.properties[argName].oneOf}
 																	bind:format={schema.properties[argName].format}
 																	contentEncoding={schema.properties[argName].contentEncoding}

frontend/src/lib/components/TemplateEditor.svelte

@@ -420,8 +420,8 @@
 
 	let initialized = false
 
-	let jsLoader: number | undefined = undefined
-	let timeoutModel: number | undefined = undefined
+	let jsLoader: NodeJS.Timeout | undefined = undefined
+	let timeoutModel: NodeJS.Timeout | undefined = undefined
 	async function loadMonaco() {
 		setMonacoJsonOptions()
 		await initializeVscode('templateEditor')
@@ -598,7 +598,7 @@
 	}
 
 	let mounted = false
-	let loadTimeout: number | undefined = undefined
+	let loadTimeout: NodeJS.Timeout | undefined = undefined
 	onMount(async () => {
 		try {
 			if (BROWSER) {

frontend/src/lib/components/common/popup/PopupV2.svelte

@@ -2,6 +2,7 @@
 	import Portal from '$lib/components/Portal.svelte'
 	import { clickOutside } from '$lib/utils'
 	import { createFloatingActions, type ComputeConfig } from 'svelte-floating-ui'
+	import { fly } from 'svelte/transition'
 
 	interface Props {
 		floatingConfig?: ComputeConfig
@@ -58,6 +59,7 @@
 			class="border rounded-lg shadow-lg bg-surface z5000"
 			style="position:absolute"
 			use:floatingContent
+			transition:fly={{ duration: 100, y: -16 }}
 		>
 			<!-- svelte-ignore event_directive_deprecated -->
 			<div

frontend/src/lib/components/copilot/FlowCopilotInputsModal.svelte

@@ -2,6 +2,7 @@
 	import { createEventDispatcher } from 'svelte'
 	import { Button, Badge } from '../common'
 	import Modal from '../common/modal/Modal.svelte'
+	import Portal from '../Portal.svelte'
 
 	export let open = false
 
@@ -10,30 +11,32 @@
 	const dispatch = createEventDispatcher()
 </script>
 
-<Modal
-	bind:open
-	on:confirmed={() => {
-		open = false
-		dispatch('confirmed')
-	}}
-	on:canceled
-	title="Windmill AI wants to add the following inputs to the flow:"
->
-	<ul class=" list-disc pl-5">
-		{#each inputs as input}
-			<li>{input}</li>
-		{/each}
-	</ul>
-
-	<Button
-		slot="actions"
-		on:click={() => {
+<Portal>
+	<Modal
+		bind:open
+		on:confirmed={() => {
 			open = false
 			dispatch('confirmed')
 		}}
-		color="light"
-		size="sm"
+		on:canceled
+		title="Windmill AI wants to add the following inputs to the flow:"
 	>
-		<span class="inline-flex gap-2">Add <Badge color="dark-green">Enter</Badge></span>
-	</Button>
-</Modal>
+		<ul class=" list-disc pl-5">
+			{#each inputs as input}
+				<li>{input}</li>
+			{/each}
+		</ul>
+
+		<Button
+			slot="actions"
+			on:click={() => {
+				open = false
+				dispatch('confirmed')
+			}}
+			color="light"
+			size="sm"
+		>
+			<span class="inline-flex gap-2">Add <Badge color="dark-green">Enter</Badge></span>
+		</Button>
+	</Modal>
+</Portal>

frontend/src/lib/components/flows/map/InsertModuleInner.svelte

@@ -6,8 +6,6 @@
 	import ToggleHubWorkspaceQuick from '$lib/components/ToggleHubWorkspaceQuick.svelte'
 	import TopLevelNode from '../pickers/TopLevelNode.svelte'
 
-	// import type { Writable } from 'svelte/store'
-
 	const dispatch = createEventDispatcher()
 	interface Props {
 		stop?: boolean
@@ -32,26 +30,14 @@
 		$state(kind)
 	let preFilter: 'all' | 'workspace' | 'hub' = $state('all')
 	let loading = $state(false)
-	let small = $state(false)
+	let small = $derived(kind === 'preprocessor' || kind === 'failure')
 
 	let width = $state(0)
 	let height = $state(0)
 
 	let displayPath = $derived(width > 650 || height > 400)
-
-	$effect(() => {
-		small = kind === 'preprocessor' || kind === 'failure'
-	})
 </script>
 
-<!-- <Menu transitionDuration={0} pointerDown bind:show={open} noMinW {placement} let:close> -->
-
-<!-- {floatingConfig}
-floatingClasses="mt-2"
-containerClasses="border rounded-lg shadow-lg  bg-surface"
-noTransition
-shouldUsePortal={true} -->
-
 <div
 	id="flow-editor-insert-module"
 	class="flex flex-col h-[400px] {small

frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte

@@ -1,11 +1,34 @@
+<script module lang="ts">
+	let listHubIntegrationsCached = createCache(
+		(params: { kind: HubScriptKind & string }) => IntegrationService.listHubIntegrations(params),
+		{ initial: { kind: 'script' } }
+	)
+	let listHubScriptsCached = createCache(
+		async ({
+			filter,
+			kind,
+			appFilter
+		}: {
+			filter: string
+			kind: HubScriptKind & string
+			appFilter: string | undefined
+		}) =>
+			filter.length > 0
+				? await ScriptService.queryHubScripts({ text: filter, limit: 40, kind })
+				: ((await ScriptService.getTopHubScripts({ limit: 40, kind, app: appFilter })).asks ?? []),
+		{ initial: { filter: '', kind: 'script', appFilter: undefined } }
+	)
+</script>
+
 <script lang="ts">
 	import { createEventDispatcher, untrack } from 'svelte'
 	import { Skeleton } from '$lib/components/common'
-	import { classNames } from '$lib/utils'
+	import { classNames, createCache } from '$lib/utils'
 	import { APP_TO_ICON_COMPONENT } from '$lib/components/icons'
 	import { IntegrationService, ScriptService, type HubScriptKind } from '$lib/gen'
 	import { Circle } from 'lucide-svelte'
 	import Popover from '$lib/components/Popover.svelte'
+	import { usePromise } from '$lib/svelte5Utils.svelte'
 
 	let hubNotAvailable = $state(false)
 
@@ -45,11 +68,7 @@
 	async function getAllApps(filterKind: typeof kind) {
 		try {
 			hubNotAvailable = false
-			allApps = (
-				await IntegrationService.listHubIntegrations({
-					kind: filterKind
-				})
-			).map((x) => x.name)
+			allApps = (await listHubIntegrationsCached({ kind: filterKind })).map((x) => x.name)
 			apps = allApps
 		} catch (err) {
 			console.error('Hub is not available')
@@ -59,34 +78,21 @@
 		}
 	}
 
-	let startTs = 0
-	async function applyFilter(
-		filter: string,
-		filterKind: typeof kind,
-		appFilter: string | undefined
-	) {
-		try {
-			loading = true
-			hubNotAvailable = false
-			const ts = Date.now()
-			startTs = ts
-			await new Promise((resolved, rejected) => setTimeout(resolved, 200))
-			if (ts < startTs) return
-			const scripts =
-				filter.length > 0
-					? await ScriptService.queryHubScripts({
-							text: `${filter}`,
-							limit: 40,
-							kind: filterKind
-						})
-					: ((
-							await ScriptService.getTopHubScripts({
-								limit: 40,
-								kind: filterKind,
-								app: appFilter
-							})
-						).asks ?? [])
-
+	let hubScriptsFilteredPromise = usePromise(
+		() => listHubScriptsCached({ appFilter, filter, kind }),
+		{ loadInit: false }
+	)
+	$effect(() => {
+		;[filter, kind, appFilter]
+		hubScriptsFilteredPromise.refresh()
+	})
+	$effect(() => {
+		// TODO: these should be derived ...
+		loading = hubScriptsFilteredPromise.status === 'loading'
+		hubNotAvailable = !!hubScriptsFilteredPromise.error
+		const scripts = hubScriptsFilteredPromise.value
+		untrack(() => {
+			if (!scripts) return
 			const mappedItems = scripts.map(
 				(x: {
 					summary: string
@@ -108,18 +114,8 @@
 			}
 
 			items = appFilter ? mappedItems.filter((x) => x.app === appFilter) : mappedItems
-
-			if (ts === startTs) {
-				loading = false
-			}
-
-			hubNotAvailable = false
-		} catch (err) {
-			hubNotAvailable = true
-			console.error('Hub not available')
-			loading = false
-		}
-	}
+		})
+	})
 
 	function onKeyDown(e: KeyboardEvent) {
 		if (
@@ -134,12 +130,6 @@
 			dispatch('pickScript', item)
 		}
 	}
-	$effect(() => {
-		;[filter, kind, appFilter]
-		untrack(() => {
-			applyFilter(filter, kind, appFilter)
-		})
-	})
 	$effect(() => {
 		kind
 		untrack(() => {

frontend/src/lib/components/flows/pickers/WorkspaceScriptPickerQuick.svelte

@@ -1,13 +1,32 @@
+<script module lang="ts">
+	let loadItemsCached = createCache(
+		({
+			workspace,
+			kind,
+			isTemplate
+		}: {
+			workspace: string
+			kind?: string
+			isTemplate?: boolean
+		}) =>
+			kind == 'flow'
+				? FlowService.listFlows({ workspace })
+				: ScriptService.listScripts({ workspace, kinds: kind, isTemplate }),
+		{ initial: { workspace: 'data-pipeline-demo', kind: 'script' } }
+	)
+</script>
+
 <script lang="ts">
 	import { workspaceStore } from '$lib/stores'
 	import { createEventDispatcher, untrack } from 'svelte'
 	import { FlowService, ScriptService } from '$lib/gen'
 	import SearchItems from '$lib/components/SearchItems.svelte'
 	import { Skeleton } from '$lib/components/common'
-	import { emptyString } from '$lib/utils'
+	import { createCache, emptyString } from '$lib/utils'
 	import { Code2 } from 'lucide-svelte'
 	import BarsStaggered from '$lib/components/icons/BarsStaggered.svelte'
 	import Popover from '$lib/components/Popover.svelte'
+	import { usePromise } from '$lib/svelte5Utils.svelte'
 
 	type Item = {
 		path: string
@@ -16,21 +35,13 @@
 		hash?: string
 	}
 
-	let items: Item[] | undefined = $state(undefined)
+	let items = usePromise(
+		async () => await loadItemsCached({ workspace: $workspaceStore!, kind, isTemplate }),
+		{ loadInit: false }
+	)
 
 	let filteredItems: (Item & { marked?: string })[] | undefined = $state(undefined)
 
-	async function loadItems(): Promise<void> {
-		items =
-			kind == 'flow'
-				? await FlowService.listFlows({ workspace: $workspaceStore! })
-				: await ScriptService.listScripts({
-						workspace: $workspaceStore!,
-						kinds: kind,
-						isTemplate
-					})
-	}
-
 	interface Props {
 		kind?: 'script' | 'trigger' | 'approval' | 'failure' | 'flow' | 'preprocessor'
 		isTemplate?: boolean | undefined
@@ -76,7 +87,7 @@
 		}
 	}
 	$effect(() => {
-		$workspaceStore && kind && untrack(() => loadItems())
+		$workspaceStore && kind && untrack(() => items.refresh())
 	})
 	$effect(() => {
 		if ($workspaceStore) {
@@ -106,7 +117,7 @@
 
 <SearchItems
 	{filter}
-	{items}
+	items={items.value}
 	bind:filteredItems
 	f={(x) => (emptyString(x.summary) ? x.path : x.summary + ' (' + x.path + ')')}
 />

frontend/src/lib/components/flows/propPicker/OutputPicker.svelte

@@ -158,6 +158,7 @@
 		<div class="flex flex-row items-center justify-center w-full h-full">
 			{#if showInput}
 				<Popover
+					enableFlyTransition
 					floatingConfig={{
 						placement: 'bottom',
 						gutter: 0,
@@ -200,6 +201,7 @@
 			{/if}
 
 			<Popover
+				enableFlyTransition
 				floatingConfig={{
 					placement: 'bottom',
 					gutter: 0,

frontend/src/lib/components/meltComponents/Popover.svelte

@@ -8,7 +8,7 @@
 
 <script lang="ts">
 	import { createPopover, createSync, melt } from '@melt-ui/svelte'
-	import { fade } from 'svelte/transition'
+	import { fly } from 'svelte/transition'
 	import { X, Minimize2, Maximize2 } from 'lucide-svelte'
 	import type { Placement } from '@floating-ui/core'
 	import { debounce, pointerDownOutside } from '$lib/utils'
@@ -38,6 +38,7 @@
 	export let documentationLink: string | undefined = undefined
 	export let disableFocusTrap: boolean = false
 	export let escapeBehavior: EscapeBehaviorType = 'close'
+	export let enableFlyTransition: boolean = false
 
 	let fullScreen = false
 	const dispatch = createEventDispatcher()
@@ -168,7 +169,7 @@
 		}}
 		on:mouseleave={debounceClose}
 		use:melt={$content}
-		transition:fade={{ duration: 0 }}
+		transition:fly={{ duration: enableFlyTransition ? 100 : 0, y: -16 }}
 		class={twMerge(
 			'relative border rounded-md bg-surface shadow-lg',
 			fullScreen

frontend/src/lib/components/triggers/AddTriggersButton.svelte

@@ -118,6 +118,7 @@
 {/snippet}
 
 <DropdownV2
+	enableFlyTransition
 	bind:this={dropdown}
 	items={addTriggerItems}
 	{placement}

frontend/src/lib/utils.ts

@@ -1582,3 +1582,32 @@ export function assert(msg: string, condition: boolean, value?: any) {
 		console.error(m)
 	}
 }
+
+export function createCache<Keys extends Record<string, any>, T, InitialKeys extends Keys = Keys>(
+	compute: (keys: Keys) => T,
+	params?: { maxSize?: number; initial?: InitialKeys }
+): (keys: Keys) => T {
+	let cache = new Map<string, T>()
+	const maxSize = params?.maxSize ?? 15
+
+	if (params?.initial) {
+		let key = JSON.stringify(params.initial, Object.keys(params.initial).sort())
+		let value = compute(params.initial)
+		cache.set(key, value)
+	}
+
+	return (keys: Keys) => {
+		let key = JSON.stringify(keys, Object.keys(keys).sort())
+		if (!cache.get(key)) {
+			let value = compute(keys)
+			cache.set(key, value)
+
+			if (cache.size > maxSize) {
+				// remove the oldest entry (first inserted)
+				const oldestKey = cache.keys().next().value!
+				cache.delete(oldestKey)
+			}
+		}
+		return cache.get(key)!
+	}
+}