feat: hub path scripts + nested inputs glue (#2668)

frontend/src/lib/components/FlowBuilder.svelte

@@ -9,7 +9,8 @@
 		ScriptService,
 		Script,
 		type HubScriptKind,
-		type OpenFlow
+		type OpenFlow,
+		type RawScript
 	} from '$lib/gen'
 	import { initHistory, push, redo, undo } from '$lib/history'
 	import {
@@ -57,7 +58,7 @@
 	import FlowCopilotDrawer from './copilot/FlowCopilotDrawer.svelte'
 	import FlowCopilotStatus from './copilot/FlowCopilotStatus.svelte'
 	import { fade } from 'svelte/transition'
-	import { loadFlowModuleState } from './flows/flowStateUtils'
+	import { loadFlowModuleState, pickScript } from './flows/flowStateUtils'
 	import FlowCopilotInputsModal from './copilot/FlowCopilotInputsModal.svelte'
 	import { snakeCase } from 'lodash'
 	import FlowBuilderTutorials from './FlowBuilderTutorials.svelte'
@@ -472,8 +473,7 @@
 				})
 			).map((s) => ({
 				...s,
-				path: `hub/${s.version_id}/${s.app}/${s.summary.toLowerCase().replaceAll(/\s+/g, '_')}`,
-				summary: `${s.summary} (${s.app})`
+				path: `hub/${s.version_id}/${s.app}/${s.summary.toLowerCase().replaceAll(/\s+/g, '_')}`
 			}))
 			if (ts < doneTs) return
 			doneTs = ts
@@ -544,7 +544,7 @@
 
 	function clearFlowInputsFromStep(id: string | undefined) {
 		const module: FlowModule | undefined = dfs(id, $flowStore)[0]
-		if (module?.value.type === 'rawscript') {
+		if (module?.value.type === 'rawscript' || module?.value.type === 'script') {
 			// clear step inputs that start with flow_input. but not flow_input.iter
 			for (const key in module.value.input_transforms) {
 				const input = module.value.input_transforms[key]
@@ -607,23 +607,9 @@
 				$scheduleStore.enabled = true
 			}
 
-			let hubScript:
-				| {
-						content: string
-						lockfile?: string | undefined
-						schema?: any
-						language: string
-						summary?: string | undefined
-				  }
-				| undefined = undefined
-
-			if (module.source === 'hub' && module.selectedCompletion) {
-				hubScript = await ScriptService.getHubScriptByPath({
-					path: module.selectedCompletion.path
-				})
-			}
-
-			const flowModule = {
+			const flowModule: FlowModule & {
+				value: RawScript | PathScript
+			} = {
 				id: module.id,
 				stop_after_if:
 					module.type === 'trigger'
@@ -635,13 +621,28 @@
 				value: {
 					input_transforms: {},
 					content: '',
-					language: (hubScript ? hubScript.language : module.lang ?? 'bun') as Script.language,
-					type: 'rawscript' as const
+					language: (module.lang ?? 'bun') as Script.language,
+					type: 'rawscript'
 				},
-				summary: module.selectedCompletion?.summary ?? module.description
+				summary: module.description
+			}
+
+			let isHubStep = false
+			if (module.source === 'hub' && module.selectedCompletion) {
+				isHubStep = true
+				const [hubScriptModule, hubScriptState] = await pickScript(
+					module.selectedCompletion.path,
+					`${module.selectedCompletion.summary} (${module.selectedCompletion.app})`,
+					module.id,
+					undefined
+				)
+				flowModule.value = hubScriptModule.value
+				flowModule.summary = hubScriptModule.summary
+				$flowStateStore[module.id] = hubScriptState
+			} else {
+				$flowStateStore[module.id] = emptyFlowModuleState()
 			}
 
-			$flowStateStore[module.id] = emptyFlowModuleState()
 			if (stepOnly) {
 				flowModules.splice(idx, 0, flowModule)
 			} else if (idx === 1 && $copilotModulesStore[idx - 1].type === 'trigger') {
@@ -665,6 +666,7 @@
 			}
 
 			$copilotDrawerStore?.closeDrawer()
+			await tick()
 			select(module.id)
 			await tick()
 			await tick()
@@ -678,12 +680,14 @@
 			) {
 				isFirstInLoop = true
 			}
-			const prevNodeId = getPreviousIds(module.id, $flowStore, false)[0]
-			const pastModule: FlowModule | undefined = dfs(prevNodeId, $flowStore, false)[0]
+			const prevNodeId = getPreviousIds(module.id, $flowStore, false)[0] as string | undefined
+			const pastModule = dfs(prevNodeId, $flowStore, false)[0] as FlowModule | undefined
 
-			if (hubScript) {
-				module.editor?.setCode(hubScript.content)
-			} else if (module.source === 'custom') {
+			if (!module.source) {
+				throw new Error('Invalid copilot module source')
+			}
+
+			if (module.source === 'custom') {
 				const deltaStore = writable<string>('')
 				const unsubscribe = deltaStore.subscribe(async (delta) => {
 					module.editor?.append(delta)
@@ -693,35 +697,38 @@
 				await stepCopilot(
 					module,
 					deltaStore,
-					pastModule?.value.type === 'rawscript' ? pastModule.value.content : '',
-					pastModule?.value.type === 'rawscript' ? pastModule.value.language : undefined,
-					pastModule === undefined,
+					$workspaceStore!,
+					pastModule?.value.type === 'rawscript' || pastModule?.value.type === 'script'
+						? (pastModule as FlowModule & {
+								value: RawScript | PathScript
+						  })
+						: undefined,
 					isFirstInLoop,
 					abortController
 				)
 				unsubscribe()
-			} else {
-				throw new Error('Invalid copilot module source')
 			}
 
 			copilotStatus = "Generating inputs for step '" + module.id + "'..."
 			await sleep(500) // make sure code was parsed
 
 			try {
-				if (flowModule.value.type === 'rawscript') {
+				if (
+					(flowModule.value.type === 'rawscript' || flowModule.value.type === 'script') &&
+					(pastModule === undefined ||
+						pastModule.value.type === 'rawscript' ||
+						pastModule.value.type === 'script')
+				) {
 					const stepSchema: Schema = JSON.parse(JSON.stringify($flowStateStore[module.id].schema)) // deep copy
-					if (
-						module.source === 'hub' &&
-						pastModule !== undefined &&
-						$copilotInfo.exists_openai_resource_path
-					) {
+					if (isHubStep && pastModule !== undefined && $copilotInfo.exists_openai_resource_path) {
 						// ask AI to set step inputs
 						abortController = new AbortController()
-						const inputs = await glueCopilot(
-							Object.keys(flowModule.value.input_transforms),
-							pastModule.value.type === 'rawscript' ? pastModule.value.content : '',
-							pastModule.value.type === 'rawscript' ? pastModule.value.language : undefined,
-							prevNodeId,
+						const { inputs, allExprs } = await glueCopilot(
+							flowModule.value.input_transforms,
+							$workspaceStore!,
+							pastModule as FlowModule & {
+								value: RawScript | PathScript
+							},
 							isFirstInLoop,
 							abortController
 						)
@@ -729,20 +736,34 @@
 						// create flow inputs used by AI for autocompletion
 						copilotFlowInputs = {}
 						copilotFlowRequiredInputs = []
-						Object.entries(inputs).forEach(([key, expr]) => {
-							const snakeKey = snakeCase(key)
-							if (
-								key in stepSchema.properties &&
-								expr.includes('flow_input.') &&
-								!expr.includes('flow_input.iter') &&
-								(!$flowStore.schema || !(snakeKey in $flowStore.schema.properties)) // prevent overriding flow inputs
-							) {
-								copilotFlowInputs[snakeKey] = stepSchema.properties[snakeKey]
-								if (stepSchema.required.includes(snakeKey)) {
-									copilotFlowRequiredInputs.push(snakeKey)
+						Object.entries(allExprs).forEach(([key, expr]) => {
+							if (expr.includes('flow_input.') && !expr.includes('flow_input.iter.')) {
+								const flowInputKey = expr.match(/flow_input\.([A-Za-z0-9_]+)/)?.[1]
+								if (
+									flowInputKey !== undefined &&
+									(!$flowStore.schema || !(flowInputKey in $flowStore.schema.properties)) // prevent overriding flow inputs
+								) {
+									if (key in stepSchema.properties) {
+										copilotFlowInputs[flowInputKey] = stepSchema.properties[key]
+										if (stepSchema.required.includes(key)) {
+											copilotFlowRequiredInputs.push(flowInputKey)
+										}
+									} else {
+										// when the key is nested (e.g. body.content)
+										const [firstKey, ...rest] = key.split('.')
+										const restKey = rest.join('.')
+										const firstKeyProperties = stepSchema.properties[firstKey]?.properties
+										if (firstKeyProperties !== undefined && restKey in firstKeyProperties) {
+											copilotFlowInputs[flowInputKey] = firstKeyProperties[restKey]
+											if (firstKeyProperties[restKey].required?.includes(flowInputKey)) {
+												copilotFlowRequiredInputs.push(flowInputKey)
+											}
+										}
+									}
 								}
 							}
 						})
+
 						if (!stepOnly) {
 							applyCopilotFlowInputs()
 						}
@@ -751,13 +772,13 @@
 						Object.entries(inputs).forEach(([key, expr]) => {
 							flowModule.value.input_transforms[key] = {
 								type: 'javascript',
-								expr: expr.replaceAll(/flow_input\.([A-Za-z0-9_]+)/g, (_, p1) => 'flow_input.' + p1)
+								expr
 							}
 							$shouldUpdatePropertyType[key] = 'javascript'
 						})
 					} else {
 						if (
-							module.source === 'hub' &&
+							isHubStep &&
 							pastModule !== undefined &&
 							!$copilotInfo.exists_openai_resource_path
 						) {
@@ -811,6 +832,19 @@
 					}
 
 					$flowStore = $flowStore // force rerendering
+				} else {
+					if (
+						pastModule !== undefined &&
+						pastModule.value.type !== 'rawscript' &&
+						pastModule.value.type !== 'script'
+					) {
+						sendUserToast(
+							`Linking to previous step ${pastModule.id} of type ${pastModule.value.type} is not yet supported`,
+							true
+						)
+					} else {
+						sendUserToast('Something went wrong, could not generate step inputs', true)
+					}
 				}
 			} catch (err) {
 				console.error(err)
@@ -852,45 +886,6 @@
 		copilotLoading = true
 		select('Input')
 		$copilotCurrentStepStore = 'Input'
-		copilotStatus = 'Setting flow inputs...'
-
-		// filter out unused flow inputs
-		const flowInputs: Record<string, SchemaProperty> = {}
-		const required = new Set<string>()
-		function getFlowInputs(modules: FlowModule[]) {
-			for (const module of modules) {
-				if (module.value.type === 'rawscript') {
-					for (const moduleAttr of Object.keys(module.value.input_transforms)) {
-						const input = module.value.input_transforms[moduleAttr]
-						if (
-							input.type === 'javascript' &&
-							input.expr.includes('flow_input.') &&
-							!input.expr.includes('flow_input.iter')
-						) {
-							const flowAttr = input.expr.split('.')[1]
-							const schema = $flowStateStore[module.id].schema
-							const schemaProperty = Object.entries(schema.properties).find(
-								(x) => x[0] === moduleAttr
-							)?.[1]
-							if (schemaProperty) {
-								flowInputs[flowAttr] = schemaProperty
-								required.add(flowAttr)
-							}
-						}
-					}
-				} else if (module.value.type === 'forloopflow') {
-					getFlowInputs(module.value.modules)
-				}
-			}
-		}
-		getFlowInputs($flowStore.value.modules)
-
-		$flowStore.schema = {
-			$schema: 'https://json-schema.org/draft/2020-12/schema',
-			properties: flowInputs,
-			required: Array.from(required),
-			type: 'object'
-		}
 
 		copilotStatus = "Done! Just check the flow's inputs and you're good to go!"
 		$copilotCurrentStepStore = undefined

frontend/src/lib/components/copilot/FlowCopilotDrawer.svelte

@@ -119,7 +119,10 @@
 									<div class="w-full text-left font-normal">
 										<div class="text-primary flex-wrap text-sm font-medium">
 											{copilotModule.source === 'hub' && copilotModule.selectedCompletion
-												? copilotModule.selectedCompletion.summary
+												? copilotModule.selectedCompletion.summary +
+												  ' (' +
+												  copilotModule.selectedCompletion.app +
+												  ')'
 												: `Generate "${copilotModule.description}" in ${
 														copilotModule.lang === 'bun' ? 'TypeScript' : 'Python'
 												  }`}
@@ -248,7 +251,7 @@
 
 													<div class="text-left font-normal text-sm">
 														<div class="text-primary font-medium">
-															{item.summary ?? ''}
+															{(item.summary ?? '') + ' (' + item['app'] + ')'}
 														</div>
 													</div>
 												</div>

frontend/src/lib/components/copilot/flow.ts

@@ -1,4 +1,12 @@
-import type { Script, FlowModule, HubScriptKind } from '$lib/gen'
+import {
+	type Script,
+	type FlowModule,
+	type HubScriptKind,
+	ScriptService,
+	RawScript,
+	type PathScript,
+	type InputTransform
+} from '$lib/gen'
 import { addResourceTypes, deltaCodeCompletion, getNonStreamingCompletion } from './lib'
 import type { Writable } from 'svelte/store'
 import type Editor from '../Editor.svelte'
@@ -56,7 +64,7 @@ const additionalInfos: {
 	python3: string
 } = {
 	bun: `<contextual_information>
-You have to write TypeScript code and export a "main" function like this: "export async function main(...)" and specify the parameter types but do not call it.
+You have to write TypeScript code and export a "main" function like this: "export async function main(...)" and specify the parameter types but do not call it. You should generally return the result.
 The fetch standard method is available globally.
 You can take as parameters resources which are dictionaries containing credentials or configuration information. For Windmill to correctly detect the resources to be passed, the resource type name has to be exactly as specified in the following list:
 <resourceTypes>
@@ -66,7 +74,7 @@ You need to define the type of the resources that are needed before the main fun
 The resource type name has to be exactly as specified (no resource suffix). If the type name conflicts with any imported methods, you have to rename the imported method with the conflicting name.
 </contextual_information>`,
 	python3: `<contextual_information>
-You have to write a function in Python called "main". Specify the parameter types. Do not call the main function.
+You have to write a function in Python called "main". Specify the parameter types. Do not call the main function. You should generally return the result.
 You can take as parameters resources which are dictionaries containing credentials or configuration information. For Windmill to correctly detect the resources to be passed, the resource type name has to be exactly as specified in the following list:
 <resourceTypes>
 {resourceTypes}
@@ -114,32 +122,67 @@ Return the script's output.
 {additionalInformation}`
 
 const inferTypeGluePrompt =
-	"Infer its type from the previous's step code: ```{codeLang}\n{prevCode}\n```"
+	"Infer its properties from the previous's step code: ```{codeLang}\n{prevCode}\n```"
 
 const loopGluePrompt = `I'm building a workflow which is a sequence of script steps. 
 My current step code has the following inputs: {inputs}. 
-Determine what to pass as inputs. You can only use the following:
-- \`flow_input\` (javascript object): general inputs that are passed to the workflow, you can assume any object properties.
+Determine for each input, what to pass from the following:
+- \`flow_input\` (javascript object): general inputs that are passed to the workflow, you can assume any object properties (snake case).
 - \`flow_input.iter.value\` (javascript object): it is ONE ELEMENT of the output of the previous step. {inferTypeGluePrompt}
 
-Reply in the following format:
+Reply with the most probable answer, do not explain or discuss.
+Your answer has to be in the following format (one line per input):
 input_name: expr`
 
 const gluePrompt = `I'm building a workflow which is a sequence of script steps. 
 My current step code has the following inputs: {inputs}. 
-Determine what to pass as inputs. You can only use the following:
-- \`flow_input\` (javascript object): general inputs that are passed to the workflow, you can assume any object properties.
-- \`results.{prevId}\` (javascript object): previous output is the output of the previous step. {inferTypeGluePrompt}
+Determine for each input, what to pass from the following:
+- \`flow_input\` (javascript object): general inputs that are passed to the workflow, you can assume any object properties (snake case).
+- \`results.{prevId}\` (javascript object): output of the previous step. {inferTypeGluePrompt}
 
-Reply in the following format:
+Reply with the most probable answer, do not explain or discuss.
+Your answer has to be in the following format (one line per input):
 input_name: expr`
 
+async function getPreviousStepContent(
+	pastModule: FlowModule & {
+		value: RawScript | PathScript
+	},
+	workspace: string
+) {
+	if (pastModule.value.type === 'rawscript') {
+		return { prevCode: pastModule.value.content, prevLang: pastModule.value.language }
+	} else {
+		if (pastModule.value.path.startsWith('hub/')) {
+			const script = await ScriptService.getHubScriptByPath({
+				path: pastModule.value.path
+			})
+			return { prevCode: script.content, prevLang: script.language as Script.language }
+		} else if (pastModule.value.hash) {
+			const script = await ScriptService.getScriptByHash({
+				workspace,
+				hash: pastModule.value.hash
+			})
+			return { prevCode: script.content, prevLang: script.language }
+		} else {
+			const script = await ScriptService.getScriptByPath({
+				workspace,
+				path: pastModule.value.path
+			})
+			return { prevCode: script.content, prevLang: script.language }
+		}
+	}
+}
+
 export async function stepCopilot(
 	module: FlowCopilotModule,
 	deltaCodeStore: Writable<string>,
-	prevCode: string,
-	prevLang: Script.language | undefined,
-	isFirstAction: boolean,
+	workspace: string,
+	pastModule:
+		| (FlowModule & {
+				value: RawScript | PathScript
+		  })
+		| undefined,
 	isFirstInLoop: boolean,
 	abortController: AbortController
 ) {
@@ -151,16 +194,20 @@ export async function stepCopilot(
 	let prompt =
 		module.type === 'trigger'
 			? triggerPrompts[lang]
-			: isFirstAction
+			: pastModule === undefined
 			? firstActionPrompt
 			: isFirstInLoop
 			? loopActionPrompt
 			: actionPrompt
+
+	const { prevCode, prevLang } = pastModule
+		? await getPreviousStepContent(pastModule, workspace)
+		: { prevCode: undefined, prevLang: undefined }
 	prompt = prompt
 		.replace('{codeLang}', codeLang)
 		.replace(
 			'{inferTypePrompt}',
-			prevCode.length > 0 && prevLang
+			prevCode && prevLang
 				? (isFirstInLoop ? inferTypeLoopPrompt : inferTypePrompt)
 						.replace('{prevCode}', prevCode)
 						.replace('{codeLang}', scriptLangToEditorLang(prevLang))
@@ -195,41 +242,75 @@ export async function stepCopilot(
 }
 
 export async function glueCopilot(
-	inputs: string[],
-	prevCode: string,
-	prevLang: Script.language | undefined,
-	prevId: string,
+	inputs: Record<string, InputTransform>,
+	workspace: string,
+	pastModule: FlowModule & {
+		value: RawScript | PathScript
+	},
 	isFirstInLoop: boolean,
 	abortController: AbortController
 ) {
+	const { prevCode, prevLang } = await getPreviousStepContent(pastModule, workspace)
+
+	const stringInputs: string[] = []
+	for (const inputName in inputs) {
+		const input = inputs[inputName]
+		if (
+			input.type === 'static' &&
+			input.value &&
+			typeof input.value === 'object' &&
+			!Array.isArray(input.value)
+		) {
+			// nested object
+			stringInputs.push(`${inputName} (${Object.keys(input.value).join(', ')})`)
+		} else {
+			stringInputs.push(inputName)
+		}
+	}
+
 	let response = await getNonStreamingCompletion(
 		[
 			{
 				role: 'user',
 				content: (isFirstInLoop ? loopGluePrompt : gluePrompt)
-					.replace('{inputs}', inputs.join(', '))
-					.replace('{prevId}', prevId)
+					.replace('{inputs}', stringInputs.join(', '))
+					.replace('{prevId}', pastModule.id)
 					.replace(
 						'{inferTypeGluePrompt}',
-						prevCode.length > 0 && prevLang
-							? inferTypeGluePrompt
-									.replace('{prevCode}', prevCode)
-									.replace('{codeLang}', scriptLangToEditorLang(prevLang))
-							: ''
+						inferTypeGluePrompt
+							.replace('{prevCode}', prevCode)
+							.replace('{codeLang}', scriptLangToEditorLang(prevLang))
 					)
 			}
 		],
 		abortController
 	)
 
-	const matches = response.matchAll(/([a-zA-Z_0-9]+): (.+)/g)
+	const matches = response.matchAll(/([a-zA-Z_0-9.]+): (.+)/g)
 
 	const result: Record<string, string> = {}
+	const allExprs: Record<string, string> = {}
 	for (const match of matches) {
 		const inputName = match[1]
 		const inputExpr = match[2].replace(',', '')
-		result[inputName] = inputExpr
+
+		allExprs[inputName] = inputExpr
+		if (inputName.includes('.')) {
+			// nested key returned by copilot (e.g. body.content: ...)
+			const [firstKey, ...rest] = inputName.split('.')
+			const restStr = rest.join('.')
+			if (!result[firstKey]) {
+				result[firstKey] = `{\n  "${restStr}": ${inputExpr}\n}`
+			} else {
+				result[firstKey] = result[firstKey].replace('\n}', `,\n  "${restStr}": ${inputExpr}\n}`)
+			}
+		} else {
+			result[inputName] = inputExpr
+		}
 	}
 
-	return result
+	return {
+		inputs: result,
+		allExprs
+	}
 }