From d7d5bce499fb65091692926d65e47fadcc6c7bb0 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 13 Jun 2023 19:49:53 +0200 Subject: [PATCH] feat: add dataflow view for workflows --- .../src/lib/components/FlowBuilder.svelte | 4 +- .../src/lib/components/flows/flowExplorer.ts | 157 ++++++++++++++++++ .../flows/map/FlowModuleSchemaMap.svelte | 2 +- .../lib/components/flows/previousResults.ts | 74 +-------- .../src/lib/components/graph/FlowGraph.svelte | 95 +++++++++-- .../svelvet/container/views/GraphView.svelte | 23 +++ .../svelvet/container/views/Svelvet.svelte | 16 +- .../graph/svelvet/edges/models/Edge.ts | 3 +- .../edges/views/Edges/SimpleBezierEdge.svelte | 16 +- .../graph/svelvet/store/controllers/util.ts | 3 +- .../components/graph/svelvet/types/types.ts | 91 +++++----- 11 files changed, 338 insertions(+), 146 deletions(-) create mode 100644 frontend/src/lib/components/flows/flowExplorer.ts diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index a5626a4eae..c3efc0013b 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -31,7 +31,7 @@ import { loadHubScripts } from '$lib/scripts' import { createEventDispatcher } from 'svelte' import Awareness from './Awareness.svelte' - import { getAllModules } from './flows/previousResults' + import { getAllModules } from './flows/flowExplorer' export let initialPath: string = '' export let selectedId: string | undefined @@ -103,7 +103,7 @@ export function computeUnlockedSteps(flow: Flow) { return Object.fromEntries( - getAllModules(flow) + getAllModules(flow.value.modules, flow.value.failure_module) .filter((m) => m.value.type == 'script' && m.value.hash == null) .map((m) => [m.id, (m.value as PathScript).path]) ) diff --git a/frontend/src/lib/components/flows/flowExplorer.ts b/frontend/src/lib/components/flows/flowExplorer.ts new file mode 100644 index 0000000000..96c2464e78 --- /dev/null +++ b/frontend/src/lib/components/flows/flowExplorer.ts @@ -0,0 +1,157 @@ +import type { Flow, FlowModule, InputTransform } from '$lib/gen' + +type ModuleBranches = FlowModule[][] + +export function getSubModules(flowModule: FlowModule): ModuleBranches { + if (flowModule.value.type === 'forloopflow') { + return [flowModule.value.modules] + } else if (flowModule.value.type === 'branchall') { + return flowModule.value.branches.map((branch) => branch.modules) + } else if (flowModule.value.type == 'branchone') { + return [...flowModule.value.branches.map((branch) => branch.modules), flowModule.value.default] + } + return [] +} + +export function getAllSubmodules(flowModule: FlowModule): ModuleBranches { + return getSubModules(flowModule).map((modules) => { + return modules + .map((module) => { + return [module, ...getAllSubmodules(module).flat()] + }) + .flat() + }) +} + +export function getAllModules( + flow_modules: FlowModule[], + failure_module?: FlowModule +): FlowModule[] { + let modules = [ + ...flow_modules, + ...flow_modules.map((x) => getAllSubmodules(x).flat()), + ...(failure_module ? [failure_module] : []) + ].flat() + return modules +} + +function getExpr(x: InputTransform | undefined) { + if (x == undefined) return [] + return x.type === 'javascript' ? [x.expr] : [] +} + +function exprsOfInputTransforms(x: Record): string[] { + return Object.values(x) + .map((x) => getExpr(x)) + .flat() +} + +export function getDependentComponents(id: string, flow: Flow): Record { + let modules = getAllModules(flow.value.modules, flow.value.failure_module) + return filterDependentComponents(modules, id) +} + +function filterDependentComponents(modules: FlowModule[], id: string): Record { + return id == 'Input' + ? Object.fromEntries( + modules + .map((mod) => [mod.id, getModuleExprs(mod).filter((expr) => expr.includes(`flow_input`))]) + .filter((x) => x[1].length > 0) + ) + : Object.fromEntries( + modules + .map((mod) => [ + mod.id, + getModuleExprs(mod).filter((expr) => expr.includes(`results.${id}`)) + ]) + .filter((x) => x[1].length > 0) + ) +} + +function getModuleExprs(x: FlowModule): string[] { + let exprs: string[] = [] + if (x.value.type === 'forloopflow') { + exprs.push(...getExpr(x.value.iterator)) + } else if (x.value.type === 'branchone') { + x.value.branches.map((branch) => { + exprs.push(branch.expr) + }) + } else if (x.value.type === 'flow' || x.value.type === 'script' || x.value.type == 'rawscript') { + exprs.push(...exprsOfInputTransforms(x.value.input_transforms)) + exprs.push(...getExpr(x.sleep)) + if (x.stop_after_if?.expr) { + exprs.push(x.stop_after_if.expr) + } + exprs.push(...getExpr(x.sleep)) + } + return exprs +} + +export function getDependeeAndDependentComponents( + id: string, + modules: FlowModule[], + failure_module: FlowModule | undefined +): { dependees: Record; dependents: Record } { + let all_modules = getAllModules(modules, failure_module) + let module = all_modules.find((x) => x.id === id) + let allIds: [string, string][] = [ + ['Input', 'flow_input'], + ...modules.map((x) => [x.id, `results.${x.id}`] as [string, string]) + ] + let dependees = {} + if (module) { + getModuleExprs(module).forEach((expr) => { + allIds.forEach((y) => { + if (expr.includes(y[1])) { + dependees[y[0]] = dependees[y[0]] ?? [] + dependees[y[0]].push(expr) + } + }) + }) + } + let dependents = filterDependentComponents(all_modules, id) + return { dependees, dependents } +} + +// export function getAllDependencies( +// flow_modules: FlowModule[], +// failure_module: FlowModule | undefined +// ): Record { +// let modules = getAllModules(flow_modules, failure_module) +// let allIds: [string, string][] = [ +// ['flow_input', 'flow_input'], +// ...modules.map((x) => [x.id, `results.${x.id}`] as [string, string]) +// ] +// let deps: Record = {} +// function filterExprs(source, ...exprs: string[]) { +// exprs.forEach((x) => { +// let f = allIds.find((y) => x.includes(y[1])) +// if (f) { +// deps[source] = deps[source] ?? [] +// deps[source].push(f[0]) +// } +// }) +// } + +// modules.forEach((x) => { +// if (x.value.type === 'forloopflow') { +// filterExprs(x.id, ...getExpr(x.value.iterator)) +// } else if (x.value.type === 'branchone') { +// x.value.branches.map((branch) => { +// filterExprs(x.id, branch.expr) +// }) +// } else if ( +// x.value.type === 'flow' || +// x.value.type === 'script' || +// x.value.type == 'rawscript' +// ) { +// filterExprs(x.id, ...exprsOfInputTransforms(x.value.input_transforms)) +// filterExprs(x.id, ...getExpr(x.sleep)) +// if (x.stop_after_if?.expr) { +// filterExprs(x.id, x.stop_after_if.expr) +// } +// filterExprs(x.id, ...getExpr(x.sleep)) +// } +// }) +// return deps +// } diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte index 09a6fd5a94..244d1a05d0 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte @@ -18,8 +18,8 @@ import FlowErrorHandlerItem from './FlowErrorHandlerItem.svelte' import { push } from '$lib/history' import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte' - import { getDependentComponents } from '../previousResults' import Portal from 'svelte-portal' + import { getDependentComponents } from '../flowExplorer' export let modules: FlowModule[] | undefined export let sidebarSize: number | undefined = undefined diff --git a/frontend/src/lib/components/flows/previousResults.ts b/frontend/src/lib/components/flows/previousResults.ts index f9ff6fc791..5903fa37fe 100644 --- a/frontend/src/lib/components/flows/previousResults.ts +++ b/frontend/src/lib/components/flows/previousResults.ts @@ -1,6 +1,7 @@ import type { Schema } from '$lib/common' -import type { Flow, FlowModule, InputTransform } from '$lib/gen' +import type { Flow, FlowModule } from '$lib/gen' import { schemaToObject } from '$lib/schema' +import { getAllSubmodules, getSubModules } from './flowExplorer' import type { FlowState } from './flowState' export type PickableProperties = { @@ -17,27 +18,6 @@ type StepPropPicker = { type ModuleBranches = FlowModule[][] -function getSubModules(flowModule: FlowModule): ModuleBranches { - if (flowModule.value.type === 'forloopflow') { - return [flowModule.value.modules] - } else if (flowModule.value.type === 'branchall') { - return flowModule.value.branches.map((branch) => branch.modules) - } else if (flowModule.value.type == 'branchone') { - return [...flowModule.value.branches.map((branch) => branch.modules), flowModule.value.default] - } - return [] -} - -function getAllSubmodules(flowModule: FlowModule): ModuleBranches { - return getSubModules(flowModule).map((modules) => { - return modules - .map((module) => { - return [module, ...getAllSubmodules(module).flat()] - }) - .flat() - }) -} - function dfs(id: string | undefined, flow: Flow, getParents: boolean = true): FlowModule[] { if (id === undefined) { return [] @@ -100,56 +80,6 @@ function getFlowInput( } } -export function getAllModules(flow: Flow): FlowModule[] { - let modules = [ - ...flow.value.modules, - ...flow.value.modules.map((x) => getAllSubmodules(x).flat()), - ...(flow.value.failure_module ? [flow.value.failure_module] : []) - ].flat() - return modules -} - -function getExpr(x: InputTransform | undefined) { - if (x == undefined) return [] - return x.type === 'javascript' ? [x.expr] : [] -} - -function exprsOfInputTransforms(x: Record): string[] { - return Object.values(x) - .map((x) => getExpr(x)) - .flat() -} -export function getDependentComponents(id: string, flow: Flow): Record { - let modules = getAllModules(flow) - return Object.fromEntries( - modules - .map((x) => { - let exprs: string[] = [] - if (x.value.type === 'forloopflow') { - exprs.push(...getExpr(x.value.iterator)) - } else if (x.value.type === 'branchone') { - x.value.branches.map((branch) => { - exprs.push(branch.expr) - }) - } else if ( - x.value.type === 'flow' || - x.value.type === 'script' || - x.value.type == 'rawscript' - ) { - exprs.push(...exprsOfInputTransforms(x.value.input_transforms)) - exprs.push(...getExpr(x.sleep)) - if (x.stop_after_if?.expr) { - exprs.push(x.stop_after_if.expr) - } - exprs.push(...getExpr(x.sleep)) - } - exprs = exprs.filter((x) => x.includes(`results.${id}`)) - return [x.id, exprs] - }) - .filter((x) => x[1].length > 0) - ) -} - export function getStepPropPicker( flowState: FlowState, parentModule: FlowModule | undefined, diff --git a/frontend/src/lib/components/graph/FlowGraph.svelte b/frontend/src/lib/components/graph/FlowGraph.svelte index 2d0964b8bd..91614a0cff 100644 --- a/frontend/src/lib/components/graph/FlowGraph.svelte +++ b/frontend/src/lib/components/graph/FlowGraph.svelte @@ -21,6 +21,8 @@ import MapItem from '../flows/map/MapItem.svelte' import VirtualItem from '../flows/map/VirtualItem.svelte' import { writable, type Writable } from 'svelte/store' + import { getDependeeAndDependentComponents } from '../flows/flowExplorer' + import { deepEqual } from 'fast-equals' export let success: boolean | undefined = undefined export let modules: FlowModule[] | undefined = [] @@ -49,20 +51,40 @@ let fullWidth: number let errorHandlers: Record = {} + $: showDataflow = + $selectedId != undefined && + !$selectedId.startsWith('constants') && + !$selectedId.startsWith('settings') && + $selectedId !== 'failure' && + $selectedId !== 'Result' + let dataflow = false + let dispatch = createEventDispatcher() $: { - rebuildOnChange + dataflow moving width && height && minHeight && $selectedId && flowModuleStates - nodes = edges = [] - errorHandlers = {} createGraph() } + $: rebuildOnChange && triggerRebuild() + + let oldRebuildOnChange = JSON.parse(JSON.stringify(rebuildOnChange)) + + function triggerRebuild() { + if (!deepEqual(oldRebuildOnChange, rebuildOnChange)) { + oldRebuildOnChange = JSON.parse(JSON.stringify(rebuildOnChange)) + createGraph() + } + } + async function createGraph() { // console.log(JSON.stringify(modules)) // return + nodes = [] + edges = [] + errorHandlers = {} if (modules) { idGenerator = createIdGenerator() @@ -83,7 +105,9 @@ 0, 0, true, - undefined + undefined, + undefined, + 'Input' ) ) @@ -109,6 +133,8 @@ 0, modules.length, true, + undefined, + undefined, undefined ) ) @@ -130,7 +156,46 @@ let hfull = Math.max(layered.height, minHeight) fullWidth = layered.width height = fullSize ? hfull : Math.min(hfull, maxHeight ?? window.innerHeight - 100) - edges = createEdges(nodes) + + let useDataflow = dataflow && showDataflow + edges = useDataflow ? [] : createEdges(nodes) + + if (useDataflow && $selectedId) { + let deps = getDependeeAndDependentComponents($selectedId, modules ?? [], failureModule) + if (deps) { + Object.entries(deps.dependees).forEach((x, i) => { + let pid = x[0] + edges.push({ + id: `dep-${pid}-${$selectedId}`, + source: pid, + target: $selectedId!, + labelBgColor: 'white', + arrow: false, + animate: true, + noHandle: true, + label: pid, + type: 'bezier', + offset: i * 20 + }) + }) + + Object.entries(deps.dependents).forEach((x, i) => { + let pid = x[0] + edges.push({ + id: `dep-${pid}-${$selectedId}`, + source: $selectedId!, + target: pid, + labelBgColor: 'white', + arrow: false, + animate: true, + noHandle: true, + label: pid, + type: 'bezier', + offset: i * 10 + }) + }) + } + } } function getConvertedFlowModule( @@ -323,6 +388,7 @@ 0, true, undefined, + undefined, undefined ) ) @@ -349,7 +415,8 @@ modules.findIndex((m) => m.id == module.id) + 1, true, undefined, - module.id + module.id, + undefined ) ) return loop @@ -388,6 +455,8 @@ loopDepth, 0, false, + undefined, + undefined, undefined ) ]) @@ -406,7 +475,9 @@ loopDepth, 0, false, - removable ? { module, index: i } : undefined + removable ? { module, index: i } : undefined, + undefined, + undefined ) ) if (modules.length) { @@ -439,7 +510,8 @@ modules.findIndex((m) => m.id == module.id) + 1, true, undefined, - module.id + module.id, + undefined ), items: bitems } @@ -563,9 +635,10 @@ index: number, selectable: boolean, deleteBranch: { module: FlowModule; index: number } | undefined, - mid: string | undefined = undefined + mid: string | undefined, + fixed_id: string | undefined ): Node { - const id = -idGenerator.next().value - 2 + (offset ?? 0) + const id = fixed_id ?? -idGenerator.next().value - 2 + (offset ?? 0) return { type: 'node', id: id.toString(), @@ -664,11 +737,13 @@ {download} highlightEdges={false} locked + bind:dataflow {nodes} width={fullSize ? fullWidth : width} {edges} {height} {scroll} + nodeSelected={showDataflow} background={false} bgColor="rgb(249 250 251)" /> diff --git a/frontend/src/lib/components/graph/svelvet/container/views/GraphView.svelte b/frontend/src/lib/components/graph/svelvet/container/views/GraphView.svelte index 5c80188e5a..8bc1ca64b0 100644 --- a/frontend/src/lib/components/graph/svelvet/container/views/GraphView.svelte +++ b/frontend/src/lib/components/graph/svelvet/container/views/GraphView.svelte @@ -13,6 +13,7 @@ import { onDestroy } from 'svelte' import { Expand, Minus, Plus } from 'lucide-svelte' + import Toggle from '$lib/components/Toggle.svelte' //these are typscripted as any, however they have been transformed inside of store.ts export let canvasId: string @@ -22,7 +23,10 @@ export let boundary = false export let scroll = false + export let dataflow = false export let download = false + export let showDataflowToggle: boolean = false + // here we lookup the store using the unique key const store = findStore(canvasId) const { @@ -198,6 +202,19 @@ +{#if showDataflowToggle} +
+ +
+{/if} +
+ {#if download}