feat: add dataflow view for workflows

This commit is contained in:
Ruben Fiszel
2023-06-13 19:49:53 +02:00
parent cbf66348c4
commit d7d5bce499
11 changed files with 338 additions and 146 deletions
@@ -31,7 +31,7 @@
import { loadHubScripts } from '$lib/scripts'
import { createEventDispatcher } from 'svelte'
import Awareness from './Awareness.svelte'
import { getAllModules } from './flows/previousResults'
import { getAllModules } from './flows/flowExplorer'
export let initialPath: string = ''
export let selectedId: string | undefined
@@ -103,7 +103,7 @@
export function computeUnlockedSteps(flow: Flow) {
return Object.fromEntries(
getAllModules(flow)
getAllModules(flow.value.modules, flow.value.failure_module)
.filter((m) => m.value.type == 'script' && m.value.hash == null)
.map((m) => [m.id, (m.value as PathScript).path])
)
@@ -0,0 +1,157 @@
import type { Flow, FlowModule, InputTransform } from '$lib/gen'
type ModuleBranches = FlowModule[][]
export function getSubModules(flowModule: FlowModule): ModuleBranches {
if (flowModule.value.type === 'forloopflow') {
return [flowModule.value.modules]
} else if (flowModule.value.type === 'branchall') {
return flowModule.value.branches.map((branch) => branch.modules)
} else if (flowModule.value.type == 'branchone') {
return [...flowModule.value.branches.map((branch) => branch.modules), flowModule.value.default]
}
return []
}
export function getAllSubmodules(flowModule: FlowModule): ModuleBranches {
return getSubModules(flowModule).map((modules) => {
return modules
.map((module) => {
return [module, ...getAllSubmodules(module).flat()]
})
.flat()
})
}
export function getAllModules(
flow_modules: FlowModule[],
failure_module?: FlowModule
): FlowModule[] {
let modules = [
...flow_modules,
...flow_modules.map((x) => getAllSubmodules(x).flat()),
...(failure_module ? [failure_module] : [])
].flat()
return modules
}
function getExpr(x: InputTransform | undefined) {
if (x == undefined) return []
return x.type === 'javascript' ? [x.expr] : []
}
function exprsOfInputTransforms(x: Record<string, InputTransform>): string[] {
return Object.values(x)
.map((x) => getExpr(x))
.flat()
}
export function getDependentComponents(id: string, flow: Flow): Record<string, string[]> {
let modules = getAllModules(flow.value.modules, flow.value.failure_module)
return filterDependentComponents(modules, id)
}
function filterDependentComponents(modules: FlowModule[], id: string): Record<string, string[]> {
return id == 'Input'
? Object.fromEntries(
modules
.map((mod) => [mod.id, getModuleExprs(mod).filter((expr) => expr.includes(`flow_input`))])
.filter((x) => x[1].length > 0)
)
: Object.fromEntries(
modules
.map((mod) => [
mod.id,
getModuleExprs(mod).filter((expr) => expr.includes(`results.${id}`))
])
.filter((x) => x[1].length > 0)
)
}
function getModuleExprs(x: FlowModule): string[] {
let exprs: string[] = []
if (x.value.type === 'forloopflow') {
exprs.push(...getExpr(x.value.iterator))
} else if (x.value.type === 'branchone') {
x.value.branches.map((branch) => {
exprs.push(branch.expr)
})
} else if (x.value.type === 'flow' || x.value.type === 'script' || x.value.type == 'rawscript') {
exprs.push(...exprsOfInputTransforms(x.value.input_transforms))
exprs.push(...getExpr(x.sleep))
if (x.stop_after_if?.expr) {
exprs.push(x.stop_after_if.expr)
}
exprs.push(...getExpr(x.sleep))
}
return exprs
}
export function getDependeeAndDependentComponents(
id: string,
modules: FlowModule[],
failure_module: FlowModule | undefined
): { dependees: Record<string, string[]>; dependents: Record<string, string[]> } {
let all_modules = getAllModules(modules, failure_module)
let module = all_modules.find((x) => x.id === id)
let allIds: [string, string][] = [
['Input', 'flow_input'],
...modules.map((x) => [x.id, `results.${x.id}`] as [string, string])
]
let dependees = {}
if (module) {
getModuleExprs(module).forEach((expr) => {
allIds.forEach((y) => {
if (expr.includes(y[1])) {
dependees[y[0]] = dependees[y[0]] ?? []
dependees[y[0]].push(expr)
}
})
})
}
let dependents = filterDependentComponents(all_modules, id)
return { dependees, dependents }
}
// export function getAllDependencies(
// flow_modules: FlowModule[],
// failure_module: FlowModule | undefined
// ): Record<string, string[]> {
// let modules = getAllModules(flow_modules, failure_module)
// let allIds: [string, string][] = [
// ['flow_input', 'flow_input'],
// ...modules.map((x) => [x.id, `results.${x.id}`] as [string, string])
// ]
// let deps: Record<string, string[]> = {}
// function filterExprs(source, ...exprs: string[]) {
// exprs.forEach((x) => {
// let f = allIds.find((y) => x.includes(y[1]))
// if (f) {
// deps[source] = deps[source] ?? []
// deps[source].push(f[0])
// }
// })
// }
// modules.forEach((x) => {
// if (x.value.type === 'forloopflow') {
// filterExprs(x.id, ...getExpr(x.value.iterator))
// } else if (x.value.type === 'branchone') {
// x.value.branches.map((branch) => {
// filterExprs(x.id, branch.expr)
// })
// } else if (
// x.value.type === 'flow' ||
// x.value.type === 'script' ||
// x.value.type == 'rawscript'
// ) {
// filterExprs(x.id, ...exprsOfInputTransforms(x.value.input_transforms))
// filterExprs(x.id, ...getExpr(x.sleep))
// if (x.stop_after_if?.expr) {
// filterExprs(x.id, x.stop_after_if.expr)
// }
// filterExprs(x.id, ...getExpr(x.sleep))
// }
// })
// return deps
// }
@@ -18,8 +18,8 @@
import FlowErrorHandlerItem from './FlowErrorHandlerItem.svelte'
import { push } from '$lib/history'
import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte'
import { getDependentComponents } from '../previousResults'
import Portal from 'svelte-portal'
import { getDependentComponents } from '../flowExplorer'
export let modules: FlowModule[] | undefined
export let sidebarSize: number | undefined = undefined
@@ -1,6 +1,7 @@
import type { Schema } from '$lib/common'
import type { Flow, FlowModule, InputTransform } from '$lib/gen'
import type { Flow, FlowModule } from '$lib/gen'
import { schemaToObject } from '$lib/schema'
import { getAllSubmodules, getSubModules } from './flowExplorer'
import type { FlowState } from './flowState'
export type PickableProperties = {
@@ -17,27 +18,6 @@ type StepPropPicker = {
type ModuleBranches = FlowModule[][]
function getSubModules(flowModule: FlowModule): ModuleBranches {
if (flowModule.value.type === 'forloopflow') {
return [flowModule.value.modules]
} else if (flowModule.value.type === 'branchall') {
return flowModule.value.branches.map((branch) => branch.modules)
} else if (flowModule.value.type == 'branchone') {
return [...flowModule.value.branches.map((branch) => branch.modules), flowModule.value.default]
}
return []
}
function getAllSubmodules(flowModule: FlowModule): ModuleBranches {
return getSubModules(flowModule).map((modules) => {
return modules
.map((module) => {
return [module, ...getAllSubmodules(module).flat()]
})
.flat()
})
}
function dfs(id: string | undefined, flow: Flow, getParents: boolean = true): FlowModule[] {
if (id === undefined) {
return []
@@ -100,56 +80,6 @@ function getFlowInput(
}
}
export function getAllModules(flow: Flow): FlowModule[] {
let modules = [
...flow.value.modules,
...flow.value.modules.map((x) => getAllSubmodules(x).flat()),
...(flow.value.failure_module ? [flow.value.failure_module] : [])
].flat()
return modules
}
function getExpr(x: InputTransform | undefined) {
if (x == undefined) return []
return x.type === 'javascript' ? [x.expr] : []
}
function exprsOfInputTransforms(x: Record<string, InputTransform>): string[] {
return Object.values(x)
.map((x) => getExpr(x))
.flat()
}
export function getDependentComponents(id: string, flow: Flow): Record<string, string[]> {
let modules = getAllModules(flow)
return Object.fromEntries(
modules
.map((x) => {
let exprs: string[] = []
if (x.value.type === 'forloopflow') {
exprs.push(...getExpr(x.value.iterator))
} else if (x.value.type === 'branchone') {
x.value.branches.map((branch) => {
exprs.push(branch.expr)
})
} else if (
x.value.type === 'flow' ||
x.value.type === 'script' ||
x.value.type == 'rawscript'
) {
exprs.push(...exprsOfInputTransforms(x.value.input_transforms))
exprs.push(...getExpr(x.sleep))
if (x.stop_after_if?.expr) {
exprs.push(x.stop_after_if.expr)
}
exprs.push(...getExpr(x.sleep))
}
exprs = exprs.filter((x) => x.includes(`results.${id}`))
return [x.id, exprs]
})
.filter((x) => x[1].length > 0)
)
}
export function getStepPropPicker(
flowState: FlowState,
parentModule: FlowModule | undefined,
@@ -21,6 +21,8 @@
import MapItem from '../flows/map/MapItem.svelte'
import VirtualItem from '../flows/map/VirtualItem.svelte'
import { writable, type Writable } from 'svelte/store'
import { getDependeeAndDependentComponents } from '../flows/flowExplorer'
import { deepEqual } from 'fast-equals'
export let success: boolean | undefined = undefined
export let modules: FlowModule[] | undefined = []
@@ -49,20 +51,40 @@
let fullWidth: number
let errorHandlers: Record<string, string> = {}
$: showDataflow =
$selectedId != undefined &&
!$selectedId.startsWith('constants') &&
!$selectedId.startsWith('settings') &&
$selectedId !== 'failure' &&
$selectedId !== 'Result'
let dataflow = false
let dispatch = createEventDispatcher()
$: {
rebuildOnChange
dataflow
moving
width && height && minHeight && $selectedId && flowModuleStates
nodes = edges = []
errorHandlers = {}
createGraph()
}
$: rebuildOnChange && triggerRebuild()
let oldRebuildOnChange = JSON.parse(JSON.stringify(rebuildOnChange))
function triggerRebuild() {
if (!deepEqual(oldRebuildOnChange, rebuildOnChange)) {
oldRebuildOnChange = JSON.parse(JSON.stringify(rebuildOnChange))
createGraph()
}
}
async function createGraph() {
// console.log(JSON.stringify(modules))
// return
nodes = []
edges = []
errorHandlers = {}
if (modules) {
idGenerator = createIdGenerator()
@@ -83,7 +105,9 @@
0,
0,
true,
undefined
undefined,
undefined,
'Input'
)
)
@@ -109,6 +133,8 @@
0,
modules.length,
true,
undefined,
undefined,
undefined
)
)
@@ -130,7 +156,46 @@
let hfull = Math.max(layered.height, minHeight)
fullWidth = layered.width
height = fullSize ? hfull : Math.min(hfull, maxHeight ?? window.innerHeight - 100)
edges = createEdges(nodes)
let useDataflow = dataflow && showDataflow
edges = useDataflow ? [] : createEdges(nodes)
if (useDataflow && $selectedId) {
let deps = getDependeeAndDependentComponents($selectedId, modules ?? [], failureModule)
if (deps) {
Object.entries(deps.dependees).forEach((x, i) => {
let pid = x[0]
edges.push({
id: `dep-${pid}-${$selectedId}`,
source: pid,
target: $selectedId!,
labelBgColor: 'white',
arrow: false,
animate: true,
noHandle: true,
label: pid,
type: 'bezier',
offset: i * 20
})
})
Object.entries(deps.dependents).forEach((x, i) => {
let pid = x[0]
edges.push({
id: `dep-${pid}-${$selectedId}`,
source: $selectedId!,
target: pid,
labelBgColor: 'white',
arrow: false,
animate: true,
noHandle: true,
label: pid,
type: 'bezier',
offset: i * 10
})
})
}
}
}
function getConvertedFlowModule(
@@ -323,6 +388,7 @@
0,
true,
undefined,
undefined,
undefined
)
)
@@ -349,7 +415,8 @@
modules.findIndex((m) => m.id == module.id) + 1,
true,
undefined,
module.id
module.id,
undefined
)
)
return loop
@@ -388,6 +455,8 @@
loopDepth,
0,
false,
undefined,
undefined,
undefined
)
])
@@ -406,7 +475,9 @@
loopDepth,
0,
false,
removable ? { module, index: i } : undefined
removable ? { module, index: i } : undefined,
undefined,
undefined
)
)
if (modules.length) {
@@ -439,7 +510,8 @@
modules.findIndex((m) => m.id == module.id) + 1,
true,
undefined,
module.id
module.id,
undefined
),
items: bitems
}
@@ -563,9 +635,10 @@
index: number,
selectable: boolean,
deleteBranch: { module: FlowModule; index: number } | undefined,
mid: string | undefined = undefined
mid: string | undefined,
fixed_id: string | undefined
): Node {
const id = -idGenerator.next().value - 2 + (offset ?? 0)
const id = fixed_id ?? -idGenerator.next().value - 2 + (offset ?? 0)
return {
type: 'node',
id: id.toString(),
@@ -664,11 +737,13 @@
{download}
highlightEdges={false}
locked
bind:dataflow
{nodes}
width={fullSize ? fullWidth : width}
{edges}
{height}
{scroll}
nodeSelected={showDataflow}
background={false}
bgColor="rgb(249 250 251)"
/>
@@ -13,6 +13,7 @@
import { onDestroy } from 'svelte'
import { Expand, Minus, Plus } from 'lucide-svelte'
import Toggle from '$lib/components/Toggle.svelte'
//these are typscripted as any, however they have been transformed inside of store.ts
export let canvasId: string
@@ -22,7 +23,10 @@
export let boundary = false
export let scroll = false
export let dataflow = false
export let download = false
export let showDataflowToggle: boolean = false
// here we lookup the store using the unique key
const store = findStore(canvasId)
const {
@@ -198,6 +202,19 @@
</g>
</svg>
</div>
{#if showDataflowToggle}
<div id="dataflow_toggle">
<Toggle
textClass="!text-gray-600"
size="xs"
bind:checked={dataflow}
options={{
right: 'dataflow'
}}
/>
</div>
{/if}
<div id="buttons">
<button title="Zoom In" id="zoom_in">
<Plus size="14" class="flex justify-start m-1" />
@@ -205,6 +222,7 @@
<button title="Zoom Out" id="zoom_out">
<Minus size="14" class="flex justify-start m-1" />
</button>
{#if download}
<button on:click={() => dispatch('expand')}>
<Expand size="14" class="flex justify-start m-1" />
@@ -213,6 +231,11 @@
</div>
<style>
#dataflow_toggle {
position: absolute;
top: 3px;
left: 8px;
}
#buttons {
position: absolute;
top: 8px;
@@ -31,7 +31,8 @@
export let highlightEdges: boolean = true
export let scroll: boolean = false
export let download: boolean = false
export let dataflow: boolean = false
export let nodeSelected: boolean = false
const fullHeight = settings?.fullHeight ?? false
// generates a unique string for each svelvet component's unique store instance
// creates a store that uses the unique sting as the key to create and look up the corresponding store
@@ -48,7 +49,6 @@
let output = sanitizeUserNodesAndEdges(nodes, edges)
const userNodes = output['userNodes']
const userEdges = output['userEdges']
// set canvas related stores. you need to do this before setting node/edge related stores because
// initializing nodes/edges might read relevant options from the store.
store.widthStore.set(width)
@@ -120,7 +120,17 @@
{#if error != ''}
<div class="error text-red-600 center-center p-4">{error}</div>
{:else}
<GraphView on:expand {download} {scroll} {canvasId} {width} {height} {boundary} />
<GraphView
showDataflowToggle={nodeSelected}
bind:dataflow
on:expand
{download}
{scroll}
{canvasId}
{width}
{height}
{boundary}
/>
{/if}
</div>
@@ -38,7 +38,8 @@ export class Edge implements EdgeType {
public noHandle: boolean,
public arrow: boolean,
public clickCallback: Function,
public className: string
public className: string,
public offset?: number
) {}
/**
@@ -1,16 +1,6 @@
<script lang="ts">
import BaseEdge from './BaseEdge.svelte'
// import { Position } from '../types/utils';
// // enumerable values (static) set for Position
// export var Position;
// (function (Position) {
// Position["Left"] = "left";
// Position["Right"] = "right";
// Position["Top"] = "top";
// Position["Bottom"] = "bottom";
// })(Position || (Position = {}));
// //
// // export type CoordinateExtent = [[number, number], [number, number]];
const Position = { Left: 'left', Right: 'right', Top: 'top', Bottom: 'bottom' }
import { findStore } from '../../../store/controllers/storeApi'
@@ -133,10 +123,10 @@
const targetAnchor = getAnchorFromEdge(store, edge.id, 'target')
const mapAngle = { 0: 'right', 90: 'top', 180: 'left', 270: 'bottom' }
params = {
sourceX: edge.sourceX,
sourceX: edge.sourceX + (edge.offset ?? 0),
sourceY: edge.sourceY,
sourcePosition: mapAngle[sourceAnchor.angle],
targetX: edge.targetX,
targetX: edge.targetX + (edge.offset ?? 0),
targetY: edge.targetY,
targetPosition: mapAngle[targetAnchor.angle],
curvature: 0.25
@@ -111,7 +111,8 @@ export function populateEdgesStore(store: StoreType, edges: UserEdgeType[], canv
userEdge.noHandle === undefined ? false : userEdge.noHandle,
userEdge.arrow === undefined ? false : userEdge.arrow,
userEdge.clickCallback === undefined ? () => {} : userEdge.clickCallback,
userEdge.className === undefined ? '' : userEdge.className
userEdge.className === undefined ? '' : userEdge.className,
userEdge.offset
)
}
store.edgesStore.set(edgesStore)
@@ -1,55 +1,60 @@
export interface UserNodeType {
id: string;
width: number;
height: number;
bgColor?: string;
data: { html?: any, custom?: { component: any, props?: any, cb?: (e: string, detail: any) => void }, img?: any }
position: { x: number; y: number };
borderColor?: string | undefined;
image?: boolean;
src?: string;
textColor?: string;
targetPosition?: 'left' | 'right' | 'top' | 'bottom';
sourcePosition?: 'left' | 'right' | 'top' | 'bottom';
borderRadius?: number;
childNodes?: string[];
className?: string;
clickCallback?: Function;
id: string
width: number
height: number
bgColor?: string
data: {
html?: any
custom?: { component: any; props?: any; cb?: (e: string, detail: any) => void }
img?: any
}
position: { x: number; y: number }
borderColor?: string | undefined
image?: boolean
src?: string
textColor?: string
targetPosition?: 'left' | 'right' | 'top' | 'bottom'
sourcePosition?: 'left' | 'right' | 'top' | 'bottom'
borderRadius?: number
childNodes?: string[]
className?: string
clickCallback?: Function
}
export interface UserEdgeType {
id: string;
source: string;
target: string;
sourceAnchorCb?: Function;
targetAnchorCb?: Function;
label?: string;
labelBgColor?: string;
labelTextColor?: string;
edgeColor?: string;
type?: 'straight' | 'smoothstep' | 'step' | 'bezier' | undefined;
animate?: boolean;
noHandle?: boolean;
arrow?: boolean;
clickCallback?: Function;
className?: string;
id: string
source: string
target: string
sourceAnchorCb?: Function
targetAnchorCb?: Function
label?: string
labelBgColor?: string
labelTextColor?: string
edgeColor?: string
type?: 'straight' | 'smoothstep' | 'step' | 'bezier' | undefined
animate?: boolean
noHandle?: boolean
arrow?: boolean
clickCallback?: Function
className?: string
offset?: number
}
import { findStore } from '../store/controllers/storeApi';
import { get } from 'svelte/store';
import { findStore } from '../store/controllers/storeApi'
import { get } from 'svelte/store'
export function getD3PositionX(canvasId: string) {
const store = findStore(canvasId);
const width = get(store.widthStore);
const x = width / 2 - get(store.d3ZoomParameters).x; // user input is shifted so that x=0, y=0 occurs in the center
return x;
const store = findStore(canvasId)
const width = get(store.widthStore)
const x = width / 2 - get(store.d3ZoomParameters).x // user input is shifted so that x=0, y=0 occurs in the center
return x
}
export function getD3PositionY(canvasId: string) {
const store = findStore(canvasId);
const height = get(store.heightStore);
const y = height / 2 - get(store.d3ZoomParameters).y; // user input is shifted so that x=0, y=0 occurs in the center
return y;
const store = findStore(canvasId)
const height = get(store.heightStore)
const y = height / 2 - get(store.d3ZoomParameters).y // user input is shifted so that x=0, y=0 occurs in the center
return y
}
export function getD3Zoom(canvasId: string) {
const store = findStore(canvasId);
return get(store.d3ZoomParameters).k;
const store = findStore(canvasId)
return get(store.d3ZoomParameters).k
}