improve job loader in case of concurrent requests

This commit is contained in:
Ruben Fiszel
2023-01-05 05:57:22 +01:00
parent aaac41761c
commit 3fba91084d
9 changed files with 330 additions and 321 deletions
+6 -5
View File
@@ -45,14 +45,15 @@
</div>
</div>
</div>
{#if duration}
{#if isLoading}
<Loader2 class="animate-spin absolute top-2 left-2" />
{:else if duration}
<span class="absolute text-xs text-gray-500 top-2 left-2">took {duration}ms</span>
{/if}
<pre class="whitespace-pre-wrap break-words bg-gray-50 text-xs w-full p-2"
>{#if content}<span>{content}</span>{:else if isLoading}
<Loader2 class="animate-spin" />
{:else}<span class="text-gray-600">No logs are available yet</span>{/if}</pre
>{#if content}<span>{content}</span>{:else if !isLoading}<span class="text-gray-600"
>No logs are available yet</span
>{/if}</pre
>
</div>
</div>
@@ -11,7 +11,6 @@
import { flowStateStore } from './flows/flowState'
import { flowStore } from './flows/flowStore'
import { workspaceStore } from '$lib/stores'
import { Icon } from 'svelte-awesome'
import { Loader2 } from 'lucide-svelte'
export let mod: FlowModule
+86 -84
View File
@@ -140,91 +140,93 @@
{/if}
</div>
</div>
<SplitPanesWrapper panesClass="!overflow-visible">
<Pane size={60} minSize={10} class="!overflow-visible">
<div
class="pl-2 h-full !overflow-visible"
on:mouseleave={() => {
inferSchema(code)
}}
>
<Editor
bind:code
bind:websocketAlive
bind:this={editor}
on:change={(e) => {
inferSchema(e.detail)
<SplitPanesWrapper>
<Splitpanes class="!overflow-visible">
<Pane size={60} minSize={10} class="!overflow-visible">
<div
class="pl-2 h-full !overflow-visible"
on:mouseleave={() => {
inferSchema(code)
}}
cmdEnterAction={async () => {
await inferSchema(code)
runTest()
}}
formatAction={async () => {
await inferSchema(code)
try {
localStorage.setItem(path ?? 'last_save', code)
} catch (e) {
console.error('Could not save last_save to local storage', e)
}
lastSave = code
}}
class="flex flex-1 h-full !overflow-visible"
lang={scriptLangToEditorLang(lang)}
automaticLayout={true}
{fixedOverflowWidgets}
/>
</div>
</Pane>
<Pane size={40} minSize={10}>
<div class="flex flex-col h-full">
<div class="px-2 w-full border-b py-1">
{#if testIsLoading}
<Button on:click={testJobLoader?.cancelJob} btnClasses="w-full" color="red" size="xs">
<WindmillIcon
white={true}
class="animate-[spin_5s_linear_infinite] mr-2 text-white"
height="20px"
width="20px"
/>
Cancel
</Button>
{:else}
<Button
on:click={runTest}
btnClasses="w-full"
size="xs"
startIcon={{
icon: faPlay,
classes: 'animate-none'
}}
>
{#if testIsLoading}
Running
{:else}
Test <Kbd class="ml-4 text-5xs -my-0.5">Ctrl+Enter</Kbd>
{/if}
</Button>
{/if}
>
<Editor
bind:code
bind:websocketAlive
bind:this={editor}
on:change={(e) => {
inferSchema(e.detail)
}}
cmdEnterAction={async () => {
await inferSchema(code)
runTest()
}}
formatAction={async () => {
await inferSchema(code)
try {
localStorage.setItem(path ?? 'last_save', code)
} catch (e) {
console.error('Could not save last_save to local storage', e)
}
lastSave = code
}}
class="flex flex-1 h-full !overflow-visible"
lang={scriptLangToEditorLang(lang)}
automaticLayout={true}
{fixedOverflowWidgets}
/>
</div>
<Splitpanes horizontal class="!max-h-[calc(100%-43px)]">
<Pane size={33}>
<div class="px-2">
<div class="break-words relative font-sans">
<SchemaForm compact {schema} bind:args bind:isValid />
</Pane>
<Pane size={40} minSize={10}>
<div class="flex flex-col h-full">
<div class="px-2 w-full border-b py-1">
{#if testIsLoading}
<Button on:click={testJobLoader?.cancelJob} btnClasses="w-full" color="red" size="xs">
<WindmillIcon
white={true}
class="animate-[spin_5s_linear_infinite] mr-2 text-white"
height="20px"
width="20px"
/>
Cancel
</Button>
{:else}
<Button
on:click={runTest}
btnClasses="w-full"
size="xs"
startIcon={{
icon: faPlay,
classes: 'animate-none'
}}
>
{#if testIsLoading}
Running
{:else}
Test <Kbd class="ml-4 text-5xs -my-0.5">Ctrl+Enter</Kbd>
{/if}
</Button>
{/if}
</div>
<Splitpanes horizontal class="!max-h-[calc(100%-43px)]">
<Pane size={33}>
<div class="px-2">
<div class="break-words relative font-sans">
<SchemaForm compact {schema} bind:args bind:isValid />
</div>
</div>
</div>
</Pane>
<Pane size={67}>
<LogPanel
{path}
{lang}
previewJob={testJob}
{pastPreviews}
previewIsLoading={testIsLoading}
bind:lastSave
/>
</Pane>
</Splitpanes>
</div>
</Pane>
</Pane>
<Pane size={67}>
<LogPanel
{path}
{lang}
previewJob={testJob}
{pastPreviews}
previewIsLoading={testIsLoading}
bind:lastSave
/>
</Pane>
</Splitpanes>
</div>
</Pane>
</Splitpanes>
</SplitPanesWrapper>
@@ -4,11 +4,6 @@
import { onDestroy } from 'svelte'
import type { Preview } from '$lib/gen/models/Preview'
import { createEventDispatcher } from 'svelte'
import {
setIntervalAsync,
clearIntervalAsync,
type SetIntervalAsyncTimer
} from 'set-interval-async'
export let isLoading = false
export let job: Job | undefined = undefined
@@ -18,7 +13,6 @@
const dispatch = createEventDispatcher()
$: workspace = workspaceOverride ?? $workspaceStore
let intervalId: SetIntervalAsyncTimer<unknown[]> | undefined = undefined
let syncIteration: number = 0
let errorIteration = 0
@@ -26,19 +20,32 @@
let ITERATIONS_BEFORE_SLOW_REFRESH = 10
let ITERATIONS_BEFORE_SUPER_SLOW_REFRESH = 100
let stopCurrentIteration = false
let lastStartedAt: number = Date.now()
let currentId: string | undefined = undefined
$: isLoading = currentId !== undefined
export async function abstractRun(fn: () => Promise<string>) {
try {
await clearCurrentJob()
isLoading = true
clearCurrentJob()
const startedAt = Date.now()
const testId = await fn()
if (testId) {
await watchJob(testId)
if (lastStartedAt < startedAt) {
lastStartedAt = startedAt
if (testId) {
try {
await watchJob(testId)
} catch {
if (currentId === testId) {
currentId = undefined
}
}
}
}
return testId
} catch (err) {
isLoading = false
throw err
}
}
@@ -89,108 +96,102 @@
}
export async function cancelJob() {
try {
await JobService.cancelQueuedJob({
workspace: $workspaceStore ?? '',
id: job?.id ?? '',
requestBody: {}
})
} catch (err) {
console.error(err)
const id = currentId
if (id) {
currentId = undefined
try {
await JobService.cancelQueuedJob({
workspace: $workspaceStore ?? '',
id,
requestBody: {}
})
} catch (err) {
console.error(err)
}
}
isLoading = false
console.log('cancelled')
console.debug('cancelled')
}
export async function clearCurrentJob() {
if (intervalId) {
const interval = intervalId
intervalId = undefined
stopCurrentIteration = true
if (isLoading && job) {
try {
await JobService.cancelQueuedJob({
workspace: workspace!,
id: job.id,
requestBody: {}
})
} catch {}
}
await clearIntervalAsync(interval)
if (currentId) {
console.debug('clear')
job = undefined
await cancelJob()
}
stopCurrentIteration = false
job = undefined
isLoading = false
}
export async function watchJob(testId: string) {
syncIteration = 0
errorIteration = 0
currentId = testId
job = undefined
const isCompleted = await loadTestJob(testId)
if (!isCompleted) {
isLoading = true
intervalId = setIntervalAsync(async () => {
await syncer(testId)
setTimeout(() => {
syncer(testId)
}, 50)
}
}
async function loadTestJob(id: string): Promise<boolean> {
let isCompleted = false
try {
if (job && `running` in job) {
let previewJobUpdates = await JobService.getJobUpdates({
workspace: workspace!,
id,
running: job.running,
logOffset: job.logs?.length ? job.logs?.length + 1 : 0
})
if (currentId === id) {
try {
if (job && `running` in job) {
let previewJobUpdates = await JobService.getJobUpdates({
workspace: workspace!,
id,
running: job.running,
logOffset: job.logs?.length ? job.logs?.length + 1 : 0
})
if (previewJobUpdates.new_logs) {
job.logs = (job?.logs ?? '').concat(previewJobUpdates.new_logs)
}
if ((previewJobUpdates.running ?? false) || (previewJobUpdates.completed ?? false)) {
if (previewJobUpdates.new_logs) {
job.logs = (job?.logs ?? '').concat(previewJobUpdates.new_logs)
}
if ((previewJobUpdates.running ?? false) || (previewJobUpdates.completed ?? false)) {
job = await JobService.getJob({ workspace: workspace!, id })
}
} else {
job = await JobService.getJob({ workspace: workspace!, id })
}
} else {
job = await JobService.getJob({ workspace: workspace!, id })
}
job = await JobService.getJob({ workspace: workspace ?? '', id })
if (job?.type === 'CompletedJob') {
//only CompletedJob has success property
isCompleted = true
intervalId && clearIntervalAsync(intervalId!)
if (isLoading) {
dispatch('done', job)
isLoading = false
if (job?.type === 'CompletedJob') {
//only CompletedJob has success property
isCompleted = true
if (currentId === id) {
dispatch('done', job)
currentId = undefined
}
}
notfound = false
} catch (err) {
errorIteration += 1
if (errorIteration == 5) {
notfound = true
await clearCurrentJob()
}
console.warn(err)
}
notfound = false
} catch (err) {
errorIteration += 1
if (errorIteration == 5) {
notfound = true
await clearCurrentJob()
}
console.warn(err)
return isCompleted
} else {
return true
}
return isCompleted
}
async function syncer(id: string): Promise<void> {
syncIteration++
if (syncIteration == ITERATIONS_BEFORE_SLOW_REFRESH) {
intervalId && clearIntervalAsync(intervalId!)
intervalId = setIntervalAsync(async () => await syncer(id), 500)
} else if (syncIteration == ITERATIONS_BEFORE_SUPER_SLOW_REFRESH) {
intervalId && clearIntervalAsync(intervalId!)
intervalId = setIntervalAsync(async () => await syncer(id), 2000)
}
if (stopCurrentIteration) {
if (currentId != id) {
console.debug('stop')
return
}
syncIteration++
await loadTestJob(id)
let nextIteration = 50
if (syncIteration == ITERATIONS_BEFORE_SLOW_REFRESH) {
nextIteration = 500
} else if (syncIteration == ITERATIONS_BEFORE_SUPER_SLOW_REFRESH) {
nextIteration = 2000
}
setTimeout(() => syncer(id), nextIteration)
}
onDestroy(async () => {
@@ -187,13 +187,15 @@
let field = fields[k]
if (field?.type == 'static' && fields[k]) {
staticRunnableInputs[k] = field.value
} else if (field?.type == 'user') {
nonStaticRunnableInputs[k] = args[k]
} else {
nonStaticRunnableInputs[k] = runnableInputValues[k]
}
})
const requestBody = {
args: { ...nonStaticRunnableInputs, ...args },
args: nonStaticRunnableInputs,
force_viewer_static_fields: !isEditor ? undefined : staticRunnableInputs
}
@@ -239,7 +241,7 @@
<TestJobLoader
workspaceOverride={workspace}
on:done={() => {
on:done={(e) => {
if (testJob && outputs) {
const startedAt = new Date(testJob.started_at).getTime()
if (startedAt > lastStartedAt) {
@@ -2,7 +2,7 @@
import SplitPanesWrapper from '$lib/components/splitPanes/SplitPanesWrapper.svelte'
import { onMount, setContext } from 'svelte'
import { Pane } from 'svelte-splitpanes'
import { Pane, Splitpanes } from 'svelte-splitpanes'
import { writable } from 'svelte/store'
import { buildWorld, type World } from '../rx'
import type {
@@ -129,87 +129,91 @@
{context}
/>
{:else}
<SplitPanesWrapper class="max-w-full overflow-hidden">
<Pane size={$connectingInput?.opened ? 40 : 15} minSize={5} maxSize={33}>
<ContextPanel />
</Pane>
<Pane size={64}>
<SplitPanesWrapper horizontal>
<Pane size={$connectingInput?.opened ? 100 : 70}>
<div
class="bg-gray-100 relative w-full h-full overflow-auto {app.fullscreen
? ''
: 'max-w-6xl'}"
>
{#if $appStore.grid}
<div class={classNames('p-4 mx-auto', width)}>
<GridEditor {policy} />
<SplitPanesWrapper>
<Splitpanes class="max-w-full overflow-hidden">
<Pane size={$connectingInput?.opened ? 40 : 15} minSize={5} maxSize={33}>
<ContextPanel />
</Pane>
<Pane size={64}>
<SplitPanesWrapper>
<Splitpanes horizontal>
<Pane size={$connectingInput?.opened ? 100 : 70}>
<div
class="bg-gray-100 relative w-full h-full overflow-auto {app.fullscreen
? ''
: 'max-w-6xl'}"
>
{#if $appStore.grid}
<div class={classNames('p-4 mx-auto', width)}>
<GridEditor {policy} />
</div>
{/if}
</div>
{/if}
</div>
</Pane>
<Pane size={$connectingInput?.opened ? 0 : 30}>
<div class="relative h-full w-full">
<InlineScriptsPanel />
</div>
</Pane>
</SplitPanesWrapper>
</Pane>
<Pane size={21} minSize={5} maxSize={33}>
<div class="relative flex flex-col h-full">
<Tabs bind:selected={selectedTab}>
<Tab value="insert" size="xs">
<div class="m-1 flex flex-row gap-2">
<Icon data={faPlus} />
<span>Insert</span>
</div>
</Tab>
<Tab value="settings" size="xs">
<div class="m-1 flex flex-row gap-2">
<Icon data={faSliders} />
<span>Settings</span>
</div>
</Tab>
<svelte:fragment slot="content">
<TabContent class="overflow-auto" value="settings">
{#if $selectedComponent !== undefined}
<SettingsPanel />
{:else}
<div class="p-2 min-w-[150px] text-sm">No component selected.</div>
{/if}
</TabContent>
<TabContent value="insert">
<ComponentList />
</TabContent>
</svelte:fragment>
</Tabs>
{#if $connectingInput.opened}
<div
class="fixed top-32 p-2 z-50 flex justify-center items-center"
transition:fly={{ duration: 100, y: -100 }}
>
<Alert title="Connecting" type="info">
<div class="flex gap-2 flex-col">
Click on the output of the component you want to connect to on the left panel.
<div>
<Button
color="blue"
variant="border"
size="xs"
on:click={() => {
$connectingInput.opened = false
$connectingInput.input = undefined
}}
>
Stop connecting
</Button>
</Pane>
<Pane size={$connectingInput?.opened ? 0 : 30}>
<div class="relative h-full w-full">
<InlineScriptsPanel />
</div>
</Pane>
</Splitpanes>
</SplitPanesWrapper>
</Pane>
<Pane size={21} minSize={5} maxSize={33}>
<div class="relative flex flex-col h-full">
<Tabs bind:selected={selectedTab}>
<Tab value="insert" size="xs">
<div class="m-1 flex flex-row gap-2">
<Icon data={faPlus} />
<span>Insert</span>
</div>
</Tab>
<Tab value="settings" size="xs">
<div class="m-1 flex flex-row gap-2">
<Icon data={faSliders} />
<span>Settings</span>
</div>
</Tab>
<svelte:fragment slot="content">
<TabContent class="overflow-auto" value="settings">
{#if $selectedComponent !== undefined}
<SettingsPanel />
{:else}
<div class="p-2 min-w-[150px] text-sm">No component selected.</div>
{/if}
</TabContent>
<TabContent value="insert">
<ComponentList />
</TabContent>
</svelte:fragment>
</Tabs>
{#if $connectingInput.opened}
<div
class="fixed top-32 p-2 z-50 flex justify-center items-center"
transition:fly={{ duration: 100, y: -100 }}
>
<Alert title="Connecting" type="info">
<div class="flex gap-2 flex-col">
Click on the output of the component you want to connect to on the left panel.
<div>
<Button
color="blue"
variant="border"
size="xs"
on:click={() => {
$connectingInput.opened = false
$connectingInput.input = undefined
}}
>
Stop connecting
</Button>
</div>
</div>
</div>
</Alert>
</div>
{/if}
</div>
</Pane>
</Alert>
</div>
{/if}
</div>
</Pane>
</Splitpanes>
</SplitPanesWrapper>
{/if}
{:else}
@@ -2,7 +2,7 @@
import { getContext } from 'svelte'
import type { AppEditorContext } from '../../types'
import SplitPanesWrapper from '$lib/components/splitPanes/SplitPanesWrapper.svelte'
import { Pane } from 'svelte-splitpanes'
import { Pane, Splitpanes } from 'svelte-splitpanes'
import InlineScriptsPanelList from './InlineScriptsPanelList.svelte'
import InlineScriptEditorPanel from './InlineScriptEditorPanel.svelte'
import InlineScriptEditor from './InlineScriptEditor.svelte'
@@ -12,43 +12,45 @@
let selectedScriptComponentId: string | undefined = undefined
</script>
<SplitPanesWrapper panesClass="!overflow-visible">
<Pane size={25}>
<InlineScriptsPanelList bind:selectedScriptComponentId />
</Pane>
<Pane size={75}>
{#each $lazyGrid as gridComponent, index (index)}
{#if gridComponent.data.id === selectedScriptComponentId}
<InlineScriptEditorPanel
id={gridComponent.data.id}
bind:componentInput={gridComponent.data.componentInput}
/>
{/if}
<SplitPanesWrapper>
<Splitpanes class="!overflow-visible">
<Pane size={25}>
<InlineScriptsPanelList bind:selectedScriptComponentId />
</Pane>
<Pane size={75}>
{#each $lazyGrid as gridComponent, index (index)}
{#if gridComponent.data.id === selectedScriptComponentId}
<InlineScriptEditorPanel
id={gridComponent.data.id}
bind:componentInput={gridComponent.data.componentInput}
/>
{/if}
{#if gridComponent.data.type === 'tablecomponent'}
{#each gridComponent.data.actionButtons as actionButton, index (index)}
{#if actionButton.id === selectedScriptComponentId}
<InlineScriptEditorPanel
id={actionButton.id}
bind:componentInput={actionButton.componentInput}
/>
{/if}
{/each}
{/if}
{/each}
{#each $app.unusedInlineScripts as unusedInlineScript, index (index)}
{#if `unused-${index}` === selectedScriptComponentId}
<InlineScriptEditor
id={`unused-${index}`}
bind:name={unusedInlineScript.name}
bind:inlineScript={unusedInlineScript.inlineScript}
on:delete={() => {
// remove the script from the array at the index
$app.unusedInlineScripts.splice(index, 1)
$app.unusedInlineScripts = [...$app.unusedInlineScripts]
}}
/>
{/if}
{/each}
</Pane>
{#if gridComponent.data.type === 'tablecomponent'}
{#each gridComponent.data.actionButtons as actionButton, index (index)}
{#if actionButton.id === selectedScriptComponentId}
<InlineScriptEditorPanel
id={actionButton.id}
bind:componentInput={actionButton.componentInput}
/>
{/if}
{/each}
{/if}
{/each}
{#each $app.unusedInlineScripts as unusedInlineScript, index (index)}
{#if `unused-${index}` === selectedScriptComponentId}
<InlineScriptEditor
id={`unused-${index}`}
bind:name={unusedInlineScript.name}
bind:inlineScript={unusedInlineScript.inlineScript}
on:delete={() => {
// remove the script from the array at the index
$app.unusedInlineScripts.splice(index, 1)
$app.unusedInlineScripts = [...$app.unusedInlineScripts]
}}
/>
{/if}
{/each}
</Pane>
</Splitpanes>
</SplitPanesWrapper>
@@ -16,7 +16,7 @@
import DrawerContent from '../common/drawer/DrawerContent.svelte'
import HighlightCode from '../HighlightCode.svelte'
import LogViewer from '../LogViewer.svelte'
import { Pane } from 'svelte-splitpanes'
import { Pane, Splitpanes } from 'svelte-splitpanes'
import SplitPanesWrapper from '../splitPanes/SplitPanesWrapper.svelte'
import { Loader2 } from 'lucide-svelte'
@@ -77,29 +77,31 @@
because SplitPanesWrapper uses the parent element as a reference point.
-->
{#if selectedTab === 'logs'}
<SplitPanesWrapper horizontal>
<Pane class="relative">
<LogViewer
duration={previewJob?.['duration_ms']}
content={previewJob?.logs}
isLoading={previewIsLoading}
/>
</Pane>
<Pane>
{#if previewJob != undefined && 'result' in previewJob && previewJob.result != undefined}
<pre class="relative w-full h-full p-2"
><DisplayResult result={previewJob.result} />
<SplitPanesWrapper>
<Splitpanes horizontal>
<Pane class="relative">
<LogViewer
duration={previewJob?.['duration_ms']}
content={previewJob?.logs}
isLoading={previewIsLoading}
/>
</Pane>
<Pane>
{#if previewJob != undefined && 'result' in previewJob && previewJob.result != undefined}
<pre class="relative w-full h-full p-2"
><DisplayResult result={previewJob.result} />
</pre>
{:else}
<div class="text-sm text-gray-600 p-2">
{#if previewIsLoading}
<Loader2 class="animate-spin" />
{:else}
Test to see the result here'
{/if}
</div>
{/if}
</Pane>
{:else}
<div class="text-sm text-gray-600 p-2">
{#if previewIsLoading}
<Loader2 class="animate-spin" />
{:else}
Test to see the result here'
{/if}
</div>
{/if}
</Pane>
</Splitpanes>
</SplitPanesWrapper>
{/if}
<TabContent value="history" class="p-2">
@@ -1,6 +1,5 @@
<script lang="ts">
import { afterUpdate } from 'svelte'
import { Splitpanes } from 'svelte-splitpanes'
/**
* This component should be used instead of `Splitpanes` if the wrapper `Splitpanes`
@@ -10,7 +9,6 @@
/** This element will act as the reference point to the `Splitpanes`
* and the top difference will be calculated from it. */
export let refElement: HTMLElement | undefined = undefined
export let panesClass = ''
let wrapper: HTMLDivElement
let gap = 0
@@ -33,7 +31,5 @@
class="h-full {$$props.class || ''}"
style="max-height: calc(100% - {gap}px) !important;"
>
<Splitpanes class={panesClass} {...$$restProps}>
<slot />
</Splitpanes>
<slot />
</div>