feat: bundle data-pipeline drafts into the DB-backed user draft system

Pipeline drafts were browser-only (localStorage `pipeline-<folder>`), so they
didn't sync across devices, weren't server-visible, and never showed in the
drafts list. Store them instead as one per-user `draft` row of a new
`data_pipeline` kind, keyed at the folder (`f/<folder>/data_pipeline`), holding
the same `{ drafts, activeDraftPath }` bundle.

Stage 1 — backend kind: add `data_pipeline` to DRAFT_KIND (migration) and
`UserDraftItemKind` (deployed_table=None, private). The list/update handlers
and folder-path access check already cover a backing-table-less kind.

Stage 2 — sync: add `GET /drafts/get_own/{kind}/{path}` so an editor with no
deployed-overlay GET can load its own draft. The pipeline page now hydrates
from the DB on mount (one-time localStorage import for in-flight drafts) and
persists via UserDraftDbSyncer (debounce + optimistic-concurrency), keeping a
localStorage crash mirror.

Stage 3 — surface: the drafts review page renders the bundle as a "pipeline"
row that opens `/pipeline/<folder>` (open-only; excluded from bulk deploy).

Verified end-to-end in-browser: DB-seeded draft hydrates to "Edit (1)", edits
persist back, and the row shows with Open pipeline / Discard.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
2026-06-16 11:49:23 +02:00
parent 4cf44280ce
commit 1cd2cc5fbd
13 changed files with 341 additions and 66 deletions
@@ -0,0 +1,64 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value as \"value!: sqlx::types::Json<Box<serde_json::value::RawValue>>\", created_at\n FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email = $4",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value!: sqlx::types::Json<Box<serde_json::value::RawValue>>",
"type_info": "Json"
},
{
"ordinal": 1,
"name": "created_at",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "draft_kind",
"kind": {
"Enum": [
"script",
"flow",
"app",
"raw_app",
"resource",
"variable",
"trigger_schedule",
"trigger_webhook",
"trigger_default_email",
"trigger_email",
"trigger_http",
"trigger_websocket",
"trigger_postgres",
"trigger_kafka",
"trigger_nats",
"trigger_mqtt",
"trigger_sqs",
"trigger_gcp",
"trigger_azure",
"trigger_poll",
"trigger_cli",
"trigger_nextcloud",
"trigger_google",
"trigger_github",
"data_pipeline"
]
}
}
},
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "6b9348e60cc1ce158314a93fc7aa55a9f8fa854b29edcea83710a9170124edf0"
}
@@ -0,0 +1,2 @@
-- Postgres cannot drop a single enum value; leaving 'data_pipeline' in
-- DRAFT_KIND is harmless on rollback.
@@ -0,0 +1,6 @@
-- A `data_pipeline` draft bundles every unsaved pipeline script of a folder
-- into a single row keyed at the folder path (typ has no deployed backing
-- table — see UserDraftItemKind::deployed_table). Lets the asset-graph view
-- store its in-flight drafts in the per-user DB draft sync instead of
-- browser-local storage.
ALTER TYPE DRAFT_KIND ADD VALUE IF NOT EXISTS 'data_pipeline';
+30
View File
@@ -7965,6 +7965,35 @@ paths:
"404":
description: no draft for that owner at that path
/w/{workspace}/drafts/get_own/{kind}/{path}:
get:
summary: fetch the current user's own draft content at a path (any kind)
operationId: getOwnDraft
tags:
- draft
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: kind
in: path
required: true
schema:
$ref: "#/components/schemas/UserDraftItemKind"
- $ref: "#/components/parameters/ScriptPath"
responses:
"200":
description: the user's draft content, or null when none exists
content:
application/json:
schema:
nullable: true
type: object
properties:
value: {}
created_at:
type: string
format: date-time
required: [value, created_at]
/w/{workspace}/drafts/update/{kind}/{path}:
post:
summary: upsert (or clear) the current user's draft at a path
@@ -21463,6 +21492,7 @@ components:
- trigger_nextcloud
- trigger_google
- trigger_github
- data_pipeline
# Do not change next line. It is used by python-client for pre-processing
# -- INLINE START --
OpenFlow:
+32
View File
@@ -25,6 +25,7 @@ pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_drafts))
.route("/get/{kind}/{*path}", get(get_draft_for_user))
.route("/get_own/{kind}/{*path}", get(get_own_draft))
.route("/update/{kind}/{*path}", post(update_draft))
}
@@ -393,6 +394,37 @@ async fn get_draft_for_user(
})
}
/// Fetch the AUTHED user's OWN draft at a path, for any kind — including
/// private kinds (`shares_drafts_across_users() == false`). Backs editors with
/// no deployed-item GET to overlay a draft onto: the `data_pipeline` bundle is
/// keyed at a folder path with no runnable to hang `get_draft` on, so it loads
/// its in-flight state from here. Returns `null` (200) when the user has no
/// draft there, so a fresh pipeline isn't a 404. Secret-variable values come
/// back `$encrypted:`-prefixed, same as `get_draft_for_user` — variable editors
/// use their own overlay GET, not this route.
async fn get_own_draft(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((w_id, kind, path)): Path<(String, UserDraftItemKind, windmill_common::utils::StripPath)>,
) -> Result<Json<Option<DraftForUser>>> {
let path = path.to_path();
require_can_read_path(&authed, &user_db, &w_id, kind, path).await?;
let row = sqlx::query_as!(
DraftForUser,
r#"SELECT value as "value!: sqlx::types::Json<Box<serde_json::value::RawValue>>", created_at
FROM draft
WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email = $4"#,
&w_id,
path,
kind as UserDraftItemKind,
&authed.email,
)
.fetch_optional(&db)
.await?;
Ok(Json(row))
}
/// The deployed table RLS resolves item-level `extra_perms` against.
/// Delegates to `UserDraftItemKind::deployed_table()` (the shared single
/// source); `None` kinds fall through to the path-only access check.
+10 -1
View File
@@ -53,6 +53,10 @@ pub enum UserDraftItemKind {
TriggerNextcloud,
TriggerGoogle,
TriggerGithub,
/// All unsaved scripts of one data pipeline, bundled into a single draft
/// keyed at the pipeline's folder path. Not a runnable: it has no deployed
/// backing table and is private to its owner.
DataPipeline,
}
impl UserDraftItemKind {
@@ -84,12 +88,13 @@ impl UserDraftItemKind {
UserDraftItemKind::TriggerNextcloud => "trigger_nextcloud",
UserDraftItemKind::TriggerGoogle => "trigger_google",
UserDraftItemKind::TriggerGithub => "trigger_github",
UserDraftItemKind::DataPipeline => "data_pipeline",
}
}
/// Every variant, for code that must enumerate kinds (e.g. generating
/// the `draft_only` existence SQL).
pub const ALL: [UserDraftItemKind; 24] = [
pub const ALL: [UserDraftItemKind; 25] = [
UserDraftItemKind::Script,
UserDraftItemKind::Flow,
UserDraftItemKind::App,
@@ -114,6 +119,7 @@ impl UserDraftItemKind {
UserDraftItemKind::TriggerNextcloud,
UserDraftItemKind::TriggerGoogle,
UserDraftItemKind::TriggerGithub,
UserDraftItemKind::DataPipeline,
];
/// The deployed table backing this kind, keyed by `(workspace_id, path)`.
@@ -144,6 +150,9 @@ impl UserDraftItemKind {
TriggerEmail | TriggerDefaultEmail => Some("email_trigger"),
TriggerWebhook | TriggerPoll | TriggerCli | TriggerNextcloud | TriggerGoogle
| TriggerGithub => None,
// Keyed at a folder path, not a runnable; access falls back to the
// path-only (folder write) check.
DataPipeline => None,
}
}
@@ -172,6 +172,13 @@
})
})
// A data-pipeline bundle isn't deployable from this page — deploy happens
// per-script inside the pipeline view. Exclude it from every selection path
// so the bulk "Deploy N drafts" never tries to deploy a bundle.
function isRowDeployable(i: { key: string; draftKind: Row['draftKind'] }): boolean {
return deploymentStatus[i.key]?.status !== 'deployed' && i.draftKind !== 'data_pipeline'
}
let selectedItems = $state<string[]>([])
let deploying = $state(false)
// Select all on the first non-empty load (deploy-all is the common intent);
@@ -198,9 +205,7 @@
$effect(() => {
if (!hasAutoSelected && items.length > 0) {
selectedItems = items
.filter((i) => deploymentStatus[i.key]?.status !== 'deployed')
.map((i) => i.key)
selectedItems = items.filter(isRowDeployable).map((i) => i.key)
hasAutoSelected = true
}
})
@@ -210,16 +215,12 @@
// Drafts resource: deploy/discard drop items, and stale keys left in
// selectedItems are simply ignored here (and by deploySelected).
let selectedCount = $derived(
items.filter(
(i) => selectedItems.includes(i.key) && deploymentStatus[i.key]?.status !== 'deployed'
).length
items.filter((i) => selectedItems.includes(i.key) && isRowDeployable(i)).length
)
let allSelected = $derived(
items.length > 0 &&
items
.filter((i) => deploymentStatus[i.key]?.status !== 'deployed')
.every((i) => selectedItems.includes(i.key))
items.filter(isRowDeployable).every((i) => selectedItems.includes(i.key))
)
function toggleItem(item: { key: string }) {
@@ -231,9 +232,7 @@
}
function selectAll() {
selectedItems = items
.filter((i) => deploymentStatus[i.key]?.status !== 'deployed')
.map((i) => i.key)
selectedItems = items.filter(isRowDeployable).map((i) => i.key)
}
function deselectAll() {
@@ -342,7 +341,19 @@
trigger_azure: '/azure_triggers',
trigger_email: '/email_triggers'
}
// A data-pipeline bundle is keyed at `f/<folder>/data_pipeline`; its editor
// is the pipeline view of that folder.
function pipelineFolderFromPath(path: string): string | undefined {
const segs = path.split('/')
return segs[0] === 'f' && segs.length >= 2 ? segs[1] : undefined
}
function draftEditUrl(d: Row): string | undefined {
if (d.draftKind === 'data_pipeline') {
const folder = pipelineFolderFromPath(d.path)
return folder
? `/pipeline/${encodeURIComponent(folder)}?workspace=${encodeURIComponent(currentWorkspaceId)}`
: undefined
}
const listPage = LIST_PAGE_FOR_KIND[d.draftKind]
if (listPage) {
return `${listPage}?workspace=${encodeURIComponent(currentWorkspaceId)}#${d.path}`
@@ -365,6 +376,12 @@
// auto-generated `draft_{uuid}` path so it isn't shown in bold (the row still
// shows the storage path in its secondary line).
function displayPath(d: Row): string {
// The pipeline bundle's storage path (`f/<folder>/data_pipeline`) is an
// implementation detail — show the folder it belongs to.
if (d.draftKind === 'data_pipeline') {
const folder = pipelineFolderFromPath(d.path)
return folder ? `f/${folder}` : d.path
}
const path = d.draft_path ?? d.path
if (AUTO_GEN_DRAFT_RE.test(path)) return ''
const segs = path.split('/')
@@ -379,6 +396,7 @@
// draft and a script draft at the same path are indistinguishable.
function kindLabel(kind: Row['draftKind']): string {
if (kind === 'raw_app') return 'app'
if (kind === 'data_pipeline') return 'pipeline'
if (kind === 'trigger_schedule') return 'schedule'
if (kind.startsWith('trigger_')) return `${kind.slice('trigger_'.length)} trigger`
return kind
@@ -392,7 +410,7 @@
{selectedItems}
{deploymentStatus}
{allSelected}
selectablePredicate={(item) => deploymentStatus[item.key]?.status !== 'deployed'}
selectablePredicate={(item) => isRowDeployable(item as unknown as Row)}
onToggleItem={toggleItem}
onSelectAll={selectAll}
onDeselectAll={deselectAll}
@@ -465,14 +483,25 @@
</Tooltip>
{/if}
{#if deploymentStatus[draftItem.key]?.status !== 'deployed'}
<Button
unifiedSize="xs"
variant="subtle"
startIcon={{ icon: DiffIcon }}
onClick={() => showDiff(draftItem)}
>
Show diff
</Button>
{#if draftItem.draftKind === 'data_pipeline'}
<!-- A bundle isn't diffable/deployable here — its scripts deploy
individually inside the pipeline view. -->
{@const openUrl = draftEditUrl(draftItem)}
{#if openUrl}
<Button unifiedSize="xs" variant="subtle" startIcon={{ icon: ArrowRight }} href={openUrl}>
Open pipeline
</Button>
{/if}
{:else}
<Button
unifiedSize="xs"
variant="subtle"
startIcon={{ icon: DiffIcon }}
onClick={() => showDiff(draftItem)}
>
Show diff
</Button>
{/if}
<Button
unifiedSize="xs"
variant="subtle"
@@ -43,6 +43,7 @@
| 'gcp_trigger'
| 'azure_trigger'
| 'email_trigger'
| 'data_pipeline'
triggerKind?: string | undefined
summary?: string | undefined
path: string
@@ -16,7 +16,8 @@
LayoutDashboard,
Mail,
Route,
Unplug
Unplug,
Workflow
} from 'lucide-svelte'
interface Props {
@@ -51,6 +52,7 @@
| 'gcp_trigger'
| 'azure_trigger'
| 'email_trigger'
| 'data_pipeline'
/** For 'trigger' kind, specifies the specific trigger type (routes, schedules, etc.) */
triggerKind?: string | undefined
size?: number
@@ -117,6 +119,8 @@
<Mail {size} class="text-gray-400" />
{:else if effectiveKind === 'trigger'}
<Calendar {size} class="text-gray-400" />
{:else if effectiveKind === 'data_pipeline'}
<Workflow {size} class="text-indigo-500" />
{:else}
<div style="width: {size}px;"></div>
{/if}
+2 -1
View File
@@ -34,7 +34,8 @@ export const USER_DRAFT_ITEM_KINDS = [
'trigger_cli',
'trigger_nextcloud',
'trigger_google',
'trigger_github'
'trigger_github',
'data_pipeline'
] as const satisfies readonly UserDraftItemKind[]
// Reverse direction: every union member must appear in the array above.
+2 -1
View File
@@ -43,7 +43,8 @@ const ITEM_KINDS = [
'trigger_cli',
'trigger_nextcloud',
'trigger_google',
'trigger_github'
'trigger_github',
'data_pipeline'
] as const satisfies readonly UserDraftItemKind[]
type _Exhaustive =
+3
View File
@@ -43,6 +43,9 @@ export type Kind =
// Legacy generic kind used by the cross-workspace `DeployWorkspace` UI,
// which carries the trigger sub-kind in `additionalInformation`.
| 'trigger'
// A data-pipeline draft bundle (drafts list only — not a deployable item;
// opens the pipeline view instead).
| 'data_pipeline'
export const ALL_DEPLOYABLE: WorkspaceDeployUISettings = {
include_path: [],
@@ -45,6 +45,8 @@
type DraftTriggerSource
} from '$lib/components/assets/AssetGraph/pipelineTemplates'
import { decodeState, encodeState } from '$lib/utils'
import { DraftService } from '$lib/gen'
import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte'
import { onMount, tick, untrack } from 'svelte'
import { aiChatManager } from '$lib/components/copilot/chat/AIChatManager.svelte'
import {
@@ -248,43 +250,120 @@
}
})
// Per-folder localStorage key. Matches the flow-builder pattern so
// reloading /pipeline/<folder> restores in-flight drafts. We serialize
// the drafts map as an entry array (Map doesn't survive JSON.stringify).
// All of this folder's in-flight drafts live in ONE per-user DB draft
// (typ `data_pipeline`) keyed at the folder, so they sync across devices
// and surface in the global drafts list — replacing the prior
// browser-only blob. The `f/<folder>/...` path drives the backend's
// folder-write access check. localStorage is kept as a synchronous crash
// mirror (no size cap, survives a hard close inside the debounce window)
// but is only ever READ for the one-time import below; the DB is the
// source of truth on load.
const PIPELINE_DRAFT_KIND = 'data_pipeline' as const
let pipelineDraftPath = $derived(`f/${folder}/data_pipeline`)
let storageKey = $derived(`pipeline-${folder}`)
type PipelineDraftBundle = { drafts: Array<[string, Draft]>; activeDraftPath?: string }
onMount(() => {
if (typeof localStorage === 'undefined') return
// Gate the persist effect until the initial DB load resolves so empty
// pre-hydration state can't clobber the server copy. `lastPersistedBundle`
// holds the last value we pushed so an unchanged re-render (and the
// just-loaded value itself) isn't re-POSTed.
let draftsHydrated = $state(false)
let lastPersistedBundle: string | undefined = undefined
function restoreBundle(bundle: PipelineDraftBundle) {
if (Array.isArray(bundle.drafts)) {
const loaded = new Map<string, Draft>()
for (const entry of bundle.drafts) {
if (entry && typeof entry[0] === 'string' && entry[1]?.script) {
const d = entry[1] as Draft
// Backfill localId for state persisted by older builds.
if (typeof d.localId !== 'string' || d.localId === '') {
d.localId = newDraftLocalId()
}
loaded.set(entry[0], d)
}
}
if (loaded.size > 0) drafts = loaded
}
if (typeof bundle.activeDraftPath === 'string') {
activeDraftPath = bundle.activeDraftPath
}
}
// The pre-DB localStorage blob, for the one-time migration when the user
// has no DB draft yet.
function readLocalBundle(): PipelineDraftBundle | undefined {
if (typeof localStorage === 'undefined') return undefined
const raw = localStorage.getItem(`pipeline-${folder}`)
if (!raw) return
if (!raw) return undefined
try {
const state = decodeState(raw)
if (Array.isArray(state?.drafts)) {
const loaded = new Map<string, Draft>()
for (const entry of state.drafts) {
if (entry && typeof entry[0] === 'string' && entry[1]?.script) {
const d = entry[1] as Draft
// Backfill localId for state persisted by older builds.
if (typeof d.localId !== 'string' || d.localId === '') {
d.localId = newDraftLocalId()
}
loaded.set(entry[0], d)
}
if (state && (Array.isArray(state.drafts) || typeof state.activeDraftPath === 'string')) {
return {
drafts: Array.isArray(state.drafts) ? state.drafts : [],
activeDraftPath: typeof state.activeDraftPath === 'string' ? state.activeDraftPath : undefined
}
if (loaded.size > 0) drafts = loaded
}
if (typeof state?.activeDraftPath === 'string') {
activeDraftPath = state.activeDraftPath
}
} catch (e) {
console.warn('failed to restore pipeline state', e)
console.warn('failed to read local pipeline state', e)
}
return undefined
}
onMount(() => {
void hydrateDrafts()
})
// Debounced persist: reruns whenever drafts / activeDraftPath change.
// 500 ms matches FlowBuilder.saveSessionDraft; balances typing churn
// against losing state to a crash.
let persistTimer: number | undefined = undefined
async function hydrateDrafts() {
const ws = $workspaceStore
const path = pipelineDraftPath
try {
let bundle: PipelineDraftBundle | undefined
let serverSavedAt: string | undefined
if (ws) {
const row = await DraftService.getOwnDraft({
workspace: ws,
kind: PIPELINE_DRAFT_KIND,
path
})
if (row?.value) {
bundle = row.value as PipelineDraftBundle
serverSavedAt = row.created_at
}
}
// One-time migration: no DB draft yet, but an older build left a
// localStorage blob — adopt it and let the persist effect push it up.
let migratedFromLocal = false
if (!bundle) {
const local = readLocalBundle()
if (local) {
bundle = local
migratedFromLocal = true
}
}
if (bundle) restoreBundle(bundle)
// Seed the conflict baseline: server timestamp when loaded from the
// DB, none otherwise (first save omits last_sync → backend first-push
// branch). A local migration counts as "nothing server-side yet".
UserDraftDbSyncer.recordRemoteSync(
{ workspace: ws ?? '', itemKind: PIPELINE_DRAFT_KIND, path },
migratedFromLocal ? undefined : serverSavedAt
)
// Record what we loaded so the first persist run is a no-op — UNLESS
// we migrated from localStorage, which must push to the DB once.
if (!migratedFromLocal) {
lastPersistedBundle = bundle ? JSON.stringify(bundle) : undefined
}
} catch (e) {
console.warn('failed to load pipeline drafts', e)
} finally {
draftsHydrated = true
}
}
// Persist on change: debounced DB sync (UserDraftDbSyncer handles the
// debounce + optimistic-concurrency) plus a synchronous localStorage
// mirror for crash recovery before the first DB confirm.
$effect(() => {
// Track deps explicitly so Svelte 5 re-runs on mutation.
// For the active draft, also snapshot the latest live body writes
@@ -309,23 +388,37 @@
})
const activePath = activeDraftPath
const key = storageKey
const ws = $workspaceStore
const path = pipelineDraftPath
const hydrated = draftsHydrated
untrack(() => {
if (typeof localStorage === 'undefined') return
if (persistTimer != undefined) clearTimeout(persistTimer)
persistTimer = window.setTimeout(() => {
try {
if (serialized.length === 0 && !activePath) {
localStorage.removeItem(key)
} else {
localStorage.setItem(
key,
encodeState({ drafts: serialized, activeDraftPath: activePath })
)
}
} catch (e) {
console.warn('failed to persist pipeline state', e)
// Don't touch storage until the initial load settled.
if (!hydrated) return
const isEmpty = serialized.length === 0 && !activePath
const bundle: PipelineDraftBundle | undefined = isEmpty
? undefined
: { drafts: serialized, activeDraftPath: activePath }
// localStorage crash mirror — synchronous, no debounce, no size cap.
try {
if (typeof localStorage !== 'undefined') {
if (isEmpty) localStorage.removeItem(key)
else localStorage.setItem(key, encodeState({ drafts: serialized, activeDraftPath: activePath }))
}
}, 500)
} catch (e) {
console.warn('failed to mirror pipeline state', e)
}
const serializedBundle = bundle ? JSON.stringify(bundle) : undefined
if (serializedBundle === lastPersistedBundle) return
lastPersistedBundle = serializedBundle
if (!ws) return
void UserDraftDbSyncer.save({
workspace: ws,
itemKind: PIPELINE_DRAFT_KIND,
path,
// `null` deletes the bundle once the last draft is gone.
value: bundle ?? null,
auto: true
})
})
})