diff --git a/.claude/scheduled_tasks.lock b/.claude/scheduled_tasks.lock deleted file mode 100644 index 3e22b93d1f..0000000000 --- a/.claude/scheduled_tasks.lock +++ /dev/null @@ -1 +0,0 @@ -{"sessionId":"2cfbff54-4fc8-4390-ab4d-83b737f1fb6b","pid":59389,"procStart":"864605","acquiredAt":1778411865080} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 5f733611de..e80b4f32ca 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,5 @@ backend/chrome_profiler.json .fast-check/ __pycache__/ .playwright-mcp/ -.codex \ No newline at end of file +.codex +.claude/scheduled_tasks.lock diff --git a/after-typing.png b/after-typing.png deleted file mode 100644 index e56bcdd06e..0000000000 Binary files a/after-typing.png and /dev/null differ diff --git a/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json b/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json new file mode 100644 index 0000000000..cb20ec2ffb --- /dev/null +++ b/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job\n SET args = CASE\n WHEN args ? 'partition'\n THEN $1 || jsonb_build_object('partition', args -> 'partition')\n ELSE $1\n END,\n preprocessed = TRUE\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b" +} diff --git a/backend/.sqlx/query-751f836dc8f78c330387456dd68a8803972c7b3e2b6a2b95c27f15068bed2ca5.json b/backend/.sqlx/query-751f836dc8f78c330387456dd68a8803972c7b3e2b6a2b95c27f15068bed2ca5.json new file mode 100644 index 0000000000..a784094d72 --- /dev/null +++ b/backend/.sqlx/query-751f836dc8f78c330387456dd68a8803972c7b3e2b6a2b95c27f15068bed2ca5.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "751f836dc8f78c330387456dd68a8803972c7b3e2b6a2b95c27f15068bed2ca5" +} diff --git a/backend/.sqlx/query-e3ee812acd5bb9d5af39ca7dc61481ddd56a641e8e6f6c7a145f2cd5f3dc4602.json b/backend/.sqlx/query-e3ee812acd5bb9d5af39ca7dc61481ddd56a641e8e6f6c7a145f2cd5f3dc4602.json deleted file mode 100644 index 2acb3c771e..0000000000 --- a/backend/.sqlx/query-e3ee812acd5bb9d5af39ca7dc61481ddd56a641e8e6f6c7a145f2cd5f3dc4602.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "e3ee812acd5bb9d5af39ca7dc61481ddd56a641e8e6f6c7a145f2cd5f3dc4602" -} diff --git a/backend/windmill-queue/src/asset_dispatch.rs b/backend/windmill-queue/src/asset_dispatch.rs index 43e05fed0b..21fe0d292b 100644 --- a/backend/windmill-queue/src/asset_dispatch.rs +++ b/backend/windmill-queue/src/asset_dispatch.rs @@ -351,16 +351,49 @@ fn is_partition_bearing_ref(trigger_ref: &str) -> bool { trigger_ref.contains(PARTITION_TOKEN) } -/// Distinct partition-bearing asset inputs an AND subscriber declares -/// (its `{partition}`-token `// on` lines). Reference inputs (no token) -/// and non-asset triggers are presence-only and do not gate the join in -/// v1, so they are excluded from the required set. -async fn count_required_join_inputs( +/// Record an AND input arrival for `(subscriber, partition)` and report +/// whether every partition-bearing input is now present for that +/// partition. The record -> count -> clear sequence runs in one +/// transaction guarded by a transaction-scoped advisory lock keyed on the +/// slot: the same subscriber's last two partition-bearing inputs can +/// complete concurrently on different workers, and a check-then-act on a +/// pooled connection would let both observe "complete" and dispatch +/// twice. Serializing per `(workspace, subscriber, partition)` makes the +/// gate fire exactly once; the lock is released on commit/rollback. +/// Idempotent per input (PK conflict ignored); the slot is cleared on +/// fire so later writes re-accumulate and can re-materialize. +/// +/// `required` = the distinct partition-bearing inputs the subscriber +/// declares (its `{partition}`-token `// on` lines). Reference inputs +/// (no token) and non-asset triggers are presence-only and excluded. +async fn record_and_check_join_slot( db: &DB, workspace_id: &str, subscriber_path: &str, -) -> Result { - let n = sqlx::query_scalar!( + partition: &str, + trigger_ref: &str, +) -> Result { + let mut tx = db.begin().await?; + let lock_key = format!("{workspace_id}|{subscriber_path}|{partition}"); + sqlx::query!( + "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", + lock_key, + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + r#"INSERT INTO join_pending_inputs + (workspace_id, subscriber_path, partition, trigger_ref) + VALUES ($1, $2, $3, $4) + ON CONFLICT DO NOTHING"#, + workspace_id, + subscriber_path, + partition, + trigger_ref, + ) + .execute(&mut *tx) + .await?; + let required = sqlx::query_scalar!( r#"SELECT count(DISTINCT trigger_ref) AS "n!" FROM script_trigger WHERE workspace_id = $1 @@ -372,36 +405,8 @@ async fn count_required_join_inputs( subscriber_path, PARTITION_TOKEN, ) - .fetch_one(db) + .fetch_one(&mut *tx) .await?; - Ok(n) -} - -/// Record an AND input arrival for `(subscriber, partition)` and report -/// whether every partition-bearing input is now present for that -/// partition. Idempotent per input (PK conflict ignored), so a re-fired -/// upstream doesn't double-count. On completion the slot is cleared so -/// later writes re-accumulate and can re-materialize the partition. -async fn record_and_check_join_slot( - db: &DB, - workspace_id: &str, - subscriber_path: &str, - partition: &str, - trigger_ref: &str, -) -> Result { - sqlx::query!( - r#"INSERT INTO join_pending_inputs - (workspace_id, subscriber_path, partition, trigger_ref) - VALUES ($1, $2, $3, $4) - ON CONFLICT DO NOTHING"#, - workspace_id, - subscriber_path, - partition, - trigger_ref, - ) - .execute(db) - .await?; - let required = count_required_join_inputs(db, workspace_id, subscriber_path).await?; let received = sqlx::query_scalar!( r#"SELECT count(DISTINCT trigger_ref) AS "n!" FROM join_pending_inputs @@ -410,9 +415,10 @@ async fn record_and_check_join_slot( subscriber_path, partition, ) - .fetch_one(db) + .fetch_one(&mut *tx) .await?; - if required > 0 && received >= required { + let fire = required > 0 && received >= required; + if fire { sqlx::query!( r#"DELETE FROM join_pending_inputs WHERE workspace_id = $1 AND subscriber_path = $2 AND partition = $3"#, @@ -420,12 +426,11 @@ async fn record_and_check_join_slot( subscriber_path, partition, ) - .execute(db) + .execute(&mut *tx) .await?; - Ok(true) - } else { - Ok(false) } + tx.commit().await?; + Ok(fire) } async fn push_subscriber( diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index d070800131..8b68e436ec 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -763,9 +763,20 @@ pub async fn process_completed_job( )) })?; } else if let Some(preprocessed_args) = preprocessed_args { - // Update script args to preprocessed args + // Update script args to preprocessed args, but preserve a + // resolved pipeline `partition` (injected before the body ran + // by resolve_partition_for_job). Run identity is immutable — + // the preprocessor must not change or drop it, or the asset + // cascade would read no partition for this producer. sqlx::query!( - "UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2", + "UPDATE v2_job + SET args = CASE + WHEN args ? 'partition' + THEN $1 || jsonb_build_object('partition', args -> 'partition') + ELSE $1 + END, + preprocessed = TRUE + WHERE id = $2", Json(preprocessed_args) as Json>>, job.id ) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 97450836d1..ac84191f44 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -4193,8 +4193,11 @@ async fn resolve_partition_for_job( use windmill_common::partition::{resolve_partition, PARTITION_ARG}; use windmill_parser::asset_parser::PartitionKind; - // Only deployed scripts participate in asset pipelines. - if !matches!(job.kind, JobKind::Script) { + // Only deployed scripts participate in asset pipelines. Cheap + // substring guard so the overwhelming majority of script jobs (no + // `// partitioned` line) skip the full annotation scan on the hot + // path; a false positive only costs one extra parse, never wrong. + if !matches!(job.kind, JobKind::Script) || !code.contains("partitioned") { return Ok(None); } let Some(spec) = windmill_parser::asset_parser::parse_pipeline_annotations(code).partition diff --git a/btn-cut.png b/btn-cut.png deleted file mode 100644 index d34fecb125..0000000000 Binary files a/btn-cut.png and /dev/null differ diff --git a/btn-fixed.png b/btn-fixed.png deleted file mode 100644 index ea85c2a43c..0000000000 Binary files a/btn-fixed.png and /dev/null differ diff --git a/draft-open.png b/draft-open.png deleted file mode 100644 index 82c7df22ea..0000000000 Binary files a/draft-open.png and /dev/null differ diff --git a/frontend/e2e/asset-runs-flicker-debug.mjs b/frontend/e2e/asset-runs-flicker-debug.mjs deleted file mode 100644 index 5de76b98db..0000000000 --- a/frontend/e2e/asset-runs-flicker-debug.mjs +++ /dev/null @@ -1,102 +0,0 @@ -// One-off repro for the asset-runs panel re-fetch flicker. Logs in, opens -// /pipeline/km, clicks the first s3 asset node, and counts list_jobs -// requests over a 10 s idle window. > 2 == reactivity bug still present. -import { chromium } from 'playwright' - -const baseURL = process.env.BASE_URL ?? 'http://localhost:3000' -const folder = process.env.PIPELINE_FOLDER ?? 'km' - -const browser = await chromium.launch({ headless: true }) -const ctx = await browser.newContext() -const page = await ctx.newPage() - -// Counters -const counters = { listJobs: 0, others: new Map() } -page.on('request', (req) => { - const u = req.url() - if (u.includes('/list_jobs') || u.includes('/concurrency_groups/list_jobs')) { - counters.listJobs++ - console.log(`[${new Date().toISOString().slice(11, 23)}] LIST_JOBS #${counters.listJobs} ${u}`) - } -}) - -// Console capture -page.on('console', (m) => { - const t = m.type() - if (t === 'error' || t === 'warning') console.log(`[console.${t}]`, m.text()) -}) - -console.log('→ navigate to', baseURL) -await page.goto(baseURL) - -// Login -await page.locator('input#email[type="email"]').fill('admin@windmill.dev') -await page.locator('input#password[type="password"]').fill('changeme') -await page.locator('button:has-text("Sign in")').click() -await page.waitForURL(/\/user\/(first-time|workspaces|$)|\/$/, { timeout: 15000 }) - -// First-time skip / workspace pick -if (page.url().includes('/user/first-time')) { - await page.locator('button:has-text("Skip")').click() -} -if (page.url().includes('/user/workspaces')) { - const ws = page.locator('text=Admins').first() - await ws.waitFor({ state: 'visible' }) - await ws.click() - await page.waitForURL(/\/$/) -} - -console.log('→ navigate to /pipeline/' + folder) -await page.goto(`${baseURL}/pipeline/${folder}`) -await page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {}) - -// Try to click the first asset node on the canvas. Asset nodes use -// AssetGenericIcon; a stable signal is the formatted-asset-path text rendered -// inside the node. Fallback: click any element with class containing 'asset'. -console.log('→ wait for canvas') -await page - .locator('.svelte-flow__node-asset') - .first() - .waitFor({ state: 'visible', timeout: 15000 }) - -const beforeClick = counters.listJobs -console.log(`baseline list_jobs before asset click: ${beforeClick}`) - -console.log('→ click first asset node') -await page.locator('.svelte-flow__node-asset').first().click() - -// Wait briefly for the runs panel to mount and fire its initial fetch. -await page.waitForTimeout(1500) -console.log(`list_jobs after click + 1.5s: ${counters.listJobs}`) -const initial = counters.listJobs - -// Idle window — no clicks, no input. Anything that fires here is the bug. -const idleSeconds = 10 -console.log(`→ idle for ${idleSeconds}s`) -const tickStart = counters.listJobs -const start = Date.now() -while (Date.now() - start < idleSeconds * 1000) { - await page.waitForTimeout(500) -} -const tickEnd = counters.listJobs -console.log(`list_jobs during idle: ${tickEnd - tickStart} (total: ${tickEnd})`) - -// Try typing in the editor pane (if present) — this is what was triggering -// the flicker via graphWithDraft re-derivation in earlier reports. We only -// want to verify the fix protects against it. -const editor = page.locator('.monaco-editor').first() -if (await editor.isVisible().catch(() => false)) { - console.log('→ focus + type a couple chars to test draft churn') - await editor.click() - const beforeType = counters.listJobs - for (let i = 0; i < 6; i++) { - await page.keyboard.type('x') - await page.waitForTimeout(200) - } - console.log(`list_jobs while typing: ${counters.listJobs - beforeType}`) -} - -console.log('=== final tally ===') -console.log({ total_list_jobs: counters.listJobs }) - -await browser.close() diff --git a/looptest.png b/looptest.png deleted file mode 100644 index 6bd42e15a8..0000000000 Binary files a/looptest.png and /dev/null differ diff --git a/picker-arrow-down.png b/picker-arrow-down.png deleted file mode 100644 index c8d580c9c7..0000000000 Binary files a/picker-arrow-down.png and /dev/null differ diff --git a/picker-arrow-right.png b/picker-arrow-right.png deleted file mode 100644 index fa85862b6a..0000000000 Binary files a/picker-arrow-right.png and /dev/null differ diff --git a/picker-default-duckdb.png b/picker-default-duckdb.png deleted file mode 100644 index ecf5363af3..0000000000 Binary files a/picker-default-duckdb.png and /dev/null differ diff --git a/pipeline-insert-menu-bun-selected.png b/pipeline-insert-menu-bun-selected.png deleted file mode 100644 index 5013d2f7eb..0000000000 Binary files a/pipeline-insert-menu-bun-selected.png and /dev/null differ diff --git a/pipeline-insert-menu-multikind.png b/pipeline-insert-menu-multikind.png deleted file mode 100644 index 1bc6d8820e..0000000000 Binary files a/pipeline-insert-menu-multikind.png and /dev/null differ diff --git a/pipeline-insert-menu-path-stage.png b/pipeline-insert-menu-path-stage.png deleted file mode 100644 index c8659727b2..0000000000 Binary files a/pipeline-insert-menu-path-stage.png and /dev/null differ diff --git a/pipeline-insert-menu-pg-selected.png b/pipeline-insert-menu-pg-selected.png deleted file mode 100644 index 31a9fa6a49..0000000000 Binary files a/pipeline-insert-menu-pg-selected.png and /dev/null differ diff --git a/pipeline-insert-menu-singlekind.png b/pipeline-insert-menu-singlekind.png deleted file mode 100644 index 1c1d1d969d..0000000000 Binary files a/pipeline-insert-menu-singlekind.png and /dev/null differ diff --git a/pipeline-km-existing.png b/pipeline-km-existing.png deleted file mode 100644 index a2b4d709a9..0000000000 Binary files a/pipeline-km-existing.png and /dev/null differ diff --git a/pipeline-km-hidden.png b/pipeline-km-hidden.png deleted file mode 100644 index 4e1515ca20..0000000000 Binary files a/pipeline-km-hidden.png and /dev/null differ diff --git a/pipeline-km-initial.png b/pipeline-km-initial.png deleted file mode 100644 index 515d519b8b..0000000000 Binary files a/pipeline-km-initial.png and /dev/null differ diff --git a/pipeline-km-no-selection.png b/pipeline-km-no-selection.png deleted file mode 100644 index 151e978e76..0000000000 Binary files a/pipeline-km-no-selection.png and /dev/null differ diff --git a/pipeline-km-restored.png b/pipeline-km-restored.png deleted file mode 100644 index 044cb190db..0000000000 Binary files a/pipeline-km-restored.png and /dev/null differ diff --git a/pipeline-km-wide.png b/pipeline-km-wide.png deleted file mode 100644 index 7d2c7ed395..0000000000 Binary files a/pipeline-km-wide.png and /dev/null differ diff --git a/pipeline-page.png b/pipeline-page.png deleted file mode 100644 index de97bfd2d4..0000000000 Binary files a/pipeline-page.png and /dev/null differ diff --git a/pipeline-template-preview.png b/pipeline-template-preview.png deleted file mode 100644 index e4046d17d5..0000000000 Binary files a/pipeline-template-preview.png and /dev/null differ diff --git a/test-after-click.png b/test-after-click.png deleted file mode 100644 index e91e9e68cc..0000000000 Binary files a/test-after-click.png and /dev/null differ diff --git a/test-after-enter.png b/test-after-enter.png deleted file mode 100644 index affeb79be6..0000000000 Binary files a/test-after-enter.png and /dev/null differ diff --git a/test-after-input.png b/test-after-input.png deleted file mode 100644 index d2be4c109b..0000000000 Binary files a/test-after-input.png and /dev/null differ diff --git a/testbed.png b/testbed.png deleted file mode 100644 index 72a91465f1..0000000000 Binary files a/testbed.png and /dev/null differ diff --git a/testing-edit.png b/testing-edit.png deleted file mode 100644 index 9afc7d3ea0..0000000000 Binary files a/testing-edit.png and /dev/null differ diff --git a/v2-hidden-fullwidth.png b/v2-hidden-fullwidth.png deleted file mode 100644 index 6e75f9ee1e..0000000000 Binary files a/v2-hidden-fullwidth.png and /dev/null differ diff --git a/v2-with-draft.png b/v2-with-draft.png deleted file mode 100644 index 5408d4afe3..0000000000 Binary files a/v2-with-draft.png and /dev/null differ diff --git a/v2-with-existing.png b/v2-with-existing.png deleted file mode 100644 index 369443d8c2..0000000000 Binary files a/v2-with-existing.png and /dev/null differ diff --git a/v3-area-zoom.png b/v3-area-zoom.png deleted file mode 100644 index 2a71d4541e..0000000000 Binary files a/v3-area-zoom.png and /dev/null differ diff --git a/v3-area.png b/v3-area.png deleted file mode 100644 index 55ae44eeca..0000000000 Binary files a/v3-area.png and /dev/null differ diff --git a/v3-button-test.png b/v3-button-test.png deleted file mode 100644 index f74db7b63d..0000000000 Binary files a/v3-button-test.png and /dev/null differ diff --git a/v3-fixed.png b/v3-fixed.png deleted file mode 100644 index f119bc1001..0000000000 Binary files a/v3-fixed.png and /dev/null differ diff --git a/v3-with-red-bg.png b/v3-with-red-bg.png deleted file mode 100644 index 6e1066c770..0000000000 Binary files a/v3-with-red-bg.png and /dev/null differ diff --git a/v3-zoom.png b/v3-zoom.png deleted file mode 100644 index 793e076a7e..0000000000 Binary files a/v3-zoom.png and /dev/null differ diff --git a/v4-current.png b/v4-current.png deleted file mode 100644 index 956b00d442..0000000000 Binary files a/v4-current.png and /dev/null differ diff --git a/v4-final-zoom.png b/v4-final-zoom.png deleted file mode 100644 index 923c3e1106..0000000000 Binary files a/v4-final-zoom.png and /dev/null differ diff --git a/v4-final.png b/v4-final.png deleted file mode 100644 index 1934788b79..0000000000 Binary files a/v4-final.png and /dev/null differ diff --git a/v4-test-zoom.png b/v4-test-zoom.png deleted file mode 100644 index 633a733623..0000000000 Binary files a/v4-test-zoom.png and /dev/null differ diff --git a/v4-test.png b/v4-test.png deleted file mode 100644 index 80c9b23827..0000000000 Binary files a/v4-test.png and /dev/null differ