perf: drive schedule already-exists check from an indexed queue lookup (#11783)

* perf: drive schedule already-exists check from v2_job_queue

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* perf: index v2_job_queue by workspace and scheduled_for

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

backend/.sqlx/query-250ac2b81a8d9a6900d8b54322fe21ef62eddd307717b24aa4fc03c0297e8f59.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT EXISTS (\n            SELECT 1 FROM v2_job_queue q\n            CROSS JOIN LATERAL (\n                SELECT trigger_kind, trigger, runnable_path, parent_job\n                FROM v2_job WHERE id = q.id OFFSET 0\n            ) j\n            WHERE q.workspace_id = $1 AND q.scheduled_for = $3\n                AND j.trigger_kind = 'schedule' AND j.trigger = $2 AND j.runnable_path = $4\n                AND j.parent_job IS NULL\n        )",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "exists",
+        "type_info": "Bool"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Text",
+        "Timestamptz",
+        "Text"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "250ac2b81a8d9a6900d8b54322fe21ef62eddd307717b24aa4fc03c0297e8f59"
+}

backend/.sqlx/query-3a47d8ec0f4ba1644951f0c88504c9f769ffe5004871a049e4232e0092532062.json

@@ -1,25 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT EXISTS (\n            SELECT 1 FROM v2_job j JOIN v2_job_queue USING (id)\n            WHERE j.workspace_id = $1 AND trigger_kind = 'schedule' AND trigger = $2 AND runnable_path = $4\n                AND parent_job IS NULL\n                AND scheduled_for = $3\n        )",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "exists",
-        "type_info": "Bool"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text",
-        "Timestamptz",
-        "Text"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "3a47d8ec0f4ba1644951f0c88504c9f769ffe5004871a049e4232e0092532062"
-}

backend/migrations/20261015114712_queue_workspace_scheduled_for_index.down.sql

@@ -0,0 +1 @@
+DROP INDEX IF EXISTS v2_job_queue_workspace_scheduled_for;

backend/migrations/20261015114712_queue_workspace_scheduled_for_index.up.sql

@@ -0,0 +1,10 @@
+-- Serves push_scheduled_job's already-exists check in windmill-queue/src/schedule.rs, which
+-- looks up the queued jobs of one workspace at one occurrence. Without it that check scans the
+-- whole queue on every schedule push.
+-- Dropped before it is built because the OVERRIDDEN_MIGRATIONS rewrite in
+-- windmill-api/src/db.rs runs both CONCURRENTLY: an interrupted concurrent build leaves an
+-- invalid index that IF NOT EXISTS would skip rebuilding.
+DROP INDEX IF EXISTS v2_job_queue_workspace_scheduled_for;
+
+CREATE INDEX IF NOT EXISTS v2_job_queue_workspace_scheduled_for
+    ON v2_job_queue (workspace_id, scheduled_for);

backend/windmill-api/src/db.rs

@@ -116,6 +116,9 @@ lazy_static::lazy_static! {
                     (20261009173546, include_str!(
                         "../../migrations/20261009173546_agent_owner_prefix_index.up.sql"
                     ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY")),
+                    (20261015114712, include_str!(
+                        "../../migrations/20261015114712_queue_workspace_scheduled_for_index.up.sql"
+                    ).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")),
                     // The widening fails every query the previous release's still-running
                     // servers prepared on the column; `variables::widen_variable_value` does it
                     // once they are gone.

backend/windmill-queue/src/schedule.rs

@@ -175,15 +175,20 @@ pub async fn push_scheduled_job<'c>(
     // Scheduled events must be stored in the database in UTC
     let next = next.with_timezone(&chrono::Utc);
     let already_exists: bool = sqlx::query_scalar!(
-        // Query plan:
-        // - use of the `ix_v2_job_root_by_path` index; hence the `parent_job IS NULL` clause.
-        // - select from `v2_job` first, then join with `v2_job_queue` to avoid a full table scan
-        //   on `scheduled_for = $3`.
+        // Must be driven by `v2_job_queue` (via `v2_job_queue_workspace_scheduled_for`), never
+        // by a `v2_job` path index, which walks the schedule's whole retained history (stats
+        // underestimate a single path's rows). The lateral reference forces the join order and
+        // `OFFSET 0` keeps the filters out of the subquery, so `v2_job` is only ever probed by
+        // primary key.
         "SELECT EXISTS (
-            SELECT 1 FROM v2_job j JOIN v2_job_queue USING (id)
-            WHERE j.workspace_id = $1 AND trigger_kind = 'schedule' AND trigger = $2 AND runnable_path = $4
-                AND parent_job IS NULL
-                AND scheduled_for = $3
+            SELECT 1 FROM v2_job_queue q
+            CROSS JOIN LATERAL (
+                SELECT trigger_kind, trigger, runnable_path, parent_job
+                FROM v2_job WHERE id = q.id OFFSET 0
+            ) j
+            WHERE q.workspace_id = $1 AND q.scheduled_for = $3
+                AND j.trigger_kind = 'schedule' AND j.trigger = $2 AND j.runnable_path = $4
+                AND j.parent_job IS NULL
         )",
         &schedule.workspace_id,
         &schedule.path,