improve indices performances

backend/.sqlx/query-7a00843921db4ac1eb20a1045e83363587c1a2cb62d2df25d5564ae78a03b097.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE\n        completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY created_at DESC LIMIT 20) AS jobs ) t\n        WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3",
+  "query": "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE\n        completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY started_at DESC LIMIT 20) AS jobs ) t\n        WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3",
   "describe": {
     "columns": [
       {
@@ -104,5 +104,5 @@
       null
     ]
   },
-  "hash": "7404fdf5ff9824fc8ebf0b9012cfe6a1204f4a67081beeb77bddda6bdf4b8987"
+  "hash": "7a00843921db4ac1eb20a1045e83363587c1a2cb62d2df25d5564ae78a03b097"
 }

backend/migrations/20230820161609_add_schedule_index.down.sql

@@ -0,0 +1 @@
+-- Add down migration script here

backend/migrations/20230820161609_add_schedule_index.up.sql

@@ -0,0 +1,25 @@
+-- Add up migration script here
+CREATE INDEX IF NOT EXISTS scheduled_root_job ON completed_job (workspace_id, schedule_path, started_at) WHERE parent_job is NULL;
+CREATE INDEX IF NOT EXISTS root_job_index_by_path ON completed_job (workspace_id, script_path, job_kind, created_at) WHERE parent_job is NULL;
+CREATE INDEX IF NOT EXISTS root_job_index ON completed_job (workspace_id, job_kind, created_at) WHERE parent_job is NULL;
+DROP INDEX IF EXISTS index_completed_on_script_hash;
+DROP INDEX IF EXISTS index_completed_on_script_path;
+DROP INDEX IF EXISTS index_completed_on_schedule_path;
+DROP INDEX IF EXISTS index_completed_on_workspace_id;
+DROP INDEX IF EXISTS index_completed_on_created_at;
+
+DROP INDEX IF EXISTS index_queue_on_script_path;
+DROP INDEX IF EXISTS index_queue_on_script_hash;
+DROP INDEX IF EXISTS index_queue_on_workspace_id;
+DROP INDEX IF EXISTS index_queue_on_scheduled_for;
+DROP INDEX IF EXISTS index_queue_on_tag;
+DROP INDEX IF EXISTS index_queue_on_running;
+DROP INDEX IF EXISTS index_queue_on_created;
+
+
+CREATE INDEX IF NOT EXISTS root_queue_index_by_path ON queue (workspace_id, created_at);
+CREATE INDEX IF NOT EXISTS root_queue_index ON queue (job_kind, tag, scheduled_for, created_at) WHERE running is false;
+CREATE INDEX IF NOT EXISTS root_queue_index_suspended ON queue (job_kind, tag, suspend_until, suspend, scheduled_for, created_at) WHERE suspend_until is not null;
+
+CREATE INDEX IF NOT EXISTS concurrency_limit_stats_queue ON queue (workspace_id, script_path, started_at) WHERE concurrent_limit is not null;
+CREATE INDEX IF NOT EXISTS concurrency_limit_stats_completed_job ON completed_job (workspace_id, script_path, started_at);

backend/windmill-api/src/schedule.rs

@@ -280,7 +280,7 @@ async fn list_schedule_with_jobs(
     let (per_page, offset) = paginate(pagination);
     let rows = sqlx::query_as!(ScheduleWJobs,
         "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE
-        completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY created_at DESC LIMIT 20) AS jobs ) t
+        completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY started_at DESC LIMIT 20) AS jobs ) t
         WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3",
         w_id,
         per_page as i64,

backend/windmill-queue/src/jobs.rs

@@ -99,11 +99,8 @@ lazy_static::lazy_static! {
         WHERE id = (
             SELECT id
             FROM queue
-            WHERE ((running = false
-                   AND scheduled_for <= now())
-               OR (suspend_until IS NOT NULL
-                   AND (   suspend <= 0
-                        OR suspend_until <= now()))) 
+            WHERE ((running = false AND scheduled_for <= now())
+               OR (suspend_until IS NOT NULL AND (suspend <= 0 OR suspend_until <= now()))) 
                 {}
             ORDER BY scheduled_for, created_at
             FOR UPDATE SKIP LOCKED

backend/windmill-worker/src/common.rs

@@ -74,7 +74,6 @@ pub async fn transform_json_value(
     workspace: &str,
     v: Value,
 ) -> error::Result<Value> {
-    tracing::info!("transform_json_value {name}", name = name);
     match v {
         Value::String(y) if y.starts_with("$var:") => {
             let path = y.strip_prefix("$var:").unwrap();