From 125a00fa8e2923f1d41bf2b5dbc349a7c145b09e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 20 Aug 2023 19:14:39 +0200 Subject: [PATCH] improve indices performances --- ...363587c1a2cb62d2df25d5564ae78a03b097.json} | 4 +-- ...20230820161609_add_schedule_index.down.sql | 1 + .../20230820161609_add_schedule_index.up.sql | 25 +++++++++++++++++++ backend/windmill-api/src/schedule.rs | 2 +- backend/windmill-queue/src/jobs.rs | 7 ++---- backend/windmill-worker/src/common.rs | 1 - 6 files changed, 31 insertions(+), 9 deletions(-) rename backend/.sqlx/{query-7404fdf5ff9824fc8ebf0b9012cfe6a1204f4a67081beeb77bddda6bdf4b8987.json => query-7a00843921db4ac1eb20a1045e83363587c1a2cb62d2df25d5564ae78a03b097.json} (94%) create mode 100644 backend/migrations/20230820161609_add_schedule_index.down.sql create mode 100644 backend/migrations/20230820161609_add_schedule_index.up.sql diff --git a/backend/.sqlx/query-7404fdf5ff9824fc8ebf0b9012cfe6a1204f4a67081beeb77bddda6bdf4b8987.json b/backend/.sqlx/query-7a00843921db4ac1eb20a1045e83363587c1a2cb62d2df25d5564ae78a03b097.json similarity index 94% rename from backend/.sqlx/query-7404fdf5ff9824fc8ebf0b9012cfe6a1204f4a67081beeb77bddda6bdf4b8987.json rename to backend/.sqlx/query-7a00843921db4ac1eb20a1045e83363587c1a2cb62d2df25d5564ae78a03b097.json index 34a96df77a..ec9f53d47c 100644 --- a/backend/.sqlx/query-7404fdf5ff9824fc8ebf0b9012cfe6a1204f4a67081beeb77bddda6bdf4b8987.json +++ b/backend/.sqlx/query-7a00843921db4ac1eb20a1045e83363587c1a2cb62d2df25d5564ae78a03b097.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE\n completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY created_at DESC LIMIT 20) AS jobs ) t\n WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3", + "query": "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE\n completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY started_at DESC LIMIT 20) AS jobs ) t\n WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -104,5 +104,5 @@ null ] }, - "hash": "7404fdf5ff9824fc8ebf0b9012cfe6a1204f4a67081beeb77bddda6bdf4b8987" + "hash": "7a00843921db4ac1eb20a1045e83363587c1a2cb62d2df25d5564ae78a03b097" } diff --git a/backend/migrations/20230820161609_add_schedule_index.down.sql b/backend/migrations/20230820161609_add_schedule_index.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20230820161609_add_schedule_index.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20230820161609_add_schedule_index.up.sql b/backend/migrations/20230820161609_add_schedule_index.up.sql new file mode 100644 index 0000000000..76d8e411d7 --- /dev/null +++ b/backend/migrations/20230820161609_add_schedule_index.up.sql @@ -0,0 +1,25 @@ +-- Add up migration script here +CREATE INDEX IF NOT EXISTS scheduled_root_job ON completed_job (workspace_id, schedule_path, started_at) WHERE parent_job is NULL; +CREATE INDEX IF NOT EXISTS root_job_index_by_path ON completed_job (workspace_id, script_path, job_kind, created_at) WHERE parent_job is NULL; +CREATE INDEX IF NOT EXISTS root_job_index ON completed_job (workspace_id, job_kind, created_at) WHERE parent_job is NULL; +DROP INDEX IF EXISTS index_completed_on_script_hash; +DROP INDEX IF EXISTS index_completed_on_script_path; +DROP INDEX IF EXISTS index_completed_on_schedule_path; +DROP INDEX IF EXISTS index_completed_on_workspace_id; +DROP INDEX IF EXISTS index_completed_on_created_at; + +DROP INDEX IF EXISTS index_queue_on_script_path; +DROP INDEX IF EXISTS index_queue_on_script_hash; +DROP INDEX IF EXISTS index_queue_on_workspace_id; +DROP INDEX IF EXISTS index_queue_on_scheduled_for; +DROP INDEX IF EXISTS index_queue_on_tag; +DROP INDEX IF EXISTS index_queue_on_running; +DROP INDEX IF EXISTS index_queue_on_created; + + +CREATE INDEX IF NOT EXISTS root_queue_index_by_path ON queue (workspace_id, created_at); +CREATE INDEX IF NOT EXISTS root_queue_index ON queue (job_kind, tag, scheduled_for, created_at) WHERE running is false; +CREATE INDEX IF NOT EXISTS root_queue_index_suspended ON queue (job_kind, tag, suspend_until, suspend, scheduled_for, created_at) WHERE suspend_until is not null; + +CREATE INDEX IF NOT EXISTS concurrency_limit_stats_queue ON queue (workspace_id, script_path, started_at) WHERE concurrent_limit is not null; +CREATE INDEX IF NOT EXISTS concurrency_limit_stats_completed_job ON completed_job (workspace_id, script_path, started_at); \ No newline at end of file diff --git a/backend/windmill-api/src/schedule.rs b/backend/windmill-api/src/schedule.rs index aa634875be..f0835218a0 100644 --- a/backend/windmill-api/src/schedule.rs +++ b/backend/windmill-api/src/schedule.rs @@ -280,7 +280,7 @@ async fn list_schedule_with_jobs( let (per_page, offset) = paginate(pagination); let rows = sqlx::query_as!(ScheduleWJobs, "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE - completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY created_at DESC LIMIT 20) AS jobs ) t + completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY started_at DESC LIMIT 20) AS jobs ) t WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3", w_id, per_page as i64, diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index f164bebff3..976e43458e 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -99,11 +99,8 @@ lazy_static::lazy_static! { WHERE id = ( SELECT id FROM queue - WHERE ((running = false - AND scheduled_for <= now()) - OR (suspend_until IS NOT NULL - AND ( suspend <= 0 - OR suspend_until <= now()))) + WHERE ((running = false AND scheduled_for <= now()) + OR (suspend_until IS NOT NULL AND (suspend <= 0 OR suspend_until <= now()))) {} ORDER BY scheduled_for, created_at FOR UPDATE SKIP LOCKED diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index f5381e56e6..01b0fa5575 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -74,7 +74,6 @@ pub async fn transform_json_value( workspace: &str, v: Value, ) -> error::Result { - tracing::info!("transform_json_value {name}", name = name); match v { Value::String(y) if y.starts_with("$var:") => { let path = y.strip_prefix("$var:").unwrap();