From e41a3fc1d27937f85e839dfdfd346fdda3840c17 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 20 Aug 2023 18:06:59 +0200 Subject: [PATCH] improve sql query performances --- ...1bcc96a0976ea0ead76049350250142d086015228d36338.json} | 7 ++++--- ...d8d43215d61268a251160db4630f0342522091668f36af0.json} | 4 ++-- ...39c3a097871526adcecdd8729f28c15e0db8cd28eaa8cf0.json} | 4 ++-- ...ebf0b9012cfe6a1204f4a67081beeb77bddda6bdf4b8987.json} | 4 ++-- ...dfd56602110b7f66903d6afb4f87fbffe0175982ea8741f.json} | 4 ++-- backend/windmill-api/src/apps.rs | 2 +- backend/windmill-api/src/flows.rs | 2 +- backend/windmill-api/src/groups.rs | 2 +- backend/windmill-api/src/resources.rs | 4 ++-- backend/windmill-api/src/schedule.rs | 2 +- backend/windmill-api/src/variables.rs | 6 +++--- backend/windmill-api/src/workspaces.rs | 2 +- backend/windmill-queue/src/jobs.rs | 5 +++-- .../lib/components/flows/map/FlowModuleSchemaMap.svelte | 2 +- .../src/routes/(root)/(logged)/run/[...run]/+page.svelte | 9 ++++++++- 15 files changed, 34 insertions(+), 25 deletions(-) rename backend/.sqlx/{query-077b925c0a9f19a3859ca7eb0bc9049fc7c303daf469c049cf03570d22310d74.json => query-09eb148c69fec39d01bcc96a0976ea0ead76049350250142d086015228d36338.json} (57%) rename backend/.sqlx/{query-c2d6cb56c1dea4498e2aab9ea9301dbbaa127602a38f57f5add4108fdc209b1a.json => query-3fdfcab1a54c166b1d8d43215d61268a251160db4630f0342522091668f36af0.json} (58%) rename backend/.sqlx/{query-7f443f5be90622eed3e878ada0ae8ca71902adb88861bc8a39713fc8942e652c.json => query-41f2c271514ee254739c3a097871526adcecdd8729f28c15e0db8cd28eaa8cf0.json} (87%) rename backend/.sqlx/{query-dc4316a8646dc642ed12c12d8673aa1ed44c551b3174c49655defacdf49880c7.json => query-7404fdf5ff9824fc8ebf0b9012cfe6a1204f4a67081beeb77bddda6bdf4b8987.json} (85%) rename backend/.sqlx/{query-f07e4fb9d3b346c6c0a93133d428691a2e17a0c7eef5c593732ad7297d366ad4.json => query-aedba9cff117a7069dfd56602110b7f66903d6afb4f87fbffe0175982ea8741f.json} (88%) diff --git a/backend/.sqlx/query-077b925c0a9f19a3859ca7eb0bc9049fc7c303daf469c049cf03570d22310d74.json b/backend/.sqlx/query-09eb148c69fec39d01bcc96a0976ea0ead76049350250142d086015228d36338.json similarity index 57% rename from backend/.sqlx/query-077b925c0a9f19a3859ca7eb0bc9049fc7c303daf469c049cf03570d22310d74.json rename to backend/.sqlx/query-09eb148c69fec39d01bcc96a0976ea0ead76049350250142d086015228d36338.json index 0d4a554a88..2e6552887e 100644 --- a/backend/.sqlx/query-077b925c0a9f19a3859ca7eb0bc9049fc7c303daf469c049cf03570d22310d74.json +++ b/backend/.sqlx/query-09eb148c69fec39d01bcc96a0976ea0ead76049350250142d086015228d36338.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COALESCE(j.min_started_at, q.min_started_at) AS min_started_at, COALESCE(completed_count, 0) + COALESCE(running_count, 0) AS total_count\n FROM\n (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as completed_count\n FROM completed_job\n WHERE script_path = $1 AND started_at + INTERVAL '1 MILLISECOND' * duration_ms > (now() - INTERVAL '1 second' * $2)\n GROUP BY script_path) as j\n FULL OUTER JOIN\n (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as running_count\n FROM queue\n WHERE script_path = $1 AND running = true\n GROUP BY script_path) as q\n ON q.script_path = j.script_path", + "query": "SELECT COALESCE(j.min_started_at, q.min_started_at) AS min_started_at, COALESCE(completed_count, 0) + COALESCE(running_count, 0) AS total_count\n FROM\n (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as completed_count\n FROM completed_job\n WHERE script_path = $1 AND started_at + INTERVAL '1 MILLISECOND' * duration_ms > (now() - INTERVAL '1 second' * $2) AND workspace_id = $3\n GROUP BY script_path) as j\n FULL OUTER JOIN\n (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as running_count\n FROM queue\n WHERE script_path = $1 AND running = true AND workspace_id = $3\n GROUP BY script_path) as q\n ON q.script_path = j.script_path", "describe": { "columns": [ { @@ -17,7 +17,8 @@ "parameters": { "Left": [ "Text", - "Float8" + "Float8", + "Text" ] }, "nullable": [ @@ -25,5 +26,5 @@ true ] }, - "hash": "077b925c0a9f19a3859ca7eb0bc9049fc7c303daf469c049cf03570d22310d74" + "hash": "09eb148c69fec39d01bcc96a0976ea0ead76049350250142d086015228d36338" } diff --git a/backend/.sqlx/query-c2d6cb56c1dea4498e2aab9ea9301dbbaa127602a38f57f5add4108fdc209b1a.json b/backend/.sqlx/query-3fdfcab1a54c166b1d8d43215d61268a251160db4630f0342522091668f36af0.json similarity index 58% rename from backend/.sqlx/query-c2d6cb56c1dea4498e2aab9ea9301dbbaa127602a38f57f5add4108fdc209b1a.json rename to backend/.sqlx/query-3fdfcab1a54c166b1d8d43215d61268a251160db4630f0342522091668f36af0.json index d2b5e6eeb4..be7e8608de 100644 --- a/backend/.sqlx/query-c2d6cb56c1dea4498e2aab9ea9301dbbaa127602a38f57f5add4108fdc209b1a.json +++ b/backend/.sqlx/query-3fdfcab1a54c166b1d8d43215d61268a251160db4630f0342522091668f36af0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT usr.username \n FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username \n WHERE group_ = $1 AND usr.workspace_id = $2 AND usr_to_group.workspace_id = $2", + "query": "SELECT usr.username \n FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username AND usr_to_group.workspace_id = $2\n WHERE group_ = $1 AND usr.workspace_id = $2 AND usr_to_group.workspace_id = $2", "describe": { "columns": [ { @@ -19,5 +19,5 @@ false ] }, - "hash": "c2d6cb56c1dea4498e2aab9ea9301dbbaa127602a38f57f5add4108fdc209b1a" + "hash": "3fdfcab1a54c166b1d8d43215d61268a251160db4630f0342522091668f36af0" } diff --git a/backend/.sqlx/query-7f443f5be90622eed3e878ada0ae8ca71902adb88861bc8a39713fc8942e652c.json b/backend/.sqlx/query-41f2c271514ee254739c3a097871526adcecdd8729f28c15e0db8cd28eaa8cf0.json similarity index 87% rename from backend/.sqlx/query-7f443f5be90622eed3e878ada0ae8ca71902adb88861bc8a39713fc8942e652c.json rename to backend/.sqlx/query-41f2c271514ee254739c3a097871526adcecdd8729f28c15e0db8cd28eaa8cf0.json index 2cbc4a5e7c..cacecd68f8 100644 --- a/backend/.sqlx/query-7f443f5be90622eed3e878ada0ae8ca71902adb88861bc8a39713fc8942e652c.json +++ b/backend/.sqlx/query-41f2c271514ee254739c3a097871526adcecdd8729f28c15e0db8cd28eaa8cf0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT resource.*, (now() > account.expires_at) as is_expired, account.refresh_token != '' as is_refreshed,\n account.refresh_error,\n variable.path IS NOT NULL as is_linked,\n variable.is_oauth as \"is_oauth?\",\n variable.account\n FROM resource\n LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = resource.workspace_id\n LEFT JOIN account ON variable.account = account.id AND account.workspace_id = resource.workspace_id\n WHERE resource.path = $1 AND resource.workspace_id = $2", + "query": "SELECT resource.*, (now() > account.expires_at) as is_expired, account.refresh_token != '' as is_refreshed,\n account.refresh_error,\n variable.path IS NOT NULL as is_linked,\n variable.is_oauth as \"is_oauth?\",\n variable.account\n FROM resource\n LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = $2\n LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $2\n WHERE resource.path = $1 AND resource.workspace_id = $2", "describe": { "columns": [ { @@ -85,5 +85,5 @@ true ] }, - "hash": "7f443f5be90622eed3e878ada0ae8ca71902adb88861bc8a39713fc8942e652c" + "hash": "41f2c271514ee254739c3a097871526adcecdd8729f28c15e0db8cd28eaa8cf0" } diff --git a/backend/.sqlx/query-dc4316a8646dc642ed12c12d8673aa1ed44c551b3174c49655defacdf49880c7.json b/backend/.sqlx/query-7404fdf5ff9824fc8ebf0b9012cfe6a1204f4a67081beeb77bddda6bdf4b8987.json similarity index 85% rename from backend/.sqlx/query-dc4316a8646dc642ed12c12d8673aa1ed44c551b3174c49655defacdf49880c7.json rename to backend/.sqlx/query-7404fdf5ff9824fc8ebf0b9012cfe6a1204f4a67081beeb77bddda6bdf4b8987.json index 1d85715647..34a96df77a 100644 --- a/backend/.sqlx/query-dc4316a8646dc642ed12c12d8673aa1ed44c551b3174c49655defacdf49880c7.json +++ b/backend/.sqlx/query-7404fdf5ff9824fc8ebf0b9012cfe6a1204f4a67081beeb77bddda6bdf4b8987.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE\n completed_job.schedule_path = schedule.path AND schedule.workspace_id = completed_job.workspace_id AND parent_job IS NULL ORDER BY created_at DESC LIMIT 20) AS jobs ) t\n WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3", + "query": "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE\n completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY created_at DESC LIMIT 20) AS jobs ) t\n WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -104,5 +104,5 @@ null ] }, - "hash": "dc4316a8646dc642ed12c12d8673aa1ed44c551b3174c49655defacdf49880c7" + "hash": "7404fdf5ff9824fc8ebf0b9012cfe6a1204f4a67081beeb77bddda6bdf4b8987" } diff --git a/backend/.sqlx/query-f07e4fb9d3b346c6c0a93133d428691a2e17a0c7eef5c593732ad7297d366ad4.json b/backend/.sqlx/query-aedba9cff117a7069dfd56602110b7f66903d6afb4f87fbffe0175982ea8741f.json similarity index 88% rename from backend/.sqlx/query-f07e4fb9d3b346c6c0a93133d428691a2e17a0c7eef5c593732ad7297d366ad4.json rename to backend/.sqlx/query-aedba9cff117a7069dfd56602110b7f66903d6afb4f87fbffe0175982ea8741f.json index 7944f0e41c..53eade2e2a 100644 --- a/backend/.sqlx/query-f07e4fb9d3b346c6c0a93133d428691a2e17a0c7eef5c593732ad7297d366ad4.json +++ b/backend/.sqlx/query-aedba9cff117a7069dfd56602110b7f66903d6afb4f87fbffe0175982ea8741f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n app.extra_perms, app_version.value, \n app_version.created_at, app_version.created_by,\n app.draft_only, draft.value as \"draft?\"\n from app\n INNER JOIN app_version ON\n app_version.id = app.versions[array_upper(app.versions, 1)]\n LEFT JOIN draft ON \n app.path = draft.path AND app.workspace_id = draft.workspace_id AND draft.typ = 'app' \n WHERE app.path = $1 AND app.workspace_id = $2", + "query": "SELECT app.id, app.path, app.summary, app.versions, app.policy,\n app.extra_perms, app_version.value, \n app_version.created_at, app_version.created_by,\n app.draft_only, draft.value as \"draft?\"\n from app\n INNER JOIN app_version ON\n app_version.id = app.versions[array_upper(app.versions, 1)]\n LEFT JOIN draft ON \n app.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'app' \n WHERE app.path = $1 AND app.workspace_id = $2", "describe": { "columns": [ { @@ -79,5 +79,5 @@ false ] }, - "hash": "f07e4fb9d3b346c6c0a93133d428691a2e17a0c7eef5c593732ad7297d366ad4" + "hash": "aedba9cff117a7069dfd56602110b7f66903d6afb4f87fbffe0175982ea8741f" } diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 93397774df..77e75f4acc 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -262,7 +262,7 @@ async fn get_app_w_draft( INNER JOIN app_version ON app_version.id = app.versions[array_upper(app.versions, 1)] LEFT JOIN draft ON - app.path = draft.path AND app.workspace_id = draft.workspace_id AND draft.typ = 'app' + app.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'app' WHERE app.path = $1 AND app.workspace_id = $2"#, path.to_owned(), &w_id diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 1da14638a0..797cc9d3b7 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -518,7 +518,7 @@ async fn get_flow_by_path_w_draft( let flow_o = sqlx::query_as::<_, FlowWDraft>( "SELECT flow.path, flow.summary, flow,description, flow.schema, flow.value, flow.extra_perms, flow.draft_only, draft.value as draft, flow.tag FROM flow LEFT JOIN draft ON - flow.path = draft.path AND flow.workspace_id = draft.workspace_id AND draft.typ = 'flow' + flow.path = draft.path AND draft.workspace_id = $2 AND draft.typ = 'flow' WHERE flow.path = $1 AND flow.workspace_id = $2", ) .bind(path) diff --git a/backend/windmill-api/src/groups.rs b/backend/windmill-api/src/groups.rs index 65a014d094..8924c6837b 100644 --- a/backend/windmill-api/src/groups.rs +++ b/backend/windmill-api/src/groups.rs @@ -339,7 +339,7 @@ async fn get_group( let members = sqlx::query_scalar!( "SELECT usr.username - FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username + FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username AND usr_to_group.workspace_id = $2 WHERE group_ = $1 AND usr.workspace_id = $2 AND usr_to_group.workspace_id = $2", name, w_id diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 9357c7b161..faed168588 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -190,8 +190,8 @@ async fn get_resource( variable.is_oauth as \"is_oauth?\", variable.account FROM resource - LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = resource.workspace_id - LEFT JOIN account ON variable.account = account.id AND account.workspace_id = resource.workspace_id + LEFT JOIN variable ON variable.path = resource.path AND variable.workspace_id = $2 + LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $2 WHERE resource.path = $1 AND resource.workspace_id = $2", path.to_owned(), &w_id diff --git a/backend/windmill-api/src/schedule.rs b/backend/windmill-api/src/schedule.rs index c3eb309d62..aa634875be 100644 --- a/backend/windmill-api/src/schedule.rs +++ b/backend/windmill-api/src/schedule.rs @@ -280,7 +280,7 @@ async fn list_schedule_with_jobs( let (per_page, offset) = paginate(pagination); let rows = sqlx::query_as!(ScheduleWJobs, "SELECT schedule.*, t.jobs FROM schedule, LATERAL ( SELECT ARRAY (SELECT json_build_object('id', id, 'success', success, 'duration_ms', duration_ms) FROM completed_job WHERE - completed_job.schedule_path = schedule.path AND schedule.workspace_id = completed_job.workspace_id AND parent_job IS NULL ORDER BY created_at DESC LIMIT 20) AS jobs ) t + completed_job.schedule_path = schedule.path AND completed_job.workspace_id = $1 AND parent_job IS NULL ORDER BY created_at DESC LIMIT 20) AS jobs ) t WHERE schedule.workspace_id = $1 ORDER BY schedule.edited_at desc LIMIT $2 OFFSET $3", w_id, per_page as i64, diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index c07ffdfe8b..8ea2e43f39 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -85,8 +85,8 @@ async fn list_variables( resource.path IS NOT NULL as is_linked, account.refresh_token != '' as is_refreshed from variable - LEFT JOIN account ON variable.account = account.id AND account.workspace_id = variable.workspace_id - LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = variable.workspace_id + LEFT JOIN account ON variable.account = account.id AND account.workspace_id = $1 + LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = $1 WHERE variable.workspace_id = $1 ORDER BY path", ) .bind(&w_id) @@ -117,7 +117,7 @@ async fn get_variable( account.refresh_token != '' as is_refreshed from variable LEFT JOIN account ON variable.account = account.id - LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = variable.workspace_id + LEFT JOIN resource ON resource.path = variable.path AND resource.workspace_id = $2 WHERE variable.path = $1 AND variable.workspace_id = $2 LIMIT 1", ) diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index d6c0f5918f..85af1db1a1 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -250,7 +250,7 @@ async fn premium_info( require_admin(authed.is_admin, &authed.username)?; let mut tx = db.begin().await?; let row = sqlx::query_as::<_, PremiumWorkspaceInfo>( - "SELECT premium, usage.usage FROM workspace LEFT JOIN usage ON workspace.id = usage.id AND usage.is_workspace IS true WHERE workspace.id = $1", + "SELECT premium, usage.usage FROM workspace LEFT JOIN usage ON workspace.id = $1 AND usage.is_workspace IS true WHERE workspace.id = $1", ) .bind(w_id) .fetch_one(&mut *tx) diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 50928a8085..f164bebff3 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -733,16 +733,17 @@ pub async fn pull( FROM (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as completed_count FROM completed_job - WHERE script_path = $1 AND started_at + INTERVAL '1 MILLISECOND' * duration_ms > (now() - INTERVAL '1 second' * $2) + WHERE script_path = $1 AND started_at + INTERVAL '1 MILLISECOND' * duration_ms > (now() - INTERVAL '1 second' * $2) AND workspace_id = $3 GROUP BY script_path) as j FULL OUTER JOIN (SELECT script_path, MIN(started_at) as min_started_at, COUNT(*) as running_count FROM queue - WHERE script_path = $1 AND running = true + WHERE script_path = $1 AND running = true AND workspace_id = $3 GROUP BY script_path) as q ON q.script_path = j.script_path", job_script_path, f64::from(job_custom_concurrency_time_window_s), + &pulled_job.workspace_id ) .fetch_one(&mut tx) .await diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte index 20755a229d..c6f7794371 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte @@ -146,7 +146,7 @@ deleteCallback = undefined }} > -
Found the following steps that will require changes after this step is deleted:
{#each Object.entries(dependents) as [k, v]} diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 64a57f1089..0b1ca40590 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -1,7 +1,7 @@