starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-18 16:02:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: improve index usage and runs page performance

Browse Source
This commit is contained in:
Ruben Fiszel
2024-07-31 12:59:15 +02:00
parent cd4a424456
commit 78c115c68c
9 changed files with 93 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.sqlx/query-061ff848f258dc880bec81d923370c905e689f37c6d931ee4559c3cfd394e168.json
Generated
-23
View File
@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT flow_status FROM completed_job WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "flow_status",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
true
]
},
"hash": "061ff848f258dc880bec81d923370c905e689f37c6d931ee4559c3cfd394e168"
}
backend/.sqlx/query-15105be6247457fc01b7d65767ccdde047d0f0c172c7a01eeabe3bd8206a3069.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO windmill_migrations (name) VALUES ($1) ON CONFLICT DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "15105be6247457fc01b7d65767ccdde047d0f0c172c7a01eeabe3bd8206a3069"
}
backend/.sqlx/query-45587eb1ea7b6e695327ab7dfbd13c96e7415d16f5fe7d92c308020696e3dd7f.json → backend/.sqlx/query-51a7e36ba1cc1616a19485eb5930cf90df758e85a8cac6037d044d5bd7440681.json
Generated
+5 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'fix_job_completed_index_2')",
"query": "SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)",
"describe": {
"columns": [
{
@@ -10,11 +10,13 @@
}
],
"parameters": {
"Left": []
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "45587eb1ea7b6e695327ab7dfbd13c96e7415d16f5fe7d92c308020696e3dd7f"
"hash": "51a7e36ba1cc1616a19485eb5930cf90df758e85a8cac6037d044d5bd7440681"
}
backend/.sqlx/query-87a38cdaec0e143ec3250cd9d68212ccb5b08c84fc75f9efe22d86842cd618c5.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO windmill_migrations (name) VALUES ('fix_job_completed_index_2') ON CONFLICT DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "87a38cdaec0e143ec3250cd9d68212ccb5b08c84fc75f9efe22d86842cd618c5"
}
backend/.sqlx/query-c8ac658002423906f2a6e43e423a9e24561dfaa6b07ce89063d0910d2342a83b.json
Generated
-30
View File
@@ -1,30 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT leaf_jobs->$1::text as leaf_jobs, parent_job FROM queue WHERE COALESCE((SELECT root_job FROM queue WHERE id = $2), $2) = id AND workspace_id = $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "leaf_jobs",
"type_info": "Jsonb"
},
{
"ordinal": 1,
"name": "parent_job",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Text",
"Uuid",
"Text"
]
},
"nullable": [
null,
true
]
},
"hash": "c8ac658002423906f2a6e43e423a9e24561dfaa6b07ce89063d0910d2342a83b"
}
backend/migrations/20240731093818_remove_foreign_key.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20240731093818_remove_foreign_key.up.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add up migration script here
alter table queue drop constraint if exists queue_workspace_id_fkey;
alter table completed_job drop constraint if exists completed_job_workspace_id_fkey;
backend/windmill-api/src/db.rs
+69 -8
View File
@@ -290,14 +290,16 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> {
// tx.commit().await?;
// }
let migration_job_name = "fix_job_completed_index_2";
let has_done_migration = sqlx::query_scalar!(
"SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = 'fix_job_completed_index_2')"
"SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)",
migration_job_name
)
.fetch_one(db)
.await?
.unwrap_or(false);
if !has_done_migration {
tracing::info!("Applying fix_job_completed_index_2 migration");
tracing::info!("Applying {migration_job_name} migration");
let mut tx = db.begin().await?;
let mut r = false;
while !r {
@@ -305,12 +307,12 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> {
.fetch_one(&mut *tx)
.await
.map_err(|e| {
tracing::error!("Error acquiring fix_job_completed_index_2 lock: {e:#}");
tracing::error!("Error acquiring {migration_job_name} lock: {e:#}");
sqlx::migrate::MigrateError::Execute(e)
})?
.unwrap_or(false);
if !r {
tracing::info!("PG fix_job_completed_index_migration_2 lock already acquired by another server or worker, retrying in 5s. (look for the advisory lock in pg_lock with granted = true)");
tracing::info!("PG {migration_job_name} lock already acquired by another server or worker, retrying in 5s. (look for the advisory lock in pg_lock with granted = true)");
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
}
}
@@ -332,14 +334,73 @@ async fn fix_job_completed_index(db: &DB) -> Result<(), Error> {
.execute(db)
.await?;
sqlx::query!("INSERT INTO windmill_migrations (name) VALUES ('fix_job_completed_index_2') ON CONFLICT DO NOTHING")
.execute(&mut *tx)
.await?;
sqlx::query!(
"INSERT INTO windmill_migrations (name) VALUES ($1) ON CONFLICT DO NOTHING",
migration_job_name
)
.execute(&mut *tx)
.await?;
let _ = sqlx::query("SELECT pg_advisory_unlock(4242)")
.execute(&mut *tx)
.await?;
tx.commit().await?;
tracing::info!("Finished applying fix_job_completed_index_2 migration");
tracing::info!("Finished applying {migration_job_name} migration");
}
let migration_job_name = "fix_job_completed_index_3";
let has_done_migration = sqlx::query_scalar!(
"SELECT EXISTS(SELECT name FROM windmill_migrations WHERE name = $1)",
migration_job_name
)
.fetch_one(db)
.await?
.unwrap_or(false);
if !has_done_migration {
tracing::info!("Applying {migration_job_name} migration");
let mut tx = db.begin().await?;
let mut r = false;
while !r {
r = sqlx::query_scalar!("SELECT pg_try_advisory_lock(4242)")
.fetch_one(&mut *tx)
.await
.map_err(|e| {
tracing::error!("Error acquiring {migration_job_name} lock: {e:#}");
sqlx::migrate::MigrateError::Execute(e)
})?
.unwrap_or(false);
if !r {
tracing::info!("PG {migration_job_name} lock already acquired by another server or worker, retrying in 5s. (look for the advisory lock in pg_lock with granted = true)");
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
}
}
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS index_completed_job_on_schedule_path")
.execute(db)
.await?;
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS concurrency_limit_stats_queue")
.execute(db)
.await?;
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS root_job_index")
.execute(db)
.await?;
sqlx::query("DROP INDEX CONCURRENTLY IF EXISTS index_completed_on_created")
.execute(db)
.await?;
sqlx::query!(
"INSERT INTO windmill_migrations (name) VALUES ($1) ON CONFLICT DO NOTHING",
migration_job_name
)
.execute(&mut *tx)
.await?;
let _ = sqlx::query("SELECT pg_advisory_unlock(4242)")
.execute(&mut *tx)
.await?;
tx.commit().await?;
tracing::info!("Finished applying {migration_job_name} migration");
}
Ok(())
frontend/src/lib/components/runs/JobLoader.svelte
+1
View File
@@ -130,6 +130,7 @@
running: success == 'running' ? true : undefined,
isSkipped: isSkipped ? undefined : false,
isFlowStep: jobKindsCat != 'all' ? false : undefined,
hasNullParent: jobKindsCat != 'all' ? false : undefined,
label: label === null || label === '' ? undefined : label,
isNotSchedule: showSchedules == false ? true : undefined,
scheduledForBeforeNow: showFutureJobs == false ? true : undefined,