From cd6e06c6385a81735e04259c3ebd77397aa17342 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 16 Sep 2026 10:46:25 +0200 Subject: [PATCH] perf: a deploy history comes a page at a time, so the diff drawer opens at once Co-Authored-By: Claude Opus 5 (1M context) --- ...29c87ce48ffd89fac070c8314556f884ce55.json} | 8 ++- ...53ccaac72871eb68822d8e296c0d747e55f35.json | 43 +++++++++++++ ...3e29b09f66090e1f9d394564c16d921d2e66c.json | 35 ----------- ...42c94445536e201b322eda6f7a90bdf38f36a.json | 35 ----------- ...7ce6482125e720fb8f53e3e9e749a69e08bd.json} | 8 ++- backend/windmill-api-flows/src/flows.rs | 18 ++++-- backend/windmill-api-scripts/src/scripts.rs | 12 +++- backend/windmill-api/openapi.yaml | 6 ++ backend/windmill-api/src/apps.rs | 12 +++- backend/windmill-common/src/utils.rs | 15 +++++ frontend/src/lib/components/DiffDrawer.svelte | 62 +++++++++++++++++-- .../src/lib/components/FlowBuilder.svelte | 19 ++++-- .../src/lib/components/ScriptBuilder.svelte | 24 ++++--- frontend/src/lib/components/diff_drawer.ts | 4 ++ .../raw_apps/RawAppEditorHeader.svelte | 20 ++++-- 15 files changed, 212 insertions(+), 109 deletions(-) rename backend/.sqlx/{query-605c575a82693bd5351cc748b31c6c44916c503837cb6d189147e184f7f41caf.json => query-044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55.json} (81%) create mode 100644 backend/.sqlx/query-7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35.json delete mode 100644 backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json delete mode 100644 backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json rename backend/.sqlx/{query-c8b369881add5574c1762431e2fb58eb04892babb4684be314f369227424e640.json => query-e7c8ccc5f1955cae39856baacafb7ce6482125e720fb8f53e3e9e749a69e08bd.json} (85%) diff --git a/backend/.sqlx/query-605c575a82693bd5351cc748b31c6c44916c503837cb6d189147e184f7f41caf.json b/backend/.sqlx/query-044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55.json similarity index 81% rename from backend/.sqlx/query-605c575a82693bd5351cc748b31c6c44916c503837cb6d189147e184f7f41caf.json rename to backend/.sqlx/query-044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55.json index 542d62cb95..be7ac83a7b 100644 --- a/backend/.sqlx/query-605c575a82693bd5351cc748b31c6c44916c503837cb6d189147e184f7f41caf.json +++ b/backend/.sqlx/query-044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC", + "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC\n LIMIT $3 OFFSET $4", "describe": { "columns": [ { @@ -27,7 +27,9 @@ "parameters": { "Left": [ "Text", - "Text" + "Text", + "Int8", + "Int8" ] }, "nullable": [ @@ -37,5 +39,5 @@ false ] }, - "hash": "605c575a82693bd5351cc748b31c6c44916c503837cb6d189147e184f7f41caf" + "hash": "044310c0e7745867a24a9d3653ff29c87ce48ffd89fac070c8314556f884ce55" } diff --git a/backend/.sqlx/query-7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35.json b/backend/.sqlx/query-7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35.json new file mode 100644 index 0000000000..6a004023a0 --- /dev/null +++ b/backend/.sqlx/query-7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version\n LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version\n WHERE flow_version.path = $1 AND flow_version.workspace_id = $2\n ORDER BY flow_version.created_at DESC\n LIMIT $3 OFFSET $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "deployment_msg", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "7ff60d155510303e307c47dcb6153ccaac72871eb68822d8e296c0d747e55f35" +} diff --git a/backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json b/backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json deleted file mode 100644 index e751b8fc7c..0000000000 --- a/backend/.sqlx/query-9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at\n FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash\n WHERE s.workspace_id = $1 AND s.path = $2\n ORDER by s.created_at DESC", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "hash", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "deployment_msg", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "created_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - true, - false - ] - }, - "hash": "9a1483a81f5b086e0765d3d69483e29b09f66090e1f9d394564c16d921d2e66c" -} diff --git a/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json b/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json deleted file mode 100644 index 3ed4f5a316..0000000000 --- a/backend/.sqlx/query-a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT flow_version.id, flow_version.created_at, deployment_metadata.deployment_msg FROM flow_version \n LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version\n WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 \n ORDER BY flow_version.created_at DESC", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "created_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 2, - "name": "deployment_msg", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "a0f1c0df6bc2f1fbca50edee90e42c94445536e201b322eda6f7a90bdf38f36a" -} diff --git a/backend/.sqlx/query-c8b369881add5574c1762431e2fb58eb04892babb4684be314f369227424e640.json b/backend/.sqlx/query-e7c8ccc5f1955cae39856baacafb7ce6482125e720fb8f53e3e9e749a69e08bd.json similarity index 85% rename from backend/.sqlx/query-c8b369881add5574c1762431e2fb58eb04892babb4684be314f369227424e640.json rename to backend/.sqlx/query-e7c8ccc5f1955cae39856baacafb7ce6482125e720fb8f53e3e9e749a69e08bd.json index a92de852d2..3262c18b6f 100644 --- a/backend/.sqlx/query-c8b369881add5574c1762431e2fb58eb04892babb4684be314f369227424e640.json +++ b/backend/.sqlx/query-e7c8ccc5f1955cae39856baacafb7ce6482125e720fb8f53e3e9e749a69e08bd.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg,\n av.created_by as created_by, av.created_at as created_at\n FROM app a LEFT JOIN app_version av ON a.id = av.app_id LEFT JOIN deployment_metadata dm ON av.id = dm.app_version\n WHERE a.workspace_id = $1 AND a.path = $2\n ORDER BY array_position(a.versions, av.id) DESC NULLS LAST, av.id DESC", + "query": "SELECT a.id as app_id, av.id as version_id, dm.deployment_msg as deployment_msg,\n av.created_by as created_by, av.created_at as created_at\n FROM app a LEFT JOIN app_version av ON a.id = av.app_id LEFT JOIN deployment_metadata dm ON av.id = dm.app_version\n WHERE a.workspace_id = $1 AND a.path = $2\n ORDER BY array_position(a.versions, av.id) DESC NULLS LAST, av.id DESC\n LIMIT $3 OFFSET $4", "describe": { "columns": [ { @@ -32,7 +32,9 @@ "parameters": { "Left": [ "Text", - "Text" + "Text", + "Int8", + "Int8" ] }, "nullable": [ @@ -43,5 +45,5 @@ false ] }, - "hash": "c8b369881add5574c1762431e2fb58eb04892babb4684be314f369227424e640" + "hash": "e7c8ccc5f1955cae39856baacafb7ce6482125e720fb8f53e3e9e749a69e08bd" } diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index 9a154ee2d1..ee063e6c8c 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -51,7 +51,10 @@ use windmill_common::{ jobs::JobPayload, schedule::Schedule, triggers::MovedNativeTrigger, - utils::{http_get_from_hub, not_found_if_none, paginate, Pagination, RunnableKind, StripPath}, + utils::{ + http_get_from_hub, not_found_if_none, paginate, paginate_with_default, Pagination, + RunnableKind, StripPath, HISTORY_PER_PAGE, + }, }; use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; @@ -900,19 +903,24 @@ async fn get_flow_history( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(pagination): Query, ) -> JsonResult> { let path = path.to_path(); check_scopes(&authed, || format!("flows:read:{}", path))?; + let (per_page, offset) = paginate_with_default(pagination, HISTORY_PER_PAGE); let mut tx = user_db.begin(&authed).await?; let flows = sqlx::query_as!( FlowVersion, - "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version + "SELECT flow_version.id, flow_version.created_at, flow_version.created_by, deployment_metadata.deployment_msg FROM flow_version LEFT JOIN deployment_metadata ON flow_version.id = deployment_metadata.flow_version - WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 - ORDER BY flow_version.created_at DESC", + WHERE flow_version.path = $1 AND flow_version.workspace_id = $2 + ORDER BY flow_version.created_at DESC + LIMIT $3 OFFSET $4", path, - w_id + w_id, + per_page as i64, + offset as i64, ) .fetch_all(&mut *tx) .await?; diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 0400cbc91d..cc98fad55c 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -73,7 +73,10 @@ use windmill_common::{ }, triggers::MovedNativeTrigger, users::username_to_permissioned_as, - utils::{not_found_if_none, query_elems_from_hub, require_admin, Pagination, StripPath}, + utils::{ + not_found_if_none, paginate_with_default, query_elems_from_hub, require_admin, Pagination, + StripPath, HISTORY_PER_PAGE, + }, worker::to_raw_value, HUB_BASE_URL, }; @@ -3088,17 +3091,22 @@ async fn get_script_history( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(pagination): Query, ) -> JsonResult> { let path = path.to_path(); check_scopes(&authed, || format!("scripts:read:{}", path))?; + let (per_page, offset) = paginate_with_default(pagination, HISTORY_PER_PAGE); let mut tx = user_db.begin(&authed).await?; let query_result = sqlx::query!( "SELECT s.hash as hash, dm.deployment_msg as deployment_msg, s.created_at as created_at, s.created_by as created_by FROM script s LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash WHERE s.workspace_id = $1 AND s.path = $2 - ORDER by s.created_at DESC", + ORDER by s.created_at DESC + LIMIT $3 OFFSET $4", w_id, path, + per_page as i64, + offset as i64, ) .fetch_all(&mut *tx) .await?; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 6d52f7b426..6b7383b0e2 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -10356,6 +10356,8 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" responses: "200": description: script history @@ -11761,6 +11763,8 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" tags: - flow responses: @@ -13295,6 +13299,8 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/ScriptPath" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" responses: "200": description: app history diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index cb3ef3dba7..a6abe01383 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -69,8 +69,9 @@ use windmill_common::{ user_drafts::{overlay_or_draft_only, DraftUserRef, UserDraftItemKind, WithDraftOverlay}, users::username_to_permissioned_as, utils::{ - http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, - strip_json_nul, Pagination, RunnableKind, StripPath, + http_get_from_hub, not_found_if_none, paginate, paginate_with_default, + query_elems_from_hub, require_admin, strip_json_nul, Pagination, RunnableKind, StripPath, + HISTORY_PER_PAGE, }, variables::{build_crypt, build_crypt_with_key_suffix, encrypt}, worker::{to_raw_value, CLOUD_HOSTED}, @@ -1237,9 +1238,11 @@ async fn get_app_history( authed: ApiAuthed, Extension(user_db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(pagination): Query, ) -> JsonResult> { let path = path.to_path(); check_scopes(&authed, || format!("apps:read:{}", &path))?; + let (per_page, offset) = paginate_with_default(pagination, HISTORY_PER_PAGE); let mut tx = user_db.begin(&authed).await?; // Newest first in the order the versions were deployed, which the picker numbers // (`v1`, `v2`, …) and reads the head off. That is the position in `app.versions`, @@ -1252,9 +1255,12 @@ async fn get_app_history( av.created_by as created_by, av.created_at as created_at FROM app a LEFT JOIN app_version av ON a.id = av.app_id LEFT JOIN deployment_metadata dm ON av.id = dm.app_version WHERE a.workspace_id = $1 AND a.path = $2 - ORDER BY array_position(a.versions, av.id) DESC NULLS LAST, av.id DESC", + ORDER BY array_position(a.versions, av.id) DESC NULLS LAST, av.id DESC + LIMIT $3 OFFSET $4", w_id, path, + per_page as i64, + offset as i64, ).fetch_all(&mut *tx).await?; tx.commit().await?; diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 62d47a34d7..64e7b73aee 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -479,6 +479,21 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) { (per_page, offset) } +/// What one page of a deploy history holds when the caller asks for no size. The +/// picker that reads these shows a screenful and fetches the next page on demand; +/// a path a pipeline deploys carries far more versions than anyone scrolls. +pub const HISTORY_PER_PAGE: usize = 20; + +/// [`paginate`] for a listing whose unasked-for page should be smaller than the generic +/// default — a version history behind a picker, where a path deployed by CI carries +/// thousands of entries and only the first screenful is ever read. +pub fn paginate_with_default(pagination: Pagination, default_per_page: usize) -> (usize, usize) { + paginate(Pagination { + per_page: Some(pagination.per_page.unwrap_or(default_per_page)), + ..pagination + }) +} + pub fn paginate_without_limits(pagination: Pagination) -> (usize, usize) { let per_page = pagination.per_page.unwrap_or(MAX_PER_PAGE); let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page; diff --git a/frontend/src/lib/components/DiffDrawer.svelte b/frontend/src/lib/components/DiffDrawer.svelte index 545785196e..79f8ce8b56 100644 --- a/frontend/src/lib/components/DiffDrawer.svelte +++ b/frontend/src/lib/components/DiffDrawer.svelte @@ -142,6 +142,42 @@ * generation is stale — a slower earlier pick, or one outlived by a drawer reset — * and neither replaces the diff nor clears the spinner, which `disabled` rides on. */ let versionLoadGeneration = 0 + /** Pages of `versions` fetched after the first. Kept beside `data` so a diff swapped in + * by a newer opening drops them along with the list they extended. */ + let extraVersions: DiffVersionOption[] = $state([]) + let moreLoader: (() => Promise) | undefined = $state(undefined) + let loadingMore = $state(false) + /** What the picker offers: the page the editor handed over plus whatever the reader has + * asked for since. Every read of the version list goes through this. */ + const pickerVersions = $derived.by(() => + data?.mode === 'normal' ? [...(data.versions ?? []), ...extraVersions] : [] + ) + + /** The editors hand over one page, so a path a pipeline has deployed thousands of times + * does not hold the drawer shut while its whole history arrives. */ + async function fetchMoreVersions() { + if (!moreLoader || loadingMore) return + loadingMore = true + const generation = versionLoadGeneration + try { + const more = await moreLoader() + // A diff swapped in while this ran owns the picker now; appending would splice + // one item's history onto another's. + if (generation !== versionLoadGeneration) return + if (more?.length) { + extraVersions = [...extraVersions, ...more] + } else { + // Nothing came back, so there is nothing further to ask for. + moreLoader = undefined + } + } catch (e: any) { + if (generation === versionLoadGeneration) { + sendUserToast(`Could not load older versions: ${e?.body ?? e?.message ?? e}`, true) + } + } finally { + loadingMore = false + } + } async function selectVersion(id: string | undefined) { if (!id || !versionLoader || !data || data.mode !== 'normal') return @@ -159,7 +195,7 @@ if (!value) sendUserToast(`Could not load version ${id}`, true) return } - const opt = data.versions?.find((v) => v.id === id) + const opt = pickerVersions.find((v) => v.id === id) data = { ...data, deployed: prepareDiff(value), @@ -181,7 +217,7 @@ * it, so "take latest" means the version the reader is looking at. */ const headShown = $derived.by(() => data?.mode === 'normal' - ? (data.versions?.find((v) => v.isHead)?.id ?? data.deployedHead) + ? (pickerVersions.find((v) => v.isHead)?.id ?? data.deployedHead) : undefined ) /** Behind as the drawer can see it. Unknown counts as not behind: offering to adopt a @@ -218,6 +254,7 @@ deployedLabel?: string versions?: DiffVersionOption[] loadVersion?: (id: string) => Promise + loadMoreVersions?: () => Promise onTakeLatest?: (head?: string) => void | Promise draftBase?: string deployedHead?: string @@ -245,6 +282,7 @@ deployedLabel, versions, loadVersion, + loadMoreVersions, onTakeLatest, draftBase, deployedHead, @@ -253,11 +291,14 @@ button } = diff versionLoader = loadVersion + moreLoader = loadMoreVersions headLabel = deployedLabel headDeployed = !deployed.draft_only ? prepareDiff(deployed) : undefined // A load still in flight belongs to the diff being replaced. versionLoadGeneration++ loadingVersion = false + extraVersions = [] + loadingMore = false selectedVersion = versions?.find((v) => v.isHead)?.id data = { mode: 'normal', @@ -297,12 +338,12 @@ - {#if data.mode === 'normal' && (data.deployedLabel || data.versions?.length)} + {#if data.mode === 'normal' && (data.deployedLabel || pickerVersions.length)}
- {#if data.versions?.length && versionLoader} + {#if pickerVersions.length && versionLoader}