perf: list audit logs by timestamp so only recent partitions are read (#12309)

* perf: list audit logs by timestamp so only recent partitions are read

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: pin audit log paging by (timestamp, id) cursor

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* chore: update ee-repo-ref to d7d509d0955c52f83afd0bf8b61a0e811162c16f

This commit updates the EE repository reference after PR #971 was merged in windmill-ee-private.

Previous ee-repo-ref: 07dcbcda7762b5efab7caa515122066fa274c039

New ee-repo-ref: d7d509d0955c52f83afd0bf8b61a0e811162c16f

Automated by sync-ee-ref workflow.

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
This commit is contained in:
Ruben Fiszel
2026-11-10 10:50:00 +01:00
committed by GitHub
co-authored by Claude Opus 5.5 windmill-internal-app[bot]
parent 2e422b497e
commit a6ec7d0c18
5 changed files with 75 additions and 17 deletions
+1 -1
View File
@@ -1 +1 @@
a2e29e58a275306fa016c6675fc8e603848fb2f0
d7d509d0955c52f83afd0bf8b61a0e811162c16f
@@ -136,6 +136,51 @@ async fn test_job_run_is_searchable_by_token_and_by_caller(
Ok(())
}
/// Paging with the `(timestamp, id)` cursor must return every row once, in `timestamp DESC, id
/// DESC` order, even where ids disagree with timestamps (an id is drawn at insert, the timestamp
/// at transaction start).
#[cfg(all(feature = "enterprise", feature = "private"))]
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_audit_list_cursor_follows_timestamp_order(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
// Hours ahead stay inside the partitions created for the next days.
sqlx::query(
"INSERT INTO audit_partitioned (workspace_id, id, timestamp, username, operation, action_kind)
SELECT 'test-workspace', id, date_trunc('second', now()) + h * interval '1 hour',
'test-user', 'cursor.probe', 'create'
FROM (VALUES (9000001, 1), (9000002, 3), (9000003, 2), (9000004, 2), (9000005, 1)) v(id, h)",
)
.execute(&db)
.await?;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let mut seen = vec![];
let mut cursor: Vec<(&str, String)> = vec![];
loop {
let url = format!("http://localhost:{port}/api/w/test-workspace/audit/list");
let page = authed(client().get(url))
.query(&[("operation", "cursor.probe"), ("per_page", "1")])
.query(&cursor)
.send()
.await?
.json::<Vec<serde_json::Value>>()
.await?;
let Some(row) = page.first() else { break };
seen.push(row["id"].as_i64().unwrap());
cursor = vec![
("before_id", row["id"].to_string()),
(
"before_id_timestamp",
row["timestamp"].as_str().unwrap().to_string(),
),
];
}
assert_eq!(seen, [9000002, 9000004, 9000003, 9000005, 9000001]);
Ok(())
}
/// The run filter must find a run's entries under each key they keep its id in: the run itself
/// (`parameters.uuid`), a cancel (`resource`) and a resume (`parameters.job_id`), and only those.
#[cfg(all(feature = "enterprise", feature = "private"))]
+12 -3
View File
@@ -271,12 +271,21 @@ paths:
- name: before_id
in: query
description: >
only return logs with an id strictly lower than this one. Logs are ordered by
descending id, so this is a keyset cursor to stream a page in several batches
without paying a growing offset.
the id half of a keyset cursor. Logs are ordered by descending timestamp then
id: pass the last row's id here and its timestamp as before_id_timestamp to
stream a page in several batches without paying a growing offset. Alone, it
only filters on id < before_id, which can skip or repeat rows across pages.
schema:
type: integer
format: int64
- name: before_id_timestamp
in: query
description: >
the timestamp of the row whose id is before_id: with it, only logs strictly
before that row in (timestamp, id) order are returned
schema:
type: string
format: date-time
- name: all_workspaces
in: query
description: get audit logs for all workspaces
+4 -2
View File
@@ -38,9 +38,11 @@ pub struct ListAuditLogQuery {
pub resource: Option<String>,
pub before: Option<chrono::DateTime<chrono::Utc>>,
pub after: Option<chrono::DateTime<chrono::Utc>>,
// Keyset cursor on the `id DESC` ordering. Lets a client stream a page in small batches
// without paying a growing OFFSET on every batch.
// Keyset cursor on the `timestamp DESC, id DESC` ordering: the id and timestamp of the last
// row seen. Lets a client stream a page in small batches without paying a growing OFFSET on
// every batch. `before_id` alone filters on the id only.
pub before_id: Option<i64>,
pub before_id_timestamp: Option<chrono::DateTime<chrono::Utc>>,
pub all_workspaces: Option<bool>,
// The entries about one run: started, viewed, cancelled, resumed, deleted...
pub job_id: Option<sqlx::types::Uuid>,
@@ -25,8 +25,8 @@ const MAX_PER_PAGE = 1000
/**
* Loads the newest audit logs, optionally streaming them in smaller batches so rows show up as
* they arrive on instances where a full load takes a long time to come back, and appends older
* ones on demand. Rows are ordered by descending id, so every batch after the first follows a
* `before_id` cursor.
* ones on demand. Rows are ordered by descending timestamp then id, so every batch after the
* first follows a cursor on the last row's (timestamp, id).
*/
export function useAuditLogsLoader(args: () => AuditLogsLoaderArgs) {
let logs: AuditLog[] | undefined = $state()
@@ -45,12 +45,14 @@ export function useAuditLogsLoader(args: () => AuditLogsLoaderArgs) {
function fetchBatch(
a: AuditLogsLoaderArgs,
limit: number,
beforeId: number | undefined
cursor: AuditLog | undefined
): CancelablePromise<AuditLog[]> {
return AuditService.listAuditLogs({
workspace: a.scope === 'instance' ? 'global' : a.workspace!,
perPage: limit,
beforeId,
beforeId: cursor?.id,
// Passed back as received: the server's microseconds would not survive a `Date`.
beforeIdTimestamp: cursor?.timestamp,
before: a.before,
after: a.after,
username: a.username === 'all' ? undefined : a.username,
@@ -105,8 +107,8 @@ export function useAuditLogsLoader(args: () => AuditLogsLoaderArgs) {
const acc: AuditLog[] = []
let slowBatchToastShown = false
function loadBatch(beforeId: number | undefined): CancelablePromise<void> {
let fetch = fetchBatch(a, size, beforeId)
function loadBatch(cursor: AuditLog | undefined): CancelablePromise<void> {
let fetch = fetchBatch(a, size, cursor)
if (isBatched && size > 1) {
fetch = CancelablePromiseUtils.onTimeout(fetch, 4000, () => {
if (slowBatchToastShown) return
@@ -132,7 +134,7 @@ export function useAuditLogsLoader(args: () => AuditLogsLoaderArgs) {
hasMore = acc.length >= total
return CancelablePromiseUtils.pure<void>(undefined)
}
return loadBatch(rows[rows.length - 1].id)
return loadBatch(rows[rows.length - 1])
})
}
@@ -221,8 +223,8 @@ export function useAuditLogsLoader(args: () => AuditLogsLoaderArgs) {
// Rows are appended as each batch lands, so stopping keeps them and the next load-more
// continues from the last one.
function extraBatch(beforeId: number): CancelablePromise<void> {
return CancelablePromiseUtils.then(fetchBatch(a, size, beforeId), (rows) => {
function extraBatch(cursor: AuditLog): CancelablePromise<void> {
return CancelablePromiseUtils.then(fetchBatch(a, size, cursor), (rows) => {
const kept = rows.slice(0, total - appended)
appended += kept.length
logs = [...(logs ?? []), ...kept]
@@ -233,11 +235,11 @@ export function useAuditLogsLoader(args: () => AuditLogsLoaderArgs) {
hasMore = false
return CancelablePromiseUtils.pure<void>(undefined)
}
return extraBatch(rows[rows.length - 1].id)
return extraBatch(rows[rows.length - 1])
})
}
let promise = CancelablePromiseUtils.onTimeout(extraBatch(last.id), 4000, () => {
let promise = CancelablePromiseUtils.onTimeout(extraBatch(last), 4000, () => {
const smaller = total > SMALL_BATCH_SIZE ? SMALL_BATCH_SIZE : 1
const offerBatches = size === total && total > 1
sendUserToast('Loading more audit logs is taking longer than expected...', 'warning', [