mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-18 16:19:57 +00:00
perf: list audit logs by timestamp so only recent partitions are read (#12309)
* perf: list audit logs by timestamp so only recent partitions are read Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * test: pin audit log paging by (timestamp, id) cursor Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore: update ee-repo-ref to d7d509d0955c52f83afd0bf8b61a0e811162c16f This commit updates the EE repository reference after PR #971 was merged in windmill-ee-private. Previous ee-repo-ref: 07dcbcda7762b5efab7caa515122066fa274c039 New ee-repo-ref: d7d509d0955c52f83afd0bf8b61a0e811162c16f Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
windmill-internal-app[bot]
parent
2e422b497e
commit
a6ec7d0c18
@@ -1 +1 @@
|
||||
a2e29e58a275306fa016c6675fc8e603848fb2f0
|
||||
d7d509d0955c52f83afd0bf8b61a0e811162c16f
|
||||
|
||||
@@ -136,6 +136,51 @@ async fn test_job_run_is_searchable_by_token_and_by_caller(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Paging with the `(timestamp, id)` cursor must return every row once, in `timestamp DESC, id
|
||||
/// DESC` order, even where ids disagree with timestamps (an id is drawn at insert, the timestamp
|
||||
/// at transaction start).
|
||||
#[cfg(all(feature = "enterprise", feature = "private"))]
|
||||
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
|
||||
async fn test_audit_list_cursor_follows_timestamp_order(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
// Hours ahead stay inside the partitions created for the next days.
|
||||
sqlx::query(
|
||||
"INSERT INTO audit_partitioned (workspace_id, id, timestamp, username, operation, action_kind)
|
||||
SELECT 'test-workspace', id, date_trunc('second', now()) + h * interval '1 hour',
|
||||
'test-user', 'cursor.probe', 'create'
|
||||
FROM (VALUES (9000001, 1), (9000002, 3), (9000003, 2), (9000004, 2), (9000005, 1)) v(id, h)",
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
let mut seen = vec![];
|
||||
let mut cursor: Vec<(&str, String)> = vec![];
|
||||
loop {
|
||||
let url = format!("http://localhost:{port}/api/w/test-workspace/audit/list");
|
||||
let page = authed(client().get(url))
|
||||
.query(&[("operation", "cursor.probe"), ("per_page", "1")])
|
||||
.query(&cursor)
|
||||
.send()
|
||||
.await?
|
||||
.json::<Vec<serde_json::Value>>()
|
||||
.await?;
|
||||
let Some(row) = page.first() else { break };
|
||||
seen.push(row["id"].as_i64().unwrap());
|
||||
cursor = vec![
|
||||
("before_id", row["id"].to_string()),
|
||||
(
|
||||
"before_id_timestamp",
|
||||
row["timestamp"].as_str().unwrap().to_string(),
|
||||
),
|
||||
];
|
||||
}
|
||||
assert_eq!(seen, [9000002, 9000004, 9000003, 9000005, 9000001]);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The run filter must find a run's entries under each key they keep its id in: the run itself
|
||||
/// (`parameters.uuid`), a cancel (`resource`) and a resume (`parameters.job_id`), and only those.
|
||||
#[cfg(all(feature = "enterprise", feature = "private"))]
|
||||
|
||||
@@ -271,12 +271,21 @@ paths:
|
||||
- name: before_id
|
||||
in: query
|
||||
description: >
|
||||
only return logs with an id strictly lower than this one. Logs are ordered by
|
||||
descending id, so this is a keyset cursor to stream a page in several batches
|
||||
without paying a growing offset.
|
||||
the id half of a keyset cursor. Logs are ordered by descending timestamp then
|
||||
id: pass the last row's id here and its timestamp as before_id_timestamp to
|
||||
stream a page in several batches without paying a growing offset. Alone, it
|
||||
only filters on id < before_id, which can skip or repeat rows across pages.
|
||||
schema:
|
||||
type: integer
|
||||
format: int64
|
||||
- name: before_id_timestamp
|
||||
in: query
|
||||
description: >
|
||||
the timestamp of the row whose id is before_id: with it, only logs strictly
|
||||
before that row in (timestamp, id) order are returned
|
||||
schema:
|
||||
type: string
|
||||
format: date-time
|
||||
- name: all_workspaces
|
||||
in: query
|
||||
description: get audit logs for all workspaces
|
||||
|
||||
@@ -38,9 +38,11 @@ pub struct ListAuditLogQuery {
|
||||
pub resource: Option<String>,
|
||||
pub before: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub after: Option<chrono::DateTime<chrono::Utc>>,
|
||||
// Keyset cursor on the `id DESC` ordering. Lets a client stream a page in small batches
|
||||
// without paying a growing OFFSET on every batch.
|
||||
// Keyset cursor on the `timestamp DESC, id DESC` ordering: the id and timestamp of the last
|
||||
// row seen. Lets a client stream a page in small batches without paying a growing OFFSET on
|
||||
// every batch. `before_id` alone filters on the id only.
|
||||
pub before_id: Option<i64>,
|
||||
pub before_id_timestamp: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub all_workspaces: Option<bool>,
|
||||
// The entries about one run: started, viewed, cancelled, resumed, deleted...
|
||||
pub job_id: Option<sqlx::types::Uuid>,
|
||||
|
||||
@@ -25,8 +25,8 @@ const MAX_PER_PAGE = 1000
|
||||
/**
|
||||
* Loads the newest audit logs, optionally streaming them in smaller batches so rows show up as
|
||||
* they arrive on instances where a full load takes a long time to come back, and appends older
|
||||
* ones on demand. Rows are ordered by descending id, so every batch after the first follows a
|
||||
* `before_id` cursor.
|
||||
* ones on demand. Rows are ordered by descending timestamp then id, so every batch after the
|
||||
* first follows a cursor on the last row's (timestamp, id).
|
||||
*/
|
||||
export function useAuditLogsLoader(args: () => AuditLogsLoaderArgs) {
|
||||
let logs: AuditLog[] | undefined = $state()
|
||||
@@ -45,12 +45,14 @@ export function useAuditLogsLoader(args: () => AuditLogsLoaderArgs) {
|
||||
function fetchBatch(
|
||||
a: AuditLogsLoaderArgs,
|
||||
limit: number,
|
||||
beforeId: number | undefined
|
||||
cursor: AuditLog | undefined
|
||||
): CancelablePromise<AuditLog[]> {
|
||||
return AuditService.listAuditLogs({
|
||||
workspace: a.scope === 'instance' ? 'global' : a.workspace!,
|
||||
perPage: limit,
|
||||
beforeId,
|
||||
beforeId: cursor?.id,
|
||||
// Passed back as received: the server's microseconds would not survive a `Date`.
|
||||
beforeIdTimestamp: cursor?.timestamp,
|
||||
before: a.before,
|
||||
after: a.after,
|
||||
username: a.username === 'all' ? undefined : a.username,
|
||||
@@ -105,8 +107,8 @@ export function useAuditLogsLoader(args: () => AuditLogsLoaderArgs) {
|
||||
const acc: AuditLog[] = []
|
||||
let slowBatchToastShown = false
|
||||
|
||||
function loadBatch(beforeId: number | undefined): CancelablePromise<void> {
|
||||
let fetch = fetchBatch(a, size, beforeId)
|
||||
function loadBatch(cursor: AuditLog | undefined): CancelablePromise<void> {
|
||||
let fetch = fetchBatch(a, size, cursor)
|
||||
if (isBatched && size > 1) {
|
||||
fetch = CancelablePromiseUtils.onTimeout(fetch, 4000, () => {
|
||||
if (slowBatchToastShown) return
|
||||
@@ -132,7 +134,7 @@ export function useAuditLogsLoader(args: () => AuditLogsLoaderArgs) {
|
||||
hasMore = acc.length >= total
|
||||
return CancelablePromiseUtils.pure<void>(undefined)
|
||||
}
|
||||
return loadBatch(rows[rows.length - 1].id)
|
||||
return loadBatch(rows[rows.length - 1])
|
||||
})
|
||||
}
|
||||
|
||||
@@ -221,8 +223,8 @@ export function useAuditLogsLoader(args: () => AuditLogsLoaderArgs) {
|
||||
|
||||
// Rows are appended as each batch lands, so stopping keeps them and the next load-more
|
||||
// continues from the last one.
|
||||
function extraBatch(beforeId: number): CancelablePromise<void> {
|
||||
return CancelablePromiseUtils.then(fetchBatch(a, size, beforeId), (rows) => {
|
||||
function extraBatch(cursor: AuditLog): CancelablePromise<void> {
|
||||
return CancelablePromiseUtils.then(fetchBatch(a, size, cursor), (rows) => {
|
||||
const kept = rows.slice(0, total - appended)
|
||||
appended += kept.length
|
||||
logs = [...(logs ?? []), ...kept]
|
||||
@@ -233,11 +235,11 @@ export function useAuditLogsLoader(args: () => AuditLogsLoaderArgs) {
|
||||
hasMore = false
|
||||
return CancelablePromiseUtils.pure<void>(undefined)
|
||||
}
|
||||
return extraBatch(rows[rows.length - 1].id)
|
||||
return extraBatch(rows[rows.length - 1])
|
||||
})
|
||||
}
|
||||
|
||||
let promise = CancelablePromiseUtils.onTimeout(extraBatch(last.id), 4000, () => {
|
||||
let promise = CancelablePromiseUtils.onTimeout(extraBatch(last), 4000, () => {
|
||||
const smaller = total > SMALL_BATCH_SIZE ? SMALL_BATCH_SIZE : 1
|
||||
const offerBatches = size === total && total > 1
|
||||
sendUserToast('Loading more audit logs is taking longer than expected...', 'warning', [
|
||||
|
||||
Reference in New Issue
Block a user