From 831370cdde8e06f4298b17baa1a0041bacdd98c7 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 31 Aug 2026 18:01:28 +0200 Subject: [PATCH] fix: harden the service log indexer's recovery and read paths (#10904) * [ee] fix: an unreadable ingest cursor should not stop the server booting Three follow-ups to #10894, all in the service log indexer: a corrupt cursor no longer takes the server down at boot, the queue's writes are covered against a real database rather than by hand, and a read skips the dedupe when the partition it reads holds a single object. * [ee] test: place the queue's rows relative to the clock the statement reads Also drops the two `.sqlx` entries the query extraction orphaned: sqlx keys on the literal including its indentation, so moving a query into a function leaves the old copy behind. * [ee] test: make the pair-exactness and rebuild-dedupe tests actually bite * [ee] docs: state the cursor and dedupe rules without their history * chore: update ee-repo-ref to 90a368362896ebcc2fcfaaf9510dc9be68c929f7 This commit updates the EE repository reference after PR #761 was merged in windmill-ee-private. Previous ee-repo-ref: e3423705aa8f2d585bc65474cfd0c4c762ec4ad5 New ee-repo-ref: 90a368362896ebcc2fcfaaf9510dc9be68c929f7 Automated by sync-ee-ref workflow. --------- Co-authored-by: windmill-internal-app[bot] --- ...f927ac4886755fd17ec755037763aa92f6c0e.json | 22 +++++++++++++++++++ ...33fa0fd05b067454f0d4686d8b155fe562ffe.json | 15 +++++++++++++ ...cdc85389c0629847d8fabb2ad0aa043957b2f.json | 22 ------------------- ...0c06924719189bec8e5c19866db8d0b87df5e.json | 15 ------------- backend/ee-repo-ref.txt | 2 +- 5 files changed, 38 insertions(+), 38 deletions(-) create mode 100644 backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json create mode 100644 backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json delete mode 100644 backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json delete mode 100644 backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json diff --git a/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json b/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json new file mode 100644 index 0000000000..98e08b8167 --- /dev/null +++ b/backend/.sqlx/query-872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH moved AS (\n UPDATE log_file SET indexed_at = CASE\n WHEN log_ts > NOW() - make_interval(secs => $1) THEN NULL\n ELSE now() END\n WHERE indexed_at = 'epoch' RETURNING 1)\n SELECT count(*) AS \"n!\" FROM moved", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "872be44352d4a27a2005f6bdc38f927ac4886755fd17ec755037763aa92f6c0e" +} diff --git a/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json b/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json new file mode 100644 index 0000000000..76456ea8fb --- /dev/null +++ b/backend/.sqlx/query-87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE log_file SET indexed_at = now()\n FROM unnest($1::text[], $2::timestamp[]) AS c(hostname, log_ts)\n WHERE log_file.hostname = c.hostname AND log_file.log_ts = c.log_ts", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TimestampArray" + ] + }, + "nullable": [] + }, + "hash": "87e8e034b1cf5ea7ce43870d77d33fa0fd05b067454f0d4686d8b155fe562ffe" +} diff --git a/backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json b/backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json deleted file mode 100644 index 96745f9485..0000000000 --- a/backend/.sqlx/query-8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH moved AS (\n UPDATE log_file SET indexed_at = CASE\n WHEN log_ts > NOW() - make_interval(secs => $1) THEN NULL\n ELSE now() END\n WHERE indexed_at = 'epoch' RETURNING 1)\n SELECT count(*) AS \"n!\" FROM moved", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "n!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Float8" - ] - }, - "nullable": [ - null - ] - }, - "hash": "8e0461855d05dc03919c8979d8acdc85389c0629847d8fabb2ad0aa043957b2f" -} diff --git a/backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json b/backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json deleted file mode 100644 index 6739d80033..0000000000 --- a/backend/.sqlx/query-947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE log_file SET indexed_at = now()\n FROM unnest($1::text[], $2::timestamp[]) AS c(hostname, log_ts)\n WHERE log_file.hostname = c.hostname AND log_file.log_ts = c.log_ts", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "TextArray", - "TimestampArray" - ] - }, - "nullable": [] - }, - "hash": "947f7ca06f6f9a3fd50f817bc9b0c06924719189bec8e5c19866db8d0b87df5e" -} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 693921efc6..5f2f41186e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -bc0c7051585194474078b6c1941a3fb73893d9e5 +90a368362896ebcc2fcfaaf9510dc9be68c929f7