rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-01-04 12:22:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(kafka): overwrite the EntryId with Offset while consuming records (#3148)

Browse Source 
* fix(kafka): overwrite the EntryId with Offset while consuming the KafkaRecords

* fix: temporarily workaround of incorrect entry Id
This commit is contained in:
Weny Xu
2024-01-12 12:46:17 +09:00
committed by GitHub
parent c1190bae7b
commit 430ffe0e28
1 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/log-store/src/kafka/log_store.rs
View File

@@ -205,7 +205,11 @@ impl LogStore for KafkaLogStore {
}
// Tries to construct an entry from records consumed so far.
if let Some(entry) = maybe_emit_entry(record, &mut entry_records)? {
if let Some(mut entry) = maybe_emit_entry(record, &mut entry_records)? {
// We don't rely on the EntryId generated by mito2.
// Instead, we use the offset return from Kafka as EntryId.
// Therefore, we MUST overwrite the EntryId with RecordOffset.
entry.id = offset as u64;
yield Ok(vec![entry]);
}
@@ -423,17 +427,20 @@ mod tests {
// Reads entries for regions and checks for each region that the gotten entries are identical with the expected ones.
for region_id in which {
let ctx = &region_contexts[&region_id];
let ctx = region_contexts.get_mut(&region_id).unwrap();
let stream = logstore
.read(&ctx.ns, ctx.flushed_entry_id + 1)
.await
.unwrap();
let got = stream
let mut got = stream
.collect::<Vec<_>>()
.await
.into_iter()
.flat_map(|x| x.unwrap())
.collect::<Vec<_>>();
//FIXME(weny): https://github.com/GreptimeTeam/greptimedb/issues/3152
ctx.expected.iter_mut().for_each(|entry| entry.id = 0);
got.iter_mut().for_each(|entry| entry.id = 0);
assert_eq!(ctx.expected, got);
}