feat(log_store): introduce the CollectionTask (#4530)

* feat: introduce the `CollectionTask`

* feat: add config of index collector

* chore: remove unused code

* feat: truncate indexes

* chore: apply suggestions from CR

* chore: update config examples

* refactor: retrieve latest offset while dumping indexes

* chore: print warn
This commit is contained in:
Weny Xu
2024-08-19 11:48:35 +08:00
committed by GitHub
parent 2a73e0937f
commit 76dc906574
24 changed files with 578 additions and 102 deletions

11
Cargo.lock generated
View File

@@ -3170,6 +3170,15 @@ dependencies = [
"uuid",
]
[[package]]
name = "delta-encoding"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f8513a5eeb3d7b9149563409dc4ab6fd9de5767fd285af5b4d0ee1b778fbce0"
dependencies = [
"num-traits",
]
[[package]]
name = "der"
version = "0.5.1"
@@ -5806,10 +5815,12 @@ dependencies = [
"common-test-util",
"common-time",
"common-wal",
"delta-encoding",
"futures",
"futures-util",
"itertools 0.10.5",
"lazy_static",
"object-store",
"pin-project",
"prometheus",
"protobuf",