feat: introduce high_watermark for remote wal logstore (#5877)

* feat: introduce high_watermark_since_flush

* test: add unit test for high watermark

* refactor: submit a request instead

* fix: send reply before submit request

* fix: no need to update twice

* feat: update high watermark in background periodically

* test: update unit tests

* fix: update high watermark periodically

* test: update unit tests

* chore: apply review comments

* chore: rename

* chore: apply review comments

* chore: clean up

* chore: apply review comments

src/store-api/src/logstore.rs

@@ -94,6 +94,9 @@ pub trait LogStore: Send + Sync + 'static + std::fmt::Debug {
         region_id: RegionId,
         provider: &Provider,
     ) -> Result<Entry, Self::Error>;
+
+    /// Returns the highest existing entry id in the log store.
+    fn high_watermark(&self, provider: &Provider) -> Result<EntryId, Self::Error>;
 }
 
 /// The response of an `append` operation.

src/store-api/src/region_engine.rs

@@ -451,6 +451,13 @@ pub struct RegionStatistic {
     /// The details of the region.
     #[serde(default)]
     pub manifest: RegionManifestInfo,
+    /// The latest entry id of the region's remote WAL since last flush.
+    /// For metric engine, there're two latest entry ids, one for data and one for metadata.
+    /// TODO(weny): remove this two fields and use single instead.
+    #[serde(default)]
+    pub data_topic_latest_entry_id: u64,
+    #[serde(default)]
+    pub metadata_topic_latest_entry_id: u64,
 }
 
 /// The manifest info of a region.