fix: obsolete wal entires while opening a migrated region (#4993)

* fix: delete obsolete wal entrie while opening a migrated region

* chore: add logs

* chore: rust fmt

* fix: fix fuzz test

src/datanode/src/heartbeat/handler/downgrade_region.rs

@@ -74,6 +74,10 @@ impl HandlerContext {
 
             // Ignores flush request
             if !writable {
+                warn!(
+                    "Region: {region_id} is not writable, flush_timeout: {:?}",
+                    flush_timeout
+                );
                 return self.downgrade_to_follower_gracefully(region_id).await;
             }
 

src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs

@@ -165,6 +165,7 @@ impl DowngradeLeaderRegion {
         match receiver.await? {
             Ok(msg) => {
                 let reply = HeartbeatMailbox::json_reply(&msg)?;
+                info!("Downgrade region reply: {:?}", reply);
                 let InstructionReply::DowngradeRegion(DowngradeRegionReply {
                     last_entry_id,
                     exists,

src/meta-srv/src/procedure/region_migration/open_candidate_region.rs

@@ -21,6 +21,7 @@ use common_meta::instruction::{Instruction, InstructionReply, OpenRegion, Simple
 use common_meta::key::datanode_table::RegionInfo;
 use common_meta::RegionIdent;
 use common_procedure::Status;
+use common_telemetry::info;
 use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt};
 
@@ -144,6 +145,7 @@ impl OpenCandidateRegion {
         match receiver.await? {
             Ok(msg) => {
                 let reply = HeartbeatMailbox::json_reply(&msg)?;
+                info!("Received open region reply: {:?}", reply);
                 let InstructionReply::OpenRegion(SimpleReply { result, error }) = reply else {
                     return error::UnexpectedInstructionReplySnafu {
                         mailbox_message: msg.to_string(),

src/mito2/src/worker/handle_catchup.rs

@@ -49,7 +49,8 @@ impl<S: LogStore> RegionWorkerLoop<S> {
         // Utilizes the short circuit evaluation.
         let region = if !is_mutable_empty || region.manifest_ctx.has_update().await? {
             let manifest_version = region.manifest_ctx.manifest_version().await;
-            info!("Reopening the region: {region_id}, empty mutable: {is_mutable_empty}, manifest version: {manifest_version}");
+            let flushed_entry_id = region.version_control.current().last_entry_id;
+            info!("Reopening the region: {region_id}, empty mutable: {is_mutable_empty}, manifest version: {manifest_version}, flushed entry id: {flushed_entry_id}");
             let reopened_region = Arc::new(
                 RegionOpener::new(
                     region_id,
@@ -111,6 +112,9 @@ impl<S: LogStore> RegionWorkerLoop<S> {
             }
         } else {
             warn!("Skips to replay memtable for region: {}", region.region_id);
+            let flushed_entry_id = region.version_control.current().last_entry_id;
+            let on_region_opened = self.wal.on_region_opened();
+            on_region_opened(region_id, flushed_entry_id, &region.provider).await?;
         }
 
         if request.set_writable {