From a3d567f0c9d9e4c14310973941583b69b9ade28c Mon Sep 17 00:00:00 2001
From: discord9 <55937128+discord9@users.noreply.github.com>
Date: Wed, 11 Sep 2024 11:31:52 +0800
Subject: [PATCH 001/128] perf(flow): use batch mode for flow (#4599)

* generic bundle trait

* feat: impl get/let

* fix: drop batch

* test: tumble batch

* feat: use batch eval flow

* fix: div use arrow::div not mul

* perf: not append batch

* perf: use bool mask for reduce

* perf: tiny opt

* perf: refactor slow path

* feat: opt if then

* fix: WIP

* perf: if then

* chore: use trace instead

* fix: reduce missing non-first batch

* perf: flow if then using interleave

* docs: add TODO

* perf: remove unnecessary eq

* chore: remove unused import

* fix: run_available no longer loop forever

* feat: blocking on high input buf

* chore: increase threhold

* chore: after rebase

* chore: per review

* chore: per review

* fix: allow empty values in reduce&test

* tests: more flow doc example tests

* chore: per review

* chore: per review
---
 src/flow/src/adapter.rs                       | 350 ++++-----
 src/flow/src/adapter/flownode_impl.rs         |   4 +-
 src/flow/src/adapter/node_context.rs          |  97 ++-
 src/flow/src/adapter/worker.rs                |  34 +-
 src/flow/src/compute/render.rs                |  91 ++-
 src/flow/src/compute/render/reduce.rs         | 422 +++++++----
 src/flow/src/compute/render/src_sink.rs       |  19 +-
 src/flow/src/compute/types.rs                 |  34 +-
 src/flow/src/expr.rs                          | 109 ++-
 src/flow/src/expr/df_func.rs                  |  16 +-
 src/flow/src/expr/error.rs                    |   9 +-
 src/flow/src/expr/func.rs                     | 278 +++++---
 src/flow/src/expr/linear.rs                   |  11 +-
 src/flow/src/expr/relation.rs                 |   1 +
 src/flow/src/expr/relation/func.rs            |  70 +-
 src/flow/src/expr/scalar.rs                   | 186 ++---
 src/flow/src/repr.rs                          |   5 +-
 src/flow/src/utils.rs                         |   8 +-
 .../standalone/common/flow/flow_basic.result  | 670 +++++++++++++++++-
 .../standalone/common/flow/flow_basic.sql     | 413 ++++++++++-
 20 files changed, 2054 insertions(+), 773 deletions(-)

diff --git a/src/flow/src/adapter.rs b/src/flow/src/adapter.rs
index 04f7fd80b3..3a8acd60e7 100644
--- a/src/flow/src/adapter.rs
+++ b/src/flow/src/adapter.rs
@@ -50,11 +50,9 @@ use crate::adapter::util::column_schemas_to_proto;
 use crate::adapter::worker::{create_worker, Worker, WorkerHandle};
 use crate::compute::ErrCollector;
 use crate::df_optimizer::sql_to_flow_plan;
-use crate::error::{ExternalSnafu, InternalSnafu, TableNotFoundSnafu, UnexpectedSnafu};
-use crate::expr::GlobalId;
-use crate::metrics::{
-    METRIC_FLOW_INPUT_BUF_SIZE, METRIC_FLOW_INSERT_ELAPSED, METRIC_FLOW_RUN_INTERVAL_MS,
-};
+use crate::error::{EvalSnafu, ExternalSnafu, InternalSnafu, TableNotFoundSnafu, UnexpectedSnafu};
+use crate::expr::{Batch, GlobalId};
+use crate::metrics::{METRIC_FLOW_INSERT_ELAPSED, METRIC_FLOW_RUN_INTERVAL_MS};
 use crate::repr::{self, DiffRow, Row, BATCH_SIZE};
 
 mod flownode_impl;
@@ -227,11 +225,24 @@ pub fn diff_row_to_request(rows: Vec<DiffRow>) -> Vec<DiffRequest> {
     reqs
 }
 
+pub fn batches_to_rows_req(batches: Vec<Batch>) -> Result<Vec<DiffRequest>, Error> {
+    let mut reqs = Vec::new();
+    for batch in batches {
+        let mut rows = Vec::with_capacity(batch.row_count());
+        for i in 0..batch.row_count() {
+            let row = batch.get_row(i).context(EvalSnafu)?;
+            rows.push((Row::new(row), 0));
+        }
+        reqs.push(DiffRequest::Insert(rows));
+    }
+    Ok(reqs)
+}
+
 /// This impl block contains methods to send writeback requests to frontend
 impl FlowWorkerManager {
     /// Return the number of requests it made
     pub async fn send_writeback_requests(&self) -> Result<usize, Error> {
-        let all_reqs = self.generate_writeback_request().await;
+        let all_reqs = self.generate_writeback_request().await?;
         if all_reqs.is_empty() || all_reqs.iter().all(|v| v.1.is_empty()) {
             return Ok(0);
         }
@@ -242,122 +253,16 @@ impl FlowWorkerManager {
             }
             let (catalog, schema) = (table_name[0].clone(), table_name[1].clone());
             let ctx = Arc::new(QueryContext::with(&catalog, &schema));
-            // TODO(discord9): instead of auto build table from request schema, actually build table
-            // before `create flow` to be able to assign pk and ts etc.
-            let (primary_keys, schema, is_ts_placeholder) = if let Some(table_id) = self
-                .table_info_source
-                .get_table_id_from_name(&table_name)
-                .await?
-            {
-                let table_info = self
-                    .table_info_source
-                    .get_table_info_value(&table_id)
-                    .await?
-                    .unwrap();
-                let meta = table_info.table_info.meta;
-                let primary_keys = meta
-                    .primary_key_indices
-                    .into_iter()
-                    .map(|i| meta.schema.column_schemas[i].name.clone())
-                    .collect_vec();
-                let schema = meta.schema.column_schemas;
-                // check if the last column is the auto created timestamp column, hence the table is auto created from
-                // flow's plan type
-                let is_auto_create = {
-                    let correct_name = schema
-                        .last()
-                        .map(|s| s.name == AUTO_CREATED_PLACEHOLDER_TS_COL)
-                        .unwrap_or(false);
-                    let correct_time_index = meta.schema.timestamp_index == Some(schema.len() - 1);
-                    correct_name && correct_time_index
-                };
-                (primary_keys, schema, is_auto_create)
-            } else {
-                // TODO(discord9): condiser remove buggy auto create by schema
 
-                let node_ctx = self.node_context.read().await;
-                let gid: GlobalId = node_ctx
-                    .table_repr
-                    .get_by_name(&table_name)
-                    .map(|x| x.1)
-                    .unwrap();
-                let schema = node_ctx
-                    .schema
-                    .get(&gid)
-                    .with_context(|| TableNotFoundSnafu {
-                        name: format!("Table name = {:?}", table_name),
-                    })?
-                    .clone();
-                // TODO(discord9): use default key from schema
-                let primary_keys = schema
-                    .typ()
-                    .keys
-                    .first()
-                    .map(|v| {
-                        v.column_indices
-                            .iter()
-                            .map(|i| {
-                                schema
-                                    .get_name(*i)
-                                    .clone()
-                                    .unwrap_or_else(|| format!("col_{i}"))
-                            })
-                            .collect_vec()
-                    })
-                    .unwrap_or_default();
-                let update_at = ColumnSchema::new(
-                    UPDATE_AT_TS_COL,
-                    ConcreteDataType::timestamp_millisecond_datatype(),
-                    true,
-                );
+            let (is_ts_placeholder, proto_schema) =
+                self.try_fetch_or_create_table(&table_name).await?;
+            let schema_len = proto_schema.len();
 
-                let original_schema = schema
-                    .typ()
-                    .column_types
-                    .clone()
-                    .into_iter()
-                    .enumerate()
-                    .map(|(idx, typ)| {
-                        let name = schema
-                            .names
-                            .get(idx)
-                            .cloned()
-                            .flatten()
-                            .unwrap_or(format!("col_{}", idx));
-                        let ret = ColumnSchema::new(name, typ.scalar_type, typ.nullable);
-                        if schema.typ().time_index == Some(idx) {
-                            ret.with_time_index(true)
-                        } else {
-                            ret
-                        }
-                    })
-                    .collect_vec();
-
-                let mut with_auto_added_col = original_schema.clone();
-                with_auto_added_col.push(update_at);
-
-                // if no time index, add one as placeholder
-                let no_time_index = schema.typ().time_index.is_none();
-                if no_time_index {
-                    let ts_col = ColumnSchema::new(
-                        AUTO_CREATED_PLACEHOLDER_TS_COL,
-                        ConcreteDataType::timestamp_millisecond_datatype(),
-                        true,
-                    )
-                    .with_time_index(true);
-                    with_auto_added_col.push(ts_col);
-                }
-
-                (primary_keys, with_auto_added_col, no_time_index)
-            };
-            let schema_len = schema.len();
-            let proto_schema = column_schemas_to_proto(schema, &primary_keys)?;
-
-            debug!(
-                "Sending {} writeback requests to table {}, reqs={:?}",
+            trace!(
+                "Sending {} writeback requests to table {}, reqs total rows={}",
                 reqs.len(),
                 table_name.join("."),
-                reqs
+                reqs.iter().map(|r| r.len()).sum::<usize>()
             );
             let now = self.tick_manager.tick();
             for req in reqs {
@@ -450,8 +355,12 @@ impl FlowWorkerManager {
     }
 
     /// Generate writeback request for all sink table
-    pub async fn generate_writeback_request(&self) -> BTreeMap<TableName, Vec<DiffRequest>> {
+    pub async fn generate_writeback_request(
+        &self,
+    ) -> Result<BTreeMap<TableName, Vec<DiffRequest>>, Error> {
+        trace!("Start to generate writeback request");
         let mut output = BTreeMap::new();
+        let mut total_row_count = 0;
         for (name, sink_recv) in self
             .node_context
             .write()
@@ -460,14 +369,133 @@ impl FlowWorkerManager {
             .iter_mut()
             .map(|(n, (_s, r))| (n, r))
         {
-            let mut rows = Vec::new();
-            while let Ok(row) = sink_recv.try_recv() {
-                rows.push(row);
+            let mut batches = Vec::new();
+            while let Ok(batch) = sink_recv.try_recv() {
+                total_row_count += batch.row_count();
+                batches.push(batch);
             }
-            let reqs = diff_row_to_request(rows);
+            let reqs = batches_to_rows_req(batches)?;
             output.insert(name.clone(), reqs);
         }
-        output
+        trace!("Prepare writeback req: total row count={}", total_row_count);
+        Ok(output)
+    }
+
+    /// Fetch table info or create table from flow's schema if not exist
+    async fn try_fetch_or_create_table(
+        &self,
+        table_name: &TableName,
+    ) -> Result<(bool, Vec<api::v1::ColumnSchema>), Error> {
+        // TODO(discord9): instead of auto build table from request schema, actually build table
+        // before `create flow` to be able to assign pk and ts etc.
+        let (primary_keys, schema, is_ts_placeholder) = if let Some(table_id) = self
+            .table_info_source
+            .get_table_id_from_name(table_name)
+            .await?
+        {
+            let table_info = self
+                .table_info_source
+                .get_table_info_value(&table_id)
+                .await?
+                .unwrap();
+            let meta = table_info.table_info.meta;
+            let primary_keys = meta
+                .primary_key_indices
+                .into_iter()
+                .map(|i| meta.schema.column_schemas[i].name.clone())
+                .collect_vec();
+            let schema = meta.schema.column_schemas;
+            // check if the last column is the auto created timestamp column, hence the table is auto created from
+            // flow's plan type
+            let is_auto_create = {
+                let correct_name = schema
+                    .last()
+                    .map(|s| s.name == AUTO_CREATED_PLACEHOLDER_TS_COL)
+                    .unwrap_or(false);
+                let correct_time_index = meta.schema.timestamp_index == Some(schema.len() - 1);
+                correct_name && correct_time_index
+            };
+            (primary_keys, schema, is_auto_create)
+        } else {
+            // TODO(discord9): condiser remove buggy auto create by schema
+
+            let node_ctx = self.node_context.read().await;
+            let gid: GlobalId = node_ctx
+                .table_repr
+                .get_by_name(table_name)
+                .map(|x| x.1)
+                .unwrap();
+            let schema = node_ctx
+                .schema
+                .get(&gid)
+                .with_context(|| TableNotFoundSnafu {
+                    name: format!("Table name = {:?}", table_name),
+                })?
+                .clone();
+            // TODO(discord9): use default key from schema
+            let primary_keys = schema
+                .typ()
+                .keys
+                .first()
+                .map(|v| {
+                    v.column_indices
+                        .iter()
+                        .map(|i| {
+                            schema
+                                .get_name(*i)
+                                .clone()
+                                .unwrap_or_else(|| format!("col_{i}"))
+                        })
+                        .collect_vec()
+                })
+                .unwrap_or_default();
+            let update_at = ColumnSchema::new(
+                UPDATE_AT_TS_COL,
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                true,
+            );
+
+            let original_schema = schema
+                .typ()
+                .column_types
+                .clone()
+                .into_iter()
+                .enumerate()
+                .map(|(idx, typ)| {
+                    let name = schema
+                        .names
+                        .get(idx)
+                        .cloned()
+                        .flatten()
+                        .unwrap_or(format!("col_{}", idx));
+                    let ret = ColumnSchema::new(name, typ.scalar_type, typ.nullable);
+                    if schema.typ().time_index == Some(idx) {
+                        ret.with_time_index(true)
+                    } else {
+                        ret
+                    }
+                })
+                .collect_vec();
+
+            let mut with_auto_added_col = original_schema.clone();
+            with_auto_added_col.push(update_at);
+
+            // if no time index, add one as placeholder
+            let no_time_index = schema.typ().time_index.is_none();
+            if no_time_index {
+                let ts_col = ColumnSchema::new(
+                    AUTO_CREATED_PLACEHOLDER_TS_COL,
+                    ConcreteDataType::timestamp_millisecond_datatype(),
+                    true,
+                )
+                .with_time_index(true);
+                with_auto_added_col.push(ts_col);
+            }
+
+            (primary_keys, with_auto_added_col, no_time_index)
+        };
+        let proto_schema = column_schemas_to_proto(schema, &primary_keys)?;
+        Ok((is_ts_placeholder, proto_schema))
     }
 }
 
@@ -498,10 +526,6 @@ impl FlowWorkerManager {
         }
     }
 
-    async fn get_buf_size(&self) -> usize {
-        self.node_context.read().await.get_send_buf_size().await
-    }
-
     /// Trigger dataflow running, and then send writeback request to the source sender
     ///
     /// note that this method didn't handle input mirror request, as this should be handled by grpc server
@@ -575,43 +599,37 @@ impl FlowWorkerManager {
     /// TODO(discord9): add flag for subgraph that have input since last run
     pub async fn run_available(&self, blocking: bool) -> Result<usize, Error> {
         let mut row_cnt = 0;
-        loop {
-            let now = self.tick_manager.tick();
-            for worker in self.worker_handles.iter() {
-                // TODO(discord9): consider how to handle error in individual worker
-                if blocking {
-                    worker.lock().await.run_available(now, blocking).await?;
-                } else if let Ok(worker) = worker.try_lock() {
-                    worker.run_available(now, blocking).await?;
-                } else {
-                    return Ok(row_cnt);
-                }
-            }
-            // check row send and rows remain in send buf
-            let (flush_res, _buf_len) = if blocking {
-                let ctx = self.node_context.read().await;
-                (ctx.flush_all_sender().await, ctx.get_send_buf_size().await)
+
+        let now = self.tick_manager.tick();
+        for worker in self.worker_handles.iter() {
+            // TODO(discord9): consider how to handle error in individual worker
+            if blocking {
+                worker.lock().await.run_available(now, blocking).await?;
+            } else if let Ok(worker) = worker.try_lock() {
+                worker.run_available(now, blocking).await?;
             } else {
-                match self.node_context.try_read() {
-                    Ok(ctx) => (ctx.flush_all_sender().await, ctx.get_send_buf_size().await),
-                    Err(_) => return Ok(row_cnt),
-                }
-            };
-            match flush_res {
-                Ok(r) => {
-                    common_telemetry::trace!("Flushed {} rows", r);
-                    row_cnt += r;
-                    // send buf is likely to be somewhere empty now, wait
-                    if r < BATCH_SIZE / 2 {
-                        break;
-                    }
-                }
-                Err(err) => {
-                    common_telemetry::error!("Flush send buf errors: {:?}", err);
-                    break;
-                }
-            };
+                return Ok(row_cnt);
+            }
         }
+        // check row send and rows remain in send buf
+        let flush_res = if blocking {
+            let ctx = self.node_context.read().await;
+            ctx.flush_all_sender().await
+        } else {
+            match self.node_context.try_read() {
+                Ok(ctx) => ctx.flush_all_sender().await,
+                Err(_) => return Ok(row_cnt),
+            }
+        };
+        match flush_res {
+            Ok(r) => {
+                common_telemetry::trace!("Total flushed {} rows", r);
+                row_cnt += r;
+            }
+            Err(err) => {
+                common_telemetry::error!("Flush send buf errors: {:?}", err);
+            }
+        };
 
         Ok(row_cnt)
     }
@@ -624,14 +642,14 @@ impl FlowWorkerManager {
     ) -> Result<(), Error> {
         let rows_len = rows.len();
         let table_id = region_id.table_id();
-        METRIC_FLOW_INPUT_BUF_SIZE.add(rows_len as _);
         let _timer = METRIC_FLOW_INSERT_ELAPSED
             .with_label_values(&[table_id.to_string().as_str()])
             .start_timer();
         self.node_context.read().await.send(table_id, rows).await?;
-        debug!(
+        trace!(
             "Handling write request for table_id={} with {} rows",
-            table_id, rows_len
+            table_id,
+            rows_len
         );
         Ok(())
     }
diff --git a/src/flow/src/adapter/flownode_impl.rs b/src/flow/src/adapter/flownode_impl.rs
index c2da7af95c..739804be93 100644
--- a/src/flow/src/adapter/flownode_impl.rs
+++ b/src/flow/src/adapter/flownode_impl.rs
@@ -23,7 +23,7 @@ use api::v1::region::InsertRequests;
 use common_error::ext::BoxedError;
 use common_meta::error::{ExternalSnafu, Result, UnexpectedSnafu};
 use common_meta::node_manager::Flownode;
-use common_telemetry::debug;
+use common_telemetry::{debug, trace};
 use itertools::Itertools;
 use snafu::{OptionExt, ResultExt};
 use store_api::storage::RegionId;
@@ -189,7 +189,7 @@ impl Flownode for FlowWorkerManager {
                     })
                     .try_collect()?;
                 if !fetch_order.iter().enumerate().all(|(i, &v)| i == v) {
-                    debug!("Reordering columns: {:?}", fetch_order)
+                    trace!("Reordering columns: {:?}", fetch_order)
                 }
                 fetch_order
             };
diff --git a/src/flow/src/adapter/node_context.rs b/src/flow/src/adapter/node_context.rs
index 99bd9f97e9..26e1a6483a 100644
--- a/src/flow/src/adapter/node_context.rs
+++ b/src/flow/src/adapter/node_context.rs
@@ -15,10 +15,10 @@
 //! Node context, prone to change with every incoming requests
 
 use std::collections::{BTreeMap, BTreeSet, HashMap};
-use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 
-use common_telemetry::debug;
+use common_telemetry::trace;
 use session::context::QueryContext;
 use snafu::{OptionExt, ResultExt};
 use table::metadata::TableId;
@@ -27,9 +27,9 @@ use tokio::sync::{broadcast, mpsc, RwLock};
 use crate::adapter::{FlowId, TableName, TableSource};
 use crate::error::{Error, EvalSnafu, TableNotFoundSnafu};
 use crate::expr::error::InternalSnafu;
-use crate::expr::GlobalId;
+use crate::expr::{Batch, GlobalId};
 use crate::metrics::METRIC_FLOW_INPUT_BUF_SIZE;
-use crate::repr::{DiffRow, RelationDesc, BROADCAST_CAP, SEND_BUF_CAP};
+use crate::repr::{DiffRow, RelationDesc, BATCH_SIZE, BROADCAST_CAP, SEND_BUF_CAP};
 
 /// A context that holds the information of the dataflow
 #[derive(Default, Debug)]
@@ -47,13 +47,8 @@ pub struct FlownodeContext {
     ///
     /// and send it back to the client, since we are mocking the sink table as a client, we should use table name as the key
     /// note that the sink receiver should only have one, and we are using broadcast as mpsc channel here
-    pub sink_receiver: BTreeMap<
-        TableName,
-        (
-            mpsc::UnboundedSender<DiffRow>,
-            mpsc::UnboundedReceiver<DiffRow>,
-        ),
-    >,
+    pub sink_receiver:
+        BTreeMap<TableName, (mpsc::UnboundedSender<Batch>, mpsc::UnboundedReceiver<Batch>)>,
     /// the schema of the table, query from metasrv or inferred from TypedPlan
     pub schema: HashMap<GlobalId, RelationDesc>,
     /// All the tables that have been registered in the worker
@@ -61,25 +56,27 @@ pub struct FlownodeContext {
     pub query_context: Option<Arc<QueryContext>>,
 }
 
-/// a simple broadcast sender with backpressure and unbound capacity
+/// a simple broadcast sender with backpressure, bounded capacity and blocking on send when send buf is full
+/// note that it wouldn't evict old data, so it's possible to block forever if the receiver is slow
 ///
 /// receiver still use tokio broadcast channel, since only sender side need to know
 /// backpressure and adjust dataflow running duration to avoid blocking
 #[derive(Debug)]
 pub struct SourceSender {
     // TODO(discord9): make it all Vec<DiffRow>?
-    sender: broadcast::Sender<DiffRow>,
-    send_buf_tx: mpsc::Sender<Vec<DiffRow>>,
-    send_buf_rx: RwLock<mpsc::Receiver<Vec<DiffRow>>>,
+    sender: broadcast::Sender<Batch>,
+    send_buf_tx: mpsc::Sender<Batch>,
+    send_buf_rx: RwLock<mpsc::Receiver<Batch>>,
     send_buf_row_cnt: AtomicUsize,
 }
 
 impl Default for SourceSender {
     fn default() -> Self {
+        // TODO(discord9): the capacity is arbitrary, we can adjust it later, might also want to limit the max number of rows in send buf
         let (send_buf_tx, send_buf_rx) = mpsc::channel(SEND_BUF_CAP);
         Self {
             // TODO(discord9): found a better way then increase this to prevent lagging and hence missing input data
-            sender: broadcast::Sender::new(BROADCAST_CAP * 2),
+            sender: broadcast::Sender::new(SEND_BUF_CAP),
             send_buf_tx,
             send_buf_rx: RwLock::new(send_buf_rx),
             send_buf_row_cnt: AtomicUsize::new(0),
@@ -90,7 +87,7 @@ impl Default for SourceSender {
 impl SourceSender {
     /// max number of iterations to try flush send buf
     const MAX_ITERATIONS: usize = 16;
-    pub fn get_receiver(&self) -> broadcast::Receiver<DiffRow> {
+    pub fn get_receiver(&self) -> broadcast::Receiver<Batch> {
         self.sender.subscribe()
     }
 
@@ -106,30 +103,27 @@ impl SourceSender {
                 break;
             }
             // TODO(discord9): send rows instead so it's just moving a point
-            if let Some(rows) = send_buf.recv().await {
-                let len = rows.len();
-                self.send_buf_row_cnt
-                    .fetch_sub(len, std::sync::atomic::Ordering::SeqCst);
-                for row in rows {
-                    self.sender
-                        .send(row)
-                        .map_err(|err| {
-                            InternalSnafu {
-                                reason: format!("Failed to send row, error = {:?}", err),
-                            }
-                            .build()
-                        })
-                        .with_context(|_| EvalSnafu)?;
-                    row_cnt += 1;
-                }
+            if let Some(batch) = send_buf.recv().await {
+                let len = batch.row_count();
+                self.send_buf_row_cnt.fetch_sub(len, Ordering::SeqCst);
+                row_cnt += len;
+                self.sender
+                    .send(batch)
+                    .map_err(|err| {
+                        InternalSnafu {
+                            reason: format!("Failed to send row, error = {:?}", err),
+                        }
+                        .build()
+                    })
+                    .with_context(|_| EvalSnafu)?;
             }
         }
         if row_cnt > 0 {
-            debug!("Send {} rows", row_cnt);
+            trace!("Source Flushed {} rows", row_cnt);
             METRIC_FLOW_INPUT_BUF_SIZE.sub(row_cnt as _);
-            debug!(
-                "Remaining Send buf.len() = {}",
-                self.send_buf_rx.read().await.len()
+            trace!(
+                "Remaining Source Send buf.len() = {}",
+                METRIC_FLOW_INPUT_BUF_SIZE.get()
             );
         }
 
@@ -138,12 +132,23 @@ impl SourceSender {
 
     /// return number of rows it actual send(including what's in the buffer)
     pub async fn send_rows(&self, rows: Vec<DiffRow>) -> Result<usize, Error> {
-        self.send_buf_tx.send(rows).await.map_err(|e| {
+        METRIC_FLOW_INPUT_BUF_SIZE.add(rows.len() as _);
+        while self.send_buf_row_cnt.load(Ordering::SeqCst) >= BATCH_SIZE * 4 {
+            tokio::task::yield_now().await;
+        }
+        // row count metrics is approx so relaxed order is ok
+        self.send_buf_row_cnt
+            .fetch_add(rows.len(), Ordering::SeqCst);
+        let batch = Batch::try_from_rows(rows.into_iter().map(|(row, _, _)| row).collect())
+            .context(EvalSnafu)?;
+        common_telemetry::trace!("Send one batch to worker with {} rows", batch.row_count());
+        self.send_buf_tx.send(batch).await.map_err(|e| {
             crate::error::InternalSnafu {
                 reason: format!("Failed to send row, error = {:?}", e),
             }
             .build()
         })?;
+
         Ok(0)
     }
 }
@@ -159,8 +164,6 @@ impl FlownodeContext {
             .with_context(|| TableNotFoundSnafu {
                 name: table_id.to_string(),
             })?;
-
-        debug!("FlownodeContext::send: trying to send {} rows", rows.len());
         sender.send_rows(rows).await
     }
 
@@ -174,16 +177,6 @@ impl FlownodeContext {
         }
         Ok(sum)
     }
-
-    /// Return the sum number of rows in all send buf
-    /// TODO(discord9): remove this since we can't get correct row cnt anyway
-    pub async fn get_send_buf_size(&self) -> usize {
-        let mut sum = 0;
-        for sender in self.source_sender.values() {
-            sum += sender.send_buf_rx.read().await.len();
-        }
-        sum
-    }
 }
 
 impl FlownodeContext {
@@ -230,7 +223,7 @@ impl FlownodeContext {
     pub fn add_sink_receiver(&mut self, table_name: TableName) {
         self.sink_receiver
             .entry(table_name)
-            .or_insert_with(mpsc::unbounded_channel::<DiffRow>);
+            .or_insert_with(mpsc::unbounded_channel);
     }
 
     pub fn get_source_by_global_id(&self, id: &GlobalId) -> Result<&SourceSender, Error> {
@@ -254,7 +247,7 @@ impl FlownodeContext {
     pub fn get_sink_by_global_id(
         &self,
         id: &GlobalId,
-    ) -> Result<mpsc::UnboundedSender<DiffRow>, Error> {
+    ) -> Result<mpsc::UnboundedSender<Batch>, Error> {
         let table_name = self
             .table_repr
             .get_by_global_id(id)
diff --git a/src/flow/src/adapter/worker.rs b/src/flow/src/adapter/worker.rs
index 72ac9df127..978d3c608c 100644
--- a/src/flow/src/adapter/worker.rs
+++ b/src/flow/src/adapter/worker.rs
@@ -27,7 +27,7 @@ use tokio::sync::{broadcast, mpsc, oneshot, Mutex};
 use crate::adapter::FlowId;
 use crate::compute::{Context, DataflowState, ErrCollector};
 use crate::error::{Error, FlowAlreadyExistSnafu, InternalSnafu, UnexpectedSnafu};
-use crate::expr::GlobalId;
+use crate::expr::{Batch, GlobalId};
 use crate::plan::TypedPlan;
 use crate::repr::{self, DiffRow};
 
@@ -89,6 +89,8 @@ impl<'subgraph> ActiveDataflowState<'subgraph> {
             err_collector: self.err_collector.clone(),
             input_collection: Default::default(),
             local_scope: Default::default(),
+            input_collection_batch: Default::default(),
+            local_scope_batch: Default::default(),
         }
     }
 
@@ -156,13 +158,13 @@ impl WorkerHandle {
     ///
     /// the returned error is unrecoverable, and the worker should be shutdown/rebooted
     pub async fn run_available(&self, now: repr::Timestamp, blocking: bool) -> Result<(), Error> {
-        common_telemetry::debug!("Running available with blocking={}", blocking);
+        common_telemetry::trace!("Running available with blocking={}", blocking);
         if blocking {
             let resp = self
                 .itc_client
                 .call_with_resp(Request::RunAvail { now, blocking })
                 .await?;
-            common_telemetry::debug!("Running available with response={:?}", resp);
+            common_telemetry::trace!("Running available with response={:?}", resp);
             Ok(())
         } else {
             self.itc_client
@@ -225,9 +227,9 @@ impl<'s> Worker<'s> {
         flow_id: FlowId,
         plan: TypedPlan,
         sink_id: GlobalId,
-        sink_sender: mpsc::UnboundedSender<DiffRow>,
+        sink_sender: mpsc::UnboundedSender<Batch>,
         source_ids: &[GlobalId],
-        src_recvs: Vec<broadcast::Receiver<DiffRow>>,
+        src_recvs: Vec<broadcast::Receiver<Batch>>,
         // TODO(discord9): set expire duration for all arrangement and compare to sys timestamp instead
         expire_after: Option<repr::Duration>,
         create_if_not_exists: bool,
@@ -249,12 +251,12 @@ impl<'s> Worker<'s> {
         {
             let mut ctx = cur_task_state.new_ctx(sink_id);
             for (source_id, src_recv) in source_ids.iter().zip(src_recvs) {
-                let bundle = ctx.render_source(src_recv)?;
-                ctx.insert_global(*source_id, bundle);
+                let bundle = ctx.render_source_batch(src_recv)?;
+                ctx.insert_global_batch(*source_id, bundle);
             }
 
-            let rendered = ctx.render_plan(plan)?;
-            ctx.render_unbounded_sink(rendered, sink_sender);
+            let rendered = ctx.render_plan_batch(plan)?;
+            ctx.render_unbounded_sink_batch(rendered, sink_sender);
         }
         self.task_states.insert(flow_id, cur_task_state);
         Ok(Some(flow_id))
@@ -370,9 +372,9 @@ pub enum Request {
         flow_id: FlowId,
         plan: TypedPlan,
         sink_id: GlobalId,
-        sink_sender: mpsc::UnboundedSender<DiffRow>,
+        sink_sender: mpsc::UnboundedSender<Batch>,
         source_ids: Vec<GlobalId>,
-        src_recvs: Vec<broadcast::Receiver<DiffRow>>,
+        src_recvs: Vec<broadcast::Receiver<Batch>>,
         expire_after: Option<repr::Duration>,
         create_if_not_exists: bool,
         err_collector: ErrCollector,
@@ -472,7 +474,7 @@ mod test {
     use super::*;
     use crate::expr::Id;
     use crate::plan::Plan;
-    use crate::repr::{RelationType, Row};
+    use crate::repr::RelationType;
 
     #[test]
     fn drop_handle() {
@@ -497,8 +499,8 @@ mod test {
         });
         let handle = rx.await.unwrap();
         let src_ids = vec![GlobalId::User(1)];
-        let (tx, rx) = broadcast::channel::<DiffRow>(1024);
-        let (sink_tx, mut sink_rx) = mpsc::unbounded_channel::<DiffRow>();
+        let (tx, rx) = broadcast::channel::<Batch>(1024);
+        let (sink_tx, mut sink_rx) = mpsc::unbounded_channel::<Batch>();
         let (flow_id, plan) = (
             1,
             TypedPlan {
@@ -523,9 +525,9 @@ mod test {
             handle.create_flow(create_reqs).await.unwrap(),
             Some(flow_id)
         );
-        tx.send((Row::empty(), 0, 0)).unwrap();
+        tx.send(Batch::empty()).unwrap();
         handle.run_available(0, true).await.unwrap();
-        assert_eq!(sink_rx.recv().await.unwrap().0, Row::empty());
+        assert_eq!(sink_rx.recv().await.unwrap(), Batch::empty());
         drop(handle);
         worker_thread_handle.join().unwrap();
     }
diff --git a/src/flow/src/compute/render.rs b/src/flow/src/compute/render.rs
index dcb2194a1d..90f0d86c73 100644
--- a/src/flow/src/compute/render.rs
+++ b/src/flow/src/compute/render.rs
@@ -49,6 +49,14 @@ pub struct Context<'referred, 'df> {
     ///
     /// TODO(discord9): consider if use Vec<(LocalId, CollectionBundle)> instead
     pub local_scope: Vec<BTreeMap<LocalId, CollectionBundle>>,
+    /// a list of all collections being used in the operator
+    ///
+    /// TODO(discord9): remove extra clone by counting usage and remove it on last usage?
+    pub input_collection_batch: BTreeMap<GlobalId, CollectionBundle<Batch>>,
+    /// used by `Get`/`Let` Plan for getting/setting local variables
+    ///
+    /// TODO(discord9): consider if use Vec<(LocalId, CollectionBundle)> instead
+    pub local_scope_batch: Vec<BTreeMap<LocalId, CollectionBundle<Batch>>>,
     // Collect all errors in this operator's evaluation
     pub err_collector: ErrCollector,
 }
@@ -67,6 +75,19 @@ impl<'referred, 'df> Drop for Context<'referred, 'df> {
             bundle.collection.into_inner().drop(self.df);
             drop(bundle.arranged);
         }
+
+        for bundle in std::mem::take(&mut self.input_collection_batch)
+            .into_values()
+            .chain(
+                std::mem::take(&mut self.local_scope_batch)
+                    .into_iter()
+                    .flat_map(|v| v.into_iter())
+                    .map(|(_k, v)| v),
+            )
+        {
+            bundle.collection.into_inner().drop(self.df);
+            drop(bundle.arranged);
+        }
         // The automatically generated "drop glue" which recursively calls the destructors of all the fields (including the now empty `input_collection`)
     }
 }
@@ -84,6 +105,19 @@ impl<'referred, 'df> Context<'referred, 'df> {
             self.local_scope.push(first);
         }
     }
+
+    pub fn insert_global_batch(&mut self, id: GlobalId, collection: CollectionBundle<Batch>) {
+        self.input_collection_batch.insert(id, collection);
+    }
+
+    pub fn insert_local_batch(&mut self, id: LocalId, collection: CollectionBundle<Batch>) {
+        if let Some(last) = self.local_scope_batch.last_mut() {
+            last.insert(id, collection);
+        } else {
+            let first = BTreeMap::from([(id, collection)]);
+            self.local_scope_batch.push(first);
+        }
+    }
 }
 
 impl<'referred, 'df> Context<'referred, 'df> {
@@ -91,14 +125,8 @@ impl<'referred, 'df> Context<'referred, 'df> {
     pub fn render_plan_batch(&mut self, plan: TypedPlan) -> Result<CollectionBundle<Batch>, Error> {
         match plan.plan {
             Plan::Constant { rows } => Ok(self.render_constant_batch(rows)),
-            Plan::Get { .. } => NotImplementedSnafu {
-                reason: "Get is still WIP in batchmode",
-            }
-            .fail(),
-            Plan::Let { .. } => NotImplementedSnafu {
-                reason: "Let is still WIP in batchmode",
-            }
-            .fail(),
+            Plan::Get { id } => self.get_batch_by_id(id),
+            Plan::Let { id, value, body } => self.eval_batch_let(id, value, body),
             Plan::Mfp { input, mfp } => self.render_mfp_batch(input, mfp),
             Plan::Reduce {
                 input,
@@ -225,6 +253,32 @@ impl<'referred, 'df> Context<'referred, 'df> {
         CollectionBundle::from_collection(Collection::from_port(recv_port))
     }
 
+    pub fn get_batch_by_id(&mut self, id: expr::Id) -> Result<CollectionBundle<Batch>, Error> {
+        let ret = match id {
+            expr::Id::Local(local) => {
+                let bundle = self
+                    .local_scope_batch
+                    .iter()
+                    .rev()
+                    .find_map(|scope| scope.get(&local))
+                    .with_context(|| InvalidQuerySnafu {
+                        reason: format!("Local variable {:?} not found", local),
+                    })?;
+                bundle.clone(self.df)
+            }
+            expr::Id::Global(id) => {
+                let bundle =
+                    self.input_collection_batch
+                        .get(&id)
+                        .with_context(|| InvalidQuerySnafu {
+                            reason: format!("Collection {:?} not found", id),
+                        })?;
+                bundle.clone(self.df)
+            }
+        };
+        Ok(ret)
+    }
+
     pub fn get_by_id(&mut self, id: expr::Id) -> Result<CollectionBundle, Error> {
         let ret = match id {
             expr::Id::Local(local) => {
@@ -251,6 +305,21 @@ impl<'referred, 'df> Context<'referred, 'df> {
         Ok(ret)
     }
 
+    /// Eval `Let` operator, useful for assigning a value to a local variable
+    pub fn eval_batch_let(
+        &mut self,
+        id: LocalId,
+        value: Box<TypedPlan>,
+        body: Box<TypedPlan>,
+    ) -> Result<CollectionBundle<Batch>, Error> {
+        let value = self.render_plan_batch(*value)?;
+
+        self.local_scope_batch.push(Default::default());
+        self.insert_local_batch(id, value);
+        let ret = self.render_plan_batch(*body)?;
+        Ok(ret)
+    }
+
     /// Eval `Let` operator, useful for assigning a value to a local variable
     pub fn eval_let(
         &mut self,
@@ -268,11 +337,11 @@ impl<'referred, 'df> Context<'referred, 'df> {
 }
 
 /// The Common argument for all `Subgraph` in the render process
-struct SubgraphArg<'a> {
+struct SubgraphArg<'a, T = Toff> {
     now: repr::Timestamp,
     err_collector: &'a ErrCollector,
     scheduler: &'a Scheduler,
-    send: &'a PortCtx<SEND, Toff>,
+    send: &'a PortCtx<SEND, T>,
 }
 
 #[cfg(test)]
@@ -345,6 +414,8 @@ mod test {
             compute_state: state,
             input_collection: BTreeMap::new(),
             local_scope: Default::default(),
+            input_collection_batch: BTreeMap::new(),
+            local_scope_batch: Default::default(),
             err_collector,
         }
     }
diff --git a/src/flow/src/compute/render/reduce.rs b/src/flow/src/compute/render/reduce.rs
index e43d3d22a4..8de6b7a17a 100644
--- a/src/flow/src/compute/render/reduce.rs
+++ b/src/flow/src/compute/render/reduce.rs
@@ -12,14 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::ops::Range;
 use std::sync::Arc;
 
+use common_telemetry::trace;
 use datatypes::data_type::ConcreteDataType;
 use datatypes::prelude::DataType;
 use datatypes::value::{ListValue, Value};
-use datatypes::vectors::NullVector;
+use datatypes::vectors::{BooleanVector, NullVector};
 use hydroflow::scheduled::graph_ext::GraphExt;
 use itertools::Itertools;
 use snafu::{ensure, OptionExt, ResultExt};
@@ -27,8 +28,8 @@ use snafu::{ensure, OptionExt, ResultExt};
 use crate::compute::render::{Context, SubgraphArg};
 use crate::compute::types::{Arranged, Collection, CollectionBundle, ErrCollector, Toff};
 use crate::error::{Error, NotImplementedSnafu, PlanSnafu};
-use crate::expr::error::{DataAlreadyExpiredSnafu, DataTypeSnafu, InternalSnafu};
-use crate::expr::{Batch, EvalError, ScalarExpr};
+use crate::expr::error::{ArrowSnafu, DataAlreadyExpiredSnafu, DataTypeSnafu, InternalSnafu};
+use crate::expr::{Accum, Accumulator, Batch, EvalError, ScalarExpr, VectorDiff};
 use crate::plan::{AccumulablePlan, AggrWithIndex, KeyValPlan, ReducePlan, TypedPlan};
 use crate::repr::{self, DiffRow, KeyValDiffRow, RelationType, Row};
 use crate::utils::{ArrangeHandler, ArrangeReader, ArrangeWriter, KeyExpiryManager};
@@ -93,152 +94,39 @@ impl<'referred, 'df> Context<'referred, 'df> {
         // TODO(discord9): better way to schedule future run
         let scheduler = self.compute_state.get_scheduler();
 
+        let scheduler_inner = scheduler.clone();
+
         let (out_send_port, out_recv_port) =
             self.df.make_edge::<_, Toff<Batch>>(Self::REDUCE_BATCH);
 
-        let subgraph =
-            self.df.add_subgraph_in_out(
-                Self::REDUCE_BATCH,
-                input.collection.into_inner(),
-                out_send_port,
-                move |_ctx, recv, send| {
-                    let now = *(now.borrow());
-                    let arrange = arrange_handler_inner.clone();
-                    // mfp only need to passively receive updates from recvs
-                    let src_data = recv
-                        .take_inner()
-                        .into_iter()
-                        .flat_map(|v| v.into_iter())
-                        .collect_vec();
+        let subgraph = self.df.add_subgraph_in_out(
+            Self::REDUCE_BATCH,
+            input.collection.into_inner(),
+            out_send_port,
+            move |_ctx, recv, send| {
+                let now = *(now.borrow());
+                let arrange = arrange_handler_inner.clone();
+                // mfp only need to passively receive updates from recvs
+                let src_data = recv
+                    .take_inner()
+                    .into_iter()
+                    .flat_map(|v| v.into_iter())
+                    .collect_vec();
 
-                    let mut key_to_many_vals = BTreeMap::<Row, Batch>::new();
-                    for batch in src_data {
-                        err_collector.run(|| {
-                            let (key_batch, val_batch) =
-                                batch_split_by_key_val(&batch, &key_val_plan, &err_collector);
-                            ensure!(
-                                key_batch.row_count() == val_batch.row_count(),
-                                InternalSnafu {
-                                    reason: format!(
-                                        "Key and val batch should have the same row count, found {} and {}", 
-                                        key_batch.row_count(),
-                                        val_batch.row_count()
-                                    )
-                                }
-                            );
-
-                            for row_idx in 0..key_batch.row_count() {
-                                let key_row = key_batch.get_row(row_idx).unwrap();
-                                let val_row = val_batch.slice(row_idx, 1)?;
-                                let val_batch =
-                                    key_to_many_vals.entry(Row::new(key_row)).or_default();
-                                val_batch.append_batch(val_row)?;
-                            }
-
-                            Ok(())
-                        });
-                    }
-
-                    // write lock the arrange for the rest of the function body
-                    // to prevent wired race condition
-                    let mut arrange = arrange.write();
-                    let mut all_arrange_updates = Vec::with_capacity(key_to_many_vals.len());
-                    let mut all_output_rows = Vec::with_capacity(key_to_many_vals.len());
-
-                    for (key, val_batch) in key_to_many_vals {
-                        err_collector.run(|| -> Result<(), _> {
-                            let (accums, _, _) = arrange.get(now, &key).unwrap_or_default();
-                            let accum_list = from_accum_values_to_live_accums(
-                                accums.unpack(),
-                                accum_plan.simple_aggrs.len(),
-                            )?;
-
-                            let mut accum_output = AccumOutput::new();
-                            for AggrWithIndex {
-                                expr,
-                                input_idx,
-                                output_idx,
-                            } in accum_plan.simple_aggrs.iter()
-                            {
-                                let cur_old_accum = accum_list.get(*output_idx).cloned().unwrap_or_default();
-                                // if batch is empty, input null instead
-                                let cur_input = val_batch.batch().get(*input_idx).cloned().unwrap_or_else(||Arc::new(NullVector::new(val_batch.row_count())));
-
-                                let (output, new_accum) =
-                                    expr.func.eval_batch(cur_old_accum, cur_input, None)?;
-
-                                accum_output.insert_accum(*output_idx, new_accum);
-                                accum_output.insert_output(*output_idx, output);
-                            }
-
-                            let (new_accums, res_val_row) = accum_output.into_accum_output()?;
-
-                            let arrange_update = ((key.clone(), Row::new(new_accums)), now, 1);
-                            all_arrange_updates.push(arrange_update);
-
-                            let mut key_val = key;
-                            key_val.extend(res_val_row);
-                            all_output_rows.push((key_val, now, 1));
-
-                            Ok(())
-                        });
-                    }
-
-                    err_collector.run(|| {
-                        arrange.apply_updates(now, all_arrange_updates)?;
-                        arrange.compact_to(now)
-                    });
-
-                    // this output part is not supposed to be resource intensive
-                    // (because for every batch there wouldn't usually be as many output row?), 
-                    // so we can do some costly operation here
-                    let output_types = all_output_rows.first().map(|(row, _, _)| {
-                        row.iter()
-                            .map(|v| v.data_type())
-                            .collect::<Vec<ConcreteDataType>>()
-                    });
-
-                    if let Some(output_types) = output_types {
-                        err_collector.run(|| {
-                            let column_cnt = output_types.len();
-                            let row_cnt = all_output_rows.len();
-
-                            let mut output_builder = output_types
-                                .into_iter()
-                                .map(|t| t.create_mutable_vector(row_cnt))
-                                .collect_vec();
-
-                            for (row, _, _) in all_output_rows {
-                                for (i, v) in row.into_iter().enumerate() {
-                                    output_builder
-                                    .get_mut(i)
-                                    .context(InternalSnafu{
-                                        reason: format!(
-                                            "Output builder should have the same length as the row, expected at most {} but got {}", 
-                                            column_cnt-1,
-                                            i
-                                        )
-                                    })?
-                                    .try_push_value_ref(v.as_value_ref())
-                                    .context(DataTypeSnafu {
-                                        msg: "Failed to push value",
-                                    })?;
-                                }
-                            }
-
-                            let output_columns = output_builder
-                                .into_iter()
-                                .map(|mut b| b.to_vector())
-                                .collect_vec();
-
-                            let output_batch = Batch::try_new(output_columns, row_cnt)?;
-                            send.give(vec![output_batch]);
-
-                            Ok(())
-                        });
-                    }
-                },
-            );
+                reduce_batch_subgraph(
+                    &arrange,
+                    src_data,
+                    &key_val_plan,
+                    &accum_plan,
+                    SubgraphArg {
+                        now,
+                        err_collector: &err_collector,
+                        scheduler: &scheduler_inner,
+                        send,
+                    },
+                )
+            },
+        );
 
         scheduler.set_cur_subgraph(subgraph);
 
@@ -461,6 +349,245 @@ fn split_rows_to_key_val(
     )
 }
 
+fn reduce_batch_subgraph(
+    arrange: &ArrangeHandler,
+    src_data: impl IntoIterator<Item = Batch>,
+    key_val_plan: &KeyValPlan,
+    accum_plan: &AccumulablePlan,
+    SubgraphArg {
+        now,
+        err_collector,
+        scheduler: _,
+        send,
+    }: SubgraphArg<Toff<Batch>>,
+) {
+    let mut key_to_many_vals = BTreeMap::<Row, Vec<Batch>>::new();
+    let mut input_row_count = 0;
+    let mut input_batch_count = 0;
+
+    for batch in src_data {
+        input_batch_count += 1;
+        input_row_count += batch.row_count();
+        err_collector.run(|| {
+            let (key_batch, val_batch) =
+                batch_split_by_key_val(&batch, key_val_plan, err_collector);
+            ensure!(
+                key_batch.row_count() == val_batch.row_count(),
+                InternalSnafu {
+                    reason: format!(
+                        "Key and val batch should have the same row count, found {} and {}",
+                        key_batch.row_count(),
+                        val_batch.row_count()
+                    )
+                }
+            );
+
+            let mut distinct_keys = BTreeSet::new();
+            for row_idx in 0..key_batch.row_count() {
+                let key_row = key_batch.get_row(row_idx)?;
+                let key_row = Row::new(key_row);
+
+                if distinct_keys.contains(&key_row) {
+                    continue;
+                } else {
+                    distinct_keys.insert(key_row.clone());
+                }
+            }
+
+            // TODO: here reduce numbers of eq to minimal by keeping slicing key/val batch
+            for key_row in distinct_keys {
+                let key_scalar_value = {
+                    let mut key_scalar_value = Vec::with_capacity(key_row.len());
+                    for key in key_row.iter() {
+                        let v =
+                            key.try_to_scalar_value(&key.data_type())
+                                .context(DataTypeSnafu {
+                                    msg: "can't convert key values to datafusion value",
+                                })?;
+                        let arrow_value =
+                            v.to_scalar().context(crate::expr::error::DatafusionSnafu {
+                                context: "can't convert key values to arrow value",
+                            })?;
+                        key_scalar_value.push(arrow_value);
+                    }
+                    key_scalar_value
+                };
+
+                // first compute equal from separate columns
+                let eq_results = key_scalar_value
+                    .into_iter()
+                    .zip(key_batch.batch().iter())
+                    .map(|(key, col)| {
+                        // TODO(discord9): this takes half of the cpu! And this is redundant amount of `eq`!
+                        arrow::compute::kernels::cmp::eq(&key, &col.to_arrow_array().as_ref() as _)
+                    })
+                    .try_collect::<_, Vec<_>, _>()
+                    .context(ArrowSnafu {
+                        context: "Failed to compare key values",
+                    })?;
+
+                // then combine all equal results to finally found equal key rows
+                let opt_eq_mask = eq_results
+                    .into_iter()
+                    .fold(None, |acc, v| match acc {
+                        Some(Ok(acc)) => Some(arrow::compute::kernels::boolean::and(&acc, &v)),
+                        Some(Err(_)) => acc,
+                        None => Some(Ok(v)),
+                    })
+                    .transpose()
+                    .context(ArrowSnafu {
+                        context: "Failed to combine key comparison results",
+                    })?;
+
+                let key_eq_mask = if let Some(eq_mask) = opt_eq_mask {
+                    BooleanVector::from(eq_mask)
+                } else {
+                    // if None, meaning key_batch's column number is zero, which means
+                    // the key is empty, so we just return a mask of all true
+                    // meaning taking all values
+                    BooleanVector::from(vec![true; key_batch.row_count()])
+                };
+                // TODO: both slice and mutate remaining batch
+
+                let cur_val_batch = val_batch.filter(&key_eq_mask)?;
+
+                key_to_many_vals
+                    .entry(key_row)
+                    .or_default()
+                    .push(cur_val_batch);
+            }
+
+            Ok(())
+        });
+    }
+
+    trace!(
+        "Reduce take {} batches, {} rows",
+        input_batch_count,
+        input_row_count
+    );
+
+    // write lock the arrange for the rest of the function body
+    // to prevent wired race condition
+    let mut arrange = arrange.write();
+    let mut all_arrange_updates = Vec::with_capacity(key_to_many_vals.len());
+
+    let mut all_output_dict = BTreeMap::new();
+
+    for (key, val_batches) in key_to_many_vals {
+        err_collector.run(|| -> Result<(), _> {
+            let (accums, _, _) = arrange.get(now, &key).unwrap_or_default();
+            let accum_list =
+                from_accum_values_to_live_accums(accums.unpack(), accum_plan.simple_aggrs.len())?;
+
+            let mut accum_output = AccumOutput::new();
+            for AggrWithIndex {
+                expr,
+                input_idx,
+                output_idx,
+            } in accum_plan.simple_aggrs.iter()
+            {
+                let cur_accum_value = accum_list.get(*output_idx).cloned().unwrap_or_default();
+                let mut cur_accum = if cur_accum_value.is_empty() {
+                    Accum::new_accum(&expr.func.clone())?
+                } else {
+                    Accum::try_into_accum(&expr.func, cur_accum_value)?
+                };
+
+                for val_batch in val_batches.iter() {
+                    // if batch is empty, input null instead
+                    let cur_input = val_batch
+                        .batch()
+                        .get(*input_idx)
+                        .cloned()
+                        .unwrap_or_else(|| Arc::new(NullVector::new(val_batch.row_count())));
+                    let len = cur_input.len();
+                    cur_accum.update_batch(&expr.func, VectorDiff::from(cur_input))?;
+
+                    trace!("Reduce accum after take {} rows: {:?}", len, cur_accum);
+                }
+                let final_output = cur_accum.eval(&expr.func)?;
+                trace!("Reduce accum final output: {:?}", final_output);
+                accum_output.insert_output(*output_idx, final_output);
+
+                let cur_accum_value = cur_accum.into_state();
+                accum_output.insert_accum(*output_idx, cur_accum_value);
+            }
+
+            let (new_accums, res_val_row) = accum_output.into_accum_output()?;
+
+            let arrange_update = ((key.clone(), Row::new(new_accums)), now, 1);
+            all_arrange_updates.push(arrange_update);
+
+            all_output_dict.insert(key, Row::from(res_val_row));
+
+            Ok(())
+        });
+    }
+
+    err_collector.run(|| {
+        arrange.apply_updates(now, all_arrange_updates)?;
+        arrange.compact_to(now)
+    });
+    // release the lock
+    drop(arrange);
+
+    // this output part is not supposed to be resource intensive
+    // (because for every batch there wouldn't usually be as many output row?),
+    // so we can do some costly operation here
+    let output_types = all_output_dict.first_entry().map(|entry| {
+        entry
+            .key()
+            .iter()
+            .chain(entry.get().iter())
+            .map(|v| v.data_type())
+            .collect::<Vec<ConcreteDataType>>()
+    });
+
+    if let Some(output_types) = output_types {
+        err_collector.run(|| {
+            let column_cnt = output_types.len();
+            let row_cnt = all_output_dict.len();
+
+            let mut output_builder = output_types
+                .into_iter()
+                .map(|t| t.create_mutable_vector(row_cnt))
+                .collect_vec();
+
+            for (key, val) in all_output_dict {
+                for (i, v) in key.into_iter().chain(val.into_iter()).enumerate() {
+                    output_builder
+                    .get_mut(i)
+                    .context(InternalSnafu{
+                        reason: format!(
+                            "Output builder should have the same length as the row, expected at most {} but got {}", 
+                            column_cnt - 1,
+                            i
+                        )
+                    })?
+                    .try_push_value_ref(v.as_value_ref())
+                    .context(DataTypeSnafu {
+                        msg: "Failed to push value",
+                    })?;
+                }
+            }
+
+            let output_columns = output_builder
+                .into_iter()
+                .map(|mut b| b.to_vector())
+                .collect_vec();
+
+            let output_batch = Batch::try_new(output_columns, row_cnt)?;
+
+            trace!("Reduce output batch: {:?}", output_batch);
+
+            send.give(vec![output_batch]);
+
+            Ok(())
+        });
+    }
+}
+
 /// reduce subgraph, reduce the input data into a single row
 /// output is concat from key and val
 fn reduce_subgraph(
@@ -856,6 +983,9 @@ impl AccumOutput {
 
     /// return (accums, output)
     fn into_accum_output(self) -> Result<(Vec<Value>, Vec<Value>), EvalError> {
+        if self.accum.is_empty() && self.output.is_empty() {
+            return Ok((vec![], vec![]));
+        }
         ensure!(
             !self.accum.is_empty() && self.accum.len() == self.output.len(),
             InternalSnafu {
diff --git a/src/flow/src/compute/render/src_sink.rs b/src/flow/src/compute/render/src_sink.rs
index 53141b8cc0..3f5bfaf345 100644
--- a/src/flow/src/compute/render/src_sink.rs
+++ b/src/flow/src/compute/render/src_sink.rs
@@ -16,7 +16,7 @@
 
 use std::collections::{BTreeMap, VecDeque};
 
-use common_telemetry::debug;
+use common_telemetry::{debug, trace};
 use hydroflow::scheduled::graph_ext::GraphExt;
 use itertools::Itertools;
 use snafu::OptionExt;
@@ -48,10 +48,13 @@ impl<'referred, 'df> Context<'referred, 'df> {
         let sub = self
             .df
             .add_subgraph_source("source_batch", send_port, move |_ctx, send| {
+                let mut total_batches = vec![];
+                let mut total_row_count = 0;
                 loop {
                     match src_recv.try_recv() {
                         Ok(batch) => {
-                            send.give(vec![batch]);
+                            total_row_count += batch.row_count();
+                            total_batches.push(batch);
                         }
                         Err(TryRecvError::Empty) => {
                             break;
@@ -78,6 +81,13 @@ impl<'referred, 'df> Context<'referred, 'df> {
                     }
                 }
 
+                trace!(
+                    "Send {} rows in {} batches",
+                    total_row_count,
+                    total_batches.len()
+                );
+                send.give(total_batches);
+
                 let now = *now.borrow();
                 // always schedule source to run at now so we can
                 // repeatedly run source if needed
@@ -185,13 +195,18 @@ impl<'referred, 'df> Context<'referred, 'df> {
             collection.into_inner(),
             move |_ctx, recv| {
                 let data = recv.take_inner();
+                let mut row_count = 0;
+                let mut batch_count = 0;
                 for batch in data.into_iter().flat_map(|i| i.into_iter()) {
+                    row_count += batch.row_count();
+                    batch_count += 1;
                     // if the sender is closed unexpectedly, stop sending
                     if sender.is_closed() || sender.send(batch).is_err() {
                         common_telemetry::error!("UnboundedSinkBatch is closed");
                         break;
                     }
                 }
+                trace!("sink send {} rows in {} batches", row_count, batch_count);
             },
         );
     }
diff --git a/src/flow/src/compute/types.rs b/src/flow/src/compute/types.rs
index f7cbebae27..00ed660a6e 100644
--- a/src/flow/src/compute/types.rs
+++ b/src/flow/src/compute/types.rs
@@ -24,7 +24,7 @@ use hydroflow::scheduled::SubgraphId;
 use itertools::Itertools;
 use tokio::sync::Mutex;
 
-use crate::expr::{EvalError, ScalarExpr};
+use crate::expr::{Batch, EvalError, ScalarExpr};
 use crate::repr::DiffRow;
 use crate::utils::ArrangeHandler;
 
@@ -123,6 +123,38 @@ pub struct CollectionBundle<T: 'static = DiffRow> {
     pub arranged: BTreeMap<Vec<ScalarExpr>, Arranged>,
 }
 
+pub trait GenericBundle {
+    fn is_batch(&self) -> bool;
+
+    fn try_as_batch(&self) -> Option<&CollectionBundle<Batch>> {
+        None
+    }
+
+    fn try_as_row(&self) -> Option<&CollectionBundle<DiffRow>> {
+        None
+    }
+}
+
+impl GenericBundle for CollectionBundle<Batch> {
+    fn is_batch(&self) -> bool {
+        true
+    }
+
+    fn try_as_batch(&self) -> Option<&CollectionBundle<Batch>> {
+        Some(self)
+    }
+}
+
+impl GenericBundle for CollectionBundle<DiffRow> {
+    fn is_batch(&self) -> bool {
+        false
+    }
+
+    fn try_as_row(&self) -> Option<&CollectionBundle<DiffRow>> {
+        Some(self)
+    }
+}
+
 impl<T: 'static> CollectionBundle<T> {
     pub fn from_collection(collection: Collection<T>) -> Self {
         Self {
diff --git a/src/flow/src/expr.rs b/src/flow/src/expr.rs
index 1f8160716a..2e6019ba4c 100644
--- a/src/flow/src/expr.rs
+++ b/src/flow/src/expr.rs
@@ -16,27 +16,29 @@
 
 mod df_func;
 pub(crate) mod error;
-mod func;
+pub(crate) mod func;
 mod id;
 mod linear;
-mod relation;
+pub(crate) mod relation;
 mod scalar;
 mod signature;
 
+use arrow::compute::FilterBuilder;
 use datatypes::prelude::DataType;
 use datatypes::value::Value;
-use datatypes::vectors::VectorRef;
+use datatypes::vectors::{BooleanVector, Helper, VectorRef};
 pub(crate) use df_func::{DfScalarFunction, RawDfScalarFn};
 pub(crate) use error::{EvalError, InvalidArgumentSnafu};
 pub(crate) use func::{BinaryFunc, UnaryFunc, UnmaterializableFunc, VariadicFunc};
 pub(crate) use id::{GlobalId, Id, LocalId};
 use itertools::Itertools;
 pub(crate) use linear::{MapFilterProject, MfpPlan, SafeMfpPlan};
-pub(crate) use relation::{AggregateExpr, AggregateFunc};
+pub(crate) use relation::{Accum, Accumulator, AggregateExpr, AggregateFunc};
 pub(crate) use scalar::{ScalarExpr, TypedExpr};
 use snafu::{ensure, ResultExt};
 
-use crate::expr::error::DataTypeSnafu;
+use crate::expr::error::{ArrowSnafu, DataTypeSnafu};
+use crate::repr::Diff;
 
 pub const TUMBLE_START: &str = "tumble_start";
 pub const TUMBLE_END: &str = "tumble_end";
@@ -179,7 +181,9 @@ impl Batch {
                 )
             }
         );
-        Ok(self.batch.iter().map(|v| v.get(idx)).collect_vec())
+        let mut ret = Vec::with_capacity(self.column_count());
+        ret.extend(self.batch.iter().map(|v| v.get(idx)));
+        Ok(ret)
     }
 
     /// Slices the `Batch`, returning a new `Batch`.
@@ -248,4 +252,97 @@ impl Batch {
         self.row_count = self_row_count + other_row_count;
         Ok(())
     }
+
+    /// filter the batch with given predicate
+    pub fn filter(&self, predicate: &BooleanVector) -> Result<Self, EvalError> {
+        let len = predicate.as_boolean_array().true_count();
+        let filter_builder = FilterBuilder::new(predicate.as_boolean_array()).optimize();
+        let filter_pred = filter_builder.build();
+        let filtered = self
+            .batch()
+            .iter()
+            .map(|col| filter_pred.filter(col.to_arrow_array().as_ref()))
+            .try_collect::<_, Vec<_>, _>()
+            .context(ArrowSnafu {
+                context: "Failed to filter val batches",
+            })?;
+        let res_vector = Helper::try_into_vectors(&filtered).context(DataTypeSnafu {
+            msg: "can't convert arrow array to vector",
+        })?;
+        Self::try_new(res_vector, len)
+    }
+}
+
+/// Vector with diff to note the insert and delete
+pub(crate) struct VectorDiff {
+    vector: VectorRef,
+    diff: Option<VectorRef>,
+}
+
+impl From<VectorRef> for VectorDiff {
+    fn from(vector: VectorRef) -> Self {
+        Self { vector, diff: None }
+    }
+}
+
+impl VectorDiff {
+    fn len(&self) -> usize {
+        self.vector.len()
+    }
+
+    fn try_new(vector: VectorRef, diff: Option<VectorRef>) -> Result<Self, EvalError> {
+        ensure!(
+            diff.as_ref()
+                .map_or(true, |diff| diff.len() == vector.len()),
+            InvalidArgumentSnafu {
+                reason: "Length of vector and diff should be the same"
+            }
+        );
+        Ok(Self { vector, diff })
+    }
+}
+
+impl IntoIterator for VectorDiff {
+    type Item = (Value, Diff);
+    type IntoIter = VectorDiffIter;
+
+    fn into_iter(self) -> Self::IntoIter {
+        VectorDiffIter {
+            vector: self.vector,
+            diff: self.diff,
+            idx: 0,
+        }
+    }
+}
+
+/// iterator for VectorDiff
+pub(crate) struct VectorDiffIter {
+    vector: VectorRef,
+    diff: Option<VectorRef>,
+    idx: usize,
+}
+
+impl std::iter::Iterator for VectorDiffIter {
+    type Item = (Value, Diff);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.idx >= self.vector.len() {
+            return None;
+        }
+        let value = self.vector.get(self.idx);
+        // +1 means insert, -1 means delete, and default to +1 insert when diff is not provided
+        let diff = if let Some(diff) = self.diff.as_ref() {
+            if let Ok(diff_at) = diff.get(self.idx).try_into() {
+                diff_at
+            } else {
+                common_telemetry::warn!("Invalid diff value at index {}", self.idx);
+                return None;
+            }
+        } else {
+            1
+        };
+
+        self.idx += 1;
+        Some((value, diff))
+    }
 }
diff --git a/src/flow/src/expr/df_func.rs b/src/flow/src/expr/df_func.rs
index b0a2648dd1..620615c9eb 100644
--- a/src/flow/src/expr/df_func.rs
+++ b/src/flow/src/expr/df_func.rs
@@ -92,12 +92,8 @@ impl DfScalarFunction {
 
         let len = rb.num_rows();
 
-        let res = self.fn_impl.evaluate(&rb).map_err(|err| {
-            EvalDatafusionSnafu {
-                raw: err,
-                context: "Failed to evaluate datafusion scalar function",
-            }
-            .build()
+        let res = self.fn_impl.evaluate(&rb).context(EvalDatafusionSnafu {
+            context: "Failed to evaluate datafusion scalar function",
         })?;
         let res = common_query::columnar_value::ColumnarValue::try_from(&res)
             .map_err(BoxedError::new)
@@ -157,12 +153,8 @@ impl DfScalarFunction {
             .into_error(err)
         })?;
 
-        let res = self.fn_impl.evaluate(&rb).map_err(|err| {
-            EvalDatafusionSnafu {
-                raw: err,
-                context: "Failed to evaluate datafusion scalar function",
-            }
-            .build()
+        let res = self.fn_impl.evaluate(&rb).context(EvalDatafusionSnafu {
+            context: "Failed to evaluate datafusion scalar function",
         })?;
         let res = common_query::columnar_value::ColumnarValue::try_from(&res)
             .map_err(BoxedError::new)
diff --git a/src/flow/src/expr/error.rs b/src/flow/src/expr/error.rs
index 6703ce2404..4b69b3df23 100644
--- a/src/flow/src/expr/error.rs
+++ b/src/flow/src/expr/error.rs
@@ -106,18 +106,19 @@ pub enum EvalError {
         location: Location,
     },
 
-    #[snafu(display("Arrow error: {raw:?}, context: {context}"))]
+    #[snafu(display("Arrow error: {error:?}, context: {context}"))]
     Arrow {
         #[snafu(source)]
-        raw: ArrowError,
+        error: ArrowError,
         context: String,
         #[snafu(implicit)]
         location: Location,
     },
 
-    #[snafu(display("DataFusion error: {raw:?}, context: {context}"))]
+    #[snafu(display("DataFusion error: {error:?}, context: {context}"))]
     Datafusion {
-        raw: DataFusionError,
+        #[snafu(source)]
+        error: DataFusionError,
         context: String,
         #[snafu(implicit)]
         location: Location,
diff --git a/src/flow/src/expr/func.rs b/src/flow/src/expr/func.rs
index 65da763e27..36a1029725 100644
--- a/src/flow/src/expr/func.rs
+++ b/src/flow/src/expr/func.rs
@@ -967,7 +967,7 @@ impl BinaryFunc {
             | Self::DivUInt32
             | Self::DivUInt64
             | Self::DivFloat32
-            | Self::DivFloat64 => arrow::compute::kernels::numeric::mul(&left, &right)
+            | Self::DivFloat64 => arrow::compute::kernels::numeric::div(&left, &right)
                 .context(ArrowSnafu { context: "div" })?,
 
             Self::ModInt16
@@ -1280,119 +1280,183 @@ where
     Ok(Value::from(left % right))
 }
 
-#[test]
-fn test_num_ops() {
-    let left = Value::from(10);
-    let right = Value::from(3);
-    let res = add::<i32>(left.clone(), right.clone()).unwrap();
-    assert_eq!(res, Value::from(13));
-    let res = sub::<i32>(left.clone(), right.clone()).unwrap();
-    assert_eq!(res, Value::from(7));
-    let res = mul::<i32>(left.clone(), right.clone()).unwrap();
-    assert_eq!(res, Value::from(30));
-    let res = div::<i32>(left.clone(), right.clone()).unwrap();
-    assert_eq!(res, Value::from(3));
-    let res = rem::<i32>(left, right).unwrap();
-    assert_eq!(res, Value::from(1));
+#[cfg(test)]
+mod test {
+    use std::sync::Arc;
 
-    let values = vec![Value::from(true), Value::from(false)];
-    let exprs = vec![ScalarExpr::Column(0), ScalarExpr::Column(1)];
-    let res = and(&values, &exprs).unwrap();
-    assert_eq!(res, Value::from(false));
-    let res = or(&values, &exprs).unwrap();
-    assert_eq!(res, Value::from(true));
-}
+    use common_time::Interval;
+    use datatypes::vectors::Vector;
+    use pretty_assertions::assert_eq;
 
-/// test if the binary function specialization works
-/// whether from direct type or from the expression that is literal
-#[test]
-fn test_binary_func_spec() {
-    assert_eq!(
-        BinaryFunc::from_str_expr_and_type(
-            "add",
-            &[ScalarExpr::Column(0), ScalarExpr::Column(0)],
-            &[
-                Some(ConcreteDataType::int32_datatype()),
-                Some(ConcreteDataType::int32_datatype())
-            ]
-        )
-        .unwrap(),
-        (BinaryFunc::AddInt32, BinaryFunc::AddInt32.signature())
-    );
+    use super::*;
 
-    assert_eq!(
-        BinaryFunc::from_str_expr_and_type(
-            "add",
-            &[ScalarExpr::Column(0), ScalarExpr::Column(0)],
-            &[Some(ConcreteDataType::int32_datatype()), None]
-        )
-        .unwrap(),
-        (BinaryFunc::AddInt32, BinaryFunc::AddInt32.signature())
-    );
+    #[test]
+    fn test_tumble_batch() {
+        let datetime_vector = DateTimeVector::from_vec(vec![1, 2, 10, 13, 14, 20, 25]);
+        let tumble_start = UnaryFunc::TumbleWindowFloor {
+            window_size: Interval::from_day_time(0, 10),
+            start_time: None,
+        };
+        let tumble_end = UnaryFunc::TumbleWindowCeiling {
+            window_size: Interval::from_day_time(0, 10),
+            start_time: None,
+        };
 
-    assert_eq!(
-        BinaryFunc::from_str_expr_and_type(
-            "add",
-            &[ScalarExpr::Column(0), ScalarExpr::Column(0)],
-            &[Some(ConcreteDataType::int32_datatype()), None]
-        )
-        .unwrap(),
-        (BinaryFunc::AddInt32, BinaryFunc::AddInt32.signature())
-    );
+        let len = datetime_vector.len();
+        let batch = Batch::try_new(vec![Arc::new(datetime_vector)], len).unwrap();
+        let arg = ScalarExpr::Column(0);
 
-    assert_eq!(
-        BinaryFunc::from_str_expr_and_type(
-            "add",
-            &[ScalarExpr::Column(0), ScalarExpr::Column(0)],
-            &[Some(ConcreteDataType::int32_datatype()), None]
-        )
-        .unwrap(),
-        (BinaryFunc::AddInt32, BinaryFunc::AddInt32.signature())
-    );
+        let start = tumble_start.eval_batch(&batch, &arg).unwrap();
+        let end = tumble_end.eval_batch(&batch, &arg).unwrap();
+        assert_eq!(
+            start.to_arrow_array().as_ref(),
+            TimestampMillisecondVector::from_vec(vec![0, 0, 10, 10, 10, 20, 20])
+                .to_arrow_array()
+                .as_ref()
+        );
 
-    assert_eq!(
-        BinaryFunc::from_str_expr_and_type(
-            "add",
-            &[
-                ScalarExpr::Literal(Value::from(1i32), ConcreteDataType::int32_datatype()),
-                ScalarExpr::Column(0)
-            ],
-            &[None, None]
-        )
-        .unwrap(),
-        (BinaryFunc::AddInt32, BinaryFunc::AddInt32.signature())
-    );
+        assert_eq!(
+            end.to_arrow_array().as_ref(),
+            TimestampMillisecondVector::from_vec(vec![10, 10, 20, 20, 20, 30, 30])
+                .to_arrow_array()
+                .as_ref()
+        );
 
-    // this testcase make sure the specialization can find actual type from expression and fill in signature
-    assert_eq!(
-        BinaryFunc::from_str_expr_and_type(
-            "equal",
-            &[
-                ScalarExpr::Literal(Value::from(1i32), ConcreteDataType::int32_datatype()),
-                ScalarExpr::Column(0)
-            ],
-            &[None, None]
-        )
-        .unwrap(),
-        (
-            BinaryFunc::Eq,
-            Signature {
-                input: smallvec![
-                    ConcreteDataType::int32_datatype(),
-                    ConcreteDataType::int32_datatype()
+        let ts_ms_vector = TimestampMillisecondVector::from_vec(vec![1, 2, 10, 13, 14, 20, 25]);
+        let batch = Batch::try_new(vec![Arc::new(ts_ms_vector)], len).unwrap();
+
+        let start = tumble_start.eval_batch(&batch, &arg).unwrap();
+        let end = tumble_end.eval_batch(&batch, &arg).unwrap();
+
+        assert_eq!(
+            start.to_arrow_array().as_ref(),
+            TimestampMillisecondVector::from_vec(vec![0, 0, 10, 10, 10, 20, 20])
+                .to_arrow_array()
+                .as_ref()
+        );
+
+        assert_eq!(
+            end.to_arrow_array().as_ref(),
+            TimestampMillisecondVector::from_vec(vec![10, 10, 20, 20, 20, 30, 30])
+                .to_arrow_array()
+                .as_ref()
+        );
+    }
+
+    #[test]
+    fn test_num_ops() {
+        let left = Value::from(10);
+        let right = Value::from(3);
+        let res = add::<i32>(left.clone(), right.clone()).unwrap();
+        assert_eq!(res, Value::from(13));
+        let res = sub::<i32>(left.clone(), right.clone()).unwrap();
+        assert_eq!(res, Value::from(7));
+        let res = mul::<i32>(left.clone(), right.clone()).unwrap();
+        assert_eq!(res, Value::from(30));
+        let res = div::<i32>(left.clone(), right.clone()).unwrap();
+        assert_eq!(res, Value::from(3));
+        let res = rem::<i32>(left, right).unwrap();
+        assert_eq!(res, Value::from(1));
+
+        let values = vec![Value::from(true), Value::from(false)];
+        let exprs = vec![ScalarExpr::Column(0), ScalarExpr::Column(1)];
+        let res = and(&values, &exprs).unwrap();
+        assert_eq!(res, Value::from(false));
+        let res = or(&values, &exprs).unwrap();
+        assert_eq!(res, Value::from(true));
+    }
+
+    /// test if the binary function specialization works
+    /// whether from direct type or from the expression that is literal
+    #[test]
+    fn test_binary_func_spec() {
+        assert_eq!(
+            BinaryFunc::from_str_expr_and_type(
+                "add",
+                &[ScalarExpr::Column(0), ScalarExpr::Column(0)],
+                &[
+                    Some(ConcreteDataType::int32_datatype()),
+                    Some(ConcreteDataType::int32_datatype())
+                ]
+            )
+            .unwrap(),
+            (BinaryFunc::AddInt32, BinaryFunc::AddInt32.signature())
+        );
+
+        assert_eq!(
+            BinaryFunc::from_str_expr_and_type(
+                "add",
+                &[ScalarExpr::Column(0), ScalarExpr::Column(0)],
+                &[Some(ConcreteDataType::int32_datatype()), None]
+            )
+            .unwrap(),
+            (BinaryFunc::AddInt32, BinaryFunc::AddInt32.signature())
+        );
+
+        assert_eq!(
+            BinaryFunc::from_str_expr_and_type(
+                "add",
+                &[ScalarExpr::Column(0), ScalarExpr::Column(0)],
+                &[Some(ConcreteDataType::int32_datatype()), None]
+            )
+            .unwrap(),
+            (BinaryFunc::AddInt32, BinaryFunc::AddInt32.signature())
+        );
+
+        assert_eq!(
+            BinaryFunc::from_str_expr_and_type(
+                "add",
+                &[ScalarExpr::Column(0), ScalarExpr::Column(0)],
+                &[Some(ConcreteDataType::int32_datatype()), None]
+            )
+            .unwrap(),
+            (BinaryFunc::AddInt32, BinaryFunc::AddInt32.signature())
+        );
+
+        assert_eq!(
+            BinaryFunc::from_str_expr_and_type(
+                "add",
+                &[
+                    ScalarExpr::Literal(Value::from(1i32), ConcreteDataType::int32_datatype()),
+                    ScalarExpr::Column(0)
                 ],
-                output: ConcreteDataType::boolean_datatype(),
-                generic_fn: GenericFn::Eq
-            }
-        )
-    );
+                &[None, None]
+            )
+            .unwrap(),
+            (BinaryFunc::AddInt32, BinaryFunc::AddInt32.signature())
+        );
 
-    matches!(
-        BinaryFunc::from_str_expr_and_type(
-            "add",
-            &[ScalarExpr::Column(0), ScalarExpr::Column(0)],
-            &[None, None]
-        ),
-        Err(Error::InvalidQuery { .. })
-    );
+        // this testcase make sure the specialization can find actual type from expression and fill in signature
+        assert_eq!(
+            BinaryFunc::from_str_expr_and_type(
+                "equal",
+                &[
+                    ScalarExpr::Literal(Value::from(1i32), ConcreteDataType::int32_datatype()),
+                    ScalarExpr::Column(0)
+                ],
+                &[None, None]
+            )
+            .unwrap(),
+            (
+                BinaryFunc::Eq,
+                Signature {
+                    input: smallvec![
+                        ConcreteDataType::int32_datatype(),
+                        ConcreteDataType::int32_datatype()
+                    ],
+                    output: ConcreteDataType::boolean_datatype(),
+                    generic_fn: GenericFn::Eq
+                }
+            )
+        );
+
+        matches!(
+            BinaryFunc::from_str_expr_and_type(
+                "add",
+                &[ScalarExpr::Column(0), ScalarExpr::Column(0)],
+                &[None, None]
+            ),
+            Err(Error::InvalidQuery { .. })
+        );
+    }
 }
diff --git a/src/flow/src/expr/linear.rs b/src/flow/src/expr/linear.rs
index 3185bdffcf..8e220f7d86 100644
--- a/src/flow/src/expr/linear.rs
+++ b/src/flow/src/expr/linear.rs
@@ -17,8 +17,9 @@
 use std::collections::{BTreeMap, BTreeSet};
 
 use arrow::array::BooleanArray;
+use arrow::buffer::BooleanBuffer;
 use arrow::compute::FilterBuilder;
-use common_telemetry::debug;
+use common_telemetry::trace;
 use datatypes::prelude::ConcreteDataType;
 use datatypes::value::Value;
 use datatypes::vectors::{BooleanVector, Helper};
@@ -500,7 +501,7 @@ impl SafeMfpPlan {
         for col in batch.batch() {
             let filtered = pred
                 .filter(col.to_arrow_array().as_ref())
-                .context(ArrowSnafu {
+                .with_context(|_| ArrowSnafu {
                     context: format!("failed to filter column for mfp operator {:?}", self),
                 })?;
             result.push(Helper::try_into_vector(filtered).context(DataTypeSnafu {
@@ -523,7 +524,9 @@ impl SafeMfpPlan {
         // mark the columns that have been evaluated and appended to the `batch`
         let mut expression = 0;
         // preds default to true and will be updated as we evaluate each predicate
-        let mut all_preds = BooleanVector::from(vec![Some(true); batch.row_count()]);
+        let buf = BooleanBuffer::new_set(batch.row_count());
+        let arr = BooleanArray::new(buf, None);
+        let mut all_preds = BooleanVector::from(arr);
 
         // to compute predicate, need to first compute all expressions used in predicates
         for (support, predicate) in self.mfp.predicates.iter() {
@@ -793,7 +796,7 @@ impl MfpPlan {
 
         if Some(lower_bound) != upper_bound && !null_eval {
             if self.mfp.mfp.projection.iter().any(|c| values.len() <= *c) {
-                debug!("values={:?}, mfp={:?}", &values, &self.mfp.mfp);
+                trace!("values={:?}, mfp={:?}", &values, &self.mfp.mfp);
                 let err = InternalSnafu {
                     reason: format!(
                         "Index out of bound for mfp={:?} and values={:?}",
diff --git a/src/flow/src/expr/relation.rs b/src/flow/src/expr/relation.rs
index 3661db4ff0..b5d7e4ef20 100644
--- a/src/flow/src/expr/relation.rs
+++ b/src/flow/src/expr/relation.rs
@@ -14,6 +14,7 @@
 
 //! Describes an aggregation function and it's input expression.
 
+pub(crate) use accum::{Accum, Accumulator};
 pub(crate) use func::AggregateFunc;
 
 use crate::expr::ScalarExpr;
diff --git a/src/flow/src/expr/relation/func.rs b/src/flow/src/expr/relation/func.rs
index afcdb7ddd1..fd72b58bb1 100644
--- a/src/flow/src/expr/relation/func.rs
+++ b/src/flow/src/expr/relation/func.rs
@@ -21,14 +21,14 @@ use datatypes::value::Value;
 use datatypes::vectors::VectorRef;
 use serde::{Deserialize, Serialize};
 use smallvec::smallvec;
-use snafu::{ensure, IntoError, OptionExt};
+use snafu::{IntoError, OptionExt};
 use strum::{EnumIter, IntoEnumIterator};
 
 use crate::error::{DatafusionSnafu, Error, InvalidQuerySnafu};
 use crate::expr::error::EvalError;
 use crate::expr::relation::accum::{Accum, Accumulator};
 use crate::expr::signature::{GenericFn, Signature};
-use crate::expr::InvalidArgumentSnafu;
+use crate::expr::VectorDiff;
 use crate::repr::Diff;
 
 /// Aggregate functions that can be applied to a group of rows.
@@ -161,72 +161,6 @@ impl AggregateFunc {
     }
 }
 
-struct VectorDiff {
-    vector: VectorRef,
-    diff: Option<VectorRef>,
-}
-
-impl VectorDiff {
-    fn len(&self) -> usize {
-        self.vector.len()
-    }
-
-    fn try_new(vector: VectorRef, diff: Option<VectorRef>) -> Result<Self, EvalError> {
-        ensure!(
-            diff.as_ref()
-                .map_or(true, |diff| diff.len() == vector.len()),
-            InvalidArgumentSnafu {
-                reason: "Length of vector and diff should be the same"
-            }
-        );
-        Ok(Self { vector, diff })
-    }
-}
-
-impl IntoIterator for VectorDiff {
-    type Item = (Value, Diff);
-    type IntoIter = VectorDiffIter;
-
-    fn into_iter(self) -> Self::IntoIter {
-        VectorDiffIter {
-            vector: self.vector,
-            diff: self.diff,
-            idx: 0,
-        }
-    }
-}
-
-struct VectorDiffIter {
-    vector: VectorRef,
-    diff: Option<VectorRef>,
-    idx: usize,
-}
-
-impl std::iter::Iterator for VectorDiffIter {
-    type Item = (Value, Diff);
-
-    fn next(&mut self) -> Option<Self::Item> {
-        if self.idx >= self.vector.len() {
-            return None;
-        }
-        let value = self.vector.get(self.idx);
-        // +1 means insert, -1 means delete, and default to +1 insert when diff is not provided
-        let diff = if let Some(diff) = self.diff.as_ref() {
-            if let Ok(diff_at) = diff.get(self.idx).try_into() {
-                diff_at
-            } else {
-                common_telemetry::warn!("Invalid diff value at index {}", self.idx);
-                return None;
-            }
-        } else {
-            1
-        };
-
-        self.idx += 1;
-        Some((value, diff))
-    }
-}
-
 /// Generate signature for each aggregate function
 macro_rules! generate_signature {
     ($value:ident,
diff --git a/src/flow/src/expr/scalar.rs b/src/flow/src/expr/scalar.rs
index 6a9cf30d95..a6e00cce5b 100644
--- a/src/flow/src/expr/scalar.rs
+++ b/src/flow/src/expr/scalar.rs
@@ -16,17 +16,20 @@
 
 use std::collections::{BTreeMap, BTreeSet};
 
+use arrow::array::{make_array, ArrayData, ArrayRef};
 use common_error::ext::BoxedError;
 use datatypes::prelude::{ConcreteDataType, DataType};
 use datatypes::value::Value;
-use datatypes::vectors::{BooleanVector, Helper, NullVector, Vector, VectorRef};
+use datatypes::vectors::{BooleanVector, Helper, VectorRef};
+use hydroflow::lattices::cc_traits::Iter;
+use itertools::Itertools;
 use snafu::{ensure, OptionExt, ResultExt};
 
 use crate::error::{
     DatafusionSnafu, Error, InvalidQuerySnafu, UnexpectedSnafu, UnsupportedTemporalFilterSnafu,
 };
 use crate::expr::error::{
-    DataTypeSnafu, EvalError, InternalSnafu, InvalidArgumentSnafu, OptimizeSnafu, TypeMismatchSnafu,
+    ArrowSnafu, DataTypeSnafu, EvalError, InvalidArgumentSnafu, OptimizeSnafu, TypeMismatchSnafu,
 };
 use crate::expr::func::{BinaryFunc, UnaryFunc, UnmaterializableFunc, VariadicFunc};
 use crate::expr::{Batch, DfScalarFunction};
@@ -222,6 +225,8 @@ impl ScalarExpr {
         }
     }
 
+    /// NOTE: this if then eval impl assume all given expr are pure, and will not change the state of the world
+    /// since it will evaluate both then and else branch and filter the result
     fn eval_if_then(
         batch: &Batch,
         cond: &ScalarExpr,
@@ -240,130 +245,69 @@ impl ScalarExpr {
             })?
             .as_boolean_array();
 
-        let mut then_input_batch = None;
-        let mut else_input_batch = None;
-        let mut null_input_batch = None;
+        let indices = bool_conds
+            .into_iter()
+            .enumerate()
+            .map(|(idx, b)| {
+                (
+                    match b {
+                        Some(true) => 0,  // then branch vector
+                        Some(false) => 1, // else branch vector
+                        None => 2,        // null vector
+                    },
+                    idx,
+                )
+            })
+            .collect_vec();
 
-        // instructions for how to reassembly result vector,
-        // iterate over (type of vec, offset, length) and append to resulting vec
-        let mut assembly_idx = vec![];
+        let then_input_vec = then.eval_batch(batch)?;
+        let else_input_vec = els.eval_batch(batch)?;
 
-        // append batch, returning appended batch's slice in (offset, length)
-        fn append_batch(
-            batch: &mut Option<Batch>,
-            to_be_append: Batch,
-        ) -> Result<(usize, usize), EvalError> {
-            let len = to_be_append.row_count();
-            if let Some(batch) = batch {
-                let offset = batch.row_count();
-                batch.append_batch(to_be_append)?;
-                Ok((offset, len))
-            } else {
-                *batch = Some(to_be_append);
-                Ok((0, len))
+        ensure!(
+            then_input_vec.data_type() == else_input_vec.data_type(),
+            TypeMismatchSnafu {
+                expected: then_input_vec.data_type(),
+                actual: else_input_vec.data_type(),
             }
+        );
+
+        ensure!(
+            then_input_vec.len() == else_input_vec.len() && then_input_vec.len() == batch.row_count(),
+            InvalidArgumentSnafu {
+                reason: format!(
+                    "then and else branch must have the same length(found {} and {}) which equals input batch's row count(which is {})",
+                    then_input_vec.len(),
+                    else_input_vec.len(),
+                    batch.row_count()
+                )
+            }
+        );
+
+        fn new_nulls(dt: &arrow_schema::DataType, len: usize) -> ArrayRef {
+            let data = ArrayData::new_null(dt, len);
+            make_array(data)
         }
 
-        let mut prev_cond: Option<Option<bool>> = None;
-        let mut prev_start_idx: Option<usize> = None;
-        // first put different conds' vector into different batches
-        for (idx, cond) in bool_conds.iter().enumerate() {
-            // if belong to same slice and not last one continue
-            if prev_cond == Some(cond) {
-                continue;
-            } else if let Some(prev_cond_idx) = prev_start_idx {
-                let prev_cond = prev_cond.unwrap();
+        let null_input_vec = new_nulls(
+            &then_input_vec.data_type().as_arrow_type(),
+            batch.row_count(),
+        );
 
-                // put a slice to corresponding batch
-                let slice_offset = prev_cond_idx;
-                let slice_length = idx - prev_cond_idx;
-                let to_be_append = batch.slice(slice_offset, slice_length)?;
+        let interleave_values = vec![
+            then_input_vec.to_arrow_array(),
+            else_input_vec.to_arrow_array(),
+            null_input_vec,
+        ];
+        let int_ref: Vec<_> = interleave_values.iter().map(|x| x.as_ref()).collect();
 
-                let to_put_back = match prev_cond {
-                    Some(true) => (
-                        Some(true),
-                        append_batch(&mut then_input_batch, to_be_append)?,
-                    ),
-                    Some(false) => (
-                        Some(false),
-                        append_batch(&mut else_input_batch, to_be_append)?,
-                    ),
-                    None => (None, append_batch(&mut null_input_batch, to_be_append)?),
-                };
-                assembly_idx.push(to_put_back);
-            }
-            prev_cond = Some(cond);
-            prev_start_idx = Some(idx);
-        }
-
-        // deal with empty and last slice case
-        if let Some(slice_offset) = prev_start_idx {
-            let prev_cond = prev_cond.unwrap();
-            let slice_length = bool_conds.len() - slice_offset;
-            let to_be_append = batch.slice(slice_offset, slice_length)?;
-            let to_put_back = match prev_cond {
-                Some(true) => (
-                    Some(true),
-                    append_batch(&mut then_input_batch, to_be_append)?,
-                ),
-                Some(false) => (
-                    Some(false),
-                    append_batch(&mut else_input_batch, to_be_append)?,
-                ),
-                None => (None, append_batch(&mut null_input_batch, to_be_append)?),
-            };
-            assembly_idx.push(to_put_back);
-        }
-
-        let then_output_vec = then_input_batch
-            .map(|batch| then.eval_batch(&batch))
-            .transpose()?;
-        let else_output_vec = else_input_batch
-            .map(|batch| els.eval_batch(&batch))
-            .transpose()?;
-        let null_output_vec = null_input_batch
-            .map(|null| NullVector::new(null.row_count()).slice(0, null.row_count()));
-
-        let dt = then_output_vec
-            .as_ref()
-            .map(|v| v.data_type())
-            .or(else_output_vec.as_ref().map(|v| v.data_type()))
-            .unwrap_or(ConcreteDataType::null_datatype());
-        let mut builder = dt.create_mutable_vector(conds.len());
-        for (cond, (offset, length)) in assembly_idx {
-            let slice = match cond {
-                Some(true) => then_output_vec.as_ref(),
-                Some(false) => else_output_vec.as_ref(),
-                None => null_output_vec.as_ref(),
-            }
-            .context(InternalSnafu {
-                reason: "Expect corresponding output vector to exist",
+        let interleave_res_arr =
+            arrow::compute::interleave(&int_ref, &indices).context(ArrowSnafu {
+                context: "Failed to interleave output arrays",
             })?;
-            // TODO(discord9): seems `extend_slice_of` doesn't support NullVector or ConstantVector
-            // consider adding it maybe?
-            if slice.data_type().is_null() {
-                builder.push_nulls(length);
-            } else if slice.is_const() {
-                let arr = slice.slice(offset, length).to_arrow_array();
-                let vector = Helper::try_into_vector(arr).context(DataTypeSnafu {
-                    msg: "Failed to convert arrow array to vector",
-                })?;
-                builder
-                    .extend_slice_of(vector.as_ref(), 0, vector.len())
-                    .context(DataTypeSnafu {
-                        msg: "Failed to build result vector for if-then expression",
-                    })?;
-            } else {
-                builder
-                    .extend_slice_of(slice.as_ref(), offset, length)
-                    .context(DataTypeSnafu {
-                        msg: "Failed to build result vector for if-then expression",
-                    })?;
-            }
-        }
-        let result_vec = builder.to_vector();
-
-        Ok(result_vec)
+        let res_vec = Helper::try_into_vector(interleave_res_arr).context(DataTypeSnafu {
+            msg: "Failed to convert arrow array to vector",
+        })?;
+        Ok(res_vec)
     }
 
     /// Eval this expression with the given values.
@@ -685,7 +629,7 @@ impl ScalarExpr {
 
 #[cfg(test)]
 mod test {
-    use datatypes::vectors::Int32Vector;
+    use datatypes::vectors::{Int32Vector, Vector};
     use pretty_assertions::assert_eq;
 
     use super::*;
@@ -781,7 +725,7 @@ mod test {
     }
 
     #[test]
-    fn test_eval_batch() {
+    fn test_eval_batch_if_then() {
         // TODO(discord9): add more tests
         {
             let expr = ScalarExpr::If {
@@ -840,7 +784,7 @@ mod test {
             let vectors = vec![Int32Vector::from(raw).slice(0, raw_len)];
 
             let batch = Batch::try_new(vectors, raw_len).unwrap();
-            let expected = NullVector::new(raw_len).slice(0, raw_len);
+            let expected = Int32Vector::from(vec![]).slice(0, raw_len);
             assert_eq!(expr.eval_batch(&batch).unwrap(), expected);
         }
     }
diff --git a/src/flow/src/repr.rs b/src/flow/src/repr.rs
index 7b57fc3ed2..acec71ebcf 100644
--- a/src/flow/src/repr.rs
+++ b/src/flow/src/repr.rs
@@ -50,12 +50,13 @@ pub type KeyValDiffRow = ((Row, Row), Timestamp, Diff);
 /// broadcast channel capacity, can be important to memory consumption, since this influence how many
 /// updates can be buffered in memory in the entire dataflow
 /// TODO(discord9): add config for this, so cpu&mem usage can be balanced and configured by this
-pub const BROADCAST_CAP: usize = 65535;
+pub const BROADCAST_CAP: usize = 1024;
 
 /// The maximum capacity of the send buffer, to prevent the buffer from growing too large
 pub const SEND_BUF_CAP: usize = BROADCAST_CAP * 2;
 
-pub const BATCH_SIZE: usize = BROADCAST_CAP / 2;
+/// Flow worker will try to at least accumulate this many rows before processing them(if one second havn't passed)
+pub const BATCH_SIZE: usize = 32 * 16384;
 
 /// Convert a value that is or can be converted to Datetime to internal timestamp
 ///
diff --git a/src/flow/src/utils.rs b/src/flow/src/utils.rs
index 269c53fa84..1cd5b3ba5c 100644
--- a/src/flow/src/utils.rs
+++ b/src/flow/src/utils.rs
@@ -18,7 +18,7 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::ops::Bound;
 use std::sync::Arc;
 
-use common_telemetry::debug;
+use common_telemetry::trace;
 use smallvec::{smallvec, SmallVec};
 use tokio::sync::RwLock;
 
@@ -235,9 +235,11 @@ impl Arrangement {
             if let Some(s) = &mut self.expire_state {
                 if let Some(expired_by) = s.get_expire_duration_and_update_event_ts(now, &key)? {
                     max_expired_by = max_expired_by.max(Some(expired_by));
-                    debug!(
+                    trace!(
                         "Expired key: {:?}, expired by: {:?} with time being now={}",
-                        key, expired_by, now
+                        key,
+                        expired_by,
+                        now
                     );
                     continue;
                 }
diff --git a/tests/cases/standalone/common/flow/flow_basic.result b/tests/cases/standalone/common/flow/flow_basic.result
index db3d3c8c3b..6aaa2c74be 100644
--- a/tests/cases/standalone/common/flow/flow_basic.result
+++ b/tests/cases/standalone/common/flow/flow_basic.result
@@ -7,10 +7,13 @@ CREATE TABLE numbers_input_basic (
 
 Affected Rows: 0
 
-CREATE FLOW test_numbers_basic
-SINK TO out_num_cnt_basic
-AS
-SELECT sum(number) FROM numbers_input_basic GROUP BY tumble(ts, '1 second', '2021-07-01 00:00:00');
+CREATE FLOW test_numbers_basic SINK TO out_num_cnt_basic AS
+SELECT
+    sum(number)
+FROM
+    numbers_input_basic
+GROUP BY
+    tumble(ts, '1 second', '2021-07-01 00:00:00');
 
 Affected Rows: 0
 
@@ -25,7 +28,8 @@ admin flush_flow('test_numbers_basic');
 +----------------------------------------+
 
 -- SQLNESS ARG restart=true
-INSERT INTO numbers_input_basic
+INSERT INTO
+    numbers_input_basic
 VALUES
     (20, "2021-07-01 00:00:00.200"),
     (22, "2021-07-01 00:00:00.600");
@@ -40,7 +44,12 @@ admin flush_flow('test_numbers_basic');
 | 1                                      |
 +----------------------------------------+
 
-SELECT "SUM(numbers_input_basic.number)", window_start, window_end FROM out_num_cnt_basic;
+SELECT
+    "SUM(numbers_input_basic.number)",
+    window_start,
+    window_end
+FROM
+    out_num_cnt_basic;
 
 +---------------------------------+---------------------+---------------------+
 | SUM(numbers_input_basic.number) | window_start        | window_end          |
@@ -56,10 +65,11 @@ admin flush_flow('test_numbers_basic');
 | 0                                      |
 +----------------------------------------+
 
-INSERT INTO numbers_input_basic
+INSERT INTO
+    numbers_input_basic
 VALUES
-    (23,"2021-07-01 00:00:01.000"),
-    (24,"2021-07-01 00:00:01.500");
+    (23, "2021-07-01 00:00:01.000"),
+    (24, "2021-07-01 00:00:01.500");
 
 Affected Rows: 2
 
@@ -72,7 +82,12 @@ admin flush_flow('test_numbers_basic');
 +----------------------------------------+
 
 -- note that this quote-unquote column is a column-name, **not** a aggregation expr, generated by datafusion
-SELECT "SUM(numbers_input_basic.number)", window_start, window_end FROM out_num_cnt_basic;
+SELECT
+    "SUM(numbers_input_basic.number)",
+    window_start,
+    window_end
+FROM
+    out_num_cnt_basic;
 
 +---------------------------------+---------------------+---------------------+
 | SUM(numbers_input_basic.number) | window_start        | window_end          |
@@ -93,6 +108,114 @@ DROP TABLE out_num_cnt_basic;
 
 Affected Rows: 0
 
+-- test distinct
+CREATE TABLE distinct_basic (
+    number INT,
+    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY(number),
+    TIME INDEX(ts)
+);
+
+Affected Rows: 0
+
+CREATE FLOW test_distinct_basic SINK TO out_distinct_basic AS
+SELECT
+    DISTINCT number as dis
+FROM
+    distinct_basic;
+
+Affected Rows: 0
+
+-- TODO(discord9): confirm if it's necessary to flush flow here?
+-- because flush_flow result is at most 1
+admin flush_flow('test_distinct_basic');
+
++-----------------------------------------+
+| ADMIN flush_flow('test_distinct_basic') |
++-----------------------------------------+
+| 0                                       |
++-----------------------------------------+
+
+-- SQLNESS ARG restart=true
+INSERT INTO
+    distinct_basic
+VALUES
+    (20, "2021-07-01 00:00:00.200"),
+    (22, "2021-07-01 00:00:00.600");
+
+Affected Rows: 2
+
+admin flush_flow('test_distinct_basic');
+
++-----------------------------------------+
+| ADMIN flush_flow('test_distinct_basic') |
++-----------------------------------------+
+| 1                                       |
++-----------------------------------------+
+
+SELECT
+    dis
+FROM
+    out_distinct_basic;
+
++-----+
+| dis |
++-----+
+| 20  |
+| 22  |
++-----+
+
+admin flush_flow('test_distinct_basic');
+
++-----------------------------------------+
+| ADMIN flush_flow('test_distinct_basic') |
++-----------------------------------------+
+| 0                                       |
++-----------------------------------------+
+
+INSERT INTO
+    distinct_basic
+VALUES
+    (23, "2021-07-01 00:00:01.000"),
+    (24, "2021-07-01 00:00:01.500");
+
+Affected Rows: 2
+
+admin flush_flow('test_distinct_basic');
+
++-----------------------------------------+
+| ADMIN flush_flow('test_distinct_basic') |
++-----------------------------------------+
+| 1                                       |
++-----------------------------------------+
+
+-- note that this quote-unquote column is a column-name, **not** a aggregation expr, generated by datafusion
+SELECT
+    dis
+FROM
+    out_distinct_basic;
+
++-----+
+| dis |
++-----+
+| 20  |
+| 22  |
+| 23  |
+| 24  |
++-----+
+
+DROP FLOW test_distinct_basic;
+
+Affected Rows: 0
+
+DROP TABLE distinct_basic;
+
+Affected Rows: 0
+
+DROP TABLE out_distinct_basic;
+
+Affected Rows: 0
+
 -- test interprete interval
 CREATE TABLE numbers_input_basic (
     number INT,
@@ -105,11 +228,20 @@ Affected Rows: 0
 
 create table out_num_cnt_basic (
     number INT,
-    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP TIME INDEX);
+    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP TIME INDEX
+);
 
 Affected Rows: 0
 
-CREATE FLOW filter_numbers_basic SINK TO out_num_cnt_basic AS SELECT INTERVAL '1 day 1 second', INTERVAL '1 month 1 day 1 second', INTERVAL '1 year 1 month' FROM numbers_input_basic where number > 10;
+CREATE FLOW filter_numbers_basic SINK TO out_num_cnt_basic AS
+SELECT
+    INTERVAL '1 day 1 second',
+    INTERVAL '1 month 1 day 1 second',
+    INTERVAL '1 year 1 month'
+FROM
+    numbers_input_basic
+where
+    number > 10;
 
 Affected Rows: 0
 
@@ -137,7 +269,8 @@ Affected Rows: 0
 
 CREATE TABLE bytes_log (
     byte INT,
-    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- event time
+    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    -- event time
     TIME INDEX(ts)
 );
 
@@ -153,16 +286,22 @@ CREATE TABLE approx_rate (
 
 Affected Rows: 0
 
-CREATE FLOW find_approx_rate
-SINK TO approx_rate
-AS
-SELECT (max(byte) - min(byte))/30.0 as rate, date_bin(INTERVAL '30 second', ts) as time_window from bytes_log GROUP BY time_window;
+CREATE FLOW find_approx_rate SINK TO approx_rate AS
+SELECT
+    (max(byte) - min(byte)) / 30.0 as rate,
+    date_bin(INTERVAL '30 second', ts) as time_window
+from
+    bytes_log
+GROUP BY
+    time_window;
 
 Affected Rows: 0
 
-INSERT INTO bytes_log VALUES
-(101, '2025-01-01 00:00:01'),
-(300, '2025-01-01 00:00:29');
+INSERT INTO
+    bytes_log
+VALUES
+    (101, '2025-01-01 00:00:01'),
+    (300, '2025-01-01 00:00:29');
 
 Affected Rows: 2
 
@@ -174,7 +313,11 @@ admin flush_flow('find_approx_rate');
 | 1                                    |
 +--------------------------------------+
 
-SELECT rate, time_window FROM approx_rate;
+SELECT
+    rate,
+    time_window
+FROM
+    approx_rate;
 
 +-------------------+---------------------+
 | rate              | time_window         |
@@ -182,9 +325,11 @@ SELECT rate, time_window FROM approx_rate;
 | 6.633333333333334 | 2025-01-01T00:00:00 |
 +-------------------+---------------------+
 
-INSERT INTO bytes_log VALUES
-(450, '2025-01-01 00:00:32'),
-(500, '2025-01-01 00:00:37');
+INSERT INTO
+    bytes_log
+VALUES
+    (450, '2025-01-01 00:00:32'),
+    (500, '2025-01-01 00:00:37');
 
 Affected Rows: 2
 
@@ -196,7 +341,11 @@ admin flush_flow('find_approx_rate');
 | 1                                    |
 +--------------------------------------+
 
-SELECT rate, time_window FROM approx_rate;
+SELECT
+    rate,
+    time_window
+FROM
+    approx_rate;
 
 +--------------------+---------------------+
 | rate               | time_window         |
@@ -217,3 +366,474 @@ DROP TABLE approx_rate;
 
 Affected Rows: 0
 
+-- input table
+CREATE TABLE ngx_access_log (
+    client STRING,
+    country STRING,
+    access_time TIMESTAMP TIME INDEX
+);
+
+Affected Rows: 0
+
+-- create flow task to calculate the distinct country
+CREATE FLOW calc_ngx_country SINK TO ngx_country AS
+SELECT
+    DISTINCT country,
+FROM
+    ngx_access_log;
+
+Affected Rows: 0
+
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", "b", 0);
+
+Affected Rows: 1
+
+ADMIN FLUSH_FLOW('calc_ngx_country');
+
++--------------------------------------+
+| ADMIN FLUSH_FLOW('calc_ngx_country') |
++--------------------------------------+
+| 1                                    |
++--------------------------------------+
+
+SELECT
+    "ngx_access_log.country"
+FROM
+    ngx_country;
+
++------------------------+
+| ngx_access_log.country |
++------------------------+
+| b                      |
++------------------------+
+
+-- making sure distinct is working
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", "b", 1);
+
+Affected Rows: 1
+
+ADMIN FLUSH_FLOW('calc_ngx_country');
+
++--------------------------------------+
+| ADMIN FLUSH_FLOW('calc_ngx_country') |
++--------------------------------------+
+| 1                                    |
++--------------------------------------+
+
+SELECT
+    "ngx_access_log.country"
+FROM
+    ngx_country;
+
++------------------------+
+| ngx_access_log.country |
++------------------------+
+| b                      |
++------------------------+
+
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", "c", 2);
+
+Affected Rows: 1
+
+ADMIN FLUSH_FLOW('calc_ngx_country');
+
++--------------------------------------+
+| ADMIN FLUSH_FLOW('calc_ngx_country') |
++--------------------------------------+
+| 1                                    |
++--------------------------------------+
+
+SELECT
+    "ngx_access_log.country"
+FROM
+    ngx_country;
+
++------------------------+
+| ngx_access_log.country |
++------------------------+
+| b                      |
+| c                      |
++------------------------+
+
+DROP FLOW calc_ngx_country;
+
+Affected Rows: 0
+
+DROP TABLE ngx_access_log;
+
+Affected Rows: 0
+
+DROP TABLE ngx_country;
+
+Affected Rows: 0
+
+CREATE TABLE ngx_access_log (
+    client STRING,
+    country STRING,
+    access_time TIMESTAMP TIME INDEX
+);
+
+Affected Rows: 0
+
+CREATE FLOW calc_ngx_country SINK TO ngx_country AS
+SELECT
+    DISTINCT country,
+    -- this distinct is not necessary, but it's a good test to see if it works
+    date_bin(INTERVAL '1 hour', access_time) as time_window,
+FROM
+    ngx_access_log
+GROUP BY
+    country,
+    time_window;
+
+Affected Rows: 0
+
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", "b", 0);
+
+Affected Rows: 1
+
+ADMIN FLUSH_FLOW('calc_ngx_country');
+
++--------------------------------------+
+| ADMIN FLUSH_FLOW('calc_ngx_country') |
++--------------------------------------+
+| 1                                    |
++--------------------------------------+
+
+SELECT
+    "ngx_access_log.country",
+    time_window
+FROM
+    ngx_country;
+
++------------------------+---------------------+
+| ngx_access_log.country | time_window         |
++------------------------+---------------------+
+| b                      | 1970-01-01T00:00:00 |
++------------------------+---------------------+
+
+-- making sure distinct is working
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", "b", 1);
+
+Affected Rows: 1
+
+ADMIN FLUSH_FLOW('calc_ngx_country');
+
++--------------------------------------+
+| ADMIN FLUSH_FLOW('calc_ngx_country') |
++--------------------------------------+
+| 1                                    |
++--------------------------------------+
+
+SELECT
+    "ngx_access_log.country",
+    time_window
+FROM
+    ngx_country;
+
++------------------------+---------------------+
+| ngx_access_log.country | time_window         |
++------------------------+---------------------+
+| b                      | 1970-01-01T00:00:00 |
++------------------------+---------------------+
+
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", "c", 2);
+
+Affected Rows: 1
+
+ADMIN FLUSH_FLOW('calc_ngx_country');
+
++--------------------------------------+
+| ADMIN FLUSH_FLOW('calc_ngx_country') |
++--------------------------------------+
+| 1                                    |
++--------------------------------------+
+
+SELECT
+    "ngx_access_log.country",
+    time_window
+FROM
+    ngx_country;
+
++------------------------+---------------------+
+| ngx_access_log.country | time_window         |
++------------------------+---------------------+
+| b                      | 1970-01-01T00:00:00 |
+| c                      | 1970-01-01T00:00:00 |
++------------------------+---------------------+
+
+DROP FLOW calc_ngx_country;
+
+Affected Rows: 0
+
+DROP TABLE ngx_access_log;
+
+Affected Rows: 0
+
+DROP TABLE ngx_country;
+
+Affected Rows: 0
+
+CREATE TABLE temp_sensor_data (
+    sensor_id INT,
+    loc STRING,
+    temperature DOUBLE,
+    ts TIMESTAMP TIME INDEX
+);
+
+Affected Rows: 0
+
+CREATE TABLE temp_alerts (
+    sensor_id INT,
+    loc STRING,
+    max_temp DOUBLE,
+    ts TIMESTAMP TIME INDEX
+);
+
+Affected Rows: 0
+
+CREATE FLOW temp_monitoring SINK TO temp_alerts AS
+SELECT
+    sensor_id,
+    loc,
+    max(temperature) as max_temp,
+FROM
+    temp_sensor_data
+GROUP BY
+    sensor_id,
+    loc
+HAVING
+    max_temp > 100;
+
+Affected Rows: 0
+
+INSERT INTO
+    temp_sensor_data
+VALUES
+    (1, "room1", 50, 0);
+
+Affected Rows: 1
+
+ADMIN FLUSH_FLOW('temp_monitoring');
+
++-------------------------------------+
+| ADMIN FLUSH_FLOW('temp_monitoring') |
++-------------------------------------+
+| 1                                   |
++-------------------------------------+
+
+-- This table should not exist yet
+SHOW TABLES LIKE 'temp_alerts';
+
++-------------+
+| Tables      |
++-------------+
+| temp_alerts |
++-------------+
+
+INSERT INTO
+    temp_sensor_data
+VALUES
+    (1, "room1", 150, 1);
+
+Affected Rows: 1
+
+ADMIN FLUSH_FLOW('temp_monitoring');
+
++-------------------------------------+
+| ADMIN FLUSH_FLOW('temp_monitoring') |
++-------------------------------------+
+| 1                                   |
++-------------------------------------+
+
+SHOW TABLES LIKE 'temp_alerts';
+
++-------------+
+| Tables      |
++-------------+
+| temp_alerts |
++-------------+
+
+SELECT
+    sensor_id,
+    loc,
+    max_temp
+FROM
+    temp_alerts;
+
++-----------+-------+----------+
+| sensor_id | loc   | max_temp |
++-----------+-------+----------+
+| 1         | room1 | 150.0    |
++-----------+-------+----------+
+
+INSERT INTO
+    temp_sensor_data
+VALUES
+    (2, "room1", 0, 2);
+
+Affected Rows: 1
+
+ADMIN FLUSH_FLOW('temp_monitoring');
+
++-------------------------------------+
+| ADMIN FLUSH_FLOW('temp_monitoring') |
++-------------------------------------+
+| 1                                   |
++-------------------------------------+
+
+SELECT
+    sensor_id,
+    loc,
+    max_temp
+FROM
+    temp_alerts;
+
++-----------+-------+----------+
+| sensor_id | loc   | max_temp |
++-----------+-------+----------+
+| 1         | room1 | 150.0    |
++-----------+-------+----------+
+
+DROP FLOW temp_monitoring;
+
+Affected Rows: 0
+
+DROP TABLE temp_sensor_data;
+
+Affected Rows: 0
+
+DROP TABLE temp_alerts;
+
+Affected Rows: 0
+
+CREATE TABLE ngx_access_log (
+    client STRING,
+    stat INT,
+    size INT,
+    access_time TIMESTAMP TIME INDEX
+);
+
+Affected Rows: 0
+
+CREATE TABLE ngx_distribution (
+    stat INT,
+    bucket_size INT,
+    total_logs BIGINT,
+    time_window TIMESTAMP TIME INDEX,
+    update_at TIMESTAMP, -- auto generated column by flow engine
+    PRIMARY KEY(stat, bucket_size)
+);
+
+Affected Rows: 0
+
+CREATE FLOW calc_ngx_distribution SINK TO ngx_distribution AS
+SELECT
+    stat,
+    trunc(size, -1)::INT as bucket_size,
+    count(client) AS total_logs,
+    date_bin(INTERVAL '1 minutes', access_time) as time_window,
+FROM
+    ngx_access_log
+GROUP BY
+    stat,
+    time_window,
+    bucket_size;
+
+Affected Rows: 0
+
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", 200, 100, 0);
+
+Affected Rows: 1
+
+ADMIN FLUSH_FLOW('calc_ngx_distribution');
+
++-------------------------------------------+
+| ADMIN FLUSH_FLOW('calc_ngx_distribution') |
++-------------------------------------------+
+| 1                                         |
++-------------------------------------------+
+
+SELECT
+    stat,
+    bucket_size,
+    total_logs,
+    time_window
+FROM
+    ngx_distribution;
+
++------+-------------+------------+---------------------+
+| stat | bucket_size | total_logs | time_window         |
++------+-------------+------------+---------------------+
+| 200  | 100         | 1          | 1970-01-01T00:00:00 |
++------+-------------+------------+---------------------+
+
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", 200, 200, 1),
+    ("cli1", 200, 205, 1),
+    ("cli1", 200, 209, 1),
+    ("cli1", 200, 210, 1),
+    ("cli2", 200, 300, 1);
+
+Affected Rows: 5
+
+ADMIN FLUSH_FLOW('calc_ngx_distribution');
+
++-------------------------------------------+
+| ADMIN FLUSH_FLOW('calc_ngx_distribution') |
++-------------------------------------------+
+| 1                                         |
++-------------------------------------------+
+
+SELECT
+    stat,
+    bucket_size,
+    total_logs,
+    time_window
+FROM
+    ngx_distribution;
+
++------+-------------+------------+---------------------+
+| stat | bucket_size | total_logs | time_window         |
++------+-------------+------------+---------------------+
+| 200  | 100         | 1          | 1970-01-01T00:00:00 |
+| 200  | 200         | 1          | 1970-01-01T00:00:00 |
+| 200  | 210         | 3          | 1970-01-01T00:00:00 |
+| 200  | 300         | 1          | 1970-01-01T00:00:00 |
++------+-------------+------------+---------------------+
+
+DROP FLOW calc_ngx_distribution;
+
+Affected Rows: 0
+
+DROP TABLE ngx_access_log;
+
+Affected Rows: 0
+
+DROP TABLE ngx_distribution;
+
+Affected Rows: 0
+
diff --git a/tests/cases/standalone/common/flow/flow_basic.sql b/tests/cases/standalone/common/flow/flow_basic.sql
index b9ccc81058..9356508aba 100644
--- a/tests/cases/standalone/common/flow/flow_basic.sql
+++ b/tests/cases/standalone/common/flow/flow_basic.sql
@@ -5,54 +5,134 @@ CREATE TABLE numbers_input_basic (
     TIME INDEX(ts)
 );
 
-CREATE FLOW test_numbers_basic
-SINK TO out_num_cnt_basic
-AS
-SELECT sum(number) FROM numbers_input_basic GROUP BY tumble(ts, '1 second', '2021-07-01 00:00:00');
+CREATE FLOW test_numbers_basic SINK TO out_num_cnt_basic AS
+SELECT
+    sum(number)
+FROM
+    numbers_input_basic
+GROUP BY
+    tumble(ts, '1 second', '2021-07-01 00:00:00');
 
 -- TODO(discord9): confirm if it's necessary to flush flow here?
 -- because flush_flow result is at most 1
 admin flush_flow('test_numbers_basic');
 
 -- SQLNESS ARG restart=true
-INSERT INTO numbers_input_basic
+INSERT INTO
+    numbers_input_basic
 VALUES
     (20, "2021-07-01 00:00:00.200"),
     (22, "2021-07-01 00:00:00.600");
 
 admin flush_flow('test_numbers_basic');
 
-SELECT "SUM(numbers_input_basic.number)", window_start, window_end FROM out_num_cnt_basic;
+SELECT
+    "SUM(numbers_input_basic.number)",
+    window_start,
+    window_end
+FROM
+    out_num_cnt_basic;
 
 admin flush_flow('test_numbers_basic');
 
-INSERT INTO numbers_input_basic
+INSERT INTO
+    numbers_input_basic
 VALUES
-    (23,"2021-07-01 00:00:01.000"),
-    (24,"2021-07-01 00:00:01.500");
+    (23, "2021-07-01 00:00:01.000"),
+    (24, "2021-07-01 00:00:01.500");
 
 admin flush_flow('test_numbers_basic');
 
 -- note that this quote-unquote column is a column-name, **not** a aggregation expr, generated by datafusion
-SELECT "SUM(numbers_input_basic.number)", window_start, window_end FROM out_num_cnt_basic;
+SELECT
+    "SUM(numbers_input_basic.number)",
+    window_start,
+    window_end
+FROM
+    out_num_cnt_basic;
 
 DROP FLOW test_numbers_basic;
+
 DROP TABLE numbers_input_basic;
+
 DROP TABLE out_num_cnt_basic;
 
--- test interprete interval
+-- test distinct
+CREATE TABLE distinct_basic (
+    number INT,
+    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY(number),
+    TIME INDEX(ts)
+);
 
+CREATE FLOW test_distinct_basic SINK TO out_distinct_basic AS
+SELECT
+    DISTINCT number as dis
+FROM
+    distinct_basic;
+
+-- TODO(discord9): confirm if it's necessary to flush flow here?
+-- because flush_flow result is at most 1
+admin flush_flow('test_distinct_basic');
+
+-- SQLNESS ARG restart=true
+INSERT INTO
+    distinct_basic
+VALUES
+    (20, "2021-07-01 00:00:00.200"),
+    (22, "2021-07-01 00:00:00.600");
+
+admin flush_flow('test_distinct_basic');
+
+SELECT
+    dis
+FROM
+    out_distinct_basic;
+
+admin flush_flow('test_distinct_basic');
+
+INSERT INTO
+    distinct_basic
+VALUES
+    (23, "2021-07-01 00:00:01.000"),
+    (24, "2021-07-01 00:00:01.500");
+
+admin flush_flow('test_distinct_basic');
+
+-- note that this quote-unquote column is a column-name, **not** a aggregation expr, generated by datafusion
+SELECT
+    dis
+FROM
+    out_distinct_basic;
+
+DROP FLOW test_distinct_basic;
+
+DROP TABLE distinct_basic;
+
+DROP TABLE out_distinct_basic;
+
+-- test interprete interval
 CREATE TABLE numbers_input_basic (
     number INT,
     ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     PRIMARY KEY(number),
     TIME INDEX(ts)
 );
+
 create table out_num_cnt_basic (
     number INT,
-    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP TIME INDEX);
+    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP TIME INDEX
+);
 
-CREATE FLOW filter_numbers_basic SINK TO out_num_cnt_basic AS SELECT INTERVAL '1 day 1 second', INTERVAL '1 month 1 day 1 second', INTERVAL '1 year 1 month' FROM numbers_input_basic where number > 10;
+CREATE FLOW filter_numbers_basic SINK TO out_num_cnt_basic AS
+SELECT
+    INTERVAL '1 day 1 second',
+    INTERVAL '1 month 1 day 1 second',
+    INTERVAL '1 year 1 month'
+FROM
+    numbers_input_basic
+where
+    number > 10;
 
 SHOW CREATE FLOW filter_numbers_basic;
 
@@ -64,7 +144,8 @@ drop table numbers_input_basic;
 
 CREATE TABLE bytes_log (
     byte INT,
-    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- event time
+    ts TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    -- event time
     TIME INDEX(ts)
 );
 
@@ -76,27 +157,307 @@ CREATE TABLE approx_rate (
     TIME INDEX(time_window)
 );
 
-CREATE FLOW find_approx_rate
-SINK TO approx_rate
-AS
-SELECT (max(byte) - min(byte))/30.0 as rate, date_bin(INTERVAL '30 second', ts) as time_window from bytes_log GROUP BY time_window;
+CREATE FLOW find_approx_rate SINK TO approx_rate AS
+SELECT
+    (max(byte) - min(byte)) / 30.0 as rate,
+    date_bin(INTERVAL '30 second', ts) as time_window
+from
+    bytes_log
+GROUP BY
+    time_window;
 
-INSERT INTO bytes_log VALUES
-(101, '2025-01-01 00:00:01'),
-(300, '2025-01-01 00:00:29');
+INSERT INTO
+    bytes_log
+VALUES
+    (101, '2025-01-01 00:00:01'),
+    (300, '2025-01-01 00:00:29');
 
 admin flush_flow('find_approx_rate');
 
-SELECT rate, time_window FROM approx_rate;
+SELECT
+    rate,
+    time_window
+FROM
+    approx_rate;
 
-INSERT INTO bytes_log VALUES
-(450, '2025-01-01 00:00:32'),
-(500, '2025-01-01 00:00:37');
+INSERT INTO
+    bytes_log
+VALUES
+    (450, '2025-01-01 00:00:32'),
+    (500, '2025-01-01 00:00:37');
 
 admin flush_flow('find_approx_rate');
 
-SELECT rate, time_window FROM approx_rate;
+SELECT
+    rate,
+    time_window
+FROM
+    approx_rate;
 
 DROP TABLE bytes_log;
+
 DROP FLOW find_approx_rate;
+
 DROP TABLE approx_rate;
+
+-- input table
+CREATE TABLE ngx_access_log (
+    client STRING,
+    country STRING,
+    access_time TIMESTAMP TIME INDEX
+);
+
+-- create flow task to calculate the distinct country
+CREATE FLOW calc_ngx_country SINK TO ngx_country AS
+SELECT
+    DISTINCT country,
+FROM
+    ngx_access_log;
+
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", "b", 0);
+
+ADMIN FLUSH_FLOW('calc_ngx_country');
+
+SELECT
+    "ngx_access_log.country"
+FROM
+    ngx_country;
+
+-- making sure distinct is working
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", "b", 1);
+
+ADMIN FLUSH_FLOW('calc_ngx_country');
+
+SELECT
+    "ngx_access_log.country"
+FROM
+    ngx_country;
+
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", "c", 2);
+
+ADMIN FLUSH_FLOW('calc_ngx_country');
+
+SELECT
+    "ngx_access_log.country"
+FROM
+    ngx_country;
+
+DROP FLOW calc_ngx_country;
+
+DROP TABLE ngx_access_log;
+
+DROP TABLE ngx_country;
+
+CREATE TABLE ngx_access_log (
+    client STRING,
+    country STRING,
+    access_time TIMESTAMP TIME INDEX
+);
+
+CREATE FLOW calc_ngx_country SINK TO ngx_country AS
+SELECT
+    DISTINCT country,
+    -- this distinct is not necessary, but it's a good test to see if it works
+    date_bin(INTERVAL '1 hour', access_time) as time_window,
+FROM
+    ngx_access_log
+GROUP BY
+    country,
+    time_window;
+
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", "b", 0);
+
+ADMIN FLUSH_FLOW('calc_ngx_country');
+
+SELECT
+    "ngx_access_log.country",
+    time_window
+FROM
+    ngx_country;
+
+-- making sure distinct is working
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", "b", 1);
+
+ADMIN FLUSH_FLOW('calc_ngx_country');
+
+SELECT
+    "ngx_access_log.country",
+    time_window
+FROM
+    ngx_country;
+
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", "c", 2);
+
+ADMIN FLUSH_FLOW('calc_ngx_country');
+
+SELECT
+    "ngx_access_log.country",
+    time_window
+FROM
+    ngx_country;
+
+DROP FLOW calc_ngx_country;
+
+DROP TABLE ngx_access_log;
+
+DROP TABLE ngx_country;
+
+CREATE TABLE temp_sensor_data (
+    sensor_id INT,
+    loc STRING,
+    temperature DOUBLE,
+    ts TIMESTAMP TIME INDEX
+);
+
+CREATE TABLE temp_alerts (
+    sensor_id INT,
+    loc STRING,
+    max_temp DOUBLE,
+    ts TIMESTAMP TIME INDEX
+);
+
+CREATE FLOW temp_monitoring SINK TO temp_alerts AS
+SELECT
+    sensor_id,
+    loc,
+    max(temperature) as max_temp,
+FROM
+    temp_sensor_data
+GROUP BY
+    sensor_id,
+    loc
+HAVING
+    max_temp > 100;
+
+INSERT INTO
+    temp_sensor_data
+VALUES
+    (1, "room1", 50, 0);
+
+ADMIN FLUSH_FLOW('temp_monitoring');
+
+-- This table should not exist yet
+SHOW TABLES LIKE 'temp_alerts';
+
+INSERT INTO
+    temp_sensor_data
+VALUES
+    (1, "room1", 150, 1);
+
+ADMIN FLUSH_FLOW('temp_monitoring');
+
+SHOW TABLES LIKE 'temp_alerts';
+
+SELECT
+    sensor_id,
+    loc,
+    max_temp
+FROM
+    temp_alerts;
+
+INSERT INTO
+    temp_sensor_data
+VALUES
+    (2, "room1", 0, 2);
+
+ADMIN FLUSH_FLOW('temp_monitoring');
+
+SELECT
+    sensor_id,
+    loc,
+    max_temp
+FROM
+    temp_alerts;
+
+DROP FLOW temp_monitoring;
+
+DROP TABLE temp_sensor_data;
+
+DROP TABLE temp_alerts;
+
+CREATE TABLE ngx_access_log (
+    client STRING,
+    stat INT,
+    size INT,
+    access_time TIMESTAMP TIME INDEX
+);
+
+CREATE TABLE ngx_distribution (
+    stat INT,
+    bucket_size INT,
+    total_logs BIGINT,
+    time_window TIMESTAMP TIME INDEX,
+    update_at TIMESTAMP, -- auto generated column by flow engine
+    PRIMARY KEY(stat, bucket_size)
+);
+
+CREATE FLOW calc_ngx_distribution SINK TO ngx_distribution AS
+SELECT
+    stat,
+    trunc(size, -1)::INT as bucket_size,
+    count(client) AS total_logs,
+    date_bin(INTERVAL '1 minutes', access_time) as time_window,
+FROM
+    ngx_access_log
+GROUP BY
+    stat,
+    time_window,
+    bucket_size;
+
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", 200, 100, 0);
+
+ADMIN FLUSH_FLOW('calc_ngx_distribution');
+
+SELECT
+    stat,
+    bucket_size,
+    total_logs,
+    time_window
+FROM
+    ngx_distribution;
+
+INSERT INTO
+    ngx_access_log
+VALUES
+    ("cli1", 200, 200, 1),
+    ("cli1", 200, 205, 1),
+    ("cli1", 200, 209, 1),
+    ("cli1", 200, 210, 1),
+    ("cli2", 200, 300, 1);
+
+ADMIN FLUSH_FLOW('calc_ngx_distribution');
+
+SELECT
+    stat,
+    bucket_size,
+    total_logs,
+    time_window
+FROM
+    ngx_distribution;
+
+DROP FLOW calc_ngx_distribution;
+
+DROP TABLE ngx_access_log;
+
+DROP TABLE ngx_distribution;

From aa03d3b11ccff692ad348cd113da46a7e758b527 Mon Sep 17 00:00:00 2001
From: zyy17 <zyylsxm@gmail.com>
Date: Wed, 11 Sep 2024 15:49:23 +0800
Subject: [PATCH 002/128] docs: use docs comment prefix and bump toml2docs
 version (#4711)

---
 Makefile                          |   2 +-
 config/config.md                  | 172 +++++++++++++++---------------
 config/datanode.example.toml      |  84 ++++++++-------
 config/flownode.example.toml      |   6 +-
 config/frontend.example.toml      |  26 ++---
 config/metasrv.example.toml       |  12 +--
 config/standalone.example.toml    |  86 ++++++++-------
 src/cmd/tests/load_config_test.rs |  11 --
 8 files changed, 200 insertions(+), 199 deletions(-)

diff --git a/Makefile b/Makefile
index acc2a73126..697a31aed6 100644
--- a/Makefile
+++ b/Makefile
@@ -221,7 +221,7 @@ config-docs: ## Generate configuration documentation from toml files.
 	docker run --rm \
     -v ${PWD}:/greptimedb \
     -w /greptimedb/config \
-    toml2docs/toml2docs:v0.1.1 \
+    toml2docs/toml2docs:v0.1.3 \
     -p '##' \
     -t ./config-docs-template.md \
     -o ./config.md
diff --git a/config/config.md b/config/config.md
index a792be5de5..235b1f86fd 100644
--- a/config/config.md
+++ b/config/config.md
@@ -14,7 +14,7 @@
 | --- | -----| ------- | ----------- |
 | `mode` | String | `standalone` | The running mode of the datanode. It can be `standalone` or `distributed`. |
 | `enable_telemetry` | Bool | `true` | Enable telemetry to collect anonymous usage data. |
-| `default_timezone` | String | `None` | The default timezone of the server. |
+| `default_timezone` | String | Unset | The default timezone of the server. |
 | `init_regions_in_background` | Bool | `false` | Initialize all regions in the background during the startup.<br/>By default, it provides services after all regions have been initialized. |
 | `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. |
 | `runtime` | -- | -- | The runtime options. |
@@ -29,8 +29,8 @@
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
 | `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. |
 | `grpc.tls.mode` | String | `disable` | TLS mode. |
-| `grpc.tls.cert_path` | String | `None` | Certificate file path. |
-| `grpc.tls.key_path` | String | `None` | Private key file path. |
+| `grpc.tls.cert_path` | String | Unset | Certificate file path. |
+| `grpc.tls.key_path` | String | Unset | Private key file path. |
 | `grpc.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload.<br/>For now, gRPC tls config does not support auto reload. |
 | `mysql` | -- | -- | MySQL server options. |
 | `mysql.enable` | Bool | `true` | Whether to enable. |
@@ -38,8 +38,8 @@
 | `mysql.runtime_size` | Integer | `2` | The number of server worker threads. |
 | `mysql.tls` | -- | -- | -- |
 | `mysql.tls.mode` | String | `disable` | TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html<br/>- `disable` (default value)<br/>- `prefer`<br/>- `require`<br/>- `verify-ca`<br/>- `verify-full` |
-| `mysql.tls.cert_path` | String | `None` | Certificate file path. |
-| `mysql.tls.key_path` | String | `None` | Private key file path. |
+| `mysql.tls.cert_path` | String | Unset | Certificate file path. |
+| `mysql.tls.key_path` | String | Unset | Private key file path. |
 | `mysql.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload |
 | `postgres` | -- | -- | PostgresSQL server options. |
 | `postgres.enable` | Bool | `true` | Whether to enable |
@@ -47,8 +47,8 @@
 | `postgres.runtime_size` | Integer | `2` | The number of server worker threads. |
 | `postgres.tls` | -- | -- | PostgresSQL server TLS options, see `mysql.tls` section. |
 | `postgres.tls.mode` | String | `disable` | TLS mode. |
-| `postgres.tls.cert_path` | String | `None` | Certificate file path. |
-| `postgres.tls.key_path` | String | `None` | Private key file path. |
+| `postgres.tls.cert_path` | String | Unset | Certificate file path. |
+| `postgres.tls.key_path` | String | Unset | Private key file path. |
 | `postgres.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload |
 | `opentsdb` | -- | -- | OpenTSDB protocol options. |
 | `opentsdb.enable` | Bool | `true` | Whether to enable OpenTSDB put in HTTP API. |
@@ -59,7 +59,7 @@
 | `prom_store.with_metric_engine` | Bool | `true` | Whether to store the data from Prometheus remote write in metric engine. |
 | `wal` | -- | -- | The WAL options. |
 | `wal.provider` | String | `raft_engine` | The provider of the WAL.<br/>- `raft_engine`: the wal is stored in the local file system by raft-engine.<br/>- `kafka`: it's remote wal that data is stored in Kafka. |
-| `wal.dir` | String | `None` | The directory to store the WAL files.<br/>**It's only used when the provider is `raft_engine`**. |
+| `wal.dir` | String | Unset | The directory to store the WAL files.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.file_size` | String | `256MB` | The size of the WAL segment file.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.purge_threshold` | String | `4GB` | The threshold of the WAL size to trigger a flush.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.purge_interval` | String | `10m` | The interval to trigger a flush.<br/>**It's only used when the provider is `raft_engine`**. |
@@ -91,22 +91,22 @@
 | `storage` | -- | -- | The data storage options. |
 | `storage.data_home` | String | `/tmp/greptimedb/` | The working home directory. |
 | `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
-| `storage.cache_path` | String | `None` | Cache configuration for object storage such as 'S3' etc.<br/>The local file cache directory. |
-| `storage.cache_capacity` | String | `None` | The local file cache capacity in bytes. |
-| `storage.bucket` | String | `None` | The S3 bucket name.<br/>**It's only used when the storage type is `S3`, `Oss` and `Gcs`**. |
-| `storage.root` | String | `None` | The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.<br/>**It's only used when the storage type is `S3`, `Oss` and `Azblob`**. |
-| `storage.access_key_id` | String | `None` | The access key id of the aws account.<br/>It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.<br/>**It's only used when the storage type is `S3` and `Oss`**. |
-| `storage.secret_access_key` | String | `None` | The secret access key of the aws account.<br/>It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.<br/>**It's only used when the storage type is `S3`**. |
-| `storage.access_key_secret` | String | `None` | The secret access key of the aliyun account.<br/>**It's only used when the storage type is `Oss`**. |
-| `storage.account_name` | String | `None` | The account key of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
-| `storage.account_key` | String | `None` | The account key of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
-| `storage.scope` | String | `None` | The scope of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
-| `storage.credential_path` | String | `None` | The credential path of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
-| `storage.credential` | String | `None` | The credential of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
-| `storage.container` | String | `None` | The container of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
-| `storage.sas_token` | String | `None` | The sas token of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
-| `storage.endpoint` | String | `None` | The endpoint of the S3 service.<br/>**It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**. |
-| `storage.region` | String | `None` | The region of the S3 service.<br/>**It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**. |
+| `storage.cache_path` | String | Unset | Cache configuration for object storage such as 'S3' etc.<br/>The local file cache directory. |
+| `storage.cache_capacity` | String | Unset | The local file cache capacity in bytes. |
+| `storage.bucket` | String | Unset | The S3 bucket name.<br/>**It's only used when the storage type is `S3`, `Oss` and `Gcs`**. |
+| `storage.root` | String | Unset | The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.<br/>**It's only used when the storage type is `S3`, `Oss` and `Azblob`**. |
+| `storage.access_key_id` | String | Unset | The access key id of the aws account.<br/>It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.<br/>**It's only used when the storage type is `S3` and `Oss`**. |
+| `storage.secret_access_key` | String | Unset | The secret access key of the aws account.<br/>It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.<br/>**It's only used when the storage type is `S3`**. |
+| `storage.access_key_secret` | String | Unset | The secret access key of the aliyun account.<br/>**It's only used when the storage type is `Oss`**. |
+| `storage.account_name` | String | Unset | The account key of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
+| `storage.account_key` | String | Unset | The account key of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
+| `storage.scope` | String | Unset | The scope of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
+| `storage.credential_path` | String | Unset | The credential path of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
+| `storage.credential` | String | Unset | The credential of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
+| `storage.container` | String | Unset | The container of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
+| `storage.sas_token` | String | Unset | The sas token of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
+| `storage.endpoint` | String | Unset | The endpoint of the S3 service.<br/>**It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**. |
+| `storage.region` | String | Unset | The region of the S3 service.<br/>**It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**. |
 | `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines. |
 | `region_engine.mito` | -- | -- | The Mito engine options. |
 | `region_engine.mito.num_workers` | Integer | `8` | Number of region workers. |
@@ -116,16 +116,16 @@
 | `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). |
 | `region_engine.mito.max_background_jobs` | Integer | `4` | Max number of running background jobs |
 | `region_engine.mito.auto_flush_interval` | String | `1h` | Interval to auto flush a region if it has not flushed yet. |
-| `region_engine.mito.global_write_buffer_size` | String | `1GB` | Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB. |
-| `region_engine.mito.global_write_buffer_reject_size` | String | `2GB` | Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size` |
-| `region_engine.mito.sst_meta_cache_size` | String | `128MB` | Cache size for SST metadata. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/32 of OS memory with a max limitation of 128MB. |
-| `region_engine.mito.vector_cache_size` | String | `512MB` | Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
-| `region_engine.mito.page_cache_size` | String | `512MB` | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/8 of OS memory. |
-| `region_engine.mito.selector_result_cache_size` | String | `512MB` | Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
+| `region_engine.mito.global_write_buffer_size` | String | Auto | Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB. |
+| `region_engine.mito.global_write_buffer_reject_size` | String | Auto | Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size`. |
+| `region_engine.mito.sst_meta_cache_size` | String | Auto | Cache size for SST metadata. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/32 of OS memory with a max limitation of 128MB. |
+| `region_engine.mito.vector_cache_size` | String | Auto | Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
+| `region_engine.mito.page_cache_size` | String | Auto | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/8 of OS memory. |
+| `region_engine.mito.selector_result_cache_size` | String | Auto | Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
 | `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache. |
 | `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/write_cache`. |
 | `region_engine.mito.experimental_write_cache_size` | String | `512MB` | Capacity for write cache. |
-| `region_engine.mito.experimental_write_cache_ttl` | String | `None` | TTL for write cache. |
+| `region_engine.mito.experimental_write_cache_ttl` | String | Unset | TTL for write cache. |
 | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. |
 | `region_engine.mito.scan_parallelism` | Integer | `0` | Parallelism to scan a region (default: 1/4 of cpu cores).<br/>- `0`: using the default value (1/4 of cpu cores).<br/>- `1`: scan in current thread.<br/>- `n`: scan in parallelism n. |
 | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. |
@@ -155,7 +155,7 @@
 | `region_engine.file` | -- | -- | Enable the file engine. |
 | `logging` | -- | -- | The logging options. |
 | `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
-| `logging.level` | String | `None` | The log level. Can be `info`/`debug`/`warn`/`error`. |
+| `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
 | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. |
 | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
@@ -165,13 +165,13 @@
 | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
-| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself<br/>You must create the database before enabling it. |
-| `export_metrics.self_import.db` | String | `None` | -- |
+| `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommended to collect metrics generated by itself<br/>You must create the database before enabling it. |
+| `export_metrics.self_import.db` | String | Unset | -- |
 | `export_metrics.remote_write` | -- | -- | -- |
 | `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
-| `tracing.tokio_console_addr` | String | `None` | The tokio console address. |
+| `tracing.tokio_console_addr` | String | Unset | The tokio console address. |
 
 
 ## Distributed Mode
@@ -180,7 +180,7 @@
 
 | Key | Type | Default | Descriptions |
 | --- | -----| ------- | ----------- |
-| `default_timezone` | String | `None` | The default timezone of the server. |
+| `default_timezone` | String | Unset | The default timezone of the server. |
 | `runtime` | -- | -- | The runtime options. |
 | `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
 | `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute the runtime for global write operations. |
@@ -197,8 +197,8 @@
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
 | `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. |
 | `grpc.tls.mode` | String | `disable` | TLS mode. |
-| `grpc.tls.cert_path` | String | `None` | Certificate file path. |
-| `grpc.tls.key_path` | String | `None` | Private key file path. |
+| `grpc.tls.cert_path` | String | Unset | Certificate file path. |
+| `grpc.tls.key_path` | String | Unset | Private key file path. |
 | `grpc.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload.<br/>For now, gRPC tls config does not support auto reload. |
 | `mysql` | -- | -- | MySQL server options. |
 | `mysql.enable` | Bool | `true` | Whether to enable. |
@@ -206,8 +206,8 @@
 | `mysql.runtime_size` | Integer | `2` | The number of server worker threads. |
 | `mysql.tls` | -- | -- | -- |
 | `mysql.tls.mode` | String | `disable` | TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html<br/>- `disable` (default value)<br/>- `prefer`<br/>- `require`<br/>- `verify-ca`<br/>- `verify-full` |
-| `mysql.tls.cert_path` | String | `None` | Certificate file path. |
-| `mysql.tls.key_path` | String | `None` | Private key file path. |
+| `mysql.tls.cert_path` | String | Unset | Certificate file path. |
+| `mysql.tls.key_path` | String | Unset | Private key file path. |
 | `mysql.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload |
 | `postgres` | -- | -- | PostgresSQL server options. |
 | `postgres.enable` | Bool | `true` | Whether to enable |
@@ -215,8 +215,8 @@
 | `postgres.runtime_size` | Integer | `2` | The number of server worker threads. |
 | `postgres.tls` | -- | -- | PostgresSQL server TLS options, see `mysql.tls` section. |
 | `postgres.tls.mode` | String | `disable` | TLS mode. |
-| `postgres.tls.cert_path` | String | `None` | Certificate file path. |
-| `postgres.tls.key_path` | String | `None` | Private key file path. |
+| `postgres.tls.cert_path` | String | Unset | Certificate file path. |
+| `postgres.tls.key_path` | String | Unset | Private key file path. |
 | `postgres.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload |
 | `opentsdb` | -- | -- | OpenTSDB protocol options. |
 | `opentsdb.enable` | Bool | `true` | Whether to enable OpenTSDB put in HTTP API. |
@@ -241,7 +241,7 @@
 | `datanode.client.tcp_nodelay` | Bool | `true` | -- |
 | `logging` | -- | -- | The logging options. |
 | `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
-| `logging.level` | String | `None` | The log level. Can be `info`/`debug`/`warn`/`error`. |
+| `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
 | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. |
 | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
@@ -252,12 +252,12 @@
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
 | `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself<br/>You must create the database before enabling it. |
-| `export_metrics.self_import.db` | String | `None` | -- |
+| `export_metrics.self_import.db` | String | Unset | -- |
 | `export_metrics.remote_write` | -- | -- | -- |
 | `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
-| `tracing.tokio_console_addr` | String | `None` | The tokio console address. |
+| `tracing.tokio_console_addr` | String | Unset | The tokio console address. |
 
 
 ### Metasrv
@@ -306,7 +306,7 @@
 | `wal.backoff_deadline` | String | `5mins` | Stop reconnecting if the total wait time reaches the deadline. If this config is missing, the reconnecting won't terminate. |
 | `logging` | -- | -- | The logging options. |
 | `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
-| `logging.level` | String | `None` | The log level. Can be `info`/`debug`/`warn`/`error`. |
+| `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
 | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. |
 | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
@@ -317,12 +317,12 @@
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
 | `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself<br/>You must create the database before enabling it. |
-| `export_metrics.self_import.db` | String | `None` | -- |
+| `export_metrics.self_import.db` | String | Unset | -- |
 | `export_metrics.remote_write` | -- | -- | -- |
 | `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
-| `tracing.tokio_console_addr` | String | `None` | The tokio console address. |
+| `tracing.tokio_console_addr` | String | Unset | The tokio console address. |
 
 
 ### Datanode
@@ -330,16 +330,16 @@
 | Key | Type | Default | Descriptions |
 | --- | -----| ------- | ----------- |
 | `mode` | String | `standalone` | The running mode of the datanode. It can be `standalone` or `distributed`. |
-| `node_id` | Integer | `None` | The datanode identifier and should be unique in the cluster. |
+| `node_id` | Integer | Unset | The datanode identifier and should be unique in the cluster. |
 | `require_lease_before_startup` | Bool | `false` | Start services after regions have obtained leases.<br/>It will block the datanode start if it can't receive leases in the heartbeat from metasrv. |
 | `init_regions_in_background` | Bool | `false` | Initialize all regions in the background during the startup.<br/>By default, it provides services after all regions have been initialized. |
 | `enable_telemetry` | Bool | `true` | Enable telemetry to collect anonymous usage data. |
 | `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. |
-| `rpc_addr` | String | `None` | Deprecated, use `grpc.addr` instead. |
-| `rpc_hostname` | String | `None` | Deprecated, use `grpc.hostname` instead. |
-| `rpc_runtime_size` | Integer | `None` | Deprecated, use `grpc.runtime_size` instead. |
-| `rpc_max_recv_message_size` | String | `None` | Deprecated, use `grpc.rpc_max_recv_message_size` instead. |
-| `rpc_max_send_message_size` | String | `None` | Deprecated, use `grpc.rpc_max_send_message_size` instead. |
+| `rpc_addr` | String | Unset | Deprecated, use `grpc.addr` instead. |
+| `rpc_hostname` | String | Unset | Deprecated, use `grpc.hostname` instead. |
+| `rpc_runtime_size` | Integer | Unset | Deprecated, use `grpc.runtime_size` instead. |
+| `rpc_max_recv_message_size` | String | Unset | Deprecated, use `grpc.rpc_max_recv_message_size` instead. |
+| `rpc_max_send_message_size` | String | Unset | Deprecated, use `grpc.rpc_max_send_message_size` instead. |
 | `http` | -- | -- | The HTTP server options. |
 | `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
 | `http.timeout` | String | `30s` | HTTP request timeout. Set to 0 to disable timeout. |
@@ -352,8 +352,8 @@
 | `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. |
 | `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. |
 | `grpc.tls.mode` | String | `disable` | TLS mode. |
-| `grpc.tls.cert_path` | String | `None` | Certificate file path. |
-| `grpc.tls.key_path` | String | `None` | Private key file path. |
+| `grpc.tls.cert_path` | String | Unset | Certificate file path. |
+| `grpc.tls.key_path` | String | Unset | Private key file path. |
 | `grpc.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload.<br/>For now, gRPC tls config does not support auto reload. |
 | `runtime` | -- | -- | The runtime options. |
 | `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
@@ -373,7 +373,7 @@
 | `meta_client.metadata_cache_tti` | String | `5m` | -- |
 | `wal` | -- | -- | The WAL options. |
 | `wal.provider` | String | `raft_engine` | The provider of the WAL.<br/>- `raft_engine`: the wal is stored in the local file system by raft-engine.<br/>- `kafka`: it's remote wal that data is stored in Kafka. |
-| `wal.dir` | String | `None` | The directory to store the WAL files.<br/>**It's only used when the provider is `raft_engine`**. |
+| `wal.dir` | String | Unset | The directory to store the WAL files.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.file_size` | String | `256MB` | The size of the WAL segment file.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.purge_threshold` | String | `4GB` | The threshold of the WAL size to trigger a flush.<br/>**It's only used when the provider is `raft_engine`**. |
 | `wal.purge_interval` | String | `10m` | The interval to trigger a flush.<br/>**It's only used when the provider is `raft_engine`**. |
@@ -395,22 +395,22 @@
 | `storage` | -- | -- | The data storage options. |
 | `storage.data_home` | String | `/tmp/greptimedb/` | The working home directory. |
 | `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
-| `storage.cache_path` | String | `None` | Cache configuration for object storage such as 'S3' etc.<br/>The local file cache directory. |
-| `storage.cache_capacity` | String | `None` | The local file cache capacity in bytes. |
-| `storage.bucket` | String | `None` | The S3 bucket name.<br/>**It's only used when the storage type is `S3`, `Oss` and `Gcs`**. |
-| `storage.root` | String | `None` | The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.<br/>**It's only used when the storage type is `S3`, `Oss` and `Azblob`**. |
-| `storage.access_key_id` | String | `None` | The access key id of the aws account.<br/>It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.<br/>**It's only used when the storage type is `S3` and `Oss`**. |
-| `storage.secret_access_key` | String | `None` | The secret access key of the aws account.<br/>It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.<br/>**It's only used when the storage type is `S3`**. |
-| `storage.access_key_secret` | String | `None` | The secret access key of the aliyun account.<br/>**It's only used when the storage type is `Oss`**. |
-| `storage.account_name` | String | `None` | The account key of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
-| `storage.account_key` | String | `None` | The account key of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
-| `storage.scope` | String | `None` | The scope of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
-| `storage.credential_path` | String | `None` | The credential path of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
-| `storage.credential` | String | `None` | The credential of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
-| `storage.container` | String | `None` | The container of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
-| `storage.sas_token` | String | `None` | The sas token of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
-| `storage.endpoint` | String | `None` | The endpoint of the S3 service.<br/>**It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**. |
-| `storage.region` | String | `None` | The region of the S3 service.<br/>**It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**. |
+| `storage.cache_path` | String | Unset | Cache configuration for object storage such as 'S3' etc.<br/>The local file cache directory. |
+| `storage.cache_capacity` | String | Unset | The local file cache capacity in bytes. |
+| `storage.bucket` | String | Unset | The S3 bucket name.<br/>**It's only used when the storage type is `S3`, `Oss` and `Gcs`**. |
+| `storage.root` | String | Unset | The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.<br/>**It's only used when the storage type is `S3`, `Oss` and `Azblob`**. |
+| `storage.access_key_id` | String | Unset | The access key id of the aws account.<br/>It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.<br/>**It's only used when the storage type is `S3` and `Oss`**. |
+| `storage.secret_access_key` | String | Unset | The secret access key of the aws account.<br/>It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.<br/>**It's only used when the storage type is `S3`**. |
+| `storage.access_key_secret` | String | Unset | The secret access key of the aliyun account.<br/>**It's only used when the storage type is `Oss`**. |
+| `storage.account_name` | String | Unset | The account key of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
+| `storage.account_key` | String | Unset | The account key of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
+| `storage.scope` | String | Unset | The scope of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
+| `storage.credential_path` | String | Unset | The credential path of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
+| `storage.credential` | String | Unset | The credential of the google cloud storage.<br/>**It's only used when the storage type is `Gcs`**. |
+| `storage.container` | String | Unset | The container of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
+| `storage.sas_token` | String | Unset | The sas token of the azure account.<br/>**It's only used when the storage type is `Azblob`**. |
+| `storage.endpoint` | String | Unset | The endpoint of the S3 service.<br/>**It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**. |
+| `storage.region` | String | Unset | The region of the S3 service.<br/>**It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**. |
 | `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines. |
 | `region_engine.mito` | -- | -- | The Mito engine options. |
 | `region_engine.mito.num_workers` | Integer | `8` | Number of region workers. |
@@ -420,16 +420,16 @@
 | `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). |
 | `region_engine.mito.max_background_jobs` | Integer | `4` | Max number of running background jobs |
 | `region_engine.mito.auto_flush_interval` | String | `1h` | Interval to auto flush a region if it has not flushed yet. |
-| `region_engine.mito.global_write_buffer_size` | String | `1GB` | Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB. |
-| `region_engine.mito.global_write_buffer_reject_size` | String | `2GB` | Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size` |
-| `region_engine.mito.sst_meta_cache_size` | String | `128MB` | Cache size for SST metadata. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/32 of OS memory with a max limitation of 128MB. |
-| `region_engine.mito.vector_cache_size` | String | `512MB` | Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
-| `region_engine.mito.page_cache_size` | String | `512MB` | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/8 of OS memory. |
-| `region_engine.mito.selector_result_cache_size` | String | `512MB` | Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
+| `region_engine.mito.global_write_buffer_size` | String | Auto | Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB. |
+| `region_engine.mito.global_write_buffer_reject_size` | String | Auto | Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size` |
+| `region_engine.mito.sst_meta_cache_size` | String | Auto | Cache size for SST metadata. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/32 of OS memory with a max limitation of 128MB. |
+| `region_engine.mito.vector_cache_size` | String | Auto | Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
+| `region_engine.mito.page_cache_size` | String | Auto | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/8 of OS memory. |
+| `region_engine.mito.selector_result_cache_size` | String | Auto | Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
 | `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache. |
 | `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/write_cache`. |
 | `region_engine.mito.experimental_write_cache_size` | String | `512MB` | Capacity for write cache. |
-| `region_engine.mito.experimental_write_cache_ttl` | String | `None` | TTL for write cache. |
+| `region_engine.mito.experimental_write_cache_ttl` | String | Unset | TTL for write cache. |
 | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. |
 | `region_engine.mito.scan_parallelism` | Integer | `0` | Parallelism to scan a region (default: 1/4 of cpu cores).<br/>- `0`: using the default value (1/4 of cpu cores).<br/>- `1`: scan in current thread.<br/>- `n`: scan in parallelism n. |
 | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. |
@@ -457,7 +457,7 @@
 | `region_engine.file` | -- | -- | Enable the file engine. |
 | `logging` | -- | -- | The logging options. |
 | `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
-| `logging.level` | String | `None` | The log level. Can be `info`/`debug`/`warn`/`error`. |
+| `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
 | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. |
 | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
@@ -468,12 +468,12 @@
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
 | `export_metrics.self_import` | -- | -- | For `standalone` mode, `self_import` is recommend to collect metrics generated by itself<br/>You must create the database before enabling it. |
-| `export_metrics.self_import.db` | String | `None` | -- |
+| `export_metrics.self_import.db` | String | Unset | -- |
 | `export_metrics.remote_write` | -- | -- | -- |
 | `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=greptime_metrics`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
-| `tracing.tokio_console_addr` | String | `None` | The tokio console address. |
+| `tracing.tokio_console_addr` | String | Unset | The tokio console address. |
 
 
 ### Flownode
@@ -481,7 +481,7 @@
 | Key | Type | Default | Descriptions |
 | --- | -----| ------- | ----------- |
 | `mode` | String | `distributed` | The running mode of the flownode. It can be `standalone` or `distributed`. |
-| `node_id` | Integer | `None` | The flownode identifier and should be unique in the cluster. |
+| `node_id` | Integer | Unset | The flownode identifier and should be unique in the cluster. |
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.addr` | String | `127.0.0.1:6800` | The address to bind the gRPC server. |
 | `grpc.hostname` | String | `127.0.0.1` | The hostname advertised to the metasrv,<br/>and used for connections from outside the host |
@@ -503,7 +503,7 @@
 | `heartbeat.retry_interval` | String | `3s` | Interval for retrying to send heartbeat messages to the metasrv. |
 | `logging` | -- | -- | The logging options. |
 | `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
-| `logging.level` | String | `None` | The log level. Can be `info`/`debug`/`warn`/`error`. |
+| `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
 | `logging.enable_otlp_tracing` | Bool | `false` | Enable OTLP tracing. |
 | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
@@ -511,4 +511,4 @@
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
-| `tracing.tokio_console_addr` | String | `None` | The tokio console address. |
+| `tracing.tokio_console_addr` | String | Unset | The tokio console address. |
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index 14fbf914e7..4388c4420f 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -2,7 +2,7 @@
 mode = "standalone"
 
 ## The datanode identifier and should be unique in the cluster.
-## +toml2docs:none-default
+## @toml2docs:none-default
 node_id = 42
 
 ## Start services after regions have obtained leases.
@@ -20,23 +20,23 @@ enable_telemetry = true
 init_regions_parallelism = 16
 
 ## Deprecated, use `grpc.addr` instead.
-## +toml2docs:none-default
+## @toml2docs:none-default
 rpc_addr = "127.0.0.1:3001"
 
 ## Deprecated, use `grpc.hostname` instead.
-## +toml2docs:none-default
+## @toml2docs:none-default
 rpc_hostname = "127.0.0.1"
 
 ## Deprecated, use `grpc.runtime_size` instead.
-## +toml2docs:none-default
+## @toml2docs:none-default
 rpc_runtime_size = 8
 
 ## Deprecated, use `grpc.rpc_max_recv_message_size` instead.
-## +toml2docs:none-default
+## @toml2docs:none-default
 rpc_max_recv_message_size = "512MB"
 
 ## Deprecated, use `grpc.rpc_max_send_message_size` instead.
-## +toml2docs:none-default
+## @toml2docs:none-default
 rpc_max_send_message_size = "512MB"
 
 
@@ -71,11 +71,11 @@ max_send_message_size = "512MB"
 mode = "disable"
 
 ## Certificate file path.
-## +toml2docs:none-default
+## @toml2docs:none-default
 cert_path = ""
 
 ## Private key file path.
-## +toml2docs:none-default
+## @toml2docs:none-default
 key_path = ""
 
 ## Watch for Certificate and key file change and auto reload.
@@ -83,11 +83,11 @@ key_path = ""
 watch = false
 
 ## The runtime options.
-[runtime]
+#+ [runtime]
 ## The number of threads to execute the runtime for global read operations.
-global_rt_size = 8
+#+ global_rt_size = 8
 ## The number of threads to execute the runtime for global write operations.
-compact_rt_size = 4
+#+ compact_rt_size = 4
 
 ## The heartbeat options.
 [heartbeat]
@@ -135,7 +135,7 @@ provider = "raft_engine"
 
 ## The directory to store the WAL files.
 ## **It's only used when the provider is `raft_engine`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 dir = "/tmp/greptimedb/wal"
 
 ## The size of the WAL segment file.
@@ -282,83 +282,83 @@ type = "File"
 
 ## Cache configuration for object storage such as 'S3' etc.
 ## The local file cache directory.
-## +toml2docs:none-default
+## @toml2docs:none-default
 cache_path = "/path/local_cache"
 
 ## The local file cache capacity in bytes.
-## +toml2docs:none-default
+## @toml2docs:none-default
 cache_capacity = "256MB"
 
 ## The S3 bucket name.
 ## **It's only used when the storage type is `S3`, `Oss` and `Gcs`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 bucket = "greptimedb"
 
 ## The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.
 ## **It's only used when the storage type is `S3`, `Oss` and `Azblob`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 root = "greptimedb"
 
 ## The access key id of the aws account.
 ## It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.
 ## **It's only used when the storage type is `S3` and `Oss`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 access_key_id = "test"
 
 ## The secret access key of the aws account.
 ## It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.
 ## **It's only used when the storage type is `S3`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 secret_access_key = "test"
 
 ## The secret access key of the aliyun account.
 ## **It's only used when the storage type is `Oss`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 access_key_secret = "test"
 
 ## The account key of the azure account.
 ## **It's only used when the storage type is `Azblob`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 account_name = "test"
 
 ## The account key of the azure account.
 ## **It's only used when the storage type is `Azblob`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 account_key = "test"
 
 ## The scope of the google cloud storage.
 ## **It's only used when the storage type is `Gcs`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 scope = "test"
 
 ## The credential path of the google cloud storage.
 ## **It's only used when the storage type is `Gcs`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 credential_path = "test"
 
 ## The credential of the google cloud storage.
 ## **It's only used when the storage type is `Gcs`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 credential = "base64-credential"
 
 ## The container of the azure account.
 ## **It's only used when the storage type is `Azblob`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 container = "greptimedb"
 
 ## The sas token of the azure account.
 ## **It's only used when the storage type is `Azblob`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 sas_token = ""
 
 ## The endpoint of the S3 service.
 ## **It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 endpoint = "https://s3.amazonaws.com"
 
 ## The region of the S3 service.
 ## **It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 region = "us-west-2"
 
 # Custom storage options
@@ -388,7 +388,7 @@ region = "us-west-2"
 [region_engine.mito]
 
 ## Number of region workers.
-num_workers = 8
+#+ num_workers = 8
 
 ## Request channel size of each worker.
 worker_channel_size = 128
@@ -409,26 +409,32 @@ max_background_jobs = 4
 auto_flush_interval = "1h"
 
 ## Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB.
-global_write_buffer_size = "1GB"
+## @toml2docs:none-default="Auto"
+#+ global_write_buffer_size = "1GB"
 
 ## Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size`
-global_write_buffer_reject_size = "2GB"
+## @toml2docs:none-default="Auto"
+#+ global_write_buffer_reject_size = "2GB"
 
 ## Cache size for SST metadata. Setting it to 0 to disable the cache.
 ## If not set, it's default to 1/32 of OS memory with a max limitation of 128MB.
-sst_meta_cache_size = "128MB"
+## @toml2docs:none-default="Auto"
+#+ sst_meta_cache_size = "128MB"
 
 ## Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.
 ## If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
-vector_cache_size = "512MB"
+## @toml2docs:none-default="Auto"
+#+ vector_cache_size = "512MB"
 
 ## Cache size for pages of SST row groups. Setting it to 0 to disable the cache.
 ## If not set, it's default to 1/8 of OS memory.
-page_cache_size = "512MB"
+## @toml2docs:none-default="Auto"
+#+ page_cache_size = "512MB"
 
 ## Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.
 ## If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
-selector_result_cache_size = "512MB"
+## @toml2docs:none-default="Auto"
+#+ selector_result_cache_size = "512MB"
 
 ## Whether to enable the experimental write cache.
 enable_experimental_write_cache = false
@@ -440,7 +446,7 @@ experimental_write_cache_path = ""
 experimental_write_cache_size = "512MB"
 
 ## TTL for write cache.
-## +toml2docs:none-default
+## @toml2docs:none-default
 experimental_write_cache_ttl = "8h"
 
 ## Buffer size for SST writing.
@@ -556,7 +562,7 @@ fork_dictionary_bytes = "1GiB"
 dir = "/tmp/greptimedb/logs"
 
 ## The log level. Can be `info`/`debug`/`warn`/`error`.
-## +toml2docs:none-default
+## @toml2docs:none-default
 level = "info"
 
 ## Enable OTLP tracing.
@@ -590,7 +596,7 @@ write_interval = "30s"
 ## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
 ## You must create the database before enabling it.
 [export_metrics.self_import]
-## +toml2docs:none-default
+## @toml2docs:none-default
 db = "greptime_metrics"
 
 [export_metrics.remote_write]
@@ -603,5 +609,5 @@ headers = { }
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
 [tracing]
 ## The tokio console address.
-## +toml2docs:none-default
+## @toml2docs:none-default
 tokio_console_addr = "127.0.0.1"
diff --git a/config/flownode.example.toml b/config/flownode.example.toml
index d5640062f7..a3a414fb98 100644
--- a/config/flownode.example.toml
+++ b/config/flownode.example.toml
@@ -2,7 +2,7 @@
 mode = "distributed"
 
 ## The flownode identifier and should be unique in the cluster.
-## +toml2docs:none-default
+## @toml2docs:none-default
 node_id = 14
 
 ## The gRPC server options.
@@ -63,7 +63,7 @@ retry_interval = "3s"
 dir = "/tmp/greptimedb/logs"
 
 ## The log level. Can be `info`/`debug`/`warn`/`error`.
-## +toml2docs:none-default
+## @toml2docs:none-default
 level = "info"
 
 ## Enable OTLP tracing.
@@ -87,6 +87,6 @@ default_ratio = 1.0
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
 [tracing]
 ## The tokio console address.
-## +toml2docs:none-default
+## @toml2docs:none-default
 tokio_console_addr = "127.0.0.1"
 
diff --git a/config/frontend.example.toml b/config/frontend.example.toml
index e5a7f5af89..eae001a2db 100644
--- a/config/frontend.example.toml
+++ b/config/frontend.example.toml
@@ -1,13 +1,13 @@
 ## The default timezone of the server.
-## +toml2docs:none-default
+## @toml2docs:none-default
 default_timezone = "UTC"
 
 ## The runtime options.
-[runtime]
+#+ [runtime]
 ## The number of threads to execute the runtime for global read operations.
-global_rt_size = 8
+#+ global_rt_size = 8
 ## The number of threads to execute the runtime for global write operations.
-compact_rt_size = 4
+#+ compact_rt_size = 4
 
 ## The heartbeat options.
 [heartbeat]
@@ -44,11 +44,11 @@ runtime_size = 8
 mode = "disable"
 
 ## Certificate file path.
-## +toml2docs:none-default
+## @toml2docs:none-default
 cert_path = ""
 
 ## Private key file path.
-## +toml2docs:none-default
+## @toml2docs:none-default
 key_path = ""
 
 ## Watch for Certificate and key file change and auto reload.
@@ -76,11 +76,11 @@ runtime_size = 2
 mode = "disable"
 
 ## Certificate file path.
-## +toml2docs:none-default
+## @toml2docs:none-default
 cert_path = ""
 
 ## Private key file path.
-## +toml2docs:none-default
+## @toml2docs:none-default
 key_path = ""
 
 ## Watch for Certificate and key file change and auto reload
@@ -101,11 +101,11 @@ runtime_size = 2
 mode = "disable"
 
 ## Certificate file path.
-## +toml2docs:none-default
+## @toml2docs:none-default
 cert_path = ""
 
 ## Private key file path.
-## +toml2docs:none-default
+## @toml2docs:none-default
 key_path = ""
 
 ## Watch for Certificate and key file change and auto reload
@@ -170,7 +170,7 @@ tcp_nodelay = true
 dir = "/tmp/greptimedb/logs"
 
 ## The log level. Can be `info`/`debug`/`warn`/`error`.
-## +toml2docs:none-default
+## @toml2docs:none-default
 level = "info"
 
 ## Enable OTLP tracing.
@@ -204,7 +204,7 @@ write_interval = "30s"
 ## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
 ## You must create the database before enabling it.
 [export_metrics.self_import]
-## +toml2docs:none-default
+## @toml2docs:none-default
 db = "greptime_metrics"
 
 [export_metrics.remote_write]
@@ -217,5 +217,5 @@ headers = { }
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
 [tracing]
 ## The tokio console address.
-## +toml2docs:none-default
+## @toml2docs:none-default
 tokio_console_addr = "127.0.0.1"
diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml
index dc5f091166..e95a9fa7f2 100644
--- a/config/metasrv.example.toml
+++ b/config/metasrv.example.toml
@@ -36,11 +36,11 @@ enable_region_failover = false
 backend = "EtcdStore"
 
 ## The runtime options.
-[runtime]
+#+ [runtime]
 ## The number of threads to execute the runtime for global read operations.
-global_rt_size = 8
+#+ global_rt_size = 8
 ## The number of threads to execute the runtime for global write operations.
-compact_rt_size = 4
+#+ compact_rt_size = 4
 
 ## Procedure storage options.
 [procedure]
@@ -157,7 +157,7 @@ backoff_deadline = "5mins"
 dir = "/tmp/greptimedb/logs"
 
 ## The log level. Can be `info`/`debug`/`warn`/`error`.
-## +toml2docs:none-default
+## @toml2docs:none-default
 level = "info"
 
 ## Enable OTLP tracing.
@@ -191,7 +191,7 @@ write_interval = "30s"
 ## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
 ## You must create the database before enabling it.
 [export_metrics.self_import]
-## +toml2docs:none-default
+## @toml2docs:none-default
 db = "greptime_metrics"
 
 [export_metrics.remote_write]
@@ -204,5 +204,5 @@ headers = { }
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
 [tracing]
 ## The tokio console address.
-## +toml2docs:none-default
+## @toml2docs:none-default
 tokio_console_addr = "127.0.0.1"
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index f7c7b2af29..040e1e62c2 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -5,7 +5,7 @@ mode = "standalone"
 enable_telemetry = true
 
 ## The default timezone of the server.
-## +toml2docs:none-default
+## @toml2docs:none-default
 default_timezone = "UTC"
 
 ## Initialize all regions in the background during the startup.
@@ -16,11 +16,11 @@ init_regions_in_background = false
 init_regions_parallelism = 16
 
 ## The runtime options.
-[runtime]
+#+ [runtime]
 ## The number of threads to execute the runtime for global read operations.
-global_rt_size = 8
+#+ global_rt_size = 8
 ## The number of threads to execute the runtime for global write operations.
-compact_rt_size = 4
+#+ compact_rt_size = 4
 
 ## The HTTP server options.
 [http]
@@ -46,11 +46,11 @@ runtime_size = 8
 mode = "disable"
 
 ## Certificate file path.
-## +toml2docs:none-default
+## @toml2docs:none-default
 cert_path = ""
 
 ## Private key file path.
-## +toml2docs:none-default
+## @toml2docs:none-default
 key_path = ""
 
 ## Watch for Certificate and key file change and auto reload.
@@ -78,11 +78,11 @@ runtime_size = 2
 mode = "disable"
 
 ## Certificate file path.
-## +toml2docs:none-default
+## @toml2docs:none-default
 cert_path = ""
 
 ## Private key file path.
-## +toml2docs:none-default
+## @toml2docs:none-default
 key_path = ""
 
 ## Watch for Certificate and key file change and auto reload
@@ -103,11 +103,11 @@ runtime_size = 2
 mode = "disable"
 
 ## Certificate file path.
-## +toml2docs:none-default
+## @toml2docs:none-default
 cert_path = ""
 
 ## Private key file path.
-## +toml2docs:none-default
+## @toml2docs:none-default
 key_path = ""
 
 ## Watch for Certificate and key file change and auto reload
@@ -139,7 +139,7 @@ provider = "raft_engine"
 
 ## The directory to store the WAL files.
 ## **It's only used when the provider is `raft_engine`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 dir = "/tmp/greptimedb/wal"
 
 ## The size of the WAL segment file.
@@ -320,83 +320,83 @@ type = "File"
 
 ## Cache configuration for object storage such as 'S3' etc.
 ## The local file cache directory.
-## +toml2docs:none-default
+## @toml2docs:none-default
 cache_path = "/path/local_cache"
 
 ## The local file cache capacity in bytes.
-## +toml2docs:none-default
+## @toml2docs:none-default
 cache_capacity = "256MB"
 
 ## The S3 bucket name.
 ## **It's only used when the storage type is `S3`, `Oss` and `Gcs`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 bucket = "greptimedb"
 
 ## The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.
 ## **It's only used when the storage type is `S3`, `Oss` and `Azblob`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 root = "greptimedb"
 
 ## The access key id of the aws account.
 ## It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.
 ## **It's only used when the storage type is `S3` and `Oss`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 access_key_id = "test"
 
 ## The secret access key of the aws account.
 ## It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.
 ## **It's only used when the storage type is `S3`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 secret_access_key = "test"
 
 ## The secret access key of the aliyun account.
 ## **It's only used when the storage type is `Oss`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 access_key_secret = "test"
 
 ## The account key of the azure account.
 ## **It's only used when the storage type is `Azblob`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 account_name = "test"
 
 ## The account key of the azure account.
 ## **It's only used when the storage type is `Azblob`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 account_key = "test"
 
 ## The scope of the google cloud storage.
 ## **It's only used when the storage type is `Gcs`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 scope = "test"
 
 ## The credential path of the google cloud storage.
 ## **It's only used when the storage type is `Gcs`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 credential_path = "test"
 
 ## The credential of the google cloud storage.
 ## **It's only used when the storage type is `Gcs`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 credential = "base64-credential"
 
 ## The container of the azure account.
 ## **It's only used when the storage type is `Azblob`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 container = "greptimedb"
 
 ## The sas token of the azure account.
 ## **It's only used when the storage type is `Azblob`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 sas_token = ""
 
 ## The endpoint of the S3 service.
 ## **It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 endpoint = "https://s3.amazonaws.com"
 
 ## The region of the S3 service.
 ## **It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**.
-## +toml2docs:none-default
+## @toml2docs:none-default
 region = "us-west-2"
 
 # Custom storage options
@@ -426,7 +426,7 @@ region = "us-west-2"
 [region_engine.mito]
 
 ## Number of region workers.
-num_workers = 8
+#+ num_workers = 8
 
 ## Request channel size of each worker.
 worker_channel_size = 128
@@ -447,26 +447,32 @@ max_background_jobs = 4
 auto_flush_interval = "1h"
 
 ## Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB.
-global_write_buffer_size = "1GB"
+## @toml2docs:none-default="Auto"
+#+ global_write_buffer_size = "1GB"
 
-## Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size`
-global_write_buffer_reject_size = "2GB"
+## Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size`.
+## @toml2docs:none-default="Auto"
+#+ global_write_buffer_reject_size = "2GB"
 
 ## Cache size for SST metadata. Setting it to 0 to disable the cache.
 ## If not set, it's default to 1/32 of OS memory with a max limitation of 128MB.
-sst_meta_cache_size = "128MB"
+## @toml2docs:none-default="Auto"
+#+ sst_meta_cache_size = "128MB"
 
 ## Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.
 ## If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
-vector_cache_size = "512MB"
+## @toml2docs:none-default="Auto"
+#+ vector_cache_size = "512MB"
 
 ## Cache size for pages of SST row groups. Setting it to 0 to disable the cache.
 ## If not set, it's default to 1/8 of OS memory.
-page_cache_size = "512MB"
+## @toml2docs:none-default="Auto"
+#+ page_cache_size = "512MB"
 
 ## Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.
 ## If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
-selector_result_cache_size = "512MB"
+## @toml2docs:none-default="Auto"
+#+ selector_result_cache_size = "512MB"
 
 ## Whether to enable the experimental write cache.
 enable_experimental_write_cache = false
@@ -478,7 +484,7 @@ experimental_write_cache_path = ""
 experimental_write_cache_size = "512MB"
 
 ## TTL for write cache.
-## +toml2docs:none-default
+## @toml2docs:none-default
 experimental_write_cache_ttl = "8h"
 
 ## Buffer size for SST writing.
@@ -600,7 +606,7 @@ fork_dictionary_bytes = "1GiB"
 dir = "/tmp/greptimedb/logs"
 
 ## The log level. Can be `info`/`debug`/`warn`/`error`.
-## +toml2docs:none-default
+## @toml2docs:none-default
 level = "info"
 
 ## Enable OTLP tracing.
@@ -631,10 +637,10 @@ enable = false
 ## The interval of export metrics.
 write_interval = "30s"
 
-## For `standalone` mode, `self_import` is recommend to collect metrics generated by itself
+## For `standalone` mode, `self_import` is recommended to collect metrics generated by itself
 ## You must create the database before enabling it.
 [export_metrics.self_import]
-## +toml2docs:none-default
+## @toml2docs:none-default
 db = "greptime_metrics"
 
 [export_metrics.remote_write]
@@ -647,5 +653,5 @@ headers = { }
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
 [tracing]
 ## The tokio console address.
-## +toml2docs:none-default
+## @toml2docs:none-default
 tokio_console_addr = "127.0.0.1"
diff --git a/src/cmd/tests/load_config_test.rs b/src/cmd/tests/load_config_test.rs
index 78d0786f7c..68d67a53a4 100644
--- a/src/cmd/tests/load_config_test.rs
+++ b/src/cmd/tests/load_config_test.rs
@@ -74,15 +74,9 @@ fn test_load_datanode_example_config() {
             },
             region_engine: vec![
                 RegionEngineConfig::Mito(MitoConfig {
-                    num_workers: 8,
                     auto_flush_interval: Duration::from_secs(3600),
                     scan_parallelism: 0,
-                    global_write_buffer_size: ReadableSize::gb(1),
                     global_write_buffer_reject_size: ReadableSize::gb(2),
-                    sst_meta_cache_size: ReadableSize::mb(128),
-                    vector_cache_size: ReadableSize::mb(512),
-                    page_cache_size: ReadableSize::mb(512),
-                    selector_result_cache_size: ReadableSize::mb(512),
                     max_background_jobs: 4,
                     experimental_write_cache_ttl: Some(Duration::from_secs(60 * 60 * 8)),
                     ..Default::default()
@@ -213,15 +207,10 @@ fn test_load_standalone_example_config() {
             }),
             region_engine: vec![
                 RegionEngineConfig::Mito(MitoConfig {
-                    num_workers: 8,
                     auto_flush_interval: Duration::from_secs(3600),
                     scan_parallelism: 0,
-                    global_write_buffer_size: ReadableSize::gb(1),
                     global_write_buffer_reject_size: ReadableSize::gb(2),
                     sst_meta_cache_size: ReadableSize::mb(128),
-                    vector_cache_size: ReadableSize::mb(512),
-                    page_cache_size: ReadableSize::mb(512),
-                    selector_result_cache_size: ReadableSize::mb(512),
                     max_background_jobs: 4,
                     experimental_write_cache_ttl: Some(Duration::from_secs(60 * 60 * 8)),
                     ..Default::default()

From 67fb3d003e3c06bfc6a98eb83f13e0cdeb8cdf2d Mon Sep 17 00:00:00 2001
From: Yohan Wal <59358312+CookiePieWw@users.noreply.github.com>
Date: Wed, 11 Sep 2024 16:17:57 +0800
Subject: [PATCH 003/128] feat: add respective get_by_path UDFs for JSON type
 (#4720)

* feat: add respectiv get_by_path udf for json type

* Apply review comments

Co-authored-by: Weny Xu <wenymedia@gmail.com>

* fix: fix compile error

* refactor: change name of UDFs, add some tests

---------

Co-authored-by: Weny Xu <wenymedia@gmail.com>
---
 src/common/function/src/scalars/json.rs       |   7 +
 .../function/src/scalars/json/json_get.rs     | 454 ++++++++++++++++++
 .../standalone/common/function/json.result    | 198 ++++++++
 .../cases/standalone/common/function/json.sql |  56 +++
 4 files changed, 715 insertions(+)
 create mode 100644 src/common/function/src/scalars/json/json_get.rs
 create mode 100644 tests/cases/standalone/common/function/json.result
 create mode 100644 tests/cases/standalone/common/function/json.sql

diff --git a/src/common/function/src/scalars/json.rs b/src/common/function/src/scalars/json.rs
index 3812b33f23..26d63d3b45 100644
--- a/src/common/function/src/scalars/json.rs
+++ b/src/common/function/src/scalars/json.rs
@@ -13,9 +13,11 @@
 // limitations under the License.
 
 use std::sync::Arc;
+mod json_get;
 mod json_to_string;
 mod to_json;
 
+use json_get::{JsonGetBool, JsonGetFloat, JsonGetInt, JsonGetString};
 use json_to_string::JsonToStringFunction;
 use to_json::ToJsonFunction;
 
@@ -27,5 +29,10 @@ impl JsonFunction {
     pub fn register(registry: &FunctionRegistry) {
         registry.register(Arc::new(JsonToStringFunction));
         registry.register(Arc::new(ToJsonFunction));
+
+        registry.register(Arc::new(JsonGetInt));
+        registry.register(Arc::new(JsonGetFloat));
+        registry.register(Arc::new(JsonGetString));
+        registry.register(Arc::new(JsonGetBool));
     }
 }
diff --git a/src/common/function/src/scalars/json/json_get.rs b/src/common/function/src/scalars/json/json_get.rs
new file mode 100644
index 0000000000..78ddc1d264
--- /dev/null
+++ b/src/common/function/src/scalars/json/json_get.rs
@@ -0,0 +1,454 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::{self, Display};
+
+use common_query::error::{InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu};
+use common_query::prelude::Signature;
+use datafusion::logical_expr::Volatility;
+use datatypes::data_type::ConcreteDataType;
+use datatypes::prelude::VectorRef;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::vectors::{
+    BooleanVectorBuilder, Float64VectorBuilder, Int64VectorBuilder, MutableVector,
+    StringVectorBuilder,
+};
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+
+fn get_json_by_path(json: &[u8], path: &str) -> Option<Vec<u8>> {
+    let json_path = jsonb::jsonpath::parse_json_path(path.as_bytes());
+    match json_path {
+        Ok(json_path) => {
+            let mut sub_jsonb = Vec::new();
+            let mut sub_offsets = Vec::new();
+            match jsonb::get_by_path(json, json_path, &mut sub_jsonb, &mut sub_offsets) {
+                Ok(_) => Some(sub_jsonb),
+                Err(_) => None,
+            }
+        }
+        _ => None,
+    }
+}
+
+/// Get the value from the JSONB by the given path and return it as specified type.
+/// If the path does not exist or the value is not the type specified, return `NULL`.
+macro_rules! json_get {
+    // e.g. name = JsonGetInt, type = Int64, rust_type = i64, doc = "Get the value from the JSONB by the given path and return it as an integer."
+    ($name: ident, $type: ident, $rust_type: ident, $doc:expr) => {
+        paste::paste! {
+            #[doc = $doc]
+            #[derive(Clone, Debug, Default)]
+            pub struct $name;
+
+            impl Function for $name {
+                fn name(&self) -> &str {
+                    stringify!([<$name:snake>])
+                }
+
+                fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+                    Ok(ConcreteDataType::[<$type:snake _datatype>]())
+                }
+
+                fn signature(&self) -> Signature {
+                    Signature::exact(
+                        vec![
+                            ConcreteDataType::json_datatype(),
+                            ConcreteDataType::string_datatype(),
+                        ],
+                        Volatility::Immutable,
+                    )
+                }
+
+                fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+                    ensure!(
+                        columns.len() == 2,
+                        InvalidFuncArgsSnafu {
+                            err_msg: format!(
+                                "The length of the args is not correct, expect exactly two, have: {}",
+                                columns.len()
+                            ),
+                        }
+                    );
+                    let jsons = &columns[0];
+                    let paths = &columns[1];
+
+                    let size = jsons.len();
+                    let datatype = jsons.data_type();
+                    let mut results = [<$type VectorBuilder>]::with_capacity(size);
+
+                    match datatype {
+                        // JSON data type uses binary vector
+                        ConcreteDataType::Binary(_) => {
+                            for i in 0..size {
+                                let json = jsons.get_ref(i);
+                                let path = paths.get_ref(i);
+
+                                let json = json.as_binary();
+                                let path = path.as_string();
+                                let result = match (json, path) {
+                                    (Ok(Some(json)), Ok(Some(path))) => {
+                                        get_json_by_path(json, path)
+                                            .and_then(|json| { jsonb::[<to_ $rust_type>](&json).ok() })
+                                    }
+                                    _ => None,
+                                };
+
+                                results.push(result);
+                            }
+                        }
+                        _ => {
+                            return UnsupportedInputDataTypeSnafu {
+                                function: stringify!([<$name:snake>]),
+                                datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
+                            }
+                            .fail();
+                        }
+                    }
+
+                    Ok(results.to_vector())
+                }
+            }
+
+            impl Display for $name {
+                fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+                    write!(f, "{}", stringify!([<$name:snake>]).to_ascii_uppercase())
+                }
+            }
+        }
+    };
+}
+
+json_get!(
+    JsonGetInt,
+    Int64,
+    i64,
+    "Get the value from the JSONB by the given path and return it as an integer."
+);
+
+json_get!(
+    JsonGetFloat,
+    Float64,
+    f64,
+    "Get the value from the JSONB by the given path and return it as a float."
+);
+
+json_get!(
+    JsonGetBool,
+    Boolean,
+    bool,
+    "Get the value from the JSONB by the given path and return it as a boolean."
+);
+
+/// Get the value from the JSONB by the given path and return it as a string.
+#[derive(Clone, Debug, Default)]
+pub struct JsonGetString;
+
+impl Function for JsonGetString {
+    fn name(&self) -> &str {
+        "json_get_string"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::string_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::exact(
+            vec![
+                ConcreteDataType::json_datatype(),
+                ConcreteDataType::string_datatype(),
+            ],
+            Volatility::Immutable,
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly two, have: {}",
+                    columns.len()
+                ),
+            }
+        );
+        let jsons = &columns[0];
+        let paths = &columns[1];
+
+        let size = jsons.len();
+        let datatype = jsons.data_type();
+        let mut results = StringVectorBuilder::with_capacity(size);
+
+        match datatype {
+            // JSON data type uses binary vector
+            ConcreteDataType::Binary(_) => {
+                for i in 0..size {
+                    let json = jsons.get_ref(i);
+                    let path = paths.get_ref(i);
+
+                    let json = json.as_binary();
+                    let path = path.as_string();
+                    let result = match (json, path) {
+                        (Ok(Some(json)), Ok(Some(path))) => {
+                            get_json_by_path(json, path).and_then(|json| jsonb::to_str(&json).ok())
+                        }
+                        _ => None,
+                    };
+
+                    results.push(result.as_deref());
+                }
+            }
+            _ => {
+                return UnsupportedInputDataTypeSnafu {
+                    function: "json_get_string",
+                    datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
+                }
+                .fail();
+            }
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+impl Display for JsonGetString {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", "json_get_string".to_ascii_uppercase())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_query::prelude::TypeSignature;
+    use datatypes::scalars::ScalarVector;
+    use datatypes::vectors::{BinaryVector, StringVector};
+
+    use super::*;
+
+    #[test]
+    fn test_json_get_int() {
+        let json_get_int = JsonGetInt;
+
+        assert_eq!("json_get_int", json_get_int.name());
+        assert_eq!(
+            ConcreteDataType::int64_datatype(),
+            json_get_int
+                .return_type(&[
+                    ConcreteDataType::json_datatype(),
+                    ConcreteDataType::string_datatype()
+                ])
+                .unwrap()
+        );
+
+        assert!(matches!(json_get_int.signature(),
+                         Signature {
+                             type_signature: TypeSignature::Exact(valid_types),
+                             volatility: Volatility::Immutable
+                         } if  valid_types == vec![ConcreteDataType::json_datatype(), ConcreteDataType::string_datatype()]
+        ));
+
+        let json_strings = [
+            r#"{"a": {"b": 2}, "b": 2, "c": 3}"#,
+            r#"{"a": 4, "b": {"c": 6}, "c": 6}"#,
+            r#"{"a": 7, "b": 8, "c": {"a": 7}}"#,
+        ];
+        let paths = vec!["$.a.b", "$.a", "$.c"];
+        let results = [Some(2), Some(4), None];
+
+        let jsonbs = json_strings
+            .iter()
+            .map(|s| {
+                let value = jsonb::parse_value(s.as_bytes()).unwrap();
+                value.to_vec()
+            })
+            .collect::<Vec<_>>();
+
+        let json_vector = BinaryVector::from_vec(jsonbs);
+        let path_vector = StringVector::from_vec(paths);
+        let args: Vec<VectorRef> = vec![Arc::new(json_vector), Arc::new(path_vector)];
+        let vector = json_get_int
+            .eval(FunctionContext::default(), &args)
+            .unwrap();
+
+        assert_eq!(3, vector.len());
+        for (i, gt) in results.iter().enumerate() {
+            let result = vector.get_ref(i);
+            let result = result.as_i64().unwrap();
+            assert_eq!(*gt, result);
+        }
+    }
+
+    #[test]
+    fn test_json_get_float() {
+        let json_get_float = JsonGetFloat;
+
+        assert_eq!("json_get_float", json_get_float.name());
+        assert_eq!(
+            ConcreteDataType::float64_datatype(),
+            json_get_float
+                .return_type(&[
+                    ConcreteDataType::json_datatype(),
+                    ConcreteDataType::string_datatype()
+                ])
+                .unwrap()
+        );
+
+        assert!(matches!(json_get_float.signature(),
+                         Signature {
+                             type_signature: TypeSignature::Exact(valid_types),
+                             volatility: Volatility::Immutable
+                         } if  valid_types == vec![ConcreteDataType::json_datatype(), ConcreteDataType::string_datatype()]
+        ));
+
+        let json_strings = [
+            r#"{"a": {"b": 2.1}, "b": 2.2, "c": 3.3}"#,
+            r#"{"a": 4.4, "b": {"c": 6.6}, "c": 6.6}"#,
+            r#"{"a": 7.7, "b": 8.8, "c": {"a": 7.7}}"#,
+        ];
+        let paths = vec!["$.a.b", "$.a", "$.c"];
+        let results = [Some(2.1), Some(4.4), None];
+
+        let jsonbs = json_strings
+            .iter()
+            .map(|s| {
+                let value = jsonb::parse_value(s.as_bytes()).unwrap();
+                value.to_vec()
+            })
+            .collect::<Vec<_>>();
+
+        let json_vector = BinaryVector::from_vec(jsonbs);
+        let path_vector = StringVector::from_vec(paths);
+        let args: Vec<VectorRef> = vec![Arc::new(json_vector), Arc::new(path_vector)];
+        let vector = json_get_float
+            .eval(FunctionContext::default(), &args)
+            .unwrap();
+
+        assert_eq!(3, vector.len());
+        for (i, gt) in results.iter().enumerate() {
+            let result = vector.get_ref(i);
+            let result = result.as_f64().unwrap();
+            assert_eq!(*gt, result);
+        }
+    }
+
+    #[test]
+    fn test_json_get_bool() {
+        let json_get_bool = JsonGetBool;
+
+        assert_eq!("json_get_bool", json_get_bool.name());
+        assert_eq!(
+            ConcreteDataType::boolean_datatype(),
+            json_get_bool
+                .return_type(&[
+                    ConcreteDataType::json_datatype(),
+                    ConcreteDataType::string_datatype()
+                ])
+                .unwrap()
+        );
+
+        assert!(matches!(json_get_bool.signature(),
+                         Signature {
+                             type_signature: TypeSignature::Exact(valid_types),
+                             volatility: Volatility::Immutable
+                         } if  valid_types == vec![ConcreteDataType::json_datatype(), ConcreteDataType::string_datatype()]
+        ));
+
+        let json_strings = [
+            r#"{"a": {"b": true}, "b": false, "c": true}"#,
+            r#"{"a": false, "b": {"c": true}, "c": false}"#,
+            r#"{"a": true, "b": false, "c": {"a": true}}"#,
+        ];
+        let paths = vec!["$.a.b", "$.a", "$.c"];
+        let results = [Some(true), Some(false), None];
+
+        let jsonbs = json_strings
+            .iter()
+            .map(|s| {
+                let value = jsonb::parse_value(s.as_bytes()).unwrap();
+                value.to_vec()
+            })
+            .collect::<Vec<_>>();
+
+        let json_vector = BinaryVector::from_vec(jsonbs);
+        let path_vector = StringVector::from_vec(paths);
+        let args: Vec<VectorRef> = vec![Arc::new(json_vector), Arc::new(path_vector)];
+        let vector = json_get_bool
+            .eval(FunctionContext::default(), &args)
+            .unwrap();
+
+        assert_eq!(3, vector.len());
+        for (i, gt) in results.iter().enumerate() {
+            let result = vector.get_ref(i);
+            let result = result.as_boolean().unwrap();
+            assert_eq!(*gt, result);
+        }
+    }
+
+    #[test]
+    fn test_json_get_string() {
+        let json_get_string = JsonGetString;
+
+        assert_eq!("json_get_string", json_get_string.name());
+        assert_eq!(
+            ConcreteDataType::string_datatype(),
+            json_get_string
+                .return_type(&[
+                    ConcreteDataType::json_datatype(),
+                    ConcreteDataType::string_datatype()
+                ])
+                .unwrap()
+        );
+
+        assert!(matches!(json_get_string.signature(),
+                         Signature {
+                             type_signature: TypeSignature::Exact(valid_types),
+                             volatility: Volatility::Immutable
+                         } if  valid_types == vec![ConcreteDataType::json_datatype(), ConcreteDataType::string_datatype()]
+        ));
+
+        let json_strings = [
+            r#"{"a": {"b": "a"}, "b": "b", "c": "c"}"#,
+            r#"{"a": "d", "b": {"c": "e"}, "c": "f"}"#,
+            r#"{"a": "g", "b": "h", "c": {"a": "g"}}"#,
+        ];
+        let paths = vec!["$.a.b", "$.a", ""];
+        let results = [Some("a"), Some("d"), None];
+
+        let jsonbs = json_strings
+            .iter()
+            .map(|s| {
+                let value = jsonb::parse_value(s.as_bytes()).unwrap();
+                value.to_vec()
+            })
+            .collect::<Vec<_>>();
+
+        let json_vector = BinaryVector::from_vec(jsonbs);
+        let path_vector = StringVector::from_vec(paths);
+        let args: Vec<VectorRef> = vec![Arc::new(json_vector), Arc::new(path_vector)];
+        let vector = json_get_string
+            .eval(FunctionContext::default(), &args)
+            .unwrap();
+
+        assert_eq!(3, vector.len());
+        for (i, gt) in results.iter().enumerate() {
+            let result = vector.get_ref(i);
+            let result = result.as_string().unwrap();
+            assert_eq!(*gt, result);
+        }
+    }
+}
diff --git a/tests/cases/standalone/common/function/json.result b/tests/cases/standalone/common/function/json.result
new file mode 100644
index 0000000000..f2a59b9d70
--- /dev/null
+++ b/tests/cases/standalone/common/function/json.result
@@ -0,0 +1,198 @@
+-- json_get functions --
+SELECT json_get_int(to_json('{"a": {"b": {"c": 1}}}'), 'a.b.c');
+
++---------------------------------------------------------------------+
+| json_get_int(to_json(Utf8("{"a": {"b": {"c": 1}}}")),Utf8("a.b.c")) |
++---------------------------------------------------------------------+
+| 1                                                                   |
++---------------------------------------------------------------------+
+
+SELECT json_get_float(to_json('{"a": {"b": {"c": 1.234}}}'), 'a:b.c');
+
++---------------------------------------------------------------------------+
+| json_get_float(to_json(Utf8("{"a": {"b": {"c": 1.234}}}")),Utf8("a:b.c")) |
++---------------------------------------------------------------------------+
+| 1.234                                                                     |
++---------------------------------------------------------------------------+
+
+SELECT json_get_string(to_json('{"a": {"b": {"c": "foo"}}}'), 'a.b:c');
+
++----------------------------------------------------------------------------+
+| json_get_string(to_json(Utf8("{"a": {"b": {"c": "foo"}}}")),Utf8("a.b:c")) |
++----------------------------------------------------------------------------+
+| foo                                                                        |
++----------------------------------------------------------------------------+
+
+SELECT json_get_bool(to_json('{"a": {"b": {"c": true}}}'), 'a.b["c"]');
+
++----------------------------------------------------------------------------+
+| json_get_bool(to_json(Utf8("{"a": {"b": {"c": true}}}")),Utf8("a.b["c"]")) |
++----------------------------------------------------------------------------+
+| true                                                                       |
++----------------------------------------------------------------------------+
+
+SELECT json_get_int(to_json('{"a": {"b": {"c": {"d": 1}}}}'), 'a.b');
+
++--------------------------------------------------------------------------+
+| json_get_int(to_json(Utf8("{"a": {"b": {"c": {"d": 1}}}}")),Utf8("a.b")) |
++--------------------------------------------------------------------------+
+|                                                                          |
++--------------------------------------------------------------------------+
+
+SELECT json_get_string(to_json('{"a": {"b": {"c": {"d": 1}}}}'), 'a.b');
+
++-----------------------------------------------------------------------------+
+| json_get_string(to_json(Utf8("{"a": {"b": {"c": {"d": 1}}}}")),Utf8("a.b")) |
++-----------------------------------------------------------------------------+
+|                                                                             |
++-----------------------------------------------------------------------------+
+
+-- test functions with table rows --
+CREATE TABLE jsons(j JSON, ts timestamp time index);
+
+Affected Rows: 0
+
+INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": 1}}}'), 1);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": 1.234}}}'), 2);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": "foo"}}}'), 3);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": true}}}'), 4);
+
+Affected Rows: 1
+
+SELECT json_get_int(j, 'a.b.c') FROM jsons;
+
++-------------------------------------+
+| json_get_int(jsons.j,Utf8("a.b.c")) |
++-------------------------------------+
+| 1                                   |
+|                                     |
+|                                     |
+| 1                                   |
++-------------------------------------+
+
+SELECT json_get_float(j, 'a["b"].c') FROM jsons;
+
++------------------------------------------+
+| json_get_float(jsons.j,Utf8("a["b"].c")) |
++------------------------------------------+
+| 1.0                                      |
+| 1.234                                    |
+|                                          |
+| 1.0                                      |
++------------------------------------------+
+
+SELECT json_get_string(j, 'a.b.c?(@ == 1)') FROM jsons;
+
++-------------------------------------------------+
+| json_get_string(jsons.j,Utf8("a.b.c?(@ == 1)")) |
++-------------------------------------------------+
+| 1                                               |
+|                                                 |
+|                                                 |
+|                                                 |
++-------------------------------------------------+
+
+SELECT json_get_bool(j, 'a.b.c') FROM jsons;
+
++--------------------------------------+
+| json_get_bool(jsons.j,Utf8("a.b.c")) |
++--------------------------------------+
+|                                      |
+|                                      |
+|                                      |
+| true                                 |
++--------------------------------------+
+
+SELECT json_get_int(j, 'a.b["c"]') FROM jsons;
+
++----------------------------------------+
+| json_get_int(jsons.j,Utf8("a.b["c"]")) |
++----------------------------------------+
+| 1                                      |
+|                                        |
+|                                        |
+| 1                                      |
++----------------------------------------+
+
+DROP TABLE jsons;
+
+Affected Rows: 0
+
+-- test functions with arrays --
+CREATE TABLE jsons(j JSON, ts timestamp time index);
+
+Affected Rows: 0
+
+INSERT INTO jsons VALUES(to_json('["a", "bcde", "", "Long time ago, there is a little pig flying in the sky"]'), 1);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(to_json('[true, false, false, false]'), 2);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(to_json('[1, 0, -2147483649, 2147483648]'), 3);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(to_json('[1.2, 3.1415926535897932384626, -3e123, 1e100]'), 4);
+
+Affected Rows: 1
+
+SELECT json_get_int(j, '[0]') FROM jsons;
+
++-----------------------------------+
+| json_get_int(jsons.j,Utf8("[0]")) |
++-----------------------------------+
+|                                   |
+| 1                                 |
+| 1                                 |
+|                                   |
++-----------------------------------+
+
+SELECT json_get_float(j, '[1]') FROM jsons;
+
++-------------------------------------+
+| json_get_float(jsons.j,Utf8("[1]")) |
++-------------------------------------+
+|                                     |
+| 0.0                                 |
+| 0.0                                 |
+| 3.141592653589793                   |
++-------------------------------------+
+
+SELECT json_get_bool(j, '[2]') FROM jsons;
+
++------------------------------------+
+| json_get_bool(jsons.j,Utf8("[2]")) |
++------------------------------------+
+|                                    |
+| false                              |
+|                                    |
+|                                    |
++------------------------------------+
+
+SELECT json_get_string(j, '[3]') FROM jsons;
+
++--------------------------------------------------------+
+| json_get_string(jsons.j,Utf8("[3]"))                   |
++--------------------------------------------------------+
+| Long time ago, there is a little pig flying in the sky |
+| false                                                  |
+| 2147483648                                             |
+| 1e100                                                  |
++--------------------------------------------------------+
+
+DROP TABLE jsons;
+
+Affected Rows: 0
+
diff --git a/tests/cases/standalone/common/function/json.sql b/tests/cases/standalone/common/function/json.sql
new file mode 100644
index 0000000000..c6214ae0f8
--- /dev/null
+++ b/tests/cases/standalone/common/function/json.sql
@@ -0,0 +1,56 @@
+-- json_get functions --
+SELECT json_get_int(to_json('{"a": {"b": {"c": 1}}}'), 'a.b.c');
+
+SELECT json_get_float(to_json('{"a": {"b": {"c": 1.234}}}'), 'a:b.c');
+
+SELECT json_get_string(to_json('{"a": {"b": {"c": "foo"}}}'), 'a.b:c');
+
+SELECT json_get_bool(to_json('{"a": {"b": {"c": true}}}'), 'a.b["c"]');
+
+SELECT json_get_int(to_json('{"a": {"b": {"c": {"d": 1}}}}'), 'a.b');
+
+SELECT json_get_string(to_json('{"a": {"b": {"c": {"d": 1}}}}'), 'a.b');
+
+-- test functions with table rows --
+CREATE TABLE jsons(j JSON, ts timestamp time index);
+
+INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": 1}}}'), 1);
+
+INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": 1.234}}}'), 2);
+
+INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": "foo"}}}'), 3);
+
+INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": true}}}'), 4);
+
+SELECT json_get_int(j, 'a.b.c') FROM jsons;
+
+SELECT json_get_float(j, 'a["b"].c') FROM jsons;
+
+SELECT json_get_string(j, 'a.b.c?(@ == 1)') FROM jsons;
+
+SELECT json_get_bool(j, 'a.b.c') FROM jsons;
+
+SELECT json_get_int(j, 'a.b["c"]') FROM jsons;
+
+DROP TABLE jsons;
+
+-- test functions with arrays --
+CREATE TABLE jsons(j JSON, ts timestamp time index);
+
+INSERT INTO jsons VALUES(to_json('["a", "bcde", "", "Long time ago, there is a little pig flying in the sky"]'), 1);
+
+INSERT INTO jsons VALUES(to_json('[true, false, false, false]'), 2);
+
+INSERT INTO jsons VALUES(to_json('[1, 0, -2147483649, 2147483648]'), 3);
+
+INSERT INTO jsons VALUES(to_json('[1.2, 3.1415926535897932384626, -3e123, 1e100]'), 4);
+
+SELECT json_get_int(j, '[0]') FROM jsons;
+
+SELECT json_get_float(j, '[1]') FROM jsons;
+
+SELECT json_get_bool(j, '[2]') FROM jsons;
+
+SELECT json_get_string(j, '[3]') FROM jsons;
+
+DROP TABLE jsons;

From 36b1bafbf00152cc89c8d4362ace432bbb1932a3 Mon Sep 17 00:00:00 2001
From: localhost <xpaomian@gmail.com>
Date: Fri, 13 Sep 2024 02:21:05 +0800
Subject: [PATCH 004/128] fix: pipeline dissert error is returned directly to
 the user, instead of printing a warn log (#4709)

* fix: pipeline dissert error is returned directly to the user, instead of printing a warn log

* chore: add more test for pipeline
---
 src/pipeline/src/etl.rs                   | 21 +++++++----
 src/pipeline/src/etl/processor/dissect.rs | 14 +++----
 src/pipeline/tests/dissect.rs             | 34 +++++++++++++++++
 src/pipeline/tests/regex.rs               | 46 +++++++++++++++++++++++
 4 files changed, 98 insertions(+), 17 deletions(-)

diff --git a/src/pipeline/src/etl.rs b/src/pipeline/src/etl.rs
index de4c544a01..f6b3efd6e6 100644
--- a/src/pipeline/src/etl.rs
+++ b/src/pipeline/src/etl.rs
@@ -21,7 +21,7 @@ pub mod value;
 
 use ahash::HashSet;
 use common_telemetry::debug;
-use itertools::{merge, Itertools};
+use itertools::Itertools;
 use processor::{Processor, ProcessorBuilder, Processors};
 use transform::{TransformBuilders, Transformer, Transforms};
 use value::Value;
@@ -91,13 +91,18 @@ where
             debug!("required_keys: {:?}", required_keys);
 
             // intermediate keys are the keys that all processor and transformer required
-            let ordered_intermediate_keys: Vec<String> =
-                merge(processors_required_keys, transforms_required_keys)
-                    .cloned()
-                    .collect::<HashSet<String>>()
-                    .into_iter()
-                    .sorted()
-                    .collect();
+            let ordered_intermediate_keys: Vec<String> = [
+                processors_required_keys,
+                transforms_required_keys,
+                processors_output_keys,
+            ]
+            .iter()
+            .flat_map(|l| l.iter())
+            .collect::<HashSet<&String>>()
+            .into_iter()
+            .sorted()
+            .cloned()
+            .collect_vec();
 
             let mut final_intermediate_keys = Vec::with_capacity(ordered_intermediate_keys.len());
             let mut intermediate_keys_exclude_original =
diff --git a/src/pipeline/src/etl/processor/dissect.rs b/src/pipeline/src/etl/processor/dissect.rs
index 9a4b8a966e..dca88d3843 100644
--- a/src/pipeline/src/etl/processor/dissect.rs
+++ b/src/pipeline/src/etl/processor/dissect.rs
@@ -817,16 +817,12 @@ impl Processor for DissectProcessor {
         for field in self.fields.iter() {
             let index = field.input_index();
             match val.get(index) {
-                Some(Value::String(val_str)) => match self.process(val_str) {
-                    Ok(r) => {
-                        for (k, v) in r {
-                            val[k] = v;
-                        }
+                Some(Value::String(val_str)) => {
+                    let r = self.process(val_str)?;
+                    for (k, v) in r {
+                        val[k] = v;
                     }
-                    Err(e) => {
-                        warn!("dissect processor: {}", e);
-                    }
-                },
+                }
                 Some(Value::Null) | None => {
                     if !self.ignore_missing {
                         return Err(format!(
diff --git a/src/pipeline/tests/dissect.rs b/src/pipeline/tests/dissect.rs
index 82ce63399c..22cf14c46b 100644
--- a/src/pipeline/tests/dissect.rs
+++ b/src/pipeline/tests/dissect.rs
@@ -247,3 +247,37 @@ transform:
         Some(StringValue("key1_key2".to_string()))
     );
 }
+
+#[test]
+fn test_parse_failure() {
+    let input_str = r#"
+{
+    "str": "key1 key2"
+}"#;
+
+    let pipeline_yaml = r#"
+processors:
+  - dissect:
+      field: str
+      patterns:
+        - "%{key1} %{key2} %{key3}"
+
+transform:
+  - fields:
+      - key1
+    type: string
+"#;
+
+    let input_value = serde_json::from_str::<serde_json::Value>(input_str).unwrap();
+
+    let yaml_content = pipeline::Content::Yaml(pipeline_yaml.into());
+    let pipeline: pipeline::Pipeline<pipeline::GreptimeTransformer> =
+        pipeline::parse(&yaml_content).expect("failed to parse pipeline");
+    let mut result = pipeline.init_intermediate_state();
+
+    pipeline.prepare(input_value, &mut result).unwrap();
+    let row = pipeline.exec_mut(&mut result);
+
+    assert!(row.is_err());
+    assert_eq!(row.err().unwrap(), "No matching pattern found");
+}
diff --git a/src/pipeline/tests/regex.rs b/src/pipeline/tests/regex.rs
index 5be60c9875..a8a7daaf5c 100644
--- a/src/pipeline/tests/regex.rs
+++ b/src/pipeline/tests/regex.rs
@@ -122,3 +122,49 @@ transform:
 
     assert_eq!(output.rows[0].values[0].value_data, None);
 }
+
+#[test]
+fn test_unuse_regex_group() {
+    let input_value_str = r#"
+  [
+    {
+      "str": "123 456"
+    }
+  ]
+"#;
+
+    let pipeline_yaml = r#"
+processors:
+- regex:
+    fields:
+      - str
+    pattern: "(?<id1>\\d+) (?<id2>\\d+)"
+
+transform:
+- field: str_id1
+  type: string
+"#;
+
+    let output = common::parse_and_exec(input_value_str, pipeline_yaml);
+
+    assert_eq!(
+        output.schema,
+        vec![
+            common::make_column_schema(
+                "str_id1".to_string(),
+                ColumnDataType::String,
+                SemanticType::Field,
+            ),
+            common::make_column_schema(
+                "greptime_timestamp".to_string(),
+                ColumnDataType::TimestampNanosecond,
+                SemanticType::Timestamp,
+            ),
+        ]
+    );
+
+    assert_eq!(
+        output.rows[0].values[0].value_data,
+        Some(StringValue("123".to_string()))
+    );
+}

From d1dfffcdaff8be61032dddf2dfabd5060f2b0b0a Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Wed, 18 Sep 2024 12:01:30 +0900
Subject: [PATCH 005/128] chore: enable fuzz test for append table (#4702)

* chore: enable fuzz test for append table

* fix: fix mysql translator
---
 tests-fuzz/src/generator/create_expr.rs         | 10 +++++++++-
 tests-fuzz/src/translator/mysql/create_expr.rs  |  2 +-
 tests-fuzz/targets/fuzz_alter_logical_table.rs  |  6 ++++++
 tests-fuzz/targets/fuzz_alter_table.rs          |  6 ++++++
 tests-fuzz/targets/fuzz_create_logical_table.rs |  6 ++++++
 tests-fuzz/targets/fuzz_create_table.rs         |  7 +++++++
 tests-fuzz/targets/fuzz_insert.rs               |  7 +++++++
 tests-fuzz/targets/fuzz_insert_logical_table.rs |  5 +++++
 8 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/tests-fuzz/src/generator/create_expr.rs b/tests-fuzz/src/generator/create_expr.rs
index 93809c06d4..200485e266 100644
--- a/tests-fuzz/src/generator/create_expr.rs
+++ b/tests-fuzz/src/generator/create_expr.rs
@@ -243,12 +243,20 @@ pub struct CreatePhysicalTableExprGenerator<R: Rng + 'static> {
     name_generator: Box<dyn Random<Ident, R>>,
     #[builder(default = "false")]
     if_not_exists: bool,
+    #[builder(default, setter(into))]
+    with_clause: HashMap<String, String>,
 }
 
 impl<R: Rng + 'static> Generator<CreateTableExpr, R> for CreatePhysicalTableExprGenerator<R> {
     type Error = Error;
 
     fn generate(&self, rng: &mut R) -> Result<CreateTableExpr> {
+        let mut options = HashMap::with_capacity(self.with_clause.len() + 1);
+        options.insert("physical_metric_table".to_string(), Value::from(""));
+        for (key, value) in &self.with_clause {
+            options.insert(key.to_string(), Value::from(value.to_string()));
+        }
+
         Ok(CreateTableExpr {
             table_name: self.name_generator.gen(rng),
             columns: vec![
@@ -266,7 +274,7 @@ impl<R: Rng + 'static> Generator<CreateTableExpr, R> for CreatePhysicalTableExpr
             if_not_exists: self.if_not_exists,
             partition: None,
             engine: "metric".to_string(),
-            options: [("physical_metric_table".to_string(), "".into())].into(),
+            options,
             primary_keys: vec![],
         })
     }
diff --git a/tests-fuzz/src/translator/mysql/create_expr.rs b/tests-fuzz/src/translator/mysql/create_expr.rs
index 3ce659bf6e..073643b059 100644
--- a/tests-fuzz/src/translator/mysql/create_expr.rs
+++ b/tests-fuzz/src/translator/mysql/create_expr.rs
@@ -151,7 +151,7 @@ impl CreateTableExprTranslator {
             for (key, value) in &input.options {
                 output.push(format!("\"{key}\" = \"{value}\""));
             }
-            format!(" with ({})", output.join("\n"))
+            format!(" with ({})", output.join(",\n"))
         }
     }
 }
diff --git a/tests-fuzz/targets/fuzz_alter_logical_table.rs b/tests-fuzz/targets/fuzz_alter_logical_table.rs
index 3ceb5b8b45..80f017a353 100644
--- a/tests-fuzz/targets/fuzz_alter_logical_table.rs
+++ b/tests-fuzz/targets/fuzz_alter_logical_table.rs
@@ -14,6 +14,7 @@
 
 #![no_main]
 
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use arbitrary::{Arbitrary, Unstructured};
@@ -76,12 +77,17 @@ impl Arbitrary<'_> for FuzzInput {
 
 fn generate_create_physical_table_expr<R: Rng + 'static>(rng: &mut R) -> Result<CreateTableExpr> {
     let physical_table_if_not_exists = rng.gen_bool(0.5);
+    let mut with_clause = HashMap::new();
+    if rng.gen_bool(0.5) {
+        with_clause.insert("append_mode".to_string(), "true".to_string());
+    }
     let create_physical_table_expr = CreatePhysicalTableExprGeneratorBuilder::default()
         .name_generator(Box::new(MappedGenerator::new(
             WordGenerator,
             merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map),
         )))
         .if_not_exists(physical_table_if_not_exists)
+        .with_clause(with_clause)
         .build()
         .unwrap();
     create_physical_table_expr.generate(rng)
diff --git a/tests-fuzz/targets/fuzz_alter_table.rs b/tests-fuzz/targets/fuzz_alter_table.rs
index 8c17612d04..7f2a809c9e 100644
--- a/tests-fuzz/targets/fuzz_alter_table.rs
+++ b/tests-fuzz/targets/fuzz_alter_table.rs
@@ -14,6 +14,7 @@
 
 #![no_main]
 
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use arbitrary::{Arbitrary, Unstructured};
@@ -71,6 +72,10 @@ enum AlterTableOption {
 fn generate_create_table_expr<R: Rng + 'static>(rng: &mut R) -> Result<CreateTableExpr> {
     let max_columns = get_gt_fuzz_input_max_columns();
     let columns = rng.gen_range(2..max_columns);
+    let mut with_clause = HashMap::new();
+    if rng.gen_bool(0.5) {
+        with_clause.insert("append_mode".to_string(), "true".to_string());
+    }
     let create_table_generator = CreateTableExprGeneratorBuilder::default()
         .name_generator(Box::new(MappedGenerator::new(
             WordGenerator,
@@ -78,6 +83,7 @@ fn generate_create_table_expr<R: Rng + 'static>(rng: &mut R) -> Result<CreateTab
         )))
         .columns(columns)
         .engine("mito")
+        .with_clause(with_clause)
         .build()
         .unwrap();
     create_table_generator.generate(rng)
diff --git a/tests-fuzz/targets/fuzz_create_logical_table.rs b/tests-fuzz/targets/fuzz_create_logical_table.rs
index 39251732f2..64ed26ba7e 100644
--- a/tests-fuzz/targets/fuzz_create_logical_table.rs
+++ b/tests-fuzz/targets/fuzz_create_logical_table.rs
@@ -14,6 +14,7 @@
 
 #![no_main]
 
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use common_telemetry::info;
@@ -68,12 +69,17 @@ async fn execute_create_logic_table(ctx: FuzzContext, input: FuzzInput) -> Resul
 
     // Create physical table
     let physical_table_if_not_exists = rng.gen_bool(0.5);
+    let mut with_clause = HashMap::new();
+    if rng.gen_bool(0.5) {
+        with_clause.insert("append_mode".to_string(), "true".to_string());
+    }
     let create_physical_table_expr = CreatePhysicalTableExprGeneratorBuilder::default()
         .name_generator(Box::new(MappedGenerator::new(
             WordGenerator,
             merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map),
         )))
         .if_not_exists(physical_table_if_not_exists)
+        .with_clause(with_clause)
         .build()
         .unwrap()
         .generate(&mut rng)?;
diff --git a/tests-fuzz/targets/fuzz_create_table.rs b/tests-fuzz/targets/fuzz_create_table.rs
index 0d2458c15e..6d03b0dffa 100644
--- a/tests-fuzz/targets/fuzz_create_table.rs
+++ b/tests-fuzz/targets/fuzz_create_table.rs
@@ -14,6 +14,8 @@
 
 #![no_main]
 
+use std::collections::HashMap;
+
 use common_telemetry::info;
 use libfuzzer_sys::arbitrary::{Arbitrary, Unstructured};
 use libfuzzer_sys::fuzz_target;
@@ -65,6 +67,10 @@ impl Arbitrary<'_> for FuzzInput {
 fn generate_expr(input: FuzzInput) -> Result<CreateTableExpr> {
     let mut rng = ChaChaRng::seed_from_u64(input.seed);
     let if_not_exists = rng.gen_bool(0.5);
+    let mut with_clause = HashMap::new();
+    if rng.gen_bool(0.5) {
+        with_clause.insert("append_mode".to_string(), "true".to_string());
+    }
 
     let create_table_generator = CreateTableExprGeneratorBuilder::default()
         .name_generator(Box::new(MappedGenerator::new(
@@ -74,6 +80,7 @@ fn generate_expr(input: FuzzInput) -> Result<CreateTableExpr> {
         .columns(input.columns)
         .engine("mito")
         .if_not_exists(if_not_exists)
+        .with_clause(with_clause)
         .build()
         .unwrap();
     create_table_generator.generate(&mut rng)
diff --git a/tests-fuzz/targets/fuzz_insert.rs b/tests-fuzz/targets/fuzz_insert.rs
index a5db2bfffe..739d6af7a3 100644
--- a/tests-fuzz/targets/fuzz_insert.rs
+++ b/tests-fuzz/targets/fuzz_insert.rs
@@ -14,6 +14,7 @@
 
 #![no_main]
 
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use common_telemetry::info;
@@ -83,6 +84,11 @@ fn generate_create_expr<R: Rng + 'static>(
     input: FuzzInput,
     rng: &mut R,
 ) -> Result<CreateTableExpr> {
+    let mut with_clause = HashMap::new();
+    if rng.gen_bool(0.5) {
+        with_clause.insert("append_mode".to_string(), "true".to_string());
+    }
+
     let create_table_generator = CreateTableExprGeneratorBuilder::default()
         .name_generator(Box::new(MappedGenerator::new(
             WordGenerator,
@@ -90,6 +96,7 @@ fn generate_create_expr<R: Rng + 'static>(
         )))
         .columns(input.columns)
         .engine("mito")
+        .with_clause(with_clause)
         .ts_column_type_generator(Box::new(MySQLTsColumnTypeGenerator))
         .build()
         .unwrap();
diff --git a/tests-fuzz/targets/fuzz_insert_logical_table.rs b/tests-fuzz/targets/fuzz_insert_logical_table.rs
index 6efb6ea509..abafdef9a8 100644
--- a/tests-fuzz/targets/fuzz_insert_logical_table.rs
+++ b/tests-fuzz/targets/fuzz_insert_logical_table.rs
@@ -79,12 +79,17 @@ impl Arbitrary<'_> for FuzzInput {
 
 fn generate_create_physical_table_expr<R: Rng + 'static>(rng: &mut R) -> Result<CreateTableExpr> {
     let physical_table_if_not_exists = rng.gen_bool(0.5);
+    let mut with_clause = HashMap::new();
+    if rng.gen_bool(0.5) {
+        with_clause.insert("append_mode".to_string(), "true".to_string());
+    }
     let create_physical_table_expr = CreatePhysicalTableExprGeneratorBuilder::default()
         .name_generator(Box::new(MappedGenerator::new(
             WordGenerator,
             merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map),
         )))
         .if_not_exists(physical_table_if_not_exists)
+        .with_clause(with_clause)
         .build()
         .unwrap();
     create_physical_table_expr.generate(rng)

From 3b5b90654374cbba640d004d0871356f865efa0a Mon Sep 17 00:00:00 2001
From: Zhenchi <zhongzc_arch@outlook.com>
Date: Wed, 18 Sep 2024 11:33:55 +0800
Subject: [PATCH 006/128] feat(index): add explicit adapter between
 `RangeReader` and `AsyncRead` (#4724)

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>
---
 src/common/base/src/range_read.rs             | 35 ++++++++++++++++---
 .../src/inverted_index/format/reader/blob.rs  | 10 ++++--
 .../inverted_index/format/reader/footer.rs    | 23 ++++++------
 .../src/inverted_index/format/writer/blob.rs  |  5 +--
 .../src/sst/index/inverted_index/applier.rs   |  2 ++
 5 files changed, 54 insertions(+), 21 deletions(-)

diff --git a/src/common/base/src/range_read.rs b/src/common/base/src/range_read.rs
index 920b2e1f8c..4e7a48a8b7 100644
--- a/src/common/base/src/range_read.rs
+++ b/src/common/base/src/range_read.rs
@@ -60,21 +60,46 @@ pub trait RangeReader: Send + Unpin {
     }
 }
 
-/// Implement `RangeReader` for a type that implements `AsyncRead + AsyncSeek`.
+#[async_trait]
+impl<R: RangeReader + Send + Unpin> RangeReader for &mut R {
+    async fn metadata(&mut self) -> io::Result<Metadata> {
+        (*self).metadata().await
+    }
+    async fn read(&mut self, range: Range<u64>) -> io::Result<Bytes> {
+        (*self).read(range).await
+    }
+    async fn read_into(
+        &mut self,
+        range: Range<u64>,
+        buf: &mut (impl BufMut + Send),
+    ) -> io::Result<()> {
+        (*self).read_into(range, buf).await
+    }
+    async fn read_vec(&mut self, ranges: &[Range<u64>]) -> io::Result<Vec<Bytes>> {
+        (*self).read_vec(ranges).await
+    }
+}
+
+/// `RangeReaderAdapter` bridges `RangeReader` and `AsyncRead + AsyncSeek`.
+pub struct RangeReaderAdapter<R>(pub R);
+
+/// Implements `RangeReader` for a type that implements `AsyncRead + AsyncSeek`.
 ///
 /// TODO(zhongzc): It's a temporary solution for porting the codebase from `AsyncRead + AsyncSeek` to `RangeReader`.
 /// Until the codebase is fully ported to `RangeReader`, remove this implementation.
 #[async_trait]
-impl<R: futures::AsyncRead + futures::AsyncSeek + Send + Unpin> RangeReader for R {
+impl<R: futures::AsyncRead + futures::AsyncSeek + Send + Unpin> RangeReader
+    for RangeReaderAdapter<R>
+{
     async fn metadata(&mut self) -> io::Result<Metadata> {
-        let content_length = self.seek(io::SeekFrom::End(0)).await?;
+        let content_length = self.0.seek(io::SeekFrom::End(0)).await?;
         Ok(Metadata { content_length })
     }
 
     async fn read(&mut self, range: Range<u64>) -> io::Result<Bytes> {
         let mut buf = vec![0; (range.end - range.start) as usize];
-        self.seek(io::SeekFrom::Start(range.start)).await?;
-        self.read_exact(&mut buf).await?;
+        self.0.seek(io::SeekFrom::Start(range.start)).await?;
+        self.0.read_exact(&mut buf).await?;
         Ok(Bytes::from(buf))
     }
 }
diff --git a/src/index/src/inverted_index/format/reader/blob.rs b/src/index/src/inverted_index/format/reader/blob.rs
index 5da70e3748..f79d651e79 100644
--- a/src/index/src/inverted_index/format/reader/blob.rs
+++ b/src/index/src/inverted_index/format/reader/blob.rs
@@ -80,6 +80,7 @@ impl<R: RangeReader> InvertedIndexReader for InvertedIndexBlobReader<R> {
 #[cfg(test)]
 mod tests {
     use common_base::bit_vec::prelude::*;
+    use common_base::range_read::RangeReaderAdapter;
     use fst::MapBuilder;
     use futures::io::Cursor;
     use greptime_proto::v1::index::{InvertedIndexMeta, InvertedIndexMetas};
@@ -162,7 +163,8 @@ mod tests {
     #[tokio::test]
     async fn test_inverted_index_blob_reader_metadata() {
         let blob = create_inverted_index_blob();
-        let mut blob_reader = InvertedIndexBlobReader::new(Cursor::new(blob));
+        let cursor = RangeReaderAdapter(Cursor::new(blob));
+        let mut blob_reader = InvertedIndexBlobReader::new(cursor);
 
         let metas = blob_reader.metadata().await.unwrap();
         assert_eq!(metas.metas.len(), 2);
@@ -189,7 +191,8 @@ mod tests {
     #[tokio::test]
     async fn test_inverted_index_blob_reader_fst() {
         let blob = create_inverted_index_blob();
-        let mut blob_reader = InvertedIndexBlobReader::new(Cursor::new(blob));
+        let cursor = RangeReaderAdapter(Cursor::new(blob));
+        let mut blob_reader = InvertedIndexBlobReader::new(cursor);
 
         let metas = blob_reader.metadata().await.unwrap();
         let meta = metas.metas.get("tag0").unwrap();
@@ -221,7 +224,8 @@ mod tests {
     #[tokio::test]
     async fn test_inverted_index_blob_reader_bitmap() {
         let blob = create_inverted_index_blob();
-        let mut blob_reader = InvertedIndexBlobReader::new(Cursor::new(blob));
+        let cursor = RangeReaderAdapter(Cursor::new(blob));
+        let mut blob_reader = InvertedIndexBlobReader::new(cursor);
 
         let metas = blob_reader.metadata().await.unwrap();
         let meta = metas.metas.get("tag0").unwrap();
diff --git a/src/index/src/inverted_index/format/reader/footer.rs b/src/index/src/inverted_index/format/reader/footer.rs
index 244973669b..ffcaf9d921 100644
--- a/src/index/src/inverted_index/format/reader/footer.rs
+++ b/src/index/src/inverted_index/format/reader/footer.rs
@@ -24,18 +24,18 @@ use crate::inverted_index::error::{
 use crate::inverted_index::format::FOOTER_PAYLOAD_SIZE_SIZE;
 
 /// InvertedIndeFooterReader is for reading the footer section of the blob.
-pub struct InvertedIndeFooterReader<'a, R> {
-    source: &'a mut R,
+pub struct InvertedIndeFooterReader<R> {
+    source: R,
     blob_size: u64,
 }
 
-impl<'a, R> InvertedIndeFooterReader<'a, R> {
-    pub fn new(source: &'a mut R, blob_size: u64) -> Self {
+impl<R> InvertedIndeFooterReader<R> {
+    pub fn new(source: R, blob_size: u64) -> Self {
         Self { source, blob_size }
     }
 }
 
-impl<'a, R: RangeReader> InvertedIndeFooterReader<'a, R> {
+impl<R: RangeReader> InvertedIndeFooterReader<R> {
     pub async fn metadata(&mut self) -> Result<InvertedIndexMetas> {
         let payload_size = self.read_payload_size().await?;
         let metas = self.read_payload(payload_size).await?;
@@ -113,6 +113,7 @@ impl<'a, R: RangeReader> InvertedIndeFooterReader<'a, R> {
 
 #[cfg(test)]
 mod tests {
+    use common_base::range_read::RangeReaderAdapter;
     use futures::io::Cursor;
     use prost::Message;
 
@@ -142,8 +143,8 @@ mod tests {
 
         let payload_buf = create_test_payload(meta);
         let blob_size = payload_buf.len() as u64;
-        let mut cursor = Cursor::new(payload_buf);
-        let mut reader = InvertedIndeFooterReader::new(&mut cursor, blob_size);
+        let cursor = RangeReaderAdapter(Cursor::new(payload_buf));
+        let mut reader = InvertedIndeFooterReader::new(cursor, blob_size);
 
         let payload_size = reader.read_payload_size().await.unwrap();
         let metas = reader.read_payload(payload_size).await.unwrap();
@@ -163,8 +164,8 @@ mod tests {
         let mut payload_buf = create_test_payload(meta);
         payload_buf.push(0xff); // Add an extra byte to corrupt the footer
         let blob_size = payload_buf.len() as u64;
-        let mut cursor = Cursor::new(payload_buf);
-        let mut reader = InvertedIndeFooterReader::new(&mut cursor, blob_size);
+        let cursor = RangeReaderAdapter(Cursor::new(payload_buf));
+        let mut reader = InvertedIndeFooterReader::new(cursor, blob_size);
 
         let payload_size_result = reader.read_payload_size().await;
         assert!(payload_size_result.is_err());
@@ -181,8 +182,8 @@ mod tests {
 
         let payload_buf = create_test_payload(meta);
         let blob_size = payload_buf.len() as u64;
-        let mut cursor = Cursor::new(payload_buf);
-        let mut reader = InvertedIndeFooterReader::new(&mut cursor, blob_size);
+        let cursor = RangeReaderAdapter(Cursor::new(payload_buf));
+        let mut reader = InvertedIndeFooterReader::new(cursor, blob_size);
 
         let payload_size = reader.read_payload_size().await.unwrap();
         let payload_result = reader.read_payload(payload_size).await;
diff --git a/src/index/src/inverted_index/format/writer/blob.rs b/src/index/src/inverted_index/format/writer/blob.rs
index 767a7a3412..26d3fb26d9 100644
--- a/src/index/src/inverted_index/format/writer/blob.rs
+++ b/src/index/src/inverted_index/format/writer/blob.rs
@@ -99,6 +99,7 @@ impl<W: AsyncWrite + Send + Unpin> InvertedIndexBlobWriter<W> {
 
 #[cfg(test)]
 mod tests {
+    use common_base::range_read::RangeReaderAdapter;
     use futures::io::Cursor;
     use futures::stream;
 
@@ -119,7 +120,7 @@ mod tests {
             .await
             .unwrap();
 
-        let cursor = Cursor::new(blob);
+        let cursor = RangeReaderAdapter(Cursor::new(blob));
         let mut reader = InvertedIndexBlobReader::new(cursor);
         let metadata = reader.metadata().await.unwrap();
         assert_eq!(metadata.total_row_count, 8);
@@ -160,7 +161,7 @@ mod tests {
             .await
             .unwrap();
 
-        let cursor = Cursor::new(blob);
+        let cursor = RangeReaderAdapter(Cursor::new(blob));
         let mut reader = InvertedIndexBlobReader::new(cursor);
         let metadata = reader.metadata().await.unwrap();
         assert_eq!(metadata.total_row_count, 8);
diff --git a/src/mito2/src/sst/index/inverted_index/applier.rs b/src/mito2/src/sst/index/inverted_index/applier.rs
index cac3ffedd7..a3482cc075 100644
--- a/src/mito2/src/sst/index/inverted_index/applier.rs
+++ b/src/mito2/src/sst/index/inverted_index/applier.rs
@@ -16,6 +16,7 @@ pub mod builder;
 
 use std::sync::Arc;
 
+use common_base::range_read::RangeReaderAdapter;
 use common_telemetry::warn;
 use index::inverted_index::format::reader::InvertedIndexBlobReader;
 use index::inverted_index::search::index_apply::{
@@ -108,6 +109,7 @@ impl InvertedIndexApplier {
                 self.remote_blob_reader(file_id).await?
             }
         };
+        let blob = RangeReaderAdapter(blob);
 
         if let Some(index_cache) = &self.inverted_index_cache {
             let mut index_reader = CachedInvertedIndexBlobReader::new(

From c014e875f36d1d8c5c91ed582583a3493837e986 Mon Sep 17 00:00:00 2001
From: shuiyisong <113876041+shuiyisong@users.noreply.github.com>
Date: Wed, 18 Sep 2024 12:32:00 +0800
Subject: [PATCH 007/128] chore: add auto-decompression layer for otlp http
 request (#4723)

* chore: add auto-decompression for http request

* test: otlp
---
 Cargo.lock                         |   3 +
 Cargo.toml                         |   1 +
 src/frontend/src/instance/otlp.rs  |   2 +-
 src/servers/src/http.rs            |   5 +
 src/servers/src/http/otlp.rs       |  32 ++----
 tests-integration/Cargo.toml       |   1 +
 tests-integration/src/test_util.rs |   1 +
 tests-integration/tests/http.rs    | 152 ++++++++++++++++++++++++++---
 8 files changed, 158 insertions(+), 39 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index c5b8fc016a..0e9db00527 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7228,9 +7228,11 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3a8fddc9b68f5b80dae9d6f510b88e02396f006ad48cac349411fbecc80caae4"
 dependencies = [
+ "hex",
  "opentelemetry 0.22.0",
  "opentelemetry_sdk 0.22.1",
  "prost 0.12.6",
+ "serde",
  "tonic 0.11.0",
 ]
 
@@ -11775,6 +11777,7 @@ dependencies = [
  "datanode",
  "datatypes",
  "dotenv",
+ "flate2",
  "flow",
  "frontend",
  "futures",
diff --git a/Cargo.toml b/Cargo.toml
index d412bf7e97..6720671402 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -136,6 +136,7 @@ opentelemetry-proto = { version = "0.5", features = [
     "gen-tonic",
     "metrics",
     "trace",
+    "with-serde",
 ] }
 parquet = { version = "51.0.0", default-features = false, features = ["arrow", "async", "object_store"] }
 paste = "1.0"
diff --git a/src/frontend/src/instance/otlp.rs b/src/frontend/src/instance/otlp.rs
index 8d707d12bd..09335af080 100644
--- a/src/frontend/src/instance/otlp.rs
+++ b/src/frontend/src/instance/otlp.rs
@@ -87,7 +87,7 @@ impl OpenTelemetryProtocolHandler for Instance {
 
         OTLP_TRACES_ROWS.inc_by(rows as u64);
 
-        self.handle_row_inserts(requests, ctx)
+        self.handle_log_inserts(requests, ctx)
             .await
             .map_err(BoxedError::new)
             .context(error::ExecuteGrpcQuerySnafu)
diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs
index 5ac52157ea..2313d19bbe 100644
--- a/src/servers/src/http.rs
+++ b/src/servers/src/http.rs
@@ -866,6 +866,11 @@ impl HttpServer {
         Router::new()
             .route("/v1/metrics", routing::post(otlp::metrics))
             .route("/v1/traces", routing::post(otlp::traces))
+            .layer(
+                ServiceBuilder::new()
+                    .layer(HandleErrorLayer::new(handle_error))
+                    .layer(RequestDecompressionLayer::new()),
+            )
             .with_state(otlp_handler)
     }
 
diff --git a/src/servers/src/http/otlp.rs b/src/servers/src/http/otlp.rs
index a04d1d42a0..3efdaeec96 100644
--- a/src/servers/src/http/otlp.rs
+++ b/src/servers/src/http/otlp.rs
@@ -14,12 +14,12 @@
 
 use std::sync::Arc;
 
-use axum::extract::{RawBody, State};
+use axum::extract::State;
 use axum::http::header;
 use axum::response::IntoResponse;
 use axum::Extension;
+use bytes::Bytes;
 use common_telemetry::tracing;
-use hyper::Body;
 use opentelemetry_proto::tonic::collector::metrics::v1::{
     ExportMetricsServiceRequest, ExportMetricsServiceResponse,
 };
@@ -39,7 +39,7 @@ use crate::query_handler::OpenTelemetryProtocolHandlerRef;
 pub async fn metrics(
     State(handler): State<OpenTelemetryProtocolHandlerRef>,
     Extension(mut query_ctx): Extension<QueryContext>,
-    RawBody(body): RawBody,
+    bytes: Bytes,
 ) -> Result<OtlpMetricsResponse> {
     let db = query_ctx.get_db_string();
     query_ctx.set_channel(Channel::Otlp);
@@ -47,7 +47,8 @@ pub async fn metrics(
     let _timer = crate::metrics::METRIC_HTTP_OPENTELEMETRY_METRICS_ELAPSED
         .with_label_values(&[db.as_str()])
         .start_timer();
-    let request = parse_metrics_body(body).await?;
+    let request =
+        ExportMetricsServiceRequest::decode(bytes).context(error::DecodeOtlpRequestSnafu)?;
 
     handler
         .metrics(request, query_ctx)
@@ -60,15 +61,6 @@ pub async fn metrics(
         })
 }
 
-async fn parse_metrics_body(body: Body) -> Result<ExportMetricsServiceRequest> {
-    hyper::body::to_bytes(body)
-        .await
-        .context(error::HyperSnafu)
-        .and_then(|buf| {
-            ExportMetricsServiceRequest::decode(&buf[..]).context(error::DecodeOtlpRequestSnafu)
-        })
-}
-
 pub struct OtlpMetricsResponse {
     resp_body: ExportMetricsServiceResponse,
     write_cost: usize,
@@ -88,7 +80,7 @@ impl IntoResponse for OtlpMetricsResponse {
 pub async fn traces(
     State(handler): State<OpenTelemetryProtocolHandlerRef>,
     Extension(mut query_ctx): Extension<QueryContext>,
-    RawBody(body): RawBody,
+    bytes: Bytes,
 ) -> Result<OtlpTracesResponse> {
     let db = query_ctx.get_db_string();
     query_ctx.set_channel(Channel::Otlp);
@@ -96,7 +88,8 @@ pub async fn traces(
     let _timer = crate::metrics::METRIC_HTTP_OPENTELEMETRY_TRACES_ELAPSED
         .with_label_values(&[db.as_str()])
         .start_timer();
-    let request = parse_traces_body(body).await?;
+    let request =
+        ExportTraceServiceRequest::decode(bytes).context(error::DecodeOtlpRequestSnafu)?;
     handler
         .traces(request, query_ctx)
         .await
@@ -108,15 +101,6 @@ pub async fn traces(
         })
 }
 
-async fn parse_traces_body(body: Body) -> Result<ExportTraceServiceRequest> {
-    hyper::body::to_bytes(body)
-        .await
-        .context(error::HyperSnafu)
-        .and_then(|buf| {
-            ExportTraceServiceRequest::decode(&buf[..]).context(error::DecodeOtlpRequestSnafu)
-        })
-}
-
 pub struct OtlpTracesResponse {
     resp_body: ExportTraceServiceResponse,
     write_cost: usize,
diff --git a/tests-integration/Cargo.toml b/tests-integration/Cargo.toml
index aa5f74540d..997214bca0 100644
--- a/tests-integration/Cargo.toml
+++ b/tests-integration/Cargo.toml
@@ -40,6 +40,7 @@ common-wal.workspace = true
 datanode = { workspace = true }
 datatypes.workspace = true
 dotenv.workspace = true
+flate2 = "1.0"
 flow.workspace = true
 frontend = { workspace = true, features = ["testing"] }
 futures.workspace = true
diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs
index 07237c8bc1..cf125a5776 100644
--- a/tests-integration/src/test_util.rs
+++ b/tests-integration/src/test_util.rs
@@ -425,6 +425,7 @@ pub async fn setup_test_http_app_with_frontend_and_user_provider(
             Some(instance.instance.clone()),
         )
         .with_log_ingest_handler(instance.instance.clone(), None)
+        .with_otlp_handler(instance.instance.clone())
         .with_greptime_config_options(instance.opts.to_toml().unwrap());
 
     if let Some(user_provider) = user_provider {
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index fe28387cd6..d467e42dd4 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -13,11 +13,17 @@
 // limitations under the License.
 
 use std::collections::BTreeMap;
+use std::io::Write;
 
 use api::prom_store::remote::WriteRequest;
 use auth::user_provider_from_option;
 use axum::http::{HeaderName, StatusCode};
 use common_error::status_code::StatusCode as ErrorCode;
+use flate2::write::GzEncoder;
+use flate2::Compression;
+use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest;
+use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
+use opentelemetry_proto::tonic::metrics::v1::ResourceMetrics;
 use prost::Message;
 use serde_json::{json, Value};
 use servers::http::error_result::ErrorResponse;
@@ -26,7 +32,7 @@ use servers::http::handler::HealthResponse;
 use servers::http::header::{GREPTIME_DB_HEADER_NAME, GREPTIME_TIMEZONE_HEADER_NAME};
 use servers::http::influxdb_result_v1::{InfluxdbOutput, InfluxdbV1Response};
 use servers::http::prometheus::{PrometheusJsonResponse, PrometheusResponse};
-use servers::http::test_helpers::TestClient;
+use servers::http::test_helpers::{TestClient, TestResponse};
 use servers::http::GreptimeQueryOutput;
 use servers::prom_store;
 use tests_integration::test_util::{
@@ -80,6 +86,9 @@ macro_rules! http_tests {
                 test_pipeline_api,
                 test_test_pipeline_api,
                 test_plain_text_ingestion,
+
+                test_otlp_metrics,
+                test_otlp_traces,
             );
         )*
     };
@@ -1391,19 +1400,7 @@ transform:
     assert_eq!(res.status(), StatusCode::OK);
     let resp = res.text().await;
 
-    let resp: Value = serde_json::from_str(&resp).unwrap();
-    let v = resp
-        .get("output")
-        .unwrap()
-        .as_array()
-        .unwrap()
-        .first()
-        .unwrap()
-        .get("records")
-        .unwrap()
-        .get("rows")
-        .unwrap()
-        .to_string();
+    let v = get_rows_from_output(&resp);
 
     assert_eq!(
         v,
@@ -1412,3 +1409,130 @@ transform:
 
     guard.remove_all().await;
 }
+
+pub async fn test_otlp_metrics(store_type: StorageType) {
+    // init
+    common_telemetry::init_default_ut_logging();
+    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_otlp_metrics").await;
+
+    let content = r#"
+{"resource":{"attributes":[],"droppedAttributesCount":0},"scopeMetrics":[{"scope":{"name":"","version":"","attributes":[],"droppedAttributesCount":0},"metrics":[{"name":"gen","description":"","unit":"","data":{"gauge":{"dataPoints":[{"attributes":[],"startTimeUnixNano":0,"timeUnixNano":1726053452870391000,"exemplars":[],"flags":0,"value":{"asInt":9471}}]}}}],"schemaUrl":""}],"schemaUrl":"https://opentelemetry.io/schemas/1.13.0"}
+    "#;
+
+    let metrics: ResourceMetrics = serde_json::from_str(content).unwrap();
+    let req = ExportMetricsServiceRequest {
+        resource_metrics: vec![metrics],
+    };
+    let body = req.encode_to_vec();
+
+    // handshake
+    let client = TestClient::new(app);
+
+    // write metrics data
+    let res = send_req(&client, "/v1/otlp/v1/metrics", body.clone(), false).await;
+    assert_eq!(StatusCode::OK, res.status());
+
+    // select metrics data
+    let expected = r#"[[1726053452870391000,9471.0]]"#;
+    validate_data(&client, "select * from gen;", expected).await;
+
+    // drop table
+    let res = client.get("/v1/sql?sql=drop table gen;").send().await;
+    assert_eq!(res.status(), StatusCode::OK);
+
+    // write metrics data with gzip
+    let res = send_req(&client, "/v1/otlp/v1/metrics", body.clone(), true).await;
+    assert_eq!(StatusCode::OK, res.status());
+
+    // select metrics data again
+    validate_data(&client, "select * from gen;", expected).await;
+
+    guard.remove_all().await;
+}
+
+pub async fn test_otlp_traces(store_type: StorageType) {
+    // init
+    common_telemetry::init_default_ut_logging();
+    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_otlp_traces").await;
+
+    let content = r#"
+{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"telemetrygen"}}],"droppedAttributesCount":0},"scopeSpans":[{"scope":{"name":"telemetrygen","version":"","attributes":[],"droppedAttributesCount":0},"spans":[{"traceId":"b5e5fb572cf0a3335dd194a14145fef5","spanId":"74c82efa6f628e80","traceState":"","parentSpanId":"3364d2da58c9fd2b","flags":0,"name":"okey-dokey-0","kind":2,"startTimeUnixNano":1726631197820927000,"endTimeUnixNano":1726631197821050000,"attributes":[{"key":"net.peer.ip","value":{"stringValue":"1.2.3.4"}},{"key":"peer.service","value":{"stringValue":"telemetrygen-client"}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"links":[],"droppedLinksCount":0,"status":{"message":"","code":0}},{"traceId":"b5e5fb572cf0a3335dd194a14145fef5","spanId":"3364d2da58c9fd2b","traceState":"","parentSpanId":"","flags":0,"name":"lets-go","kind":3,"startTimeUnixNano":1726631197820927000,"endTimeUnixNano":1726631197821050000,"attributes":[{"key":"net.peer.ip","value":{"stringValue":"1.2.3.4"}},{"key":"peer.service","value":{"stringValue":"telemetrygen-server"}}],"droppedAttributesCount":0,"events":[],"droppedEventsCount":0,"links":[],"droppedLinksCount":0,"status":{"message":"","code":0}}],"schemaUrl":""}],"schemaUrl":"https://opentelemetry.io/schemas/1.4.0"}]}
+    "#;
+
+    let req: ExportTraceServiceRequest = serde_json::from_str(content).unwrap();
+    let body = req.encode_to_vec();
+
+    // handshake
+    let client = TestClient::new(app);
+
+    // write traces data
+    let res = send_req(&client, "/v1/otlp/v1/traces", body.clone(), false).await;
+    assert_eq!(StatusCode::OK, res.status());
+
+    // select traces data
+    let expected = r#"[["b5e5fb572cf0a3335dd194a14145fef5","3364d2da58c9fd2b","","{\"service.name\":\"telemetrygen\"}","telemetrygen","","{}","","lets-go","SPAN_KIND_CLIENT","STATUS_CODE_UNSET","","{\"net.peer.ip\":\"1.2.3.4\",\"peer.service\":\"telemetrygen-server\"}","[]","[]",1726631197820927000,1726631197821050000,0.123,1726631197820927000],["b5e5fb572cf0a3335dd194a14145fef5","74c82efa6f628e80","3364d2da58c9fd2b","{\"service.name\":\"telemetrygen\"}","telemetrygen","","{}","","okey-dokey-0","SPAN_KIND_SERVER","STATUS_CODE_UNSET","","{\"net.peer.ip\":\"1.2.3.4\",\"peer.service\":\"telemetrygen-client\"}","[]","[]",1726631197820927000,1726631197821050000,0.123,1726631197820927000]]"#;
+    validate_data(&client, "select * from traces_preview_v01;", expected).await;
+
+    // drop table
+    let res = client
+        .get("/v1/sql?sql=drop table traces_preview_v01;")
+        .send()
+        .await;
+    assert_eq!(res.status(), StatusCode::OK);
+
+    // write metrics data with gzip
+    let res = send_req(&client, "/v1/otlp/v1/traces", body.clone(), true).await;
+    assert_eq!(StatusCode::OK, res.status());
+
+    // select metrics data again
+    validate_data(&client, "select * from traces_preview_v01;", expected).await;
+
+    guard.remove_all().await;
+}
+
+async fn validate_data(client: &TestClient, sql: &str, expected: &str) {
+    let res = client
+        .get(format!("/v1/sql?sql={sql}").as_str())
+        .send()
+        .await;
+    assert_eq!(res.status(), StatusCode::OK);
+    let resp = res.text().await;
+    let v = get_rows_from_output(&resp);
+
+    assert_eq!(v, expected);
+}
+
+async fn send_req(client: &TestClient, path: &str, body: Vec<u8>, with_gzip: bool) -> TestResponse {
+    let mut req = client
+        .post(path)
+        .header("content-type", "application/x-protobuf");
+
+    let mut len = body.len();
+
+    if with_gzip {
+        let encoded = compress_vec_with_gzip(body);
+        len = encoded.len();
+        req = req.header("content-encoding", "gzip").body(encoded);
+    } else {
+        req = req.body(body);
+    }
+
+    req.header("content-length", len).send().await
+}
+
+fn get_rows_from_output(output: &str) -> String {
+    let resp: Value = serde_json::from_str(output).unwrap();
+    resp.get("output")
+        .and_then(Value::as_array)
+        .and_then(|v| v.first())
+        .and_then(|v| v.get("records"))
+        .and_then(|v| v.get("rows"))
+        .unwrap()
+        .to_string()
+}
+
+fn compress_vec_with_gzip(data: Vec<u8>) -> Vec<u8> {
+    let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
+    encoder.write_all(&data).unwrap();
+    encoder.finish().unwrap()
+}

From 0847ff36ce49c3c560c7a828378c49ce010817b1 Mon Sep 17 00:00:00 2001
From: zyy17 <zyylsxm@gmail.com>
Date: Wed, 18 Sep 2024 15:41:25 +0800
Subject: [PATCH 008/128] fix: config test failed and use
 `similar_asserts::assert_eq` to replace `assert_eq` for long string compare
 (#4731)

* fix: config test failed and use 'similar_asserts::assert_eq' to replace 'assert_eq' for long string compare

* Update Cargo.toml

Co-authored-by: Yingwen <realevenyag@gmail.com>

* Update src/cmd/tests/load_config_test.rs

Co-authored-by: Yingwen <realevenyag@gmail.com>

---------

Co-authored-by: Ruihang Xia <waynestxia@gmail.com>
Co-authored-by: Yingwen <realevenyag@gmail.com>
---
 Cargo.lock                        | 39 ++++++++++++++++++++++++++++---
 Cargo.toml                        |  1 +
 src/cmd/Cargo.toml                |  1 +
 src/cmd/tests/load_config_test.rs | 37 +++++++----------------------
 tests-integration/Cargo.toml      |  1 +
 tests-integration/tests/http.rs   |  2 +-
 6 files changed, 49 insertions(+), 32 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 0e9db00527..b121698979 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1175,6 +1175,17 @@ dependencies = [
  "regex-automata 0.1.10",
 ]
 
+[[package]]
+name = "bstr"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c"
+dependencies = [
+ "memchr",
+ "regex-automata 0.4.7",
+ "serde",
+]
+
 [[package]]
 name = "btoi"
 version = "0.4.3"
@@ -1761,6 +1772,7 @@ dependencies = [
  "serde_json",
  "servers",
  "session",
+ "similar-asserts",
  "snafu 0.8.4",
  "store-api",
  "substrait 0.9.3",
@@ -9657,7 +9669,7 @@ source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b
 dependencies = [
  "ascii",
  "bitflags 1.3.2",
- "bstr",
+ "bstr 0.2.17",
  "cfg-if",
  "hexf-parse",
  "itertools 0.10.5",
@@ -9692,7 +9704,7 @@ version = "0.2.0"
 source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e"
 dependencies = [
  "bitflags 1.3.2",
- "bstr",
+ "bstr 0.2.17",
  "itertools 0.10.5",
  "lz4_flex 0.9.5",
  "num-bigint",
@@ -9845,7 +9857,7 @@ dependencies = [
  "ascii",
  "atty",
  "bitflags 1.3.2",
- "bstr",
+ "bstr 0.2.17",
  "caseless",
  "cfg-if",
  "chrono",
@@ -10613,6 +10625,26 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
 
+[[package]]
+name = "similar"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e"
+dependencies = [
+ "bstr 1.10.0",
+ "unicode-segmentation",
+]
+
+[[package]]
+name = "similar-asserts"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfe85670573cd6f0fa97940f26e7e6601213c3b0555246c24234131f88c5709e"
+dependencies = [
+ "console",
+ "similar",
+]
+
 [[package]]
 name = "simple_asn1"
 version = "0.6.2"
@@ -11801,6 +11833,7 @@ dependencies = [
  "serde_json",
  "servers",
  "session",
+ "similar-asserts",
  "snafu 0.8.4",
  "sql",
  "sqlx",
diff --git a/Cargo.toml b/Cargo.toml
index 6720671402..3854ebc084 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -169,6 +169,7 @@ shadow-rs = "0.31"
 smallvec = { version = "1", features = ["serde"] }
 snafu = "0.8"
 sysinfo = "0.30"
+similar-asserts = "1.6.0"
 # on branch v0.44.x
 sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "54a267ac89c09b11c0c88934690530807185d3e7", features = [
     "visitor",
diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml
index 71841aabec..b57d221187 100644
--- a/src/cmd/Cargo.toml
+++ b/src/cmd/Cargo.toml
@@ -70,6 +70,7 @@ serde.workspace = true
 serde_json.workspace = true
 servers.workspace = true
 session.workspace = true
+similar-asserts.workspace = true
 snafu.workspace = true
 store-api.workspace = true
 substrait.workspace = true
diff --git a/src/cmd/tests/load_config_test.rs b/src/cmd/tests/load_config_test.rs
index 68d67a53a4..7a50234431 100644
--- a/src/cmd/tests/load_config_test.rs
+++ b/src/cmd/tests/load_config_test.rs
@@ -16,12 +16,10 @@ use std::time::Duration;
 
 use cmd::options::GreptimeOptions;
 use cmd::standalone::StandaloneOptions;
-use common_base::readable_size::ReadableSize;
 use common_config::Configurable;
 use common_grpc::channel_manager::{
     DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE, DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
 };
-use common_runtime::global::RuntimeOptions;
 use common_telemetry::logging::{LoggingOptions, DEFAULT_OTLP_ENDPOINT};
 use common_wal::config::raft_engine::RaftEngineConfig;
 use common_wal::config::DatanodeWalConfig;
@@ -45,10 +43,6 @@ fn test_load_datanode_example_config() {
             .unwrap();
 
     let expected = GreptimeOptions::<DatanodeOptions> {
-        runtime: RuntimeOptions {
-            global_rt_size: 8,
-            compact_rt_size: 4,
-        },
         component: DatanodeOptions {
             node_id: Some(42),
             meta_client: Some(MetaClientOptions {
@@ -76,8 +70,6 @@ fn test_load_datanode_example_config() {
                 RegionEngineConfig::Mito(MitoConfig {
                     auto_flush_interval: Duration::from_secs(3600),
                     scan_parallelism: 0,
-                    global_write_buffer_reject_size: ReadableSize::gb(2),
-                    max_background_jobs: 4,
                     experimental_write_cache_ttl: Some(Duration::from_secs(60 * 60 * 8)),
                     ..Default::default()
                 }),
@@ -102,9 +94,10 @@ fn test_load_datanode_example_config() {
             rpc_max_send_message_size: Some(DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE),
             ..Default::default()
         },
+        ..Default::default()
     };
 
-    assert_eq!(options, expected);
+    similar_asserts::assert_eq!(options, expected);
 }
 
 #[test]
@@ -114,10 +107,6 @@ fn test_load_frontend_example_config() {
         GreptimeOptions::<FrontendOptions>::load_layered_options(example_config.to_str(), "")
             .unwrap();
     let expected = GreptimeOptions::<FrontendOptions> {
-        runtime: RuntimeOptions {
-            global_rt_size: 8,
-            compact_rt_size: 4,
-        },
         component: FrontendOptions {
             default_timezone: Some("UTC".to_string()),
             meta_client: Some(MetaClientOptions {
@@ -150,8 +139,9 @@ fn test_load_frontend_example_config() {
             },
             ..Default::default()
         },
+        ..Default::default()
     };
-    assert_eq!(options, expected);
+    similar_asserts::assert_eq!(options, expected);
 }
 
 #[test]
@@ -161,10 +151,6 @@ fn test_load_metasrv_example_config() {
         GreptimeOptions::<MetasrvOptions>::load_layered_options(example_config.to_str(), "")
             .unwrap();
     let expected = GreptimeOptions::<MetasrvOptions> {
-        runtime: RuntimeOptions {
-            global_rt_size: 8,
-            compact_rt_size: 4,
-        },
         component: MetasrvOptions {
             selector: SelectorType::default(),
             data_home: "/tmp/metasrv/".to_string(),
@@ -182,8 +168,9 @@ fn test_load_metasrv_example_config() {
             },
             ..Default::default()
         },
+        ..Default::default()
     };
-    assert_eq!(options, expected);
+    similar_asserts::assert_eq!(options, expected);
 }
 
 #[test]
@@ -193,10 +180,6 @@ fn test_load_standalone_example_config() {
         GreptimeOptions::<StandaloneOptions>::load_layered_options(example_config.to_str(), "")
             .unwrap();
     let expected = GreptimeOptions::<StandaloneOptions> {
-        runtime: RuntimeOptions {
-            global_rt_size: 8,
-            compact_rt_size: 4,
-        },
         component: StandaloneOptions {
             default_timezone: Some("UTC".to_string()),
             wal: DatanodeWalConfig::RaftEngine(RaftEngineConfig {
@@ -208,11 +191,8 @@ fn test_load_standalone_example_config() {
             region_engine: vec![
                 RegionEngineConfig::Mito(MitoConfig {
                     auto_flush_interval: Duration::from_secs(3600),
-                    scan_parallelism: 0,
-                    global_write_buffer_reject_size: ReadableSize::gb(2),
-                    sst_meta_cache_size: ReadableSize::mb(128),
-                    max_background_jobs: 4,
                     experimental_write_cache_ttl: Some(Duration::from_secs(60 * 60 * 8)),
+                    scan_parallelism: 0,
                     ..Default::default()
                 }),
                 RegionEngineConfig::File(EngineConfig {}),
@@ -234,6 +214,7 @@ fn test_load_standalone_example_config() {
             },
             ..Default::default()
         },
+        ..Default::default()
     };
-    assert_eq!(options, expected);
+    similar_asserts::assert_eq!(options, expected);
 }
diff --git a/tests-integration/Cargo.toml b/tests-integration/Cargo.toml
index 997214bca0..80fb76c55a 100644
--- a/tests-integration/Cargo.toml
+++ b/tests-integration/Cargo.toml
@@ -60,6 +60,7 @@ rstest_reuse.workspace = true
 serde_json.workspace = true
 servers = { workspace = true, features = ["testing"] }
 session.workspace = true
+similar-asserts.workspace = true
 snafu.workspace = true
 sql.workspace = true
 sqlx = { version = "0.6", features = [
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index d467e42dd4..e11060fbbd 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -888,7 +888,7 @@ write_interval = "30s"
     .trim()
     .to_string();
     let body_text = drop_lines_with_inconsistent_results(res_get.text().await);
-    assert_eq!(body_text, expected_toml_str);
+    similar_asserts::assert_eq!(body_text, expected_toml_str);
 }
 
 fn drop_lines_with_inconsistent_results(input: String) -> String {

From 50b3bb4c0ddf935a1bf6cc3e9525679c903397a4 Mon Sep 17 00:00:00 2001
From: shuiyisong <113876041+shuiyisong@users.noreply.github.com>
Date: Wed, 18 Sep 2024 17:19:05 +0800
Subject: [PATCH 009/128] fix: sort cargo toml (#4735)

---
 Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index 3854ebc084..d199caac09 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -166,10 +166,10 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["float_roundtrip"] }
 serde_with = "3"
 shadow-rs = "0.31"
+similar-asserts = "1.6.0"
 smallvec = { version = "1", features = ["serde"] }
 snafu = "0.8"
 sysinfo = "0.30"
-similar-asserts = "1.6.0"
 # on branch v0.44.x
 sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "54a267ac89c09b11c0c88934690530807185d3e7", features = [
     "visitor",

From f73fb8213309c71de271fb0060dacedddcbb2d2a Mon Sep 17 00:00:00 2001
From: Yohan Wal <59358312+CookiePieWw@users.noreply.github.com>
Date: Wed, 18 Sep 2024 19:07:30 +0800
Subject: [PATCH 010/128] feat: add respective `json_is` UDFs for JSON type
 (#4726)

* feat: add respective json_is UDFs

* refactor: rename to_json to parse_json

* chore: happy clippy

* chore: some rename

* fix: small fixes
---
 src/common/function/src/scalars/json.rs       |  18 +-
 .../function/src/scalars/json/json_get.rs     |   2 +-
 .../function/src/scalars/json/json_is.rs      | 215 ++++++++++++++++++
 .../src/scalars/json/json_to_string.rs        |   2 +-
 .../json/{to_json.rs => parse_json.rs}        |  21 +-
 .../cases/standalone/common/function/json.sql |  56 -----
 .../{json.result => json/json_get.result}     | 151 ++++++++----
 .../common/function/json/json_get.sql         |  79 +++++++
 .../common/function/json/json_is.result       | 175 ++++++++++++++
 .../common/function/json/json_is.sql          |  53 +++++
 .../standalone/common/types/json/json.result  |  30 +--
 .../standalone/common/types/json/json.sql     |  30 +--
 12 files changed, 688 insertions(+), 144 deletions(-)
 create mode 100644 src/common/function/src/scalars/json/json_is.rs
 rename src/common/function/src/scalars/json/{to_json.rs => parse_json.rs} (92%)
 delete mode 100644 tests/cases/standalone/common/function/json.sql
 rename tests/cases/standalone/common/function/{json.result => json/json_get.result} (61%)
 create mode 100644 tests/cases/standalone/common/function/json/json_get.sql
 create mode 100644 tests/cases/standalone/common/function/json/json_is.result
 create mode 100644 tests/cases/standalone/common/function/json/json_is.sql

diff --git a/src/common/function/src/scalars/json.rs b/src/common/function/src/scalars/json.rs
index 26d63d3b45..279577b495 100644
--- a/src/common/function/src/scalars/json.rs
+++ b/src/common/function/src/scalars/json.rs
@@ -14,12 +14,16 @@
 
 use std::sync::Arc;
 mod json_get;
+mod json_is;
 mod json_to_string;
-mod to_json;
+mod parse_json;
 
 use json_get::{JsonGetBool, JsonGetFloat, JsonGetInt, JsonGetString};
+use json_is::{
+    JsonIsArray, JsonIsBool, JsonIsFloat, JsonIsInt, JsonIsNull, JsonIsObject, JsonIsString,
+};
 use json_to_string::JsonToStringFunction;
-use to_json::ToJsonFunction;
+use parse_json::ParseJsonFunction;
 
 use crate::function_registry::FunctionRegistry;
 
@@ -28,11 +32,19 @@ pub(crate) struct JsonFunction;
 impl JsonFunction {
     pub fn register(registry: &FunctionRegistry) {
         registry.register(Arc::new(JsonToStringFunction));
-        registry.register(Arc::new(ToJsonFunction));
+        registry.register(Arc::new(ParseJsonFunction));
 
         registry.register(Arc::new(JsonGetInt));
         registry.register(Arc::new(JsonGetFloat));
         registry.register(Arc::new(JsonGetString));
         registry.register(Arc::new(JsonGetBool));
+
+        registry.register(Arc::new(JsonIsNull));
+        registry.register(Arc::new(JsonIsInt));
+        registry.register(Arc::new(JsonIsFloat));
+        registry.register(Arc::new(JsonIsString));
+        registry.register(Arc::new(JsonIsBool));
+        registry.register(Arc::new(JsonIsArray));
+        registry.register(Arc::new(JsonIsObject));
     }
 }
diff --git a/src/common/function/src/scalars/json/json_get.rs b/src/common/function/src/scalars/json/json_get.rs
index 78ddc1d264..d31f7a0c6e 100644
--- a/src/common/function/src/scalars/json/json_get.rs
+++ b/src/common/function/src/scalars/json/json_get.rs
@@ -47,7 +47,7 @@ fn get_json_by_path(json: &[u8], path: &str) -> Option<Vec<u8>> {
 /// If the path does not exist or the value is not the type specified, return `NULL`.
 macro_rules! json_get {
     // e.g. name = JsonGetInt, type = Int64, rust_type = i64, doc = "Get the value from the JSONB by the given path and return it as an integer."
-    ($name: ident, $type: ident, $rust_type: ident, $doc:expr) => {
+    ($name:ident, $type:ident, $rust_type:ident, $doc:expr) => {
         paste::paste! {
             #[doc = $doc]
             #[derive(Clone, Debug, Default)]
diff --git a/src/common/function/src/scalars/json/json_is.rs b/src/common/function/src/scalars/json/json_is.rs
new file mode 100644
index 0000000000..e0580ad9d4
--- /dev/null
+++ b/src/common/function/src/scalars/json/json_is.rs
@@ -0,0 +1,215 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::{self, Display};
+
+use common_query::error::{InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu};
+use common_query::prelude::Signature;
+use datafusion::logical_expr::Volatility;
+use datatypes::data_type::ConcreteDataType;
+use datatypes::prelude::VectorRef;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::vectors::{BooleanVectorBuilder, MutableVector};
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+
+/// Checks if the input is a JSON object of the given type.
+macro_rules! json_is {
+    ($name:ident, $json_type:ident, $doc:expr) => {
+        paste::paste! {
+            #[derive(Clone, Debug, Default)]
+            pub struct $name;
+
+            impl Function for $name {
+                fn name(&self) -> &str {
+                    stringify!([<$name:snake>])
+                }
+
+                fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+                    Ok(ConcreteDataType::boolean_datatype())
+                }
+
+                fn signature(&self) -> Signature {
+                    Signature::exact(vec![ConcreteDataType::json_datatype()], Volatility::Immutable)
+                }
+
+                fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+                    ensure!(
+                        columns.len() == 1,
+                        InvalidFuncArgsSnafu {
+                            err_msg: format!(
+                                "The length of the args is not correct, expect exactly one, have: {}",
+                                columns.len()
+                            ),
+                        }
+                    );
+
+                    let jsons = &columns[0];
+                    let size = jsons.len();
+                    let datatype = jsons.data_type();
+                    let mut results = BooleanVectorBuilder::with_capacity(size);
+
+                    match datatype {
+                        // JSON data type uses binary vector
+                        ConcreteDataType::Binary(_) => {
+                            for i in 0..size {
+                                let json = jsons.get_ref(i);
+                                let json = json.as_binary();
+                                let result = match json {
+                                    Ok(Some(json)) => {
+                                        Some(jsonb::[<is_ $json_type>](json))
+                                    }
+                                    _ => None,
+                                };
+                                results.push(result);
+                            }
+                        }
+                        _ => {
+                            return UnsupportedInputDataTypeSnafu {
+                                function: stringify!([<$name:snake>]),
+                                datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
+                            }
+                            .fail();
+                        }
+                    }
+
+                    Ok(results.to_vector())
+                }
+            }
+
+            impl Display for $name {
+                fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+                    write!(f, "{}", stringify!([<$name:snake>]).to_ascii_uppercase())
+                }
+            }
+        }
+    }
+}
+
+json_is!(JsonIsNull, null, "Checks if the input JSONB is null");
+json_is!(
+    JsonIsBool,
+    boolean,
+    "Checks if the input JSONB is a boolean type JSON value"
+);
+json_is!(
+    JsonIsInt,
+    i64,
+    "Checks if the input JSONB is a integer type JSON value"
+);
+json_is!(
+    JsonIsFloat,
+    number,
+    "Checks if the input JSONB is a JSON float"
+);
+json_is!(
+    JsonIsString,
+    string,
+    "Checks if the input JSONB is a JSON string"
+);
+json_is!(
+    JsonIsArray,
+    array,
+    "Checks if the input JSONB is a JSON array"
+);
+json_is!(
+    JsonIsObject,
+    object,
+    "Checks if the input JSONB is a JSON object"
+);
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use datatypes::scalars::ScalarVector;
+    use datatypes::vectors::BinaryVector;
+
+    use super::*;
+
+    #[test]
+    fn test_json_is_functions() {
+        let json_is_functions: [&dyn Function; 6] = [
+            &JsonIsBool,
+            &JsonIsInt,
+            &JsonIsFloat,
+            &JsonIsString,
+            &JsonIsArray,
+            &JsonIsObject,
+        ];
+        let expected_names = [
+            "json_is_bool",
+            "json_is_int",
+            "json_is_float",
+            "json_is_string",
+            "json_is_array",
+            "json_is_object",
+        ];
+        for (func, expected_name) in json_is_functions.iter().zip(expected_names.iter()) {
+            assert_eq!(func.name(), *expected_name);
+            assert_eq!(
+                func.return_type(&[ConcreteDataType::json_datatype()])
+                    .unwrap(),
+                ConcreteDataType::boolean_datatype()
+            );
+            assert_eq!(
+                func.signature(),
+                Signature::exact(
+                    vec![ConcreteDataType::json_datatype()],
+                    Volatility::Immutable
+                )
+            );
+        }
+
+        let json_strings = [
+            r#"true"#,
+            r#"1"#,
+            r#"1.0"#,
+            r#""The pig fly through a castle, and has been attracted by the princess.""#,
+            r#"[1, 2]"#,
+            r#"{"a": 1}"#,
+        ];
+        let expected_results = [
+            [true, false, false, false, false, false],
+            [false, true, false, false, false, false],
+            // Integers are also floats
+            [false, true, true, false, false, false],
+            [false, false, false, true, false, false],
+            [false, false, false, false, true, false],
+            [false, false, false, false, false, true],
+        ];
+
+        let jsonbs = json_strings
+            .iter()
+            .map(|s| {
+                let value = jsonb::parse_value(s.as_bytes()).unwrap();
+                value.to_vec()
+            })
+            .collect::<Vec<_>>();
+        let json_vector = BinaryVector::from_vec(jsonbs);
+        let args: Vec<VectorRef> = vec![Arc::new(json_vector)];
+
+        for (func, expected_result) in json_is_functions.iter().zip(expected_results.iter()) {
+            let vector = func.eval(FunctionContext::default(), &args).unwrap();
+            assert_eq!(vector.len(), json_strings.len());
+
+            for (i, expected) in expected_result.iter().enumerate() {
+                let result = vector.get_ref(i);
+                let result = result.as_boolean().unwrap().unwrap();
+                assert_eq!(result, *expected);
+            }
+        }
+    }
+}
diff --git a/src/common/function/src/scalars/json/json_to_string.rs b/src/common/function/src/scalars/json/json_to_string.rs
index 8a5e569a14..9873000d6e 100644
--- a/src/common/function/src/scalars/json/json_to_string.rs
+++ b/src/common/function/src/scalars/json/json_to_string.rs
@@ -119,7 +119,7 @@ mod tests {
     use super::*;
 
     #[test]
-    fn test_get_by_path_function() {
+    fn test_json_to_string_function() {
         let json_to_string = JsonToStringFunction;
 
         assert_eq!("json_to_string", json_to_string.name());
diff --git a/src/common/function/src/scalars/json/to_json.rs b/src/common/function/src/scalars/json/parse_json.rs
similarity index 92%
rename from src/common/function/src/scalars/json/to_json.rs
rename to src/common/function/src/scalars/json/parse_json.rs
index 9c3cc90b66..64300838d8 100644
--- a/src/common/function/src/scalars/json/to_json.rs
+++ b/src/common/function/src/scalars/json/parse_json.rs
@@ -27,11 +27,11 @@ use crate::function::{Function, FunctionContext};
 
 /// Parses the `String` into `JSONB`.
 #[derive(Clone, Debug, Default)]
-pub struct ToJsonFunction;
+pub struct ParseJsonFunction;
 
-const NAME: &str = "to_json";
+const NAME: &str = "parse_json";
 
-impl Function for ToJsonFunction {
+impl Function for ParseJsonFunction {
     fn name(&self) -> &str {
         NAME
     }
@@ -101,9 +101,9 @@ impl Function for ToJsonFunction {
     }
 }
 
-impl Display for ToJsonFunction {
+impl Display for ParseJsonFunction {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "TO_JSON")
+        write!(f, "PARSE_JSON")
     }
 }
 
@@ -119,17 +119,17 @@ mod tests {
 
     #[test]
     fn test_get_by_path_function() {
-        let to_json = ToJsonFunction;
+        let parse_json = ParseJsonFunction;
 
-        assert_eq!("to_json", to_json.name());
+        assert_eq!("parse_json", parse_json.name());
         assert_eq!(
             ConcreteDataType::json_datatype(),
-            to_json
+            parse_json
                 .return_type(&[ConcreteDataType::json_datatype()])
                 .unwrap()
         );
 
-        assert!(matches!(to_json.signature(),
+        assert!(matches!(parse_json.signature(),
                          Signature {
                              type_signature: TypeSignature::Exact(valid_types),
                              volatility: Volatility::Immutable
@@ -152,13 +152,12 @@ mod tests {
 
         let json_string_vector = StringVector::from_vec(json_strings.to_vec());
         let args: Vec<VectorRef> = vec![Arc::new(json_string_vector)];
-        let vector = to_json.eval(FunctionContext::default(), &args).unwrap();
+        let vector = parse_json.eval(FunctionContext::default(), &args).unwrap();
 
         assert_eq!(3, vector.len());
         for (i, gt) in jsonbs.iter().enumerate() {
             let result = vector.get_ref(i);
             let result = result.as_binary().unwrap().unwrap();
-            // remove whitespaces
             assert_eq!(gt, result);
         }
     }
diff --git a/tests/cases/standalone/common/function/json.sql b/tests/cases/standalone/common/function/json.sql
deleted file mode 100644
index c6214ae0f8..0000000000
--- a/tests/cases/standalone/common/function/json.sql
+++ /dev/null
@@ -1,56 +0,0 @@
--- json_get functions --
-SELECT json_get_int(to_json('{"a": {"b": {"c": 1}}}'), 'a.b.c');
-
-SELECT json_get_float(to_json('{"a": {"b": {"c": 1.234}}}'), 'a:b.c');
-
-SELECT json_get_string(to_json('{"a": {"b": {"c": "foo"}}}'), 'a.b:c');
-
-SELECT json_get_bool(to_json('{"a": {"b": {"c": true}}}'), 'a.b["c"]');
-
-SELECT json_get_int(to_json('{"a": {"b": {"c": {"d": 1}}}}'), 'a.b');
-
-SELECT json_get_string(to_json('{"a": {"b": {"c": {"d": 1}}}}'), 'a.b');
-
--- test functions with table rows --
-CREATE TABLE jsons(j JSON, ts timestamp time index);
-
-INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": 1}}}'), 1);
-
-INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": 1.234}}}'), 2);
-
-INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": "foo"}}}'), 3);
-
-INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": true}}}'), 4);
-
-SELECT json_get_int(j, 'a.b.c') FROM jsons;
-
-SELECT json_get_float(j, 'a["b"].c') FROM jsons;
-
-SELECT json_get_string(j, 'a.b.c?(@ == 1)') FROM jsons;
-
-SELECT json_get_bool(j, 'a.b.c') FROM jsons;
-
-SELECT json_get_int(j, 'a.b["c"]') FROM jsons;
-
-DROP TABLE jsons;
-
--- test functions with arrays --
-CREATE TABLE jsons(j JSON, ts timestamp time index);
-
-INSERT INTO jsons VALUES(to_json('["a", "bcde", "", "Long time ago, there is a little pig flying in the sky"]'), 1);
-
-INSERT INTO jsons VALUES(to_json('[true, false, false, false]'), 2);
-
-INSERT INTO jsons VALUES(to_json('[1, 0, -2147483649, 2147483648]'), 3);
-
-INSERT INTO jsons VALUES(to_json('[1.2, 3.1415926535897932384626, -3e123, 1e100]'), 4);
-
-SELECT json_get_int(j, '[0]') FROM jsons;
-
-SELECT json_get_float(j, '[1]') FROM jsons;
-
-SELECT json_get_bool(j, '[2]') FROM jsons;
-
-SELECT json_get_string(j, '[3]') FROM jsons;
-
-DROP TABLE jsons;
diff --git a/tests/cases/standalone/common/function/json.result b/tests/cases/standalone/common/function/json/json_get.result
similarity index 61%
rename from tests/cases/standalone/common/function/json.result
rename to tests/cases/standalone/common/function/json/json_get.result
index f2a59b9d70..01767387a9 100644
--- a/tests/cases/standalone/common/function/json.result
+++ b/tests/cases/standalone/common/function/json/json_get.result
@@ -1,70 +1,70 @@
 -- json_get functions --
-SELECT json_get_int(to_json('{"a": {"b": {"c": 1}}}'), 'a.b.c');
+SELECT json_get_int(parse_json('{"a": {"b": {"c": 1}}}'), 'a.b.c');
 
-+---------------------------------------------------------------------+
-| json_get_int(to_json(Utf8("{"a": {"b": {"c": 1}}}")),Utf8("a.b.c")) |
-+---------------------------------------------------------------------+
-| 1                                                                   |
-+---------------------------------------------------------------------+
++------------------------------------------------------------------------+
+| json_get_int(parse_json(Utf8("{"a": {"b": {"c": 1}}}")),Utf8("a.b.c")) |
++------------------------------------------------------------------------+
+| 1                                                                      |
++------------------------------------------------------------------------+
 
-SELECT json_get_float(to_json('{"a": {"b": {"c": 1.234}}}'), 'a:b.c');
+SELECT json_get_float(parse_json('{"a": {"b": {"c": 1.234}}}'), 'a:b.c');
 
-+---------------------------------------------------------------------------+
-| json_get_float(to_json(Utf8("{"a": {"b": {"c": 1.234}}}")),Utf8("a:b.c")) |
-+---------------------------------------------------------------------------+
-| 1.234                                                                     |
-+---------------------------------------------------------------------------+
++------------------------------------------------------------------------------+
+| json_get_float(parse_json(Utf8("{"a": {"b": {"c": 1.234}}}")),Utf8("a:b.c")) |
++------------------------------------------------------------------------------+
+| 1.234                                                                        |
++------------------------------------------------------------------------------+
 
-SELECT json_get_string(to_json('{"a": {"b": {"c": "foo"}}}'), 'a.b:c');
+SELECT json_get_string(parse_json('{"a": {"b": {"c": "foo"}}}'), 'a.b:c');
 
-+----------------------------------------------------------------------------+
-| json_get_string(to_json(Utf8("{"a": {"b": {"c": "foo"}}}")),Utf8("a.b:c")) |
-+----------------------------------------------------------------------------+
-| foo                                                                        |
-+----------------------------------------------------------------------------+
++-------------------------------------------------------------------------------+
+| json_get_string(parse_json(Utf8("{"a": {"b": {"c": "foo"}}}")),Utf8("a.b:c")) |
++-------------------------------------------------------------------------------+
+| foo                                                                           |
++-------------------------------------------------------------------------------+
 
-SELECT json_get_bool(to_json('{"a": {"b": {"c": true}}}'), 'a.b["c"]');
+SELECT json_get_bool(parse_json('{"a": {"b": {"c": true}}}'), 'a.b["c"]');
 
-+----------------------------------------------------------------------------+
-| json_get_bool(to_json(Utf8("{"a": {"b": {"c": true}}}")),Utf8("a.b["c"]")) |
-+----------------------------------------------------------------------------+
-| true                                                                       |
-+----------------------------------------------------------------------------+
++-------------------------------------------------------------------------------+
+| json_get_bool(parse_json(Utf8("{"a": {"b": {"c": true}}}")),Utf8("a.b["c"]")) |
++-------------------------------------------------------------------------------+
+| true                                                                          |
++-------------------------------------------------------------------------------+
 
-SELECT json_get_int(to_json('{"a": {"b": {"c": {"d": 1}}}}'), 'a.b');
-
-+--------------------------------------------------------------------------+
-| json_get_int(to_json(Utf8("{"a": {"b": {"c": {"d": 1}}}}")),Utf8("a.b")) |
-+--------------------------------------------------------------------------+
-|                                                                          |
-+--------------------------------------------------------------------------+
-
-SELECT json_get_string(to_json('{"a": {"b": {"c": {"d": 1}}}}'), 'a.b');
+SELECT json_get_int(parse_json('{"a": {"b": {"c": {"d": 1}}}}'), 'a.b');
 
 +-----------------------------------------------------------------------------+
-| json_get_string(to_json(Utf8("{"a": {"b": {"c": {"d": 1}}}}")),Utf8("a.b")) |
+| json_get_int(parse_json(Utf8("{"a": {"b": {"c": {"d": 1}}}}")),Utf8("a.b")) |
 +-----------------------------------------------------------------------------+
 |                                                                             |
 +-----------------------------------------------------------------------------+
 
+SELECT json_get_string(parse_json('{"a": {"b": {"c": {"d": 1}}}}'), 'a.b');
+
++--------------------------------------------------------------------------------+
+| json_get_string(parse_json(Utf8("{"a": {"b": {"c": {"d": 1}}}}")),Utf8("a.b")) |
++--------------------------------------------------------------------------------+
+|                                                                                |
++--------------------------------------------------------------------------------+
+
 -- test functions with table rows --
 CREATE TABLE jsons(j JSON, ts timestamp time index);
 
 Affected Rows: 0
 
-INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": 1}}}'), 1);
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": 1}}}'), 1);
 
 Affected Rows: 1
 
-INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": 1.234}}}'), 2);
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": 1.234}}}'), 2);
 
 Affected Rows: 1
 
-INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": "foo"}}}'), 3);
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": "foo"}}}'), 3);
 
 Affected Rows: 1
 
-INSERT INTO jsons VALUES(to_json('{"a": {"b": {"c": true}}}'), 4);
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": true}}}'), 4);
 
 Affected Rows: 1
 
@@ -132,19 +132,19 @@ CREATE TABLE jsons(j JSON, ts timestamp time index);
 
 Affected Rows: 0
 
-INSERT INTO jsons VALUES(to_json('["a", "bcde", "", "Long time ago, there is a little pig flying in the sky"]'), 1);
+INSERT INTO jsons VALUES(parse_json('["a", "bcde", "", "Long time ago, there is a little pig flying in the sky"]'), 1);
 
 Affected Rows: 1
 
-INSERT INTO jsons VALUES(to_json('[true, false, false, false]'), 2);
+INSERT INTO jsons VALUES(parse_json('[true, false, false, false]'), 2);
 
 Affected Rows: 1
 
-INSERT INTO jsons VALUES(to_json('[1, 0, -2147483649, 2147483648]'), 3);
+INSERT INTO jsons VALUES(parse_json('[1, 0, -2147483649, 2147483648]'), 3);
 
 Affected Rows: 1
 
-INSERT INTO jsons VALUES(to_json('[1.2, 3.1415926535897932384626, -3e123, 1e100]'), 4);
+INSERT INTO jsons VALUES(parse_json('[1.2, 3.1415926535897932384626, -3e123, 1e100]'), 4);
 
 Affected Rows: 1
 
@@ -196,3 +196,70 @@ DROP TABLE jsons;
 
 Affected Rows: 0
 
+-- test functions in WHERE clause --
+CREATE TABLE jsons(j JSON, ts timestamp time index);
+
+Affected Rows: 0
+
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": 1}}}'), 1);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": 1.234}}}'), 2);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": "foo"}}}'), 3);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": true}}}'), 4);
+
+Affected Rows: 1
+
+SELECT json_to_string(j) FROM jsons WHERE json_get_int(j, 'a.b.c') = 1;
+
++-------------------------+
+| json_to_string(jsons.j) |
++-------------------------+
+| {"a":{"b":{"c":1}}}     |
+| {"a":{"b":{"c":true}}}  |
++-------------------------+
+
+SELECT json_to_string(j) FROM jsons WHERE json_get_float(j, 'a.b.c') = 1.234;
+
++-------------------------+
+| json_to_string(jsons.j) |
++-------------------------+
+| {"a":{"b":{"c":1.234}}} |
++-------------------------+
+
+SELECT json_to_string(j) FROM jsons WHERE json_get_string(j, 'a.b.c') = 'foo';
+
++-------------------------+
+| json_to_string(jsons.j) |
++-------------------------+
+| {"a":{"b":{"c":"foo"}}} |
++-------------------------+
+
+SELECT json_to_string(j) FROM jsons WHERE json_get_bool(j, 'a.b.c') = true;
+
++-------------------------+
+| json_to_string(jsons.j) |
++-------------------------+
+| {"a":{"b":{"c":true}}}  |
++-------------------------+
+
+SELECT json_to_string(j) FROM jsons WHERE CAST(json_get_int(j, 'a.b.c') AS BOOLEAN);
+
++-------------------------+
+| json_to_string(jsons.j) |
++-------------------------+
+| {"a":{"b":{"c":1}}}     |
+| {"a":{"b":{"c":true}}}  |
++-------------------------+
+
+DROP TABLE jsons;
+
+Affected Rows: 0
+
diff --git a/tests/cases/standalone/common/function/json/json_get.sql b/tests/cases/standalone/common/function/json/json_get.sql
new file mode 100644
index 0000000000..3247536b07
--- /dev/null
+++ b/tests/cases/standalone/common/function/json/json_get.sql
@@ -0,0 +1,79 @@
+-- json_get functions --
+SELECT json_get_int(parse_json('{"a": {"b": {"c": 1}}}'), 'a.b.c');
+
+SELECT json_get_float(parse_json('{"a": {"b": {"c": 1.234}}}'), 'a:b.c');
+
+SELECT json_get_string(parse_json('{"a": {"b": {"c": "foo"}}}'), 'a.b:c');
+
+SELECT json_get_bool(parse_json('{"a": {"b": {"c": true}}}'), 'a.b["c"]');
+
+SELECT json_get_int(parse_json('{"a": {"b": {"c": {"d": 1}}}}'), 'a.b');
+
+SELECT json_get_string(parse_json('{"a": {"b": {"c": {"d": 1}}}}'), 'a.b');
+
+-- test functions with table rows --
+CREATE TABLE jsons(j JSON, ts timestamp time index);
+
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": 1}}}'), 1);
+
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": 1.234}}}'), 2);
+
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": "foo"}}}'), 3);
+
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": true}}}'), 4);
+
+SELECT json_get_int(j, 'a.b.c') FROM jsons;
+
+SELECT json_get_float(j, 'a["b"].c') FROM jsons;
+
+SELECT json_get_string(j, 'a.b.c?(@ == 1)') FROM jsons;
+
+SELECT json_get_bool(j, 'a.b.c') FROM jsons;
+
+SELECT json_get_int(j, 'a.b["c"]') FROM jsons;
+
+DROP TABLE jsons;
+
+-- test functions with arrays --
+CREATE TABLE jsons(j JSON, ts timestamp time index);
+
+INSERT INTO jsons VALUES(parse_json('["a", "bcde", "", "Long time ago, there is a little pig flying in the sky"]'), 1);
+
+INSERT INTO jsons VALUES(parse_json('[true, false, false, false]'), 2);
+
+INSERT INTO jsons VALUES(parse_json('[1, 0, -2147483649, 2147483648]'), 3);
+
+INSERT INTO jsons VALUES(parse_json('[1.2, 3.1415926535897932384626, -3e123, 1e100]'), 4);
+
+SELECT json_get_int(j, '[0]') FROM jsons;
+
+SELECT json_get_float(j, '[1]') FROM jsons;
+
+SELECT json_get_bool(j, '[2]') FROM jsons;
+
+SELECT json_get_string(j, '[3]') FROM jsons;
+
+DROP TABLE jsons;
+
+-- test functions in WHERE clause --
+CREATE TABLE jsons(j JSON, ts timestamp time index);
+
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": 1}}}'), 1);
+
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": 1.234}}}'), 2);
+
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": "foo"}}}'), 3);
+
+INSERT INTO jsons VALUES(parse_json('{"a": {"b": {"c": true}}}'), 4);
+
+SELECT json_to_string(j) FROM jsons WHERE json_get_int(j, 'a.b.c') = 1;
+
+SELECT json_to_string(j) FROM jsons WHERE json_get_float(j, 'a.b.c') = 1.234;
+
+SELECT json_to_string(j) FROM jsons WHERE json_get_string(j, 'a.b.c') = 'foo';
+
+SELECT json_to_string(j) FROM jsons WHERE json_get_bool(j, 'a.b.c') = true;
+
+SELECT json_to_string(j) FROM jsons WHERE CAST(json_get_int(j, 'a.b.c') AS BOOLEAN);
+
+DROP TABLE jsons;
diff --git a/tests/cases/standalone/common/function/json/json_is.result b/tests/cases/standalone/common/function/json/json_is.result
new file mode 100644
index 0000000000..d162fa1611
--- /dev/null
+++ b/tests/cases/standalone/common/function/json/json_is.result
@@ -0,0 +1,175 @@
+-- json_is functions --
+SELECT json_is_object(parse_json('{"a": 1}'));
+
++----------------------------------------------+
+| json_is_object(parse_json(Utf8("{"a": 1}"))) |
++----------------------------------------------+
+| true                                         |
++----------------------------------------------+
+
+SELECT json_is_array(parse_json('[1, 2, 3]'));
+
++----------------------------------------------+
+| json_is_array(parse_json(Utf8("[1, 2, 3]"))) |
++----------------------------------------------+
+| true                                         |
++----------------------------------------------+
+
+SELECT json_is_int(parse_json('1'));
+
++------------------------------------+
+| json_is_int(parse_json(Utf8("1"))) |
++------------------------------------+
+| true                               |
++------------------------------------+
+
+SELECT json_is_bool(parse_json('true'));
+
++----------------------------------------+
+| json_is_bool(parse_json(Utf8("true"))) |
++----------------------------------------+
+| true                                   |
++----------------------------------------+
+
+SELECT json_is_null(parse_json('null'));
+
++----------------------------------------+
+| json_is_null(parse_json(Utf8("null"))) |
++----------------------------------------+
+| true                                   |
++----------------------------------------+
+
+SELECT json_is_float(parse_json('1.2'));
+
++----------------------------------------+
+| json_is_float(parse_json(Utf8("1.2"))) |
++----------------------------------------+
+| true                                   |
++----------------------------------------+
+
+SELECT json_is_string(parse_json('"foo"'));
+
++-------------------------------------------+
+| json_is_string(parse_json(Utf8(""foo""))) |
++-------------------------------------------+
+| true                                      |
++-------------------------------------------+
+
+SELECT json_is_null(parse_json('{"a": 1}'));
+
++--------------------------------------------+
+| json_is_null(parse_json(Utf8("{"a": 1}"))) |
++--------------------------------------------+
+| false                                      |
++--------------------------------------------+
+
+SELECT json_is_string(parse_json('[1, 2, 3]'));
+
++-----------------------------------------------+
+| json_is_string(parse_json(Utf8("[1, 2, 3]"))) |
++-----------------------------------------------+
+| false                                         |
++-----------------------------------------------+
+
+SELECT json_is_float(parse_json('1'));
+
++--------------------------------------+
+| json_is_float(parse_json(Utf8("1"))) |
++--------------------------------------+
+| true                                 |
++--------------------------------------+
+
+-- test json_is functions in table rows and WHERE clause --
+CREATE TABLE jsons(j JSON, ts timestamp time index);
+
+Affected Rows: 0
+
+INSERT INTO jsons VALUES(parse_json('{"a": 1}'), 1);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(parse_json('[1, 2, 3]'), 2);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(parse_json('1'), 3);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(parse_json('true'), 4);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(parse_json('null'), 5);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(parse_json('1.2'), 6);
+
+Affected Rows: 1
+
+INSERT INTO jsons VALUES(parse_json('"foo"'), 7);
+
+Affected Rows: 1
+
+SELECT json_to_string(j) FROM jsons WHERE json_is_object(j);
+
++-------------------------+
+| json_to_string(jsons.j) |
++-------------------------+
+| {"a":1}                 |
++-------------------------+
+
+SELECT json_to_string(j) FROM jsons WHERE json_is_array(j);
+
++-------------------------+
+| json_to_string(jsons.j) |
++-------------------------+
+| [1,2,3]                 |
++-------------------------+
+
+SELECT json_to_string(j) FROM jsons WHERE json_is_int(j);
+
++-------------------------+
+| json_to_string(jsons.j) |
++-------------------------+
+| 1                       |
++-------------------------+
+
+SELECT json_to_string(j) FROM jsons WHERE json_is_bool(j);
+
++-------------------------+
+| json_to_string(jsons.j) |
++-------------------------+
+| true                    |
++-------------------------+
+
+SELECT json_to_string(j) FROM jsons WHERE json_is_null(j);
+
++-------------------------+
+| json_to_string(jsons.j) |
++-------------------------+
+| null                    |
++-------------------------+
+
+SELECT json_to_string(j) FROM jsons WHERE json_is_float(j);
+
++-------------------------+
+| json_to_string(jsons.j) |
++-------------------------+
+| 1                       |
+| 1.2                     |
++-------------------------+
+
+SELECT json_to_string(j) FROM jsons WHERE json_is_string(j);
+
++-------------------------+
+| json_to_string(jsons.j) |
++-------------------------+
+| "foo"                   |
++-------------------------+
+
+DROP TABLE jsons;
+
+Affected Rows: 0
+
diff --git a/tests/cases/standalone/common/function/json/json_is.sql b/tests/cases/standalone/common/function/json/json_is.sql
new file mode 100644
index 0000000000..72e88d1da2
--- /dev/null
+++ b/tests/cases/standalone/common/function/json/json_is.sql
@@ -0,0 +1,53 @@
+-- json_is functions --
+SELECT json_is_object(parse_json('{"a": 1}'));
+
+SELECT json_is_array(parse_json('[1, 2, 3]'));
+
+SELECT json_is_int(parse_json('1'));
+
+SELECT json_is_bool(parse_json('true'));
+
+SELECT json_is_null(parse_json('null'));
+
+SELECT json_is_float(parse_json('1.2'));
+
+SELECT json_is_string(parse_json('"foo"'));
+
+SELECT json_is_null(parse_json('{"a": 1}'));
+
+SELECT json_is_string(parse_json('[1, 2, 3]'));
+
+SELECT json_is_float(parse_json('1'));
+
+-- test json_is functions in table rows and WHERE clause --
+CREATE TABLE jsons(j JSON, ts timestamp time index);
+
+INSERT INTO jsons VALUES(parse_json('{"a": 1}'), 1);
+
+INSERT INTO jsons VALUES(parse_json('[1, 2, 3]'), 2);
+
+INSERT INTO jsons VALUES(parse_json('1'), 3);
+
+INSERT INTO jsons VALUES(parse_json('true'), 4);
+
+INSERT INTO jsons VALUES(parse_json('null'), 5);
+
+INSERT INTO jsons VALUES(parse_json('1.2'), 6);
+
+INSERT INTO jsons VALUES(parse_json('"foo"'), 7);
+
+SELECT json_to_string(j) FROM jsons WHERE json_is_object(j);
+
+SELECT json_to_string(j) FROM jsons WHERE json_is_array(j);
+
+SELECT json_to_string(j) FROM jsons WHERE json_is_int(j);
+
+SELECT json_to_string(j) FROM jsons WHERE json_is_bool(j);
+
+SELECT json_to_string(j) FROM jsons WHERE json_is_null(j);
+
+SELECT json_to_string(j) FROM jsons WHERE json_is_float(j);
+
+SELECT json_to_string(j) FROM jsons WHERE json_is_string(j);
+
+DROP TABLE jsons;
diff --git a/tests/cases/standalone/common/types/json/json.result b/tests/cases/standalone/common/types/json/json.result
index 710d0b230a..8392b4cb4a 100644
--- a/tests/cases/standalone/common/types/json/json.result
+++ b/tests/cases/standalone/common/types/json/json.result
@@ -41,19 +41,19 @@ INSERT INTO jsons VALUES('[null]', 0),
 
 Affected Rows: 12
 
-INSERT INTO jsons VALUES(to_json('[null]'), 12),
-(to_json('[true]'), 13),
-(to_json('[false]'), 14),
-(to_json('[0]'), 15),
-(to_json('["foo"]'), 16),
-(to_json('[]'), 17),
-(to_json('{}'), 18),
-(to_json('[0,1]'), 19),
-(to_json('{"foo":"bar"}'), 20),
-(to_json('{"a":null,"foo":"bar"}'), 21),
-(to_json('[-1]'), 22),
-(to_json('[-2147483648]'), 23),
-(to_json('{"entities": {
+INSERT INTO jsons VALUES(parse_json('[null]'), 12),
+(parse_json('[true]'), 13),
+(parse_json('[false]'), 14),
+(parse_json('[0]'), 15),
+(parse_json('["foo"]'), 16),
+(parse_json('[]'), 17),
+(parse_json('{}'), 18),
+(parse_json('[0,1]'), 19),
+(parse_json('{"foo":"bar"}'), 20),
+(parse_json('{"a":null,"foo":"bar"}'), 21),
+(parse_json('[-1]'), 22),
+(parse_json('[-2147483648]'), 23),
+(parse_json('{"entities": {
             "description": {
                 "urls": [
                     {
@@ -117,11 +117,11 @@ DELETE FROM jsons;
 
 Affected Rows: 25
 
-INSERT INTO jsons VALUES(to_json('{"a":1, "b":2, "c":3'), 4);
+INSERT INTO jsons VALUES(parse_json('{"a":1, "b":2, "c":3'), 4);
 
 Error: 3001(EngineExecuteQuery), DataFusion error: Invalid function args: Cannot convert the string to json, have: {"a":1, "b":2, "c":3
 
-INSERT INTO jsons VALUES(to_json('Morning my friends, have a nice day :)'), 5);
+INSERT INTO jsons VALUES(parse_json('Morning my friends, have a nice day :)'), 5);
 
 Error: 3001(EngineExecuteQuery), DataFusion error: Invalid function args: Cannot convert the string to json, have: Morning my friends, have a nice day :)
 
diff --git a/tests/cases/standalone/common/types/json/json.sql b/tests/cases/standalone/common/types/json/json.sql
index 57fce9a8ea..868edc59e8 100644
--- a/tests/cases/standalone/common/types/json/json.sql
+++ b/tests/cases/standalone/common/types/json/json.sql
@@ -37,19 +37,19 @@ INSERT INTO jsons VALUES('[null]', 0),
     }
 }}', 11);
 
-INSERT INTO jsons VALUES(to_json('[null]'), 12),
-(to_json('[true]'), 13),
-(to_json('[false]'), 14),
-(to_json('[0]'), 15),
-(to_json('["foo"]'), 16),
-(to_json('[]'), 17),
-(to_json('{}'), 18),
-(to_json('[0,1]'), 19),
-(to_json('{"foo":"bar"}'), 20),
-(to_json('{"a":null,"foo":"bar"}'), 21),
-(to_json('[-1]'), 22),
-(to_json('[-2147483648]'), 23),
-(to_json('{"entities": {
+INSERT INTO jsons VALUES(parse_json('[null]'), 12),
+(parse_json('[true]'), 13),
+(parse_json('[false]'), 14),
+(parse_json('[0]'), 15),
+(parse_json('["foo"]'), 16),
+(parse_json('[]'), 17),
+(parse_json('{}'), 18),
+(parse_json('[0,1]'), 19),
+(parse_json('{"foo":"bar"}'), 20),
+(parse_json('{"a":null,"foo":"bar"}'), 21),
+(parse_json('[-1]'), 22),
+(parse_json('[-2147483648]'), 23),
+(parse_json('{"entities": {
             "description": {
                 "urls": [
                     {
@@ -79,9 +79,9 @@ SELECT json_to_string(j), t FROM jsons;
 --Insert invalid json strings--
 DELETE FROM jsons;
 
-INSERT INTO jsons VALUES(to_json('{"a":1, "b":2, "c":3'), 4);
+INSERT INTO jsons VALUES(parse_json('{"a":1, "b":2, "c":3'), 4);
 
-INSERT INTO jsons VALUES(to_json('Morning my friends, have a nice day :)'), 5);
+INSERT INTO jsons VALUES(parse_json('Morning my friends, have a nice day :)'), 5);
 
 SELECT json_to_string(j), t FROM jsons;
 

From befb6d85f09b6eec51147257b7976344da55e1ca Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Thu, 19 Sep 2024 07:17:39 +0900
Subject: [PATCH 011/128] fix: determine region role by using is_readonly
 (#4725)

fix: correct `is_writable` behavior
---
 src/mito2/src/engine.rs | 6 +++---
 src/mito2/src/region.rs | 5 +++++
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs
index 62912b8ffb..c0655eb690 100644
--- a/src/mito2/src/engine.rs
+++ b/src/mito2/src/engine.rs
@@ -462,10 +462,10 @@ impl EngineInner {
 
     fn role(&self, region_id: RegionId) -> Option<RegionRole> {
         self.workers.get_region(region_id).map(|region| {
-            if region.is_writable() {
-                RegionRole::Leader
-            } else {
+            if region.is_readonly() {
                 RegionRole::Follower
+            } else {
+                RegionRole::Leader
             }
         })
     }
diff --git a/src/mito2/src/region.rs b/src/mito2/src/region.rs
index 086fbef7d0..e2a4801643 100644
--- a/src/mito2/src/region.rs
+++ b/src/mito2/src/region.rs
@@ -174,6 +174,11 @@ impl MitoRegion {
         self.manifest_ctx.state.load() == RegionState::Writable
     }
 
+    /// Returns whether the region is readonly.
+    pub(crate) fn is_readonly(&self) -> bool {
+        self.manifest_ctx.state.load() == RegionState::ReadOnly
+    }
+
     /// Returns the state of the region.
     pub(crate) fn state(&self) -> RegionState {
         self.manifest_ctx.state.load()

From d1e0602c7687d0bcaae8670dce7b3b08b6e62951 Mon Sep 17 00:00:00 2001
From: discord9 <55937128+discord9@users.noreply.github.com>
Date: Thu, 19 Sep 2024 12:12:13 +0800
Subject: [PATCH 012/128] fix: opensrv Use After Free update (#4732)

* chore: version skew

* fix: even more version skew

* feat: use `ring` instead of `aws-lc` for remove nasm assembler on windows

* feat: use `ring` for pgwire

* feat: change to use `aws-lc-sys` on windows instead

* feat: change back to use `ring`

* chore: provide CryptoProvider

* feat: use upstream repo

* feat: install ring crypto lib in main

* chore: use same fn to install in tests

* feat: make pgwire use `ring`
---
 Cargo.lock                                   | 101 ++++++++++++++-----
 Cargo.toml                                   |   2 +-
 src/cmd/src/bin/greptime.rs                  |   4 +-
 src/cmd/src/error.rs                         |  13 ++-
 src/servers/Cargo.toml                       |  17 +++-
 src/servers/src/lib.rs                       |  16 +++
 src/servers/src/tls.rs                       |   2 +
 src/servers/tests/mysql/mysql_server_test.rs |   2 +
 src/servers/tests/postgres/mod.rs            |   3 +
 9 files changed, 122 insertions(+), 38 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index b121698979..fdad3a09be 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -811,6 +811,33 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "aws-lc-rs"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f95446d919226d587817a7d21379e6eb099b97b45110a7f272a444ca5c54070"
+dependencies = [
+ "aws-lc-sys",
+ "mirai-annotations",
+ "paste",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-lc-sys"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5055edc4a9a1b2a917a818258cdfb86a535947feebd9981adc99667a062c6f85"
+dependencies = [
+ "bindgen",
+ "cc",
+ "cmake",
+ "dunce",
+ "fs_extra",
+ "libc",
+ "paste",
+]
+
 [[package]]
 name = "axum"
 version = "0.6.20"
@@ -986,15 +1013,18 @@ dependencies = [
  "bitflags 2.5.0",
  "cexpr",
  "clang-sys",
- "itertools 0.12.1",
+ "itertools 0.10.5",
  "lazy_static",
  "lazycell",
+ "log",
+ "prettyplease",
  "proc-macro2",
  "quote",
  "regex",
  "rustc-hash 1.1.0",
  "shlex",
  "syn 2.0.66",
+ "which",
 ]
 
 [[package]]
@@ -1247,9 +1277,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.6.0"
+version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
+checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50"
 dependencies = [
  "serde",
 ]
@@ -1390,13 +1420,13 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.99"
+version = "1.1.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695"
+checksum = "45bcde016d64c21da4be18b655631e5ab6d3107607e71a73a9f53eb48aae23fb"
 dependencies = [
  "jobserver",
  "libc",
- "once_cell",
+ "shlex",
 ]
 
 [[package]]
@@ -3527,6 +3557,12 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
 
+[[package]]
+name = "dunce"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
+
 [[package]]
 name = "duration-str"
 version = "0.11.2"
@@ -4085,6 +4121,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "fs_extra"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
+
 [[package]]
 name = "fsevent-sys"
 version = "4.1.0"
@@ -4790,7 +4832,7 @@ dependencies = [
  "httpdate",
  "itoa",
  "pin-project-lite",
- "socket2 0.5.7",
+ "socket2 0.4.10",
  "tokio",
  "tower-service",
  "tracing",
@@ -6403,6 +6445,12 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "mirai-annotations"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1"
+
 [[package]]
 name = "mito2"
 version = "0.9.3"
@@ -7153,17 +7201,17 @@ dependencies = [
 [[package]]
 name = "opensrv-mysql"
 version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4148ab944991b0a33be74d2636a815268974578812a9e4cf7dc785325e858154"
+source = "git+https://github.com/datafuselabs/opensrv?rev=6bbc3b65e6b19212c4f7fc4f40c20daf6f452deb#6bbc3b65e6b19212c4f7fc4f40c20daf6f452deb"
 dependencies = [
  "async-trait",
  "byteorder",
+ "bytes",
  "chrono",
  "mysql_common 0.32.4",
  "nom",
  "pin-project-lite",
  "tokio",
- "tokio-rustls 0.25.0",
+ "tokio-rustls 0.26.0",
 ]
 
 [[package]]
@@ -7779,29 +7827,24 @@ dependencies = [
 
 [[package]]
 name = "pgwire"
-version = "0.20.0"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c00492c52bb65e0421211b7f4c5d9de7586e53786a3b244efb00f74851206bf6"
+checksum = "3770f56e1e8a608c6de40011b9a00c6b669c14d121024411701b4bc3b2a5be99"
 dependencies = [
  "async-trait",
- "base64 0.22.1",
  "bytes",
  "chrono",
  "derive-new 0.6.0",
  "futures",
  "hex",
- "log",
  "md5",
  "postgres-types",
  "rand",
  "ring 0.17.8",
- "stringprep",
  "thiserror",
- "time",
  "tokio",
- "tokio-rustls 0.25.0",
+ "tokio-rustls 0.26.0",
  "tokio-util",
- "x509-certificate",
 ]
 
 [[package]]
@@ -8392,7 +8435,7 @@ checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
 dependencies = [
  "bytes",
  "heck 0.5.0",
- "itertools 0.12.1",
+ "itertools 0.10.5",
  "log",
  "multimap",
  "once_cell",
@@ -8444,7 +8487,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1"
 dependencies = [
  "anyhow",
- "itertools 0.12.1",
+ "itertools 0.10.5",
  "proc-macro2",
  "quote",
  "syn 2.0.66",
@@ -8604,7 +8647,7 @@ dependencies = [
  "indoc",
  "libc",
  "memoffset 0.9.1",
- "parking_lot 0.12.3",
+ "parking_lot 0.11.2",
  "portable-atomic",
  "pyo3-build-config",
  "pyo3-ffi",
@@ -9568,6 +9611,7 @@ version = "0.23.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402"
 dependencies = [
+ "aws-lc-rs",
  "log",
  "once_cell",
  "ring 0.17.8",
@@ -9631,6 +9675,7 @@ version = "0.102.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e"
 dependencies = [
+ "aws-lc-rs",
  "ring 0.17.8",
  "rustls-pki-types",
  "untrusted 0.9.0",
@@ -10403,6 +10448,7 @@ dependencies = [
  "arrow-schema",
  "async-trait",
  "auth",
+ "aws-lc-sys",
  "axum",
  "axum-macros",
  "base64 0.21.7",
@@ -10467,7 +10513,7 @@ dependencies = [
  "regex",
  "reqwest",
  "rust-embed",
- "rustls 0.22.4",
+ "rustls 0.23.10",
  "rustls-pemfile 2.1.2",
  "rustls-pki-types",
  "schemars",
@@ -10485,7 +10531,7 @@ dependencies = [
  "tokio",
  "tokio-postgres",
  "tokio-postgres-rustls",
- "tokio-rustls 0.25.0",
+ "tokio-rustls 0.26.0",
  "tokio-stream",
  "tokio-test",
  "tokio-util",
@@ -12143,16 +12189,15 @@ dependencies = [
 
 [[package]]
 name = "tokio-postgres-rustls"
-version = "0.11.1"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ea13f22eda7127c827983bdaf0d7fff9df21c8817bab02815ac277a21143677"
+checksum = "04fb792ccd6bbcd4bba408eb8a292f70fc4a3589e5d793626f45190e6454b6ab"
 dependencies = [
- "futures",
  "ring 0.17.8",
- "rustls 0.22.4",
+ "rustls 0.23.10",
  "tokio",
  "tokio-postgres",
- "tokio-rustls 0.25.0",
+ "tokio-rustls 0.26.0",
  "x509-certificate",
 ]
 
diff --git a/Cargo.toml b/Cargo.toml
index d199caac09..046bf82478 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -99,7 +99,7 @@ base64 = "0.21"
 bigdecimal = "0.4.2"
 bitflags = "2.4.1"
 bytemuck = "1.12"
-bytes = { version = "1.5", features = ["serde"] }
+bytes = { version = "1.7", features = ["serde"] }
 chrono = { version = "0.4", features = ["serde"] }
 clap = { version = "4.4", features = ["derive"] }
 config = "0.13.0"
diff --git a/src/cmd/src/bin/greptime.rs b/src/cmd/src/bin/greptime.rs
index f36d0f1331..54659833fa 100644
--- a/src/cmd/src/bin/greptime.rs
+++ b/src/cmd/src/bin/greptime.rs
@@ -15,10 +15,11 @@
 #![doc = include_str!("../../../../README.md")]
 
 use clap::{Parser, Subcommand};
-use cmd::error::Result;
+use cmd::error::{InitTlsProviderSnafu, Result};
 use cmd::options::GlobalOptions;
 use cmd::{cli, datanode, flownode, frontend, metasrv, standalone, App};
 use common_version::version;
+use servers::install_ring_crypto_provider;
 
 #[derive(Parser)]
 #[command(name = "greptime", author, version, long_version = version(), about)]
@@ -94,6 +95,7 @@ async fn main() -> Result<()> {
 
 async fn main_body() -> Result<()> {
     setup_human_panic();
+    install_ring_crypto_provider().map_err(|msg| InitTlsProviderSnafu { msg }.build())?;
     start(Command::parse()).await
 }
 
diff --git a/src/cmd/src/error.rs b/src/cmd/src/error.rs
index 08c81c414c..f042b48478 100644
--- a/src/cmd/src/error.rs
+++ b/src/cmd/src/error.rs
@@ -24,6 +24,12 @@ use snafu::{Location, Snafu};
 #[snafu(visibility(pub))]
 #[stack_trace_debug]
 pub enum Error {
+    #[snafu(display("Failed to install ring crypto provider: {}", msg))]
+    InitTlsProvider {
+        #[snafu(implicit)]
+        location: Location,
+        msg: String,
+    },
     #[snafu(display("Failed to create default catalog and schema"))]
     InitMetadata {
         #[snafu(implicit)]
@@ -369,9 +375,10 @@ impl ErrorExt for Error {
             }
             Error::SubstraitEncodeLogicalPlan { source, .. } => source.status_code(),
 
-            Error::SerdeJson { .. } | Error::FileIo { .. } | Error::SpawnThread { .. } => {
-                StatusCode::Unexpected
-            }
+            Error::SerdeJson { .. }
+            | Error::FileIo { .. }
+            | Error::SpawnThread { .. }
+            | Error::InitTlsProvider { .. } => StatusCode::Unexpected,
 
             Error::Other { source, .. } => source.status_code(),
 
diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml
index 626fdaa404..d088961e73 100644
--- a/src/servers/Cargo.toml
+++ b/src/servers/Cargo.toml
@@ -65,10 +65,12 @@ notify.workspace = true
 object-pool = "0.5"
 once_cell.workspace = true
 openmetrics-parser = "0.4"
-opensrv-mysql = "0.7.0"
+# use crates.io version after current revision is merged in next release
+# opensrv-mysql = "0.7.0"
+opensrv-mysql = { git = "https://github.com/datafuselabs/opensrv", rev = "6bbc3b65e6b19212c4f7fc4f40c20daf6f452deb" }
 opentelemetry-proto.workspace = true
 parking_lot = "0.12"
-pgwire = "0.20"
+pgwire = { version = "0.22", default-features = false, features = ["server-api-ring"] }
 pin-project = "1.0"
 pipeline.workspace = true
 postgres-types = { version = "0.2", features = ["with-chrono-0_4", "with-serde_json-1"] }
@@ -85,7 +87,7 @@ rand.workspace = true
 regex.workspace = true
 reqwest.workspace = true
 rust-embed = { version = "6.6", features = ["debug-embed"] }
-rustls = "0.22"
+rustls = { version = "0.23", default-features = false, features = ["ring", "logging", "std", "tls12"] }
 rustls-pemfile = "2.0"
 rustls-pki-types = "1.0"
 schemars.workspace = true
@@ -98,7 +100,7 @@ sql.workspace = true
 strum.workspace = true
 table.workspace = true
 tokio.workspace = true
-tokio-rustls = "0.25"
+tokio-rustls = "0.26"
 tokio-stream = { workspace = true, features = ["net"] }
 tokio-util.workspace = true
 tonic.workspace = true
@@ -130,12 +132,17 @@ session = { workspace = true, features = ["testing"] }
 table.workspace = true
 tempfile = "3.0.0"
 tokio-postgres = "0.7"
-tokio-postgres-rustls = "0.11"
+tokio-postgres-rustls = "0.12"
 tokio-test = "0.4"
 
 [target.'cfg(not(windows))'.dev-dependencies]
 pprof = { version = "0.13", features = ["criterion", "flamegraph"] }
 
+[target.'cfg(windows)'.dependencies]
+aws-lc-sys = { version = "0.21.0", features = [
+    "prebuilt-nasm",
+] } # use prebuilt nasm on windows per https://github.com/aws/aws-lc-rs/blob/main/aws-lc-sys/README.md#use-of-prebuilt-nasm-objects
+
 [build-dependencies]
 common-version.workspace = true
 
diff --git a/src/servers/src/lib.rs b/src/servers/src/lib.rs
index a8f97877bd..ff1af967fa 100644
--- a/src/servers/src/lib.rs
+++ b/src/servers/src/lib.rs
@@ -55,3 +55,19 @@ pub struct SqlPlan {
     plan: Option<LogicalPlan>,
     schema: Option<Schema>,
 }
+
+/// Install the ring crypto provider for rustls process-wide. see:
+///
+///  https://docs.rs/rustls/latest/rustls/crypto/struct.CryptoProvider.html#using-the-per-process-default-cryptoprovider
+///
+/// for more information.
+pub fn install_ring_crypto_provider() -> Result<(), String> {
+    rustls::crypto::CryptoProvider::install_default(rustls::crypto::ring::default_provider())
+        .map_err(|ret| {
+            format!(
+                "CryptoProvider already installed as: {:?}, but providing {:?}",
+                rustls::crypto::CryptoProvider::get_default(),
+                ret
+            )
+        })
+}
diff --git a/src/servers/src/tls.rs b/src/servers/src/tls.rs
index b2b3550596..70c2be5d9d 100644
--- a/src/servers/src/tls.rs
+++ b/src/servers/src/tls.rs
@@ -239,6 +239,7 @@ pub fn maybe_watch_tls_config(tls_server_config: Arc<ReloadableTlsServerConfig>)
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::install_ring_crypto_provider;
     use crate::tls::TlsMode::Disable;
 
     #[test]
@@ -392,6 +393,7 @@ mod tests {
     #[test]
     fn test_tls_file_change_watch() {
         common_telemetry::init_default_ut_logging();
+        let _ = install_ring_crypto_provider();
 
         let dir = tempfile::tempdir().unwrap();
         let cert_path = dir.path().join("serevr.crt");
diff --git a/src/servers/tests/mysql/mysql_server_test.rs b/src/servers/tests/mysql/mysql_server_test.rs
index e077409a62..ba2cdbdab2 100644
--- a/src/servers/tests/mysql/mysql_server_test.rs
+++ b/src/servers/tests/mysql/mysql_server_test.rs
@@ -28,6 +28,7 @@ use mysql_async::{Conn, Row, SslOpts};
 use rand::rngs::StdRng;
 use rand::Rng;
 use servers::error::Result;
+use servers::install_ring_crypto_provider;
 use servers::mysql::server::{MysqlServer, MysqlSpawnConfig, MysqlSpawnRef};
 use servers::server::Server;
 use servers::tls::{ReloadableTlsServerConfig, TlsOption};
@@ -45,6 +46,7 @@ struct MysqlOpts<'a> {
 }
 
 fn create_mysql_server(table: TableRef, opts: MysqlOpts<'_>) -> Result<Box<dyn Server>> {
+    let _ = install_ring_crypto_provider();
     let query_handler = create_testing_sql_query_handler(table);
     let io_runtime = RuntimeBuilder::default()
         .worker_threads(4)
diff --git a/src/servers/tests/postgres/mod.rs b/src/servers/tests/postgres/mod.rs
index ad135dd9d0..f3ff827db4 100644
--- a/src/servers/tests/postgres/mod.rs
+++ b/src/servers/tests/postgres/mod.rs
@@ -27,6 +27,7 @@ use rustls::client::danger::{ServerCertVerified, ServerCertVerifier};
 use rustls::{Error, SignatureScheme};
 use rustls_pki_types::{CertificateDer, ServerName};
 use servers::error::Result;
+use servers::install_ring_crypto_provider;
 use servers::postgres::PostgresServer;
 use servers::server::Server;
 use servers::tls::{ReloadableTlsServerConfig, TlsOption};
@@ -357,6 +358,8 @@ async fn test_extended_query() -> Result<()> {
 
 async fn start_test_server(server_tls: TlsOption) -> Result<u16> {
     common_telemetry::init_default_ut_logging();
+    let _ = install_ring_crypto_provider();
+
     let table = MemTable::default_numbers_table();
     let pg_server = create_postgres_server(table, false, server_tls, None)?;
     let listening = "127.0.0.1:0".parse::<SocketAddr>().unwrap();

From 08bd40333cb228c51f74529f6edf66b56c508bc9 Mon Sep 17 00:00:00 2001
From: Ning Sun <sunng@protonmail.com>
Date: Thu, 19 Sep 2024 12:38:41 +0800
Subject: [PATCH 013/128] feat: add an option to turn on compression for arrow
 output (#4730)

* feat: add an option to turn on compression for arrow output

* fix: typo
---
 Cargo.lock                           |  1 +
 Cargo.toml                           |  2 +-
 src/servers/src/http.rs              |  2 +-
 src/servers/src/http/arrow_result.rs | 95 ++++++++++++++++++++++++++--
 src/servers/src/http/handler.rs      |  9 ++-
 5 files changed, 100 insertions(+), 9 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index fdad3a09be..3176657706 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -443,6 +443,7 @@ dependencies = [
  "arrow-schema",
  "flatbuffers",
  "lz4_flex 0.11.3",
+ "zstd 0.13.1",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 046bf82478..c1eea12a53 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -90,7 +90,7 @@ aquamarine = "0.3"
 arrow = { version = "51.0.0", features = ["prettyprint"] }
 arrow-array = { version = "51.0.0", default-features = false, features = ["chrono-tz"] }
 arrow-flight = "51.0"
-arrow-ipc = { version = "51.0.0", default-features = false, features = ["lz4"] }
+arrow-ipc = { version = "51.0.0", default-features = false, features = ["lz4", "zstd"] }
 arrow-schema = { version = "51.0", features = ["serde"] }
 async-stream = "0.3"
 async-trait = "0.1"
diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs
index 2313d19bbe..956a650fcc 100644
--- a/src/servers/src/http.rs
+++ b/src/servers/src/http.rs
@@ -1136,7 +1136,7 @@ mod test {
                 RecordBatches::try_new(schema.clone(), vec![recordbatch.clone()]).unwrap();
             let outputs = vec![Ok(Output::new_with_record_batches(recordbatches))];
             let json_resp = match format {
-                ResponseFormat::Arrow => ArrowResponse::from_output(outputs).await,
+                ResponseFormat::Arrow => ArrowResponse::from_output(outputs, None).await,
                 ResponseFormat::Csv => CsvResponse::from_output(outputs).await,
                 ResponseFormat::Table => TableResponse::from_output(outputs).await,
                 ResponseFormat::GreptimedbV1 => GreptimedbV1Response::from_output(outputs).await,
diff --git a/src/servers/src/http/arrow_result.rs b/src/servers/src/http/arrow_result.rs
index e6d2441ee2..6a739fee04 100644
--- a/src/servers/src/http/arrow_result.rs
+++ b/src/servers/src/http/arrow_result.rs
@@ -16,7 +16,8 @@ use std::pin::Pin;
 use std::sync::Arc;
 
 use arrow::datatypes::Schema;
-use arrow_ipc::writer::FileWriter;
+use arrow_ipc::writer::{FileWriter, IpcWriteOptions};
+use arrow_ipc::CompressionType;
 use axum::http::{header, HeaderValue};
 use axum::response::{IntoResponse, Response};
 use common_error::status_code::StatusCode;
@@ -41,10 +42,15 @@ pub struct ArrowResponse {
 async fn write_arrow_bytes(
     mut recordbatches: Pin<Box<dyn RecordBatchStream + Send>>,
     schema: &Arc<Schema>,
+    compression: Option<CompressionType>,
 ) -> Result<Vec<u8>, Error> {
     let mut bytes = Vec::new();
     {
-        let mut writer = FileWriter::try_new(&mut bytes, schema).context(error::ArrowSnafu)?;
+        let options = IpcWriteOptions::default()
+            .try_with_compression(compression)
+            .context(error::ArrowSnafu)?;
+        let mut writer = FileWriter::try_new_with_options(&mut bytes, schema, options)
+            .context(error::ArrowSnafu)?;
 
         while let Some(rb) = recordbatches.next().await {
             let rb = rb.context(error::CollectRecordbatchSnafu)?;
@@ -59,8 +65,22 @@ async fn write_arrow_bytes(
     Ok(bytes)
 }
 
+fn compression_type(compression: Option<String>) -> Option<CompressionType> {
+    match compression
+        .map(|compression| compression.to_lowercase())
+        .as_deref()
+    {
+        Some("zstd") => Some(CompressionType::ZSTD),
+        Some("lz4") => Some(CompressionType::LZ4_FRAME),
+        _ => None,
+    }
+}
+
 impl ArrowResponse {
-    pub async fn from_output(mut outputs: Vec<error::Result<Output>>) -> HttpResponse {
+    pub async fn from_output(
+        mut outputs: Vec<error::Result<Output>>,
+        compression: Option<String>,
+    ) -> HttpResponse {
         if outputs.len() > 1 {
             return HttpResponse::Error(ErrorResponse::from_error_message(
                 StatusCode::InvalidArguments,
@@ -68,6 +88,8 @@ impl ArrowResponse {
             ));
         }
 
+        let compression = compression_type(compression);
+
         match outputs.pop() {
             None => HttpResponse::Arrow(ArrowResponse {
                 data: vec![],
@@ -80,7 +102,9 @@ impl ArrowResponse {
                 }),
                 OutputData::RecordBatches(batches) => {
                     let schema = batches.schema();
-                    match write_arrow_bytes(batches.as_stream(), schema.arrow_schema()).await {
+                    match write_arrow_bytes(batches.as_stream(), schema.arrow_schema(), compression)
+                        .await
+                    {
                         Ok(payload) => HttpResponse::Arrow(ArrowResponse {
                             data: payload,
                             execution_time_ms: 0,
@@ -90,7 +114,7 @@ impl ArrowResponse {
                 }
                 OutputData::Stream(batches) => {
                     let schema = batches.schema();
-                    match write_arrow_bytes(batches, schema.arrow_schema()).await {
+                    match write_arrow_bytes(batches, schema.arrow_schema(), compression).await {
                         Ok(payload) => HttpResponse::Arrow(ArrowResponse {
                             data: payload,
                             execution_time_ms: 0,
@@ -136,3 +160,64 @@ impl IntoResponse for ArrowResponse {
             .into_response()
     }
 }
+
+#[cfg(test)]
+mod test {
+    use std::io::Cursor;
+
+    use arrow_ipc::reader::FileReader;
+    use arrow_schema::DataType;
+    use common_recordbatch::{RecordBatch, RecordBatches};
+    use datatypes::prelude::*;
+    use datatypes::schema::{ColumnSchema, Schema};
+    use datatypes::vectors::{StringVector, UInt32Vector};
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_arrow_output() {
+        let column_schemas = vec![
+            ColumnSchema::new("numbers", ConcreteDataType::uint32_datatype(), false),
+            ColumnSchema::new("strings", ConcreteDataType::string_datatype(), true),
+        ];
+        let schema = Arc::new(Schema::new(column_schemas));
+        let columns: Vec<VectorRef> = vec![
+            Arc::new(UInt32Vector::from_slice(vec![1, 2, 3, 4])),
+            Arc::new(StringVector::from(vec![
+                None,
+                Some("hello"),
+                Some("greptime"),
+                None,
+            ])),
+        ];
+
+        for compression in [None, Some("zstd".to_string()), Some("lz4".to_string())].into_iter() {
+            let recordbatch = RecordBatch::new(schema.clone(), columns.clone()).unwrap();
+            let recordbatches =
+                RecordBatches::try_new(schema.clone(), vec![recordbatch.clone()]).unwrap();
+            let outputs = vec![Ok(Output::new_with_record_batches(recordbatches))];
+
+            let http_resp = ArrowResponse::from_output(outputs, compression).await;
+            match http_resp {
+                HttpResponse::Arrow(resp) => {
+                    let output = resp.data;
+                    let mut reader =
+                        FileReader::try_new(Cursor::new(output), None).expect("Arrow reader error");
+                    let schema = reader.schema();
+                    assert_eq!(schema.fields[0].name(), "numbers");
+                    assert_eq!(schema.fields[0].data_type(), &DataType::UInt32);
+                    assert_eq!(schema.fields[1].name(), "strings");
+                    assert_eq!(schema.fields[1].data_type(), &DataType::Utf8);
+
+                    let rb = reader.next().unwrap().expect("read record batch failed");
+                    assert_eq!(rb.num_columns(), 2);
+                    assert_eq!(rb.num_rows(), 4);
+                }
+                HttpResponse::Error(e) => {
+                    panic!("unexpected {:?}", e);
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+}
diff --git a/src/servers/src/http/handler.rs b/src/servers/src/http/handler.rs
index 4d5ca58461..1befc22240 100644
--- a/src/servers/src/http/handler.rs
+++ b/src/servers/src/http/handler.rs
@@ -51,7 +51,8 @@ use crate::query_handler::sql::ServerSqlQueryHandlerRef;
 pub struct SqlQuery {
     pub db: Option<String>,
     pub sql: Option<String>,
-    // (Optional) result format: [`greptimedb_v1`, `influxdb_v1`, `csv`],
+    // (Optional) result format: [`greptimedb_v1`, `influxdb_v1`, `csv`,
+    // `arrow`],
     // the default value is `greptimedb_v1`
     pub format: Option<String>,
     // Returns epoch timestamps with the specified precision.
@@ -64,6 +65,8 @@ pub struct SqlQuery {
     // param too.
     pub epoch: Option<String>,
     pub limit: Option<usize>,
+    // For arrow output
+    pub compression: Option<String>,
 }
 
 /// Handler to execute sql
@@ -128,7 +131,9 @@ pub async fn sql(
     };
 
     let mut resp = match format {
-        ResponseFormat::Arrow => ArrowResponse::from_output(outputs).await,
+        ResponseFormat::Arrow => {
+            ArrowResponse::from_output(outputs, query_params.compression).await
+        }
         ResponseFormat::Csv => CsvResponse::from_output(outputs).await,
         ResponseFormat::Table => TableResponse::from_output(outputs).await,
         ResponseFormat::GreptimedbV1 => GreptimedbV1Response::from_output(outputs).await,

From b5f7138d337ee2229c3b45633e9ccc38f8600104 Mon Sep 17 00:00:00 2001
From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com>
Date: Thu, 19 Sep 2024 12:44:02 +0800
Subject: [PATCH 014/128] refactor(tables): improve `tables` performance
 (#4737)

* chore: cherrypick 52e8eebb2dbbbe81179583c05094004a5eedd7fd

* refactor/tables: Change variable from immutable to mutable in KvBackendCatalogManager's method

* refactor/tables: Replace unbounded channel with bounded and use semaphore for concurrency control in KvBackendCatalogManager

* refactor/tables: Add common-runtime dependency and update KvBackendCatalogManager to use common_runtime::spawn_global

* refactor/tables: Await on sending error through channel in KvBackendCatalogManager
---
 Cargo.lock                                    |   3 +
 src/catalog/Cargo.toml                        |   2 +
 src/catalog/src/kvbackend/manager.rs          |  83 ++++++++----
 src/common/meta/Cargo.toml                    |   1 +
 src/common/meta/src/key/catalog_name.rs       |   3 +-
 src/common/meta/src/key/datanode_table.rs     |   3 +-
 src/common/meta/src/key/flow/flow_name.rs     |   3 +-
 src/common/meta/src/key/flow/flow_route.rs    |   3 +-
 src/common/meta/src/key/flow/flownode_flow.rs |   3 +-
 src/common/meta/src/key/flow/table_flow.rs    |   3 +-
 src/common/meta/src/key/schema_name.rs        |   3 +-
 src/common/meta/src/key/table_name.rs         |   3 +-
 src/common/meta/src/range_stream.rs           | 126 ++++--------------
 src/common/meta/src/state_store.rs            |   3 +-
 src/log-store/src/raft_engine/backend.rs      |   3 +-
 src/meta-srv/src/service/store/cached_kv.rs   |   5 +-
 16 files changed, 108 insertions(+), 142 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 3176657706..786c1c3a8b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1381,6 +1381,7 @@ dependencies = [
  "common-meta",
  "common-query",
  "common-recordbatch",
+ "common-runtime",
  "common-telemetry",
  "common-test-util",
  "common-time",
@@ -1408,6 +1409,7 @@ dependencies = [
  "store-api",
  "table",
  "tokio",
+ "tokio-stream",
 ]
 
 [[package]]
@@ -2109,6 +2111,7 @@ dependencies = [
  "anymap2",
  "api",
  "async-recursion",
+ "async-stream",
  "async-trait",
  "base64 0.21.7",
  "bytes",
diff --git a/src/catalog/Cargo.toml b/src/catalog/Cargo.toml
index a484171733..cfea5e02c6 100644
--- a/src/catalog/Cargo.toml
+++ b/src/catalog/Cargo.toml
@@ -24,6 +24,7 @@ common-macro.workspace = true
 common-meta.workspace = true
 common-query.workspace = true
 common-recordbatch.workspace = true
+common-runtime.workspace = true
 common-telemetry.workspace = true
 common-time.workspace = true
 common-version.workspace = true
@@ -48,6 +49,7 @@ sql.workspace = true
 store-api.workspace = true
 table.workspace = true
 tokio.workspace = true
+tokio-stream = "0.1"
 
 [dev-dependencies]
 cache.workspace = true
diff --git a/src/catalog/src/kvbackend/manager.rs b/src/catalog/src/kvbackend/manager.rs
index feb5e31d09..1559022514 100644
--- a/src/catalog/src/kvbackend/manager.rs
+++ b/src/catalog/src/kvbackend/manager.rs
@@ -42,6 +42,8 @@ use table::dist_table::DistTable;
 use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
 use table::table_name::TableName;
 use table::TableRef;
+use tokio::sync::Semaphore;
+use tokio_stream::wrappers::ReceiverStream;
 
 use crate::error::{
     CacheNotFoundSnafu, GetTableCacheSnafu, InvalidTableInfoInCatalogSnafu, ListCatalogsSnafu,
@@ -179,21 +181,18 @@ impl CatalogManager for KvBackendCatalogManager {
         schema: &str,
         query_ctx: Option<&QueryContext>,
     ) -> Result<Vec<String>> {
-        let stream = self
+        let mut tables = self
             .table_metadata_manager
             .table_name_manager()
-            .tables(catalog, schema);
-        let mut tables = stream
+            .tables(catalog, schema)
+            .map_ok(|(table_name, _)| table_name)
             .try_collect::<Vec<_>>()
             .await
             .map_err(BoxedError::new)
-            .context(ListTablesSnafu { catalog, schema })?
-            .into_iter()
-            .map(|(k, _)| k)
-            .collect::<Vec<_>>();
-        tables.extend_from_slice(&self.system_catalog.table_names(schema, query_ctx));
+            .context(ListTablesSnafu { catalog, schema })?;
 
-        Ok(tables.into_iter().collect())
+        tables.extend(self.system_catalog.table_names(schema, query_ctx));
+        Ok(tables)
     }
 
     async fn catalog_exists(&self, catalog: &str) -> Result<bool> {
@@ -303,36 +302,68 @@ impl CatalogManager for KvBackendCatalogManager {
             }
         });
 
-        let table_id_stream = self
-            .table_metadata_manager
-            .table_name_manager()
-            .tables(catalog, schema)
-            .map_ok(|(_, v)| v.table_id());
         const BATCH_SIZE: usize = 128;
-        let user_tables = try_stream!({
+        const CONCURRENCY: usize = 8;
+
+        let (tx, rx) = tokio::sync::mpsc::channel(64);
+        let metadata_manager = self.table_metadata_manager.clone();
+        let catalog = catalog.to_string();
+        let schema = schema.to_string();
+        let semaphore = Arc::new(Semaphore::new(CONCURRENCY));
+
+        common_runtime::spawn_global(async move {
+            let table_id_stream = metadata_manager
+                .table_name_manager()
+                .tables(&catalog, &schema)
+                .map_ok(|(_, v)| v.table_id());
             // Split table ids into chunks
             let mut table_id_chunks = table_id_stream.ready_chunks(BATCH_SIZE);
 
             while let Some(table_ids) = table_id_chunks.next().await {
-                let table_ids = table_ids
+                let table_ids = match table_ids
                     .into_iter()
                     .collect::<std::result::Result<Vec<_>, _>>()
                     .map_err(BoxedError::new)
-                    .context(ListTablesSnafu { catalog, schema })?;
+                    .context(ListTablesSnafu {
+                        catalog: &catalog,
+                        schema: &schema,
+                    }) {
+                    Ok(table_ids) => table_ids,
+                    Err(e) => {
+                        let _ = tx.send(Err(e)).await;
+                        return;
+                    }
+                };
 
-                let table_info_values = self
-                    .table_metadata_manager
-                    .table_info_manager()
-                    .batch_get(&table_ids)
-                    .await
-                    .context(TableMetadataManagerSnafu)?;
+                let metadata_manager = metadata_manager.clone();
+                let tx = tx.clone();
+                let semaphore = semaphore.clone();
+                common_runtime::spawn_global(async move {
+                    // we don't explicitly close the semaphore so just ignore the potential error.
+                    let _ = semaphore.acquire().await;
+                    let table_info_values = match metadata_manager
+                        .table_info_manager()
+                        .batch_get(&table_ids)
+                        .await
+                        .context(TableMetadataManagerSnafu)
+                    {
+                        Ok(table_info_values) => table_info_values,
+                        Err(e) => {
+                            let _ = tx.send(Err(e)).await;
+                            return;
+                        }
+                    };
 
-                for table_info_value in table_info_values.into_values() {
-                    yield build_table(table_info_value)?;
-                }
+                    for table in table_info_values.into_values().map(build_table) {
+                        if tx.send(table).await.is_err() {
+                            return;
+                        }
+                    }
+                });
             }
         });
 
+        let user_tables = ReceiverStream::new(rx);
         Box::pin(sys_tables.chain(user_tables))
     }
 }
diff --git a/src/common/meta/Cargo.toml b/src/common/meta/Cargo.toml
index 28f518190a..591055d472 100644
--- a/src/common/meta/Cargo.toml
+++ b/src/common/meta/Cargo.toml
@@ -15,6 +15,7 @@ workspace = true
 anymap2 = "0.13.0"
 api.workspace = true
 async-recursion = "1.0"
+async-stream = "0.3"
 async-trait.workspace = true
 base64.workspace = true
 bytes.workspace = true
diff --git a/src/common/meta/src/key/catalog_name.rs b/src/common/meta/src/key/catalog_name.rs
index a3a9841618..9238f95c82 100644
--- a/src/common/meta/src/key/catalog_name.rs
+++ b/src/common/meta/src/key/catalog_name.rs
@@ -147,7 +147,8 @@ impl CatalogManager {
             req,
             DEFAULT_PAGE_SIZE,
             Arc::new(catalog_decoder),
-        );
+        )
+        .into_stream();
 
         Box::pin(stream)
     }
diff --git a/src/common/meta/src/key/datanode_table.rs b/src/common/meta/src/key/datanode_table.rs
index b53060cd16..b10d536672 100644
--- a/src/common/meta/src/key/datanode_table.rs
+++ b/src/common/meta/src/key/datanode_table.rs
@@ -167,7 +167,8 @@ impl DatanodeTableManager {
             req,
             DEFAULT_PAGE_SIZE,
             Arc::new(datanode_table_value_decoder),
-        );
+        )
+        .into_stream();
 
         Box::pin(stream)
     }
diff --git a/src/common/meta/src/key/flow/flow_name.rs b/src/common/meta/src/key/flow/flow_name.rs
index 201c354f99..3a331be800 100644
--- a/src/common/meta/src/key/flow/flow_name.rs
+++ b/src/common/meta/src/key/flow/flow_name.rs
@@ -200,7 +200,8 @@ impl FlowNameManager {
             req,
             DEFAULT_PAGE_SIZE,
             Arc::new(flow_name_decoder),
-        );
+        )
+        .into_stream();
 
         Box::pin(stream)
     }
diff --git a/src/common/meta/src/key/flow/flow_route.rs b/src/common/meta/src/key/flow/flow_route.rs
index 96db01aef0..47ee94ce95 100644
--- a/src/common/meta/src/key/flow/flow_route.rs
+++ b/src/common/meta/src/key/flow/flow_route.rs
@@ -180,7 +180,8 @@ impl FlowRouteManager {
             req,
             DEFAULT_PAGE_SIZE,
             Arc::new(flow_route_decoder),
-        );
+        )
+        .into_stream();
 
         Box::pin(stream)
     }
diff --git a/src/common/meta/src/key/flow/flownode_flow.rs b/src/common/meta/src/key/flow/flownode_flow.rs
index 4b4c31a7b0..552abfcdbe 100644
--- a/src/common/meta/src/key/flow/flownode_flow.rs
+++ b/src/common/meta/src/key/flow/flownode_flow.rs
@@ -180,7 +180,8 @@ impl FlownodeFlowManager {
             req,
             DEFAULT_PAGE_SIZE,
             Arc::new(flownode_flow_key_decoder),
-        );
+        )
+        .into_stream();
 
         Box::pin(stream.map_ok(|key| (key.flow_id(), key.partition_id())))
     }
diff --git a/src/common/meta/src/key/flow/table_flow.rs b/src/common/meta/src/key/flow/table_flow.rs
index 5297734a23..c4f47cde51 100644
--- a/src/common/meta/src/key/flow/table_flow.rs
+++ b/src/common/meta/src/key/flow/table_flow.rs
@@ -207,7 +207,8 @@ impl TableFlowManager {
             req,
             DEFAULT_PAGE_SIZE,
             Arc::new(table_flow_decoder),
-        );
+        )
+        .into_stream();
 
         Box::pin(stream)
     }
diff --git a/src/common/meta/src/key/schema_name.rs b/src/common/meta/src/key/schema_name.rs
index 7c9822e18d..f9cbc4a9f2 100644
--- a/src/common/meta/src/key/schema_name.rs
+++ b/src/common/meta/src/key/schema_name.rs
@@ -230,7 +230,8 @@ impl SchemaManager {
             req,
             DEFAULT_PAGE_SIZE,
             Arc::new(schema_decoder),
-        );
+        )
+        .into_stream();
 
         Box::pin(stream)
     }
diff --git a/src/common/meta/src/key/table_name.rs b/src/common/meta/src/key/table_name.rs
index 7054ffc4e9..a632e3a233 100644
--- a/src/common/meta/src/key/table_name.rs
+++ b/src/common/meta/src/key/table_name.rs
@@ -259,7 +259,8 @@ impl TableNameManager {
             req,
             DEFAULT_PAGE_SIZE,
             Arc::new(table_decoder),
-        );
+        )
+        .into_stream();
 
         Box::pin(stream)
     }
diff --git a/src/common/meta/src/range_stream.rs b/src/common/meta/src/range_stream.rs
index b14a2bf8f1..be54865281 100644
--- a/src/common/meta/src/range_stream.rs
+++ b/src/common/meta/src/range_stream.rs
@@ -12,14 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::VecDeque;
-use std::pin::Pin;
 use std::sync::Arc;
-use std::task::{Context, Poll};
 
+use async_stream::try_stream;
 use common_telemetry::debug;
-use futures::future::BoxFuture;
-use futures::{ready, FutureExt, Stream};
+use futures::Stream;
 use snafu::ensure;
 
 use crate::error::{self, Result};
@@ -30,17 +27,6 @@ use crate::util::get_next_prefix_key;
 
 pub type KeyValueDecoderFn<T> = dyn Fn(KeyValue) -> Result<T> + Send + Sync;
 
-enum PaginationStreamState<T> {
-    /// At the start of reading.
-    Init,
-    /// Decoding key value pairs.
-    Decoding(SimpleKeyValueDecoder<T>),
-    /// Retrieving data from backend.
-    Reading(BoxFuture<'static, Result<(PaginationStreamFactory, Option<RangeResponse>)>>),
-    /// Error
-    Error,
-}
-
 /// The Range Request's default page size.
 ///
 /// It dependents on upstream KvStore server side grpc message size limitation.
@@ -65,8 +51,6 @@ struct PaginationStreamFactory {
     /// keys.
     pub range_end: Vec<u8>,
 
-    /// page_size is the pagination page size.
-    page_size: usize,
     /// keys_only when set returns only the keys and not the values.
     pub keys_only: bool,
 
@@ -89,7 +73,6 @@ impl PaginationStreamFactory {
             kv: kv.clone(),
             key,
             range_end,
-            page_size,
             keys_only,
             more,
             adaptive_page_size: if page_size == 0 {
@@ -137,7 +120,7 @@ impl PaginationStreamFactory {
         }
     }
 
-    async fn read_next(mut self) -> Result<(Self, Option<RangeResponse>)> {
+    async fn read_next(&mut self) -> Result<Option<RangeResponse>> {
         if self.more {
             let resp = self
                 .adaptive_range(RangeRequest {
@@ -151,33 +134,22 @@ impl PaginationStreamFactory {
             let key = resp
                 .kvs
                 .last()
-                .map(|kv| kv.key.clone())
-                .unwrap_or_else(Vec::new);
+                .map(|kv| kv.key.as_slice())
+                .unwrap_or_default();
 
-            let next_key = get_next_prefix_key(&key);
-
-            Ok((
-                Self {
-                    kv: self.kv,
-                    key: next_key,
-                    range_end: self.range_end,
-                    page_size: self.page_size,
-                    keys_only: self.keys_only,
-                    more: resp.more,
-                    adaptive_page_size: self.adaptive_page_size,
-                },
-                Some(resp),
-            ))
+            let next_key = get_next_prefix_key(key);
+            self.key = next_key;
+            self.more = resp.more;
+            Ok(Some(resp))
         } else {
-            Ok((self, None))
+            Ok(None)
         }
     }
 }
 
 pub struct PaginationStream<T> {
-    state: PaginationStreamState<T>,
     decoder_fn: Arc<KeyValueDecoderFn<T>>,
-    factory: Option<PaginationStreamFactory>,
+    factory: PaginationStreamFactory,
 }
 
 impl<T> PaginationStream<T> {
@@ -189,82 +161,28 @@ impl<T> PaginationStream<T> {
         decoder_fn: Arc<KeyValueDecoderFn<T>>,
     ) -> Self {
         Self {
-            state: PaginationStreamState::Init,
             decoder_fn,
-            factory: Some(PaginationStreamFactory::new(
+            factory: PaginationStreamFactory::new(
                 &kv,
                 req.key,
                 req.range_end,
                 page_size,
                 req.keys_only,
                 true,
-            )),
+            ),
         }
     }
 }
 
-struct SimpleKeyValueDecoder<T> {
-    kv: VecDeque<KeyValue>,
-    decoder: Arc<KeyValueDecoderFn<T>>,
-}
-
-impl<T> Iterator for SimpleKeyValueDecoder<T> {
-    type Item = Result<T>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        if let Some(kv) = self.kv.pop_front() {
-            Some((self.decoder)(kv))
-        } else {
-            None
-        }
-    }
-}
-
-impl<T> Stream for PaginationStream<T> {
-    type Item = Result<T>;
-
-    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-        loop {
-            match &mut self.state {
-                PaginationStreamState::Decoding(decoder) => match decoder.next() {
-                    Some(Ok(result)) => return Poll::Ready(Some(Ok(result))),
-                    Some(Err(e)) => {
-                        self.state = PaginationStreamState::Error;
-                        return Poll::Ready(Some(Err(e)));
-                    }
-                    None => self.state = PaginationStreamState::Init,
-                },
-                PaginationStreamState::Init => {
-                    let factory = self.factory.take().expect("lost factory");
-                    if !factory.more {
-                        // Ensures the factory always exists.
-                        self.factory = Some(factory);
-                        return Poll::Ready(None);
-                    }
-                    let fut = factory.read_next().boxed();
-                    self.state = PaginationStreamState::Reading(fut);
+impl<T> PaginationStream<T> {
+    pub fn into_stream(mut self) -> impl Stream<Item = Result<T>> {
+        try_stream!({
+            while let Some(resp) = self.factory.read_next().await? {
+                for kv in resp.kvs {
+                    yield (self.decoder_fn)(kv)?
                 }
-                PaginationStreamState::Reading(f) => match ready!(f.poll_unpin(cx)) {
-                    Ok((factory, Some(resp))) => {
-                        self.factory = Some(factory);
-                        let decoder = SimpleKeyValueDecoder {
-                            kv: resp.kvs.into(),
-                            decoder: self.decoder_fn.clone(),
-                        };
-                        self.state = PaginationStreamState::Decoding(decoder);
-                    }
-                    Ok((factory, None)) => {
-                        self.factory = Some(factory);
-                        self.state = PaginationStreamState::Init;
-                    }
-                    Err(e) => {
-                        self.state = PaginationStreamState::Error;
-                        return Poll::Ready(Some(Err(e)));
-                    }
-                },
-                PaginationStreamState::Error => return Poll::Ready(None), // Ends the stream as error happens.
             }
-        }
+        })
     }
 }
 
@@ -333,7 +251,8 @@ mod tests {
             },
             DEFAULT_PAGE_SIZE,
             Arc::new(decoder),
-        );
+        )
+        .into_stream();
         let kv = stream.try_collect::<Vec<_>>().await.unwrap();
 
         assert!(kv.is_empty());
@@ -374,6 +293,7 @@ mod tests {
             Arc::new(decoder),
         );
         let kv = stream
+            .into_stream()
             .try_collect::<Vec<_>>()
             .await
             .unwrap()
diff --git a/src/common/meta/src/state_store.rs b/src/common/meta/src/state_store.rs
index 1cf1ea8649..89d5dfd0ff 100644
--- a/src/common/meta/src/state_store.rs
+++ b/src/common/meta/src/state_store.rs
@@ -172,7 +172,8 @@ impl StateStore for KvStateStore {
             req,
             self.max_num_per_range_request.unwrap_or_default(),
             Arc::new(decode_kv),
-        );
+        )
+        .into_stream();
 
         let stream = stream.map(move |r| {
             let path = path.clone();
diff --git a/src/log-store/src/raft_engine/backend.rs b/src/log-store/src/raft_engine/backend.rs
index e2cd65c8fa..33cb64a2e8 100644
--- a/src/log-store/src/raft_engine/backend.rs
+++ b/src/log-store/src/raft_engine/backend.rs
@@ -264,9 +264,8 @@ impl KvBackend for RaftEngineBackend {
         let mut response = BatchGetResponse {
             kvs: Vec::with_capacity(req.keys.len()),
         };
-        let engine = self.engine.read().unwrap();
         for key in req.keys {
-            let Some(value) = engine.get(SYSTEM_NAMESPACE, &key) else {
+            let Some(value) = self.engine.read().unwrap().get(SYSTEM_NAMESPACE, &key) else {
                 continue;
             };
             response.kvs.push(KeyValue { key, value });
diff --git a/src/meta-srv/src/service/store/cached_kv.rs b/src/meta-srv/src/service/store/cached_kv.rs
index ddf7b3e516..d4b6f84f58 100644
--- a/src/meta-srv/src/service/store/cached_kv.rs
+++ b/src/meta-srv/src/service/store/cached_kv.rs
@@ -103,9 +103,10 @@ impl LeaderCachedKvBackend {
                 RangeRequest::new().with_prefix(prefix.as_bytes()),
                 DEFAULT_PAGE_SIZE,
                 Arc::new(Ok),
-            );
+            )
+            .into_stream();
 
-            let kvs = stream.try_collect::<Vec<_>>().await?.into_iter().collect();
+            let kvs = stream.try_collect::<Vec<_>>().await?;
 
             self.cache
                 .batch_put(BatchPutRequest {

From 52d627e37db26b9830024de9b005444207299436 Mon Sep 17 00:00:00 2001
From: shuiyisong <113876041+shuiyisong@users.noreply.github.com>
Date: Thu, 19 Sep 2024 13:14:47 +0800
Subject: [PATCH 015/128] chore: add log ingest interceptor (#4734)

* chore: add log ingest interceptor

* chore: rename

* chore: update interceptor signature
---
 src/frontend/src/instance/log_handler.rs  | 11 ++++-
 src/frontend/src/server.rs                |  8 ++-
 src/pipeline/src/etl/processor/dissect.rs |  8 +--
 src/servers/src/http.rs                   | 18 +++----
 src/servers/src/http/event.rs             | 10 +++-
 src/servers/src/interceptor.rs            | 59 +++++++++++++++++++++++
 tests-integration/src/test_util.rs        |  2 +-
 7 files changed, 96 insertions(+), 20 deletions(-)

diff --git a/src/frontend/src/instance/log_handler.rs b/src/frontend/src/instance/log_handler.rs
index 7edda5ccf1..441501b242 100644
--- a/src/frontend/src/instance/log_handler.rs
+++ b/src/frontend/src/instance/log_handler.rs
@@ -20,7 +20,10 @@ use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq};
 use client::Output;
 use common_error::ext::BoxedError;
 use pipeline::{GreptimeTransformer, Pipeline, PipelineInfo, PipelineVersion};
-use servers::error::{AuthSnafu, ExecuteGrpcRequestSnafu, PipelineSnafu, Result as ServerResult};
+use servers::error::{
+    AuthSnafu, Error as ServerError, ExecuteGrpcRequestSnafu, PipelineSnafu, Result as ServerResult,
+};
+use servers::interceptor::{LogIngestInterceptor, LogIngestInterceptorRef};
 use servers::query_handler::LogHandler;
 use session::context::QueryContextRef;
 use snafu::ResultExt;
@@ -40,6 +43,12 @@ impl LogHandler for Instance {
             .check_permission(ctx.current_user(), PermissionReq::LogWrite)
             .context(AuthSnafu)?;
 
+        let log = self
+            .plugins
+            .get::<LogIngestInterceptorRef<ServerError>>()
+            .as_ref()
+            .pre_ingest(log, ctx.clone())?;
+
         self.handle_log_inserts(log, ctx).await
     }
 
diff --git a/src/frontend/src/server.rs b/src/frontend/src/server.rs
index 9660b27620..115002c3ab 100644
--- a/src/frontend/src/server.rs
+++ b/src/frontend/src/server.rs
@@ -18,11 +18,13 @@ use std::sync::Arc;
 use auth::UserProviderRef;
 use common_base::Plugins;
 use common_config::{Configurable, Mode};
+use servers::error::Error as ServerError;
 use servers::grpc::builder::GrpcServerBuilder;
 use servers::grpc::greptime_handler::GreptimeRequestHandler;
 use servers::grpc::{GrpcOptions, GrpcServer, GrpcServerConfig};
 use servers::http::event::LogValidatorRef;
 use servers::http::{HttpServer, HttpServerBuilder};
+use servers::interceptor::LogIngestInterceptorRef;
 use servers::metrics_handler::MetricsHandler;
 use servers::mysql::server::{MysqlServer, MysqlSpawnConfig, MysqlSpawnRef};
 use servers::postgres::PostgresServer;
@@ -81,8 +83,10 @@ where
             Some(self.instance.clone()),
         );
 
-        builder = builder
-            .with_log_ingest_handler(self.instance.clone(), self.plugins.get::<LogValidatorRef>());
+        let validator = self.plugins.get::<LogValidatorRef>();
+        let ingest_interceptor = self.plugins.get::<LogIngestInterceptorRef<ServerError>>();
+        builder =
+            builder.with_log_ingest_handler(self.instance.clone(), validator, ingest_interceptor);
 
         if let Some(user_provider) = self.plugins.get::<UserProviderRef>() {
             builder = builder.with_user_provider(user_provider);
diff --git a/src/pipeline/src/etl/processor/dissect.rs b/src/pipeline/src/etl/processor/dissect.rs
index dca88d3843..f9925916fc 100644
--- a/src/pipeline/src/etl/processor/dissect.rs
+++ b/src/pipeline/src/etl/processor/dissect.rs
@@ -15,7 +15,6 @@
 use std::ops::Deref;
 
 use ahash::{HashMap, HashMapExt, HashSet, HashSetExt};
-use common_telemetry::warn;
 use itertools::Itertools;
 
 use crate::etl::field::{Fields, InputFieldInfo, OneInputMultiOutputField};
@@ -742,11 +741,8 @@ impl DissectProcessor {
         let chs = val.chars().collect::<Vec<char>>();
 
         for pattern in &self.patterns {
-            match self.process_pattern(&chs, pattern) {
-                Ok(map) => return Ok(map),
-                Err(e) => {
-                    warn!("dissect processor: {}", e);
-                }
+            if let Ok(map) = self.process_pattern(&chs, pattern) {
+                return Ok(map);
             }
         }
 
diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs
index 956a650fcc..18388998e7 100644
--- a/src/servers/src/http.rs
+++ b/src/servers/src/http.rs
@@ -63,6 +63,7 @@ use crate::http::prometheus::{
     build_info_query, format_query, instant_query, label_values_query, labels_query, range_query,
     series_query,
 };
+use crate::interceptor::LogIngestInterceptorRef;
 use crate::metrics::http_metrics_layer;
 use crate::metrics_handler::MetricsHandler;
 use crate::prometheus_handler::PrometheusHandlerRef;
@@ -596,11 +597,16 @@ impl HttpServerBuilder {
         self,
         handler: LogHandlerRef,
         validator: Option<LogValidatorRef>,
+        ingest_interceptor: Option<LogIngestInterceptorRef<Error>>,
     ) -> Self {
         Self {
             router: self.router.nest(
                 &format!("/{HTTP_API_VERSION}/events"),
-                HttpServer::route_log(handler, validator),
+                HttpServer::route_log(LogState {
+                    log_handler: handler,
+                    log_validator: validator,
+                    ingest_interceptor,
+                }),
             ),
             ..self
         }
@@ -739,10 +745,7 @@ impl HttpServer {
             .with_state(metrics_handler)
     }
 
-    fn route_log<S>(
-        log_handler: LogHandlerRef,
-        log_validator: Option<LogValidatorRef>,
-    ) -> Router<S> {
+    fn route_log<S>(log_state: LogState) -> Router<S> {
         Router::new()
             .route("/logs", routing::post(event::log_ingester))
             .route(
@@ -759,10 +762,7 @@ impl HttpServer {
                     .layer(HandleErrorLayer::new(handle_error))
                     .layer(RequestDecompressionLayer::new()),
             )
-            .with_state(LogState {
-                log_handler,
-                log_validator,
-            })
+            .with_state(log_state)
     }
 
     fn route_sql<S>(api_state: ApiState) -> ApiRouter<S> {
diff --git a/src/servers/src/http/event.rs b/src/servers/src/http/event.rs
index dbd7f1232a..f0a0902837 100644
--- a/src/servers/src/http/event.rs
+++ b/src/servers/src/http/event.rs
@@ -37,11 +37,13 @@ use session::context::{Channel, QueryContext, QueryContextRef};
 use snafu::{ensure, OptionExt, ResultExt};
 
 use crate::error::{
-    InvalidParameterSnafu, ParseJsonSnafu, PipelineSnafu, Result, UnsupportedContentTypeSnafu,
+    Error, InvalidParameterSnafu, ParseJsonSnafu, PipelineSnafu, Result,
+    UnsupportedContentTypeSnafu,
 };
 use crate::http::greptime_manage_resp::GreptimedbManageResponse;
 use crate::http::greptime_result_v1::GreptimedbV1Response;
 use crate::http::HttpResponse;
+use crate::interceptor::{LogIngestInterceptor, LogIngestInterceptorRef};
 use crate::metrics::{
     METRIC_FAILURE_VALUE, METRIC_HTTP_LOGS_INGESTION_COUNTER, METRIC_HTTP_LOGS_INGESTION_ELAPSED,
     METRIC_HTTP_LOGS_TRANSFORM_ELAPSED, METRIC_SUCCESS_VALUE,
@@ -378,6 +380,11 @@ pub async fn log_ingester(
     query_ctx.set_channel(Channel::Http);
     let query_ctx = Arc::new(query_ctx);
 
+    let value = log_state
+        .ingest_interceptor
+        .as_ref()
+        .pre_pipeline(value, query_ctx.clone())?;
+
     ingest_logs_inner(
         handler,
         pipeline_name,
@@ -506,6 +513,7 @@ pub type LogValidatorRef = Arc<dyn LogValidator + 'static>;
 pub struct LogState {
     pub log_handler: LogHandlerRef,
     pub log_validator: Option<LogValidatorRef>,
+    pub ingest_interceptor: Option<LogIngestInterceptorRef<Error>>,
 }
 
 #[cfg(test)]
diff --git a/src/servers/src/interceptor.rs b/src/servers/src/interceptor.rs
index e4aceeb442..d3478a56ea 100644
--- a/src/servers/src/interceptor.rs
+++ b/src/servers/src/interceptor.rs
@@ -23,6 +23,7 @@ use common_error::ext::ErrorExt;
 use common_query::Output;
 use query::parser::PromQuery;
 use query::plan::LogicalPlan;
+use serde_json::Value;
 use session::context::QueryContextRef;
 use sql::statements::statement::Statement;
 
@@ -397,3 +398,61 @@ impl<E: ErrorExt> PromStoreProtocolInterceptor for Option<PromStoreProtocolInter
         }
     }
 }
+
+/// LogIngestInterceptor can track life cycle of a log ingestion request
+/// and customize or abort its execution at given point.
+pub trait LogIngestInterceptor {
+    type Error: ErrorExt;
+
+    /// Called before pipeline execution.
+    fn pre_pipeline(
+        &self,
+        values: Vec<Value>,
+        _query_ctx: QueryContextRef,
+    ) -> Result<Vec<Value>, Self::Error> {
+        Ok(values)
+    }
+
+    /// Called before insertion.
+    fn pre_ingest(
+        &self,
+        request: RowInsertRequests,
+        _query_ctx: QueryContextRef,
+    ) -> Result<RowInsertRequests, Self::Error> {
+        Ok(request)
+    }
+}
+
+pub type LogIngestInterceptorRef<E> =
+    Arc<dyn LogIngestInterceptor<Error = E> + Send + Sync + 'static>;
+
+impl<E> LogIngestInterceptor for Option<&LogIngestInterceptorRef<E>>
+where
+    E: ErrorExt,
+{
+    type Error = E;
+
+    fn pre_pipeline(
+        &self,
+        values: Vec<Value>,
+        query_ctx: QueryContextRef,
+    ) -> Result<Vec<Value>, Self::Error> {
+        if let Some(this) = self {
+            this.pre_pipeline(values, query_ctx)
+        } else {
+            Ok(values)
+        }
+    }
+
+    fn pre_ingest(
+        &self,
+        request: RowInsertRequests,
+        query_ctx: QueryContextRef,
+    ) -> Result<RowInsertRequests, Self::Error> {
+        if let Some(this) = self {
+            this.pre_ingest(request, query_ctx)
+        } else {
+            Ok(request)
+        }
+    }
+}
diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs
index cf125a5776..a055527e2b 100644
--- a/tests-integration/src/test_util.rs
+++ b/tests-integration/src/test_util.rs
@@ -424,7 +424,7 @@ pub async fn setup_test_http_app_with_frontend_and_user_provider(
             ServerSqlQueryHandlerAdapter::arc(instance.instance.clone()),
             Some(instance.instance.clone()),
         )
-        .with_log_ingest_handler(instance.instance.clone(), None)
+        .with_log_ingest_handler(instance.instance.clone(), None, None)
         .with_otlp_handler(instance.instance.clone())
         .with_greptime_config_options(instance.opts.to_toml().unwrap());
 

From 8786624515b85cdcf238ae88f92b7cfbb80abab0 Mon Sep 17 00:00:00 2001
From: Ning Sun <sunng@protonmail.com>
Date: Thu, 19 Sep 2024 13:30:56 +0800
Subject: [PATCH 016/128] feat: improve support for postgres extended protocol
 (#4721)

* feat: improve support for postgres extended protocol

* fix: lint fix

* fix: test code

* fix: adopt upstream

* refactor: remove dup code

* refactor: avoid copy on error message
---
 src/query/src/datafusion.rs          |  18 +-
 src/servers/src/postgres/fixtures.rs |  22 +-
 src/servers/src/postgres/handler.rs  |  49 ++-
 src/servers/src/postgres/types.rs    | 471 ++++++++++++++-------------
 4 files changed, 313 insertions(+), 247 deletions(-)

diff --git a/src/query/src/datafusion.rs b/src/query/src/datafusion.rs
index 03eadfde97..6ed5844de0 100644
--- a/src/query/src/datafusion.rs
+++ b/src/query/src/datafusion.rs
@@ -398,11 +398,19 @@ impl QueryEngine for DatafusionQueryEngine {
         query_ctx: QueryContextRef,
     ) -> Result<DescribeResult> {
         let ctx = self.engine_context(query_ctx);
-        let optimised_plan = self.optimize(&ctx, &plan)?;
-        Ok(DescribeResult {
-            schema: optimised_plan.schema()?,
-            logical_plan: optimised_plan,
-        })
+        if let Ok(optimised_plan) = self.optimize(&ctx, &plan) {
+            Ok(DescribeResult {
+                schema: optimised_plan.schema()?,
+                logical_plan: optimised_plan,
+            })
+        } else {
+            // Table's like those in information_schema cannot be optimized when
+            // it contains parameters. So we fallback to original plans.
+            Ok(DescribeResult {
+                schema: plan.schema()?,
+                logical_plan: plan,
+            })
+        }
     }
 
     async fn execute(&self, plan: LogicalPlan, query_ctx: QueryContextRef) -> Result<Output> {
diff --git a/src/servers/src/postgres/fixtures.rs b/src/servers/src/postgres/fixtures.rs
index 5b02480da9..18c3661b93 100644
--- a/src/servers/src/postgres/fixtures.rs
+++ b/src/servers/src/postgres/fixtures.rs
@@ -54,17 +54,19 @@ static SET_TRANSACTION_PATTERN: Lazy<Regex> =
 static TRANSACTION_PATTERN: Lazy<Regex> =
     Lazy::new(|| Regex::new("(?i)^(BEGIN|ROLLBACK|COMMIT);?").unwrap());
 
+/// Test if given query statement matches the patterns
+pub(crate) fn matches(query: &str) -> bool {
+    TRANSACTION_PATTERN.captures(query).is_some()
+        || SHOW_PATTERN.captures(query).is_some()
+        || SET_TRANSACTION_PATTERN.is_match(query)
+}
+
 /// Process unsupported SQL and return fixed result as a compatibility solution
-pub(crate) fn process<'a>(
-    query: &str,
-    _query_ctx: QueryContextRef,
-) -> Option<PgWireResult<Vec<Response<'a>>>> {
+pub(crate) fn process<'a>(query: &str, _query_ctx: QueryContextRef) -> Option<Vec<Response<'a>>> {
     // Transaction directives:
     if let Some(tx) = TRANSACTION_PATTERN.captures(query) {
         let tx_tag = &tx[1];
-        Some(Ok(vec![Response::Execution(Tag::new(
-            &tx_tag.to_uppercase(),
-        ))]))
+        Some(vec![Response::Execution(Tag::new(&tx_tag.to_uppercase()))])
     } else if let Some(show_var) = SHOW_PATTERN.captures(query) {
         let show_var = show_var[1].to_lowercase();
         if let Some(value) = VAR_VALUES.get(&show_var.as_ref()) {
@@ -81,12 +83,12 @@ pub(crate) fn process<'a>(
                 vec![vec![value.to_string()]],
             ));
 
-            Some(Ok(vec![Response::Query(QueryResponse::new(schema, data))]))
+            Some(vec![Response::Query(QueryResponse::new(schema, data))])
         } else {
             None
         }
     } else if SET_TRANSACTION_PATTERN.is_match(query) {
-        Some(Ok(vec![Response::Execution(Tag::new("SET"))]))
+        Some(vec![Response::Execution(Tag::new("SET"))])
     } else {
         None
     }
@@ -101,7 +103,6 @@ mod test {
     fn assert_tag(q: &str, t: &str, query_context: QueryContextRef) {
         if let Response::Execution(tag) = process(q, query_context.clone())
             .unwrap_or_else(|| panic!("fail to match {}", q))
-            .expect("unexpected error")
             .remove(0)
         {
             assert_eq!(Tag::new(t), tag);
@@ -113,7 +114,6 @@ mod test {
     fn get_data<'a>(q: &str, query_context: QueryContextRef) -> QueryResponse<'a> {
         if let Response::Query(resp) = process(q, query_context.clone())
             .unwrap_or_else(|| panic!("fail to match {}", q))
-            .expect("unexpected error")
             .remove(0)
         {
             resp
diff --git a/src/servers/src/postgres/handler.rs b/src/servers/src/postgres/handler.rs
index 190684ed34..53d907d814 100644
--- a/src/servers/src/postgres/handler.rs
+++ b/src/servers/src/postgres/handler.rs
@@ -59,8 +59,13 @@ impl SimpleQueryHandler for PostgresServerHandler {
             .with_label_values(&[crate::metrics::METRIC_POSTGRES_SIMPLE_QUERY, db.as_str()])
             .start_timer();
 
+        if query.is_empty() {
+            // early return if query is empty
+            return Ok(vec![Response::EmptyQuery]);
+        }
+
         if let Some(resps) = fixtures::process(query, query_ctx.clone()) {
-            resps
+            Ok(resps)
         } else {
             let outputs = self.query_handler.do_query(query, query_ctx.clone()).await;
 
@@ -184,6 +189,16 @@ impl QueryParser for DefaultQueryParser {
     async fn parse_sql(&self, sql: &str, _types: &[Type]) -> PgWireResult<Self::Statement> {
         crate::metrics::METRIC_POSTGRES_PREPARED_COUNT.inc();
         let query_ctx = self.session.new_query_context();
+
+        // do not parse if query is empty or matches rules
+        if sql.is_empty() || fixtures::matches(sql) {
+            return Ok(SqlPlan {
+                query: sql.to_owned(),
+                plan: None,
+                schema: None,
+            });
+        }
+
         let mut stmts =
             ParserContext::create_with_dialect(sql, &PostgreSqlDialect {}, ParseOptions::default())
                 .map_err(|e| PgWireError::ApiError(Box::new(e)))?;
@@ -193,6 +208,7 @@ impl QueryParser for DefaultQueryParser {
             ))))
         } else {
             let stmt = stmts.remove(0);
+
             let describe_result = self
                 .query_handler
                 .do_describe(stmt, query_ctx)
@@ -244,6 +260,16 @@ impl ExtendedQueryHandler for PostgresServerHandler {
 
         let sql_plan = &portal.statement.statement;
 
+        if sql_plan.query.is_empty() {
+            // early return if query is empty
+            return Ok(Response::EmptyQuery);
+        }
+
+        if let Some(mut resps) = fixtures::process(&sql_plan.query, query_ctx.clone()) {
+            // if the statement matches our predefined rules, return it early
+            return Ok(resps.remove(0));
+        }
+
         let output = if let Some(plan) = &sql_plan.plan {
             let plan = plan
                 .replace_params_with_values(parameters_to_scalar_values(plan, portal)?.as_ref())
@@ -297,6 +323,17 @@ impl ExtendedQueryHandler for PostgresServerHandler {
                 .map(|fields| DescribeStatementResponse::new(param_types, fields))
                 .map_err(|e| PgWireError::ApiError(Box::new(e)))
         } else {
+            if let Some(mut resp) =
+                fixtures::process(&sql_plan.query, self.session.new_query_context())
+            {
+                if let Response::Query(query_response) = resp.remove(0) {
+                    return Ok(DescribeStatementResponse::new(
+                        param_types,
+                        (*query_response.row_schema()).clone(),
+                    ));
+                }
+            }
+
             Ok(DescribeStatementResponse::new(param_types, vec![]))
         }
     }
@@ -317,6 +354,16 @@ impl ExtendedQueryHandler for PostgresServerHandler {
                 .map(DescribePortalResponse::new)
                 .map_err(|e| PgWireError::ApiError(Box::new(e)))
         } else {
+            if let Some(mut resp) =
+                fixtures::process(&sql_plan.query, self.session.new_query_context())
+            {
+                if let Response::Query(query_response) = resp.remove(0) {
+                    return Ok(DescribePortalResponse::new(
+                        (*query_response.row_schema()).clone(),
+                    ));
+                }
+            }
+
             Ok(DescribePortalResponse::new(vec![]))
         }
     }
diff --git a/src/servers/src/postgres/types.rs b/src/servers/src/postgres/types.rs
index 2bec6c2999..9f9d94905e 100644
--- a/src/servers/src/postgres/types.rs
+++ b/src/servers/src/postgres/types.rs
@@ -239,14 +239,14 @@ pub(super) fn parameter_to_string(portal: &Portal<SqlPlan>, idx: usize) -> PgWir
             .unwrap_or_else(|| "".to_owned())),
         _ => Err(invalid_parameter_error(
             "unsupported_parameter_type",
-            Some(&param_type.to_string()),
+            Some(param_type.to_string()),
         )),
     }
 }
 
-pub(super) fn invalid_parameter_error(msg: &str, detail: Option<&str>) -> PgWireError {
+pub(super) fn invalid_parameter_error(msg: &str, detail: Option<String>) -> PgWireError {
     let mut error_info = PgErrorCode::Ec22023.to_err_info(msg.to_string());
-    error_info.detail = detail.map(|s| s.to_owned());
+    error_info.detail = detail;
     PgWireError::UserError(Box::new(error_info))
 }
 
@@ -279,303 +279,314 @@ pub(super) fn parameters_to_scalar_values(
         .get_param_types()
         .map_err(|e| PgWireError::ApiError(Box::new(e)))?;
 
-    // ensure parameter count consistent for: client parameter types, server
-    // parameter types and parameter count
-    if param_types.len() != param_count {
-        return Err(invalid_parameter_error(
-            "invalid_parameter_count",
-            Some(&format!(
-                "Expected: {}, found: {}",
-                param_types.len(),
-                param_count
-            )),
-        ));
-    }
-
     for idx in 0..param_count {
-        let server_type =
-            if let Some(Some(server_infer_type)) = param_types.get(&format!("${}", idx + 1)) {
-                server_infer_type
-            } else {
-                // at the moment we require type information inferenced by
-                // server so here we return error if the type is unknown from
-                // server-side.
-                //
-                // It might be possible to parse the parameter just using client
-                // specified type, we will implement that if there is a case.
-                return Err(invalid_parameter_error("unknown_parameter_type", None));
-            };
+        let server_type = param_types
+            .get(&format!("${}", idx + 1))
+            .and_then(|t| t.as_ref());
 
         let client_type = if let Some(client_given_type) = client_param_types.get(idx) {
             client_given_type.clone()
+        } else if let Some(server_provided_type) = &server_type {
+            type_gt_to_pg(server_provided_type).map_err(|e| PgWireError::ApiError(Box::new(e)))?
         } else {
-            type_gt_to_pg(server_type).map_err(|e| PgWireError::ApiError(Box::new(e)))?
+            return Err(invalid_parameter_error(
+                "unknown_parameter_type",
+                Some(format!(
+                    "Cannot get parameter type information for parameter {}",
+                    idx
+                )),
+            ));
         };
 
         let value = match &client_type {
             &Type::VARCHAR | &Type::TEXT => {
                 let data = portal.parameter::<String>(idx, &client_type)?;
-                match server_type {
-                    ConcreteDataType::String(_) => ScalarValue::Utf8(data),
-                    _ => {
-                        return Err(invalid_parameter_error(
-                            "invalid_parameter_type",
-                            Some(&format!(
-                                "Expected: {}, found: {}",
-                                server_type, client_type
-                            )),
-                        ))
+                if let Some(server_type) = &server_type {
+                    match server_type {
+                        ConcreteDataType::String(_) => ScalarValue::Utf8(data),
+                        _ => {
+                            return Err(invalid_parameter_error(
+                                "invalid_parameter_type",
+                                Some(format!("Expected: {}, found: {}", server_type, client_type)),
+                            ))
+                        }
                     }
+                } else {
+                    ScalarValue::Utf8(data)
                 }
             }
             &Type::BOOL => {
                 let data = portal.parameter::<bool>(idx, &client_type)?;
-                match server_type {
-                    ConcreteDataType::Boolean(_) => ScalarValue::Boolean(data),
-                    _ => {
-                        return Err(invalid_parameter_error(
-                            "invalid_parameter_type",
-                            Some(&format!(
-                                "Expected: {}, found: {}",
-                                server_type, client_type
-                            )),
-                        ))
+                if let Some(server_type) = &server_type {
+                    match server_type {
+                        ConcreteDataType::Boolean(_) => ScalarValue::Boolean(data),
+                        _ => {
+                            return Err(invalid_parameter_error(
+                                "invalid_parameter_type",
+                                Some(format!("Expected: {}, found: {}", server_type, client_type)),
+                            ))
+                        }
                     }
+                } else {
+                    ScalarValue::Boolean(data)
                 }
             }
             &Type::INT2 => {
                 let data = portal.parameter::<i16>(idx, &client_type)?;
-                match server_type {
-                    ConcreteDataType::Int8(_) => ScalarValue::Int8(data.map(|n| n as i8)),
-                    ConcreteDataType::Int16(_) => ScalarValue::Int16(data),
-                    ConcreteDataType::Int32(_) => ScalarValue::Int32(data.map(|n| n as i32)),
-                    ConcreteDataType::Int64(_) => ScalarValue::Int64(data.map(|n| n as i64)),
-                    ConcreteDataType::UInt8(_) => ScalarValue::UInt8(data.map(|n| n as u8)),
-                    ConcreteDataType::UInt16(_) => ScalarValue::UInt16(data.map(|n| n as u16)),
-                    ConcreteDataType::UInt32(_) => ScalarValue::UInt32(data.map(|n| n as u32)),
-                    ConcreteDataType::UInt64(_) => ScalarValue::UInt64(data.map(|n| n as u64)),
-                    ConcreteDataType::Timestamp(unit) => {
-                        to_timestamp_scalar_value(data, unit, server_type)?
-                    }
-                    ConcreteDataType::DateTime(_) => ScalarValue::Date64(data.map(|d| d as i64)),
-                    _ => {
-                        return Err(invalid_parameter_error(
-                            "invalid_parameter_type",
-                            Some(&format!(
-                                "Expected: {}, found: {}",
-                                server_type, client_type
-                            )),
-                        ))
+                if let Some(server_type) = &server_type {
+                    match server_type {
+                        ConcreteDataType::Int8(_) => ScalarValue::Int8(data.map(|n| n as i8)),
+                        ConcreteDataType::Int16(_) => ScalarValue::Int16(data),
+                        ConcreteDataType::Int32(_) => ScalarValue::Int32(data.map(|n| n as i32)),
+                        ConcreteDataType::Int64(_) => ScalarValue::Int64(data.map(|n| n as i64)),
+                        ConcreteDataType::UInt8(_) => ScalarValue::UInt8(data.map(|n| n as u8)),
+                        ConcreteDataType::UInt16(_) => ScalarValue::UInt16(data.map(|n| n as u16)),
+                        ConcreteDataType::UInt32(_) => ScalarValue::UInt32(data.map(|n| n as u32)),
+                        ConcreteDataType::UInt64(_) => ScalarValue::UInt64(data.map(|n| n as u64)),
+                        ConcreteDataType::Timestamp(unit) => {
+                            to_timestamp_scalar_value(data, unit, server_type)?
+                        }
+                        ConcreteDataType::DateTime(_) => {
+                            ScalarValue::Date64(data.map(|d| d as i64))
+                        }
+                        _ => {
+                            return Err(invalid_parameter_error(
+                                "invalid_parameter_type",
+                                Some(format!("Expected: {}, found: {}", server_type, client_type)),
+                            ))
+                        }
                     }
+                } else {
+                    ScalarValue::Int16(data)
                 }
             }
             &Type::INT4 => {
                 let data = portal.parameter::<i32>(idx, &client_type)?;
-                match server_type {
-                    ConcreteDataType::Int8(_) => ScalarValue::Int8(data.map(|n| n as i8)),
-                    ConcreteDataType::Int16(_) => ScalarValue::Int16(data.map(|n| n as i16)),
-                    ConcreteDataType::Int32(_) => ScalarValue::Int32(data),
-                    ConcreteDataType::Int64(_) => ScalarValue::Int64(data.map(|n| n as i64)),
-                    ConcreteDataType::UInt8(_) => ScalarValue::UInt8(data.map(|n| n as u8)),
-                    ConcreteDataType::UInt16(_) => ScalarValue::UInt16(data.map(|n| n as u16)),
-                    ConcreteDataType::UInt32(_) => ScalarValue::UInt32(data.map(|n| n as u32)),
-                    ConcreteDataType::UInt64(_) => ScalarValue::UInt64(data.map(|n| n as u64)),
-                    ConcreteDataType::Timestamp(unit) => {
-                        to_timestamp_scalar_value(data, unit, server_type)?
-                    }
-                    ConcreteDataType::DateTime(_) => ScalarValue::Date64(data.map(|d| d as i64)),
-                    _ => {
-                        return Err(invalid_parameter_error(
-                            "invalid_parameter_type",
-                            Some(&format!(
-                                "Expected: {}, found: {}",
-                                server_type, client_type
-                            )),
-                        ))
+                if let Some(server_type) = &server_type {
+                    match server_type {
+                        ConcreteDataType::Int8(_) => ScalarValue::Int8(data.map(|n| n as i8)),
+                        ConcreteDataType::Int16(_) => ScalarValue::Int16(data.map(|n| n as i16)),
+                        ConcreteDataType::Int32(_) => ScalarValue::Int32(data),
+                        ConcreteDataType::Int64(_) => ScalarValue::Int64(data.map(|n| n as i64)),
+                        ConcreteDataType::UInt8(_) => ScalarValue::UInt8(data.map(|n| n as u8)),
+                        ConcreteDataType::UInt16(_) => ScalarValue::UInt16(data.map(|n| n as u16)),
+                        ConcreteDataType::UInt32(_) => ScalarValue::UInt32(data.map(|n| n as u32)),
+                        ConcreteDataType::UInt64(_) => ScalarValue::UInt64(data.map(|n| n as u64)),
+                        ConcreteDataType::Timestamp(unit) => {
+                            to_timestamp_scalar_value(data, unit, server_type)?
+                        }
+                        ConcreteDataType::DateTime(_) => {
+                            ScalarValue::Date64(data.map(|d| d as i64))
+                        }
+                        _ => {
+                            return Err(invalid_parameter_error(
+                                "invalid_parameter_type",
+                                Some(format!("Expected: {}, found: {}", server_type, client_type)),
+                            ))
+                        }
                     }
+                } else {
+                    ScalarValue::Int32(data)
                 }
             }
             &Type::INT8 => {
                 let data = portal.parameter::<i64>(idx, &client_type)?;
-                match server_type {
-                    ConcreteDataType::Int8(_) => ScalarValue::Int8(data.map(|n| n as i8)),
-                    ConcreteDataType::Int16(_) => ScalarValue::Int16(data.map(|n| n as i16)),
-                    ConcreteDataType::Int32(_) => ScalarValue::Int32(data.map(|n| n as i32)),
-                    ConcreteDataType::Int64(_) => ScalarValue::Int64(data),
-                    ConcreteDataType::UInt8(_) => ScalarValue::UInt8(data.map(|n| n as u8)),
-                    ConcreteDataType::UInt16(_) => ScalarValue::UInt16(data.map(|n| n as u16)),
-                    ConcreteDataType::UInt32(_) => ScalarValue::UInt32(data.map(|n| n as u32)),
-                    ConcreteDataType::UInt64(_) => ScalarValue::UInt64(data.map(|n| n as u64)),
-                    ConcreteDataType::Timestamp(unit) => {
-                        to_timestamp_scalar_value(data, unit, server_type)?
-                    }
-                    ConcreteDataType::DateTime(_) => ScalarValue::Date64(data),
-                    _ => {
-                        return Err(invalid_parameter_error(
-                            "invalid_parameter_type",
-                            Some(&format!(
-                                "Expected: {}, found: {}",
-                                server_type, client_type
-                            )),
-                        ))
+                if let Some(server_type) = &server_type {
+                    match server_type {
+                        ConcreteDataType::Int8(_) => ScalarValue::Int8(data.map(|n| n as i8)),
+                        ConcreteDataType::Int16(_) => ScalarValue::Int16(data.map(|n| n as i16)),
+                        ConcreteDataType::Int32(_) => ScalarValue::Int32(data.map(|n| n as i32)),
+                        ConcreteDataType::Int64(_) => ScalarValue::Int64(data),
+                        ConcreteDataType::UInt8(_) => ScalarValue::UInt8(data.map(|n| n as u8)),
+                        ConcreteDataType::UInt16(_) => ScalarValue::UInt16(data.map(|n| n as u16)),
+                        ConcreteDataType::UInt32(_) => ScalarValue::UInt32(data.map(|n| n as u32)),
+                        ConcreteDataType::UInt64(_) => ScalarValue::UInt64(data.map(|n| n as u64)),
+                        ConcreteDataType::Timestamp(unit) => {
+                            to_timestamp_scalar_value(data, unit, server_type)?
+                        }
+                        ConcreteDataType::DateTime(_) => ScalarValue::Date64(data),
+                        _ => {
+                            return Err(invalid_parameter_error(
+                                "invalid_parameter_type",
+                                Some(format!("Expected: {}, found: {}", server_type, client_type)),
+                            ))
+                        }
                     }
+                } else {
+                    ScalarValue::Int64(data)
                 }
             }
             &Type::FLOAT4 => {
                 let data = portal.parameter::<f32>(idx, &client_type)?;
-                match server_type {
-                    ConcreteDataType::Int8(_) => ScalarValue::Int8(data.map(|n| n as i8)),
-                    ConcreteDataType::Int16(_) => ScalarValue::Int16(data.map(|n| n as i16)),
-                    ConcreteDataType::Int32(_) => ScalarValue::Int32(data.map(|n| n as i32)),
-                    ConcreteDataType::Int64(_) => ScalarValue::Int64(data.map(|n| n as i64)),
-                    ConcreteDataType::UInt8(_) => ScalarValue::UInt8(data.map(|n| n as u8)),
-                    ConcreteDataType::UInt16(_) => ScalarValue::UInt16(data.map(|n| n as u16)),
-                    ConcreteDataType::UInt32(_) => ScalarValue::UInt32(data.map(|n| n as u32)),
-                    ConcreteDataType::UInt64(_) => ScalarValue::UInt64(data.map(|n| n as u64)),
-                    ConcreteDataType::Float32(_) => ScalarValue::Float32(data),
-                    ConcreteDataType::Float64(_) => ScalarValue::Float64(data.map(|n| n as f64)),
-                    _ => {
-                        return Err(invalid_parameter_error(
-                            "invalid_parameter_type",
-                            Some(&format!(
-                                "Expected: {}, found: {}",
-                                server_type, client_type
-                            )),
-                        ))
+                if let Some(server_type) = &server_type {
+                    match server_type {
+                        ConcreteDataType::Int8(_) => ScalarValue::Int8(data.map(|n| n as i8)),
+                        ConcreteDataType::Int16(_) => ScalarValue::Int16(data.map(|n| n as i16)),
+                        ConcreteDataType::Int32(_) => ScalarValue::Int32(data.map(|n| n as i32)),
+                        ConcreteDataType::Int64(_) => ScalarValue::Int64(data.map(|n| n as i64)),
+                        ConcreteDataType::UInt8(_) => ScalarValue::UInt8(data.map(|n| n as u8)),
+                        ConcreteDataType::UInt16(_) => ScalarValue::UInt16(data.map(|n| n as u16)),
+                        ConcreteDataType::UInt32(_) => ScalarValue::UInt32(data.map(|n| n as u32)),
+                        ConcreteDataType::UInt64(_) => ScalarValue::UInt64(data.map(|n| n as u64)),
+                        ConcreteDataType::Float32(_) => ScalarValue::Float32(data),
+                        ConcreteDataType::Float64(_) => {
+                            ScalarValue::Float64(data.map(|n| n as f64))
+                        }
+                        _ => {
+                            return Err(invalid_parameter_error(
+                                "invalid_parameter_type",
+                                Some(format!("Expected: {}, found: {}", server_type, client_type)),
+                            ))
+                        }
                     }
+                } else {
+                    ScalarValue::Float32(data)
                 }
             }
             &Type::FLOAT8 => {
                 let data = portal.parameter::<f64>(idx, &client_type)?;
-                match server_type {
-                    ConcreteDataType::Int8(_) => ScalarValue::Int8(data.map(|n| n as i8)),
-                    ConcreteDataType::Int16(_) => ScalarValue::Int16(data.map(|n| n as i16)),
-                    ConcreteDataType::Int32(_) => ScalarValue::Int32(data.map(|n| n as i32)),
-                    ConcreteDataType::Int64(_) => ScalarValue::Int64(data.map(|n| n as i64)),
-                    ConcreteDataType::UInt8(_) => ScalarValue::UInt8(data.map(|n| n as u8)),
-                    ConcreteDataType::UInt16(_) => ScalarValue::UInt16(data.map(|n| n as u16)),
-                    ConcreteDataType::UInt32(_) => ScalarValue::UInt32(data.map(|n| n as u32)),
-                    ConcreteDataType::UInt64(_) => ScalarValue::UInt64(data.map(|n| n as u64)),
-                    ConcreteDataType::Float32(_) => ScalarValue::Float32(data.map(|n| n as f32)),
-                    ConcreteDataType::Float64(_) => ScalarValue::Float64(data),
-                    _ => {
-                        return Err(invalid_parameter_error(
-                            "invalid_parameter_type",
-                            Some(&format!(
-                                "Expected: {}, found: {}",
-                                server_type, client_type
-                            )),
-                        ))
+                if let Some(server_type) = &server_type {
+                    match server_type {
+                        ConcreteDataType::Int8(_) => ScalarValue::Int8(data.map(|n| n as i8)),
+                        ConcreteDataType::Int16(_) => ScalarValue::Int16(data.map(|n| n as i16)),
+                        ConcreteDataType::Int32(_) => ScalarValue::Int32(data.map(|n| n as i32)),
+                        ConcreteDataType::Int64(_) => ScalarValue::Int64(data.map(|n| n as i64)),
+                        ConcreteDataType::UInt8(_) => ScalarValue::UInt8(data.map(|n| n as u8)),
+                        ConcreteDataType::UInt16(_) => ScalarValue::UInt16(data.map(|n| n as u16)),
+                        ConcreteDataType::UInt32(_) => ScalarValue::UInt32(data.map(|n| n as u32)),
+                        ConcreteDataType::UInt64(_) => ScalarValue::UInt64(data.map(|n| n as u64)),
+                        ConcreteDataType::Float32(_) => {
+                            ScalarValue::Float32(data.map(|n| n as f32))
+                        }
+                        ConcreteDataType::Float64(_) => ScalarValue::Float64(data),
+                        _ => {
+                            return Err(invalid_parameter_error(
+                                "invalid_parameter_type",
+                                Some(format!("Expected: {}, found: {}", server_type, client_type)),
+                            ))
+                        }
                     }
+                } else {
+                    ScalarValue::Float64(data)
                 }
             }
             &Type::TIMESTAMP => {
                 let data = portal.parameter::<NaiveDateTime>(idx, &client_type)?;
-                match server_type {
-                    ConcreteDataType::Timestamp(unit) => match *unit {
-                        TimestampType::Second(_) => ScalarValue::TimestampSecond(
-                            data.map(|ts| ts.and_utc().timestamp()),
-                            None,
-                        ),
-                        TimestampType::Millisecond(_) => ScalarValue::TimestampMillisecond(
-                            data.map(|ts| ts.and_utc().timestamp_millis()),
-                            None,
-                        ),
-                        TimestampType::Microsecond(_) => ScalarValue::TimestampMicrosecond(
-                            data.map(|ts| ts.and_utc().timestamp_micros()),
-                            None,
-                        ),
-                        TimestampType::Nanosecond(_) => ScalarValue::TimestampNanosecond(
-                            data.map(|ts| ts.and_utc().timestamp_micros()),
-                            None,
-                        ),
-                    },
-                    ConcreteDataType::DateTime(_) => {
-                        ScalarValue::Date64(data.map(|d| d.and_utc().timestamp_millis()))
-                    }
-                    _ => {
-                        return Err(invalid_parameter_error(
-                            "invalid_parameter_type",
-                            Some(&format!(
-                                "Expected: {}, found: {}",
-                                server_type, client_type
-                            )),
-                        ))
+                if let Some(server_type) = &server_type {
+                    match server_type {
+                        ConcreteDataType::Timestamp(unit) => match *unit {
+                            TimestampType::Second(_) => ScalarValue::TimestampSecond(
+                                data.map(|ts| ts.and_utc().timestamp()),
+                                None,
+                            ),
+                            TimestampType::Millisecond(_) => ScalarValue::TimestampMillisecond(
+                                data.map(|ts| ts.and_utc().timestamp_millis()),
+                                None,
+                            ),
+                            TimestampType::Microsecond(_) => ScalarValue::TimestampMicrosecond(
+                                data.map(|ts| ts.and_utc().timestamp_micros()),
+                                None,
+                            ),
+                            TimestampType::Nanosecond(_) => ScalarValue::TimestampNanosecond(
+                                data.map(|ts| ts.and_utc().timestamp_micros()),
+                                None,
+                            ),
+                        },
+                        ConcreteDataType::DateTime(_) => {
+                            ScalarValue::Date64(data.map(|d| d.and_utc().timestamp_millis()))
+                        }
+                        _ => {
+                            return Err(invalid_parameter_error(
+                                "invalid_parameter_type",
+                                Some(format!("Expected: {}, found: {}", server_type, client_type)),
+                            ))
+                        }
                     }
+                } else {
+                    ScalarValue::TimestampMillisecond(
+                        data.map(|ts| ts.and_utc().timestamp_millis()),
+                        None,
+                    )
                 }
             }
             &Type::DATE => {
                 let data = portal.parameter::<NaiveDate>(idx, &client_type)?;
-                match server_type {
-                    ConcreteDataType::Date(_) => ScalarValue::Date32(data.map(|d| {
-                        (d - NaiveDate::from_ymd_opt(1970, 1, 1).unwrap()).num_days() as i32
-                    })),
-                    _ => {
-                        return Err(invalid_parameter_error(
-                            "invalid_parameter_type",
-                            Some(&format!(
-                                "Expected: {}, found: {}",
-                                server_type, client_type
-                            )),
-                        ));
+                if let Some(server_type) = &server_type {
+                    match server_type {
+                        ConcreteDataType::Date(_) => ScalarValue::Date32(data.map(|d| {
+                            (d - NaiveDate::from(NaiveDateTime::UNIX_EPOCH)).num_days() as i32
+                        })),
+                        _ => {
+                            return Err(invalid_parameter_error(
+                                "invalid_parameter_type",
+                                Some(format!("Expected: {}, found: {}", server_type, client_type)),
+                            ));
+                        }
                     }
+                } else {
+                    ScalarValue::Date32(data.map(|d| {
+                        (d - NaiveDate::from(NaiveDateTime::UNIX_EPOCH)).num_days() as i32
+                    }))
                 }
             }
             &Type::INTERVAL => {
                 let data = portal.parameter::<PgInterval>(idx, &client_type)?;
-                match server_type {
-                    ConcreteDataType::Interval(_) => {
-                        ScalarValue::IntervalMonthDayNano(data.map(|i| Interval::from(i).to_i128()))
-                    }
-                    _ => {
-                        return Err(invalid_parameter_error(
-                            "invalid_parameter_type",
-                            Some(&format!(
-                                "Expected: {}, found: {}",
-                                server_type, client_type
-                            )),
-                        ));
+                if let Some(server_type) = &server_type {
+                    match server_type {
+                        ConcreteDataType::Interval(_) => ScalarValue::IntervalMonthDayNano(
+                            data.map(|i| Interval::from(i).to_i128()),
+                        ),
+                        _ => {
+                            return Err(invalid_parameter_error(
+                                "invalid_parameter_type",
+                                Some(format!("Expected: {}, found: {}", server_type, client_type)),
+                            ));
+                        }
                     }
+                } else {
+                    ScalarValue::IntervalMonthDayNano(data.map(|i| Interval::from(i).to_i128()))
                 }
             }
             &Type::BYTEA => {
                 let data = portal.parameter::<Vec<u8>>(idx, &client_type)?;
-                match server_type {
-                    ConcreteDataType::String(_) => {
-                        ScalarValue::Utf8(data.map(|d| String::from_utf8_lossy(&d).to_string()))
-                    }
-                    ConcreteDataType::Binary(_) => ScalarValue::Binary(data),
-                    _ => {
-                        return Err(invalid_parameter_error(
-                            "invalid_parameter_type",
-                            Some(&format!(
-                                "Expected: {}, found: {}",
-                                server_type, client_type
-                            )),
-                        ));
+                if let Some(server_type) = &server_type {
+                    match server_type {
+                        ConcreteDataType::String(_) => {
+                            ScalarValue::Utf8(data.map(|d| String::from_utf8_lossy(&d).to_string()))
+                        }
+                        ConcreteDataType::Binary(_) => ScalarValue::Binary(data),
+                        _ => {
+                            return Err(invalid_parameter_error(
+                                "invalid_parameter_type",
+                                Some(format!("Expected: {}, found: {}", server_type, client_type)),
+                            ));
+                        }
                     }
+                } else {
+                    ScalarValue::Binary(data)
                 }
             }
             &Type::JSONB => {
                 let data = portal.parameter::<serde_json::Value>(idx, &client_type)?;
-                match server_type {
-                    ConcreteDataType::Binary(_) => {
-                        ScalarValue::Binary(data.map(|d| jsonb::Value::from(d).to_vec()))
-                    }
-                    _ => {
-                        return Err(invalid_parameter_error(
-                            "invalid_parameter_type",
-                            Some(&format!(
-                                "Expected: {}, found: {}",
-                                server_type, client_type
-                            )),
-                        ));
+                if let Some(server_type) = &server_type {
+                    match server_type {
+                        ConcreteDataType::Binary(_) => {
+                            ScalarValue::Binary(data.map(|d| jsonb::Value::from(d).to_vec()))
+                        }
+                        _ => {
+                            return Err(invalid_parameter_error(
+                                "invalid_parameter_type",
+                                Some(format!("Expected: {}, found: {}", server_type, client_type)),
+                            ));
+                        }
                     }
+                } else {
+                    ScalarValue::Binary(data.map(|d| jsonb::Value::from(d).to_vec()))
                 }
             }
             _ => Err(invalid_parameter_error(
                 "unsupported_parameter_value",
-                Some(&format!("Found type: {}", client_type)),
+                Some(format!("Found type: {}", client_type)),
             ))?,
         };
 

From 0295f8dbea58b2285762ace1741b71ee5b2bc2b0 Mon Sep 17 00:00:00 2001
From: Yohan Wal <59358312+CookiePieWw@users.noreply.github.com>
Date: Thu, 19 Sep 2024 13:49:10 +0800
Subject: [PATCH 017/128] docs: json datatype rfc (#4515)

* docs: json datatype rfc

* docs: turn to a jsonb proposal

* chore: fix typo

* feat: add store and query process

* fix: typo

* fix: use query nodes instead of query plans

* feat: a detailed overview of query

* fix: grammar

* fix: use independent cast function

* fix: unify cast function

* fix: refine, make statements clear

* docs: update rfc according to impl

* docs: refine

* docs: fix wrong arrows

* docs: refine

* docs: fix some errors qaq
---
 docs/rfcs/2024-08-06-json-datatype.md | 197 ++++++++++++++++++++++++++
 1 file changed, 197 insertions(+)
 create mode 100644 docs/rfcs/2024-08-06-json-datatype.md

diff --git a/docs/rfcs/2024-08-06-json-datatype.md b/docs/rfcs/2024-08-06-json-datatype.md
new file mode 100644
index 0000000000..8a617a1709
--- /dev/null
+++ b/docs/rfcs/2024-08-06-json-datatype.md
@@ -0,0 +1,197 @@
+---
+Feature Name: Json Datatype
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/4230
+Date: 2024-8-6
+Author: "Yuhan Wang <profsyb@gmail.com>"
+---
+
+# Summary
+This RFC proposes a method for storing and querying JSON data in the database.
+
+# Motivation
+JSON is widely used across various scenarios. Direct support for writing and querying JSON can significantly enhance the database's flexibility.
+
+# Details
+
+## Storage and Query
+
+GreptimeDB's type system is built on Arrow/DataFusion, where each data type in GreptimeDB corresponds to a data type in Arrow/DataFusion. The proposed JSON type will be implemented on top of the existing `Binary` type, leveraging the current `datatype::value::Value` and `datatype::vectors::BinaryVector` implementations, utilizing the JSONB format as the encoding of JSON data. JSON data is stored and processed similarly to binary data within the storage layer and query engine.
+
+This approach brings problems when dealing with insertions and queries of JSON columns.
+
+## Insertion
+
+Users commonly write JSON data as strings. Thus we need to make conversions between string and JSONB. There are 2 ways to do this:
+
+1. MySQL and PostgreSQL servers provide auto-conversions between strings and JSONB. When a string is inserted into a JSON column, the server will try to parse the string as JSON and convert it to JSONB. The non-JSON strings will be rejected.
+
+2. A function `parse_json` is provided to convert string to JSONB. If the string is not a valid JSON string, the function will return an error.
+
+For example, in MySQL client:
+```SQL
+CREATE TABLE IF NOT EXISTS test (
+    ts TIMESTAMP TIME INDEX,
+    a INT,
+    b JSON
+);
+
+INSERT INTO test VALUES(
+    0,
+    0,
+    '{
+        "name": "jHl2oDDnPc1i2OzlP5Y",
+        "timestamp": "2024-07-25T04:33:11.369386Z",
+        "attributes": { "event_attributes": 48.28667 }
+    }'
+);
+
+INSERT INTO test VALUES(
+    0,
+    0,
+    parse_json('{
+        "name": "jHl2oDDnPc1i2OzlP5Y",
+        "timestamp": "2024-07-25T04:33:11.369386Z",
+        "attributes": { "event_attributes": 48.28667 }
+    }')
+);
+```
+Are both valid.
+
+The dataflow of the insertion process is as follows:
+```
+Insert JSON strings directly through client:
+                                   Parse                       Insert
+        String(Serialized JSON)┌──────────┐Arrow Binary(JSONB)┌──────┐Arrow Binary(JSONB)
+ Client ---------------------->│  Server  │------------------>│ Mito │------------------> Storage
+                               └──────────┘                   └──────┘
+        (Server identifies JSON type and performs auto-conversion)
+
+Insert JSON strings through parse_json function:
+                                                                   Parse                     Insert
+        String(Serialized JSON)┌──────────┐String(Serialized JSON)┌─────┐Arrow Binary(JSONB)┌──────┐Arrow Binary(JSONB)
+ Client ---------------------->│  Server  │---------------------->│ UDF │------------------>│ Mito │------------------> Storage
+                               └──────────┘                       └─────┘                   └──────┘
+                                            (Conversion is performed by UDF inside Query Engine)
+```
+
+Servers identify JSON column through column schema and perform auto-conversions. But when using prepared statements and binding parameters, the corresponding cached plans in datafusion generated by prepared statements cannot identify JSON columns. Under this circumstance, the servers identify JSON columns through the given parameters and perform auto-conversions.
+
+The following is an example of inserting JSON data through prepared statements:
+```Rust
+sqlx::query(
+    "create table test(ts timestamp time index, j json)",
+)
+.execute(&pool)
+.await
+.unwrap();
+
+let json = serde_json::json!({
+    "code": 200,
+    "success": true,
+    "payload": {
+        "features": [
+            "serde",
+            "json"
+        ],
+        "homepage": null
+    }
+});
+
+// Valid, can identify serde_json::Value as JSON type
+sqlx::query("insert into test values($1, $2)")
+    .bind(i)
+    .bind(json)
+    .execute(&pool)
+    .await
+    .unwrap();
+
+// Invalid, cannot identify String as JSON type
+sqlx::query("insert into test values($1, $2)")
+    .bind(i)
+    .bind(json.to_string())
+    .execute(&pool)
+    .await
+    .unwrap();
+```
+
+## Query
+
+Correspondingly, users prefer to display JSON data as strings. Thus we need to make conversions between JSON data and strings before presenting JSON data. There are also 2 ways to do this: auto-conversions on MySQL and PostgreSQL servers, and function `json_to_string`.
+
+For example, in MySQL client:
+```SQL
+SELECT b FROM test;
+
+SELECT json_to_string(b) FROM test;
+```
+Will both return the JSON as human-readable strings.
+
+Specifically, to perform auto-conversions, we attach a message to JSON data in the `metadata` of `Field` in Arrow/Datafusion schema when scanning a JSON column. Frontend servers could identify JSON data and convert it to strings.
+
+The dataflow of the query process is as follows:
+```
+Query directly through client:
+                                  Decode                            Scan
+        String(Serialized JSON)┌──────────┐Arrow Binary(JSONB)┌──────────────┐Arrow Binary(JSONB)
+ Client <----------------------│  Server  │<------------------│ Query Engine │<----------------- Storage
+                               └──────────┘                   └──────────────┘
+(Server identifies JSON type and performs auto-conversion based on column metadata)
+
+Query through json_to_string function:
+                                                                   Scan & Decode
+        String(Serialized JSON)┌──────────┐String(Serialized JSON)┌──────────────┐Arrow Binary(JSONB)
+ Client <----------------------│  Server  │<----------------------│ Query Engine │<----------------- Storage
+                               └──────────┘                       └──────────────┘
+                                                 (Conversion is performed by UDF inside Query Engine)
+
+```
+
+However, if a function uses JSON type as its return type, the metadata method mentioned above is not applicable. Thus the functions of JSON type should specify the return type explicitly instead of returning a JSON type, such as `json_get_int` and `json_get_float` which return corresponding data of `INT` and `FLOAT` type respectively.
+
+## Functions
+Similar to the common JSON type, JSON data can be queried with functions.
+
+For example:
+```SQL
+CREATE TABLE IF NOT EXISTS test (
+    ts TIMESTAMP TIME INDEX,
+    a INT,
+    b JSON
+);
+
+INSERT INTO test VALUES(
+    0,
+    0,
+    '{
+        "name": "jHl2oDDnPc1i2OzlP5Y",
+        "timestamp": "2024-07-25T04:33:11.369386Z",
+        "attributes": { "event_attributes": 48.28667 }
+    }'
+);
+
+SELECT json_get_string(b, 'name') FROM test;
++---------------------+
+| b.name              |
++---------------------+
+| jHl2oDDnPc1i2OzlP5Y |
++---------------------+
+
+SELECT json_get_float(b, 'attributes.event_attributes') FROM test;
++--------------------------------+
+| b.attributes.event_attributes  |
++--------------------------------+
+| 48.28667                       |
++--------------------------------+
+
+```
+And more functions can be added in the future.
+
+# Drawbacks
+
+As a general purpose JSON data type, JSONB may not be as efficient as specialized data types for specific scenarios.
+
+The auto-conversion mechanism is not supported in all scenarios. We need to find workarounds for these scenarios.
+
+# Alternatives
+
+Extract and flatten JSON schema to store in a structured format through pipeline. For nested data, we can provide nested types like `STRUCT` or `ARRAY`.

From d0f5b2ad7d0cb8feea8bc303fe3ff63ba8a550bf Mon Sep 17 00:00:00 2001
From: discord9 <55937128+discord9@users.noreply.github.com>
Date: Thu, 19 Sep 2024 17:34:06 +0800
Subject: [PATCH 018/128] fix: use gcc-10 in release dev build (#4741)

---
 docker/dev-builder/ubuntu/Dockerfile | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docker/dev-builder/ubuntu/Dockerfile b/docker/dev-builder/ubuntu/Dockerfile
index 3f76d80eff..d70824e911 100644
--- a/docker/dev-builder/ubuntu/Dockerfile
+++ b/docker/dev-builder/ubuntu/Dockerfile
@@ -24,6 +24,10 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     python3.10 \
     python3.10-dev
 
+# https://github.com/GreptimeTeam/greptimedb/actions/runs/10935485852/job/30357457188#step:3:7106
+# `aws-lc-sys` require gcc >= 10.3.0 to work, hence alias to use gcc-10
+RUN alias gcc="gcc-10" && alias cc="gcc-10"
+
 # Remove Python 3.8 and install pip.
 RUN apt-get -y purge python3.8 && \
     apt-get -y autoremove && \

From 49004391d348af785aa910691e48d2037b621ffa Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Thu, 19 Sep 2024 18:40:10 +0900
Subject: [PATCH 019/128] chore(fuzz): print table name for debugging (#4738)

* chore(fuzz): print table name for debugging

* chore: apply suggestions
---
 .../targets/failover/fuzz_failover_metric_regions.rs       | 7 ++++++-
 tests-fuzz/targets/failover/fuzz_failover_mito_regions.rs  | 6 +++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/tests-fuzz/targets/failover/fuzz_failover_metric_regions.rs b/tests-fuzz/targets/failover/fuzz_failover_metric_regions.rs
index 5eaf43ab0a..147c3ead1e 100644
--- a/tests-fuzz/targets/failover/fuzz_failover_metric_regions.rs
+++ b/tests-fuzz/targets/failover/fuzz_failover_metric_regions.rs
@@ -257,7 +257,12 @@ async fn execute_failover(ctx: FuzzContext, input: FuzzInput) -> Result<()> {
     for (table_ctx, insert_expr) in tables.values() {
         let sql = format!("select count(1) as count from {}", table_ctx.name);
         let values = count_values(&ctx.greptime, &sql).await?;
-        assert_eq!(values.count as usize, insert_expr.values_list.len());
+        let expected_rows = insert_expr.values_list.len() as u64;
+        assert_eq!(
+            values.count as u64, expected_rows,
+            "Expected rows: {}, got: {}, table: {}",
+            expected_rows, values.count, table_ctx.name
+        );
     }
 
     // Clean up
diff --git a/tests-fuzz/targets/failover/fuzz_failover_mito_regions.rs b/tests-fuzz/targets/failover/fuzz_failover_mito_regions.rs
index 5fab5f2603..f456550b3f 100644
--- a/tests-fuzz/targets/failover/fuzz_failover_mito_regions.rs
+++ b/tests-fuzz/targets/failover/fuzz_failover_mito_regions.rs
@@ -323,7 +323,11 @@ async fn execute_failover(ctx: FuzzContext, input: FuzzInput) -> Result<()> {
     for (table_ctx, expected_rows) in table_ctxs.iter().zip(affected_rows) {
         let sql = format!("select count(1) as count from {}", table_ctx.name);
         let values = count_values(&ctx.greptime, &sql).await?;
-        assert_eq!(values.count as u64, expected_rows);
+        assert_eq!(
+            values.count as u64, expected_rows,
+            "Expected rows: {}, got: {}, table: {}",
+            expected_rows, values.count, table_ctx.name
+        );
     }
 
     for table_ctx in table_ctxs {

From 95787825f1f6c09839271a5e6cc6d11b062bbaad Mon Sep 17 00:00:00 2001
From: Yohan Wal <59358312+CookiePieWw@users.noreply.github.com>
Date: Thu, 19 Sep 2024 17:44:44 +0800
Subject: [PATCH 020/128] build(deps): use original jsonb repo (#4742)

---
 Cargo.lock | 2 +-
 Cargo.toml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 786c1c3a8b..88c7f7f080 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5481,7 +5481,7 @@ dependencies = [
 [[package]]
 name = "jsonb"
 version = "0.4.1"
-source = "git+https://github.com/CookiePieWw/jsonb.git?rev=d0166c130fce903bf6c58643417a3173a6172d31#d0166c130fce903bf6c58643417a3173a6172d31"
+source = "git+https://github.com/datafuselabs/jsonb.git?rev=46ad50fc71cf75afbf98eec455f7892a6387c1fc#46ad50fc71cf75afbf98eec455f7892a6387c1fc"
 dependencies = [
  "byteorder",
  "fast-float",
diff --git a/Cargo.toml b/Cargo.toml
index c1eea12a53..25cd5b5dd2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -124,7 +124,7 @@ greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", r
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"
-jsonb = { git = "https://github.com/CookiePieWw/jsonb.git", rev = "d0166c130fce903bf6c58643417a3173a6172d31", default-features = false }
+jsonb = { git = "https://github.com/datafuselabs/jsonb.git", rev = "46ad50fc71cf75afbf98eec455f7892a6387c1fc", default-features = false }
 lazy_static = "1.4"
 meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd" }
 mockall = "0.11.4"

From 1acda74c260dc989ba6c2aa010782aa6cf67cafb Mon Sep 17 00:00:00 2001
From: liyang <ly18846162402@163.com>
Date: Thu, 19 Sep 2024 19:14:34 +0800
Subject: [PATCH 021/128] fix: cannot input tag for the dev-builder image
 (#4743)

---
 .github/actions/build-dev-builder-images/action.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/actions/build-dev-builder-images/action.yml b/.github/actions/build-dev-builder-images/action.yml
index 6bb14e14f4..9c30caad35 100644
--- a/.github/actions/build-dev-builder-images/action.yml
+++ b/.github/actions/build-dev-builder-images/action.yml
@@ -50,7 +50,7 @@ runs:
           BUILDX_MULTI_PLATFORM_BUILD=all \
           IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
           IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
-          IMAGE_TAG=${{ inputs.version }}
+          DEV_BUILDER_IMAGE_TAG=${{ inputs.version }}
 
     - name: Build and push dev-builder-centos image
       shell: bash
@@ -61,7 +61,7 @@ runs:
           BUILDX_MULTI_PLATFORM_BUILD=amd64 \
           IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
           IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
-          IMAGE_TAG=${{ inputs.version }}
+          DEV_BUILDER_IMAGE_TAG=${{ inputs.version }}
 
     - name: Build and push dev-builder-android image # Only build image for amd64 platform.
       shell: bash
@@ -71,6 +71,6 @@ runs:
           BASE_IMAGE=android \
           IMAGE_REGISTRY=${{ inputs.dockerhub-image-registry }} \
           IMAGE_NAMESPACE=${{ inputs.dockerhub-image-namespace }} \
-          IMAGE_TAG=${{ inputs.version }} && \
+          DEV_BUILDER_IMAGE_TAG=${{ inputs.version }} && \
 
         docker push ${{ inputs.dockerhub-image-registry }}/${{ inputs.dockerhub-image-namespace }}/dev-builder-android:${{ inputs.version }}

From f5cf25b0dbe588e78f2c03a7cb854aeed176067e Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Thu, 19 Sep 2024 20:29:33 +0800
Subject: [PATCH 022/128] refactor: remove DfPlan wrapper  (#4733)

* refactor: remove DfPlan wrapper

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* clean up

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* remove unused errors

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix test assertion

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 Cargo.lock                                  |  12 +-
 src/cmd/src/cli/repl.rs                     |   3 +-
 src/datanode/src/region_server.rs           |   2 +-
 src/datanode/src/tests.rs                   |   2 +-
 src/flow/src/df_optimizer.rs                |   2 -
 src/flow/src/transform.rs                   |   3 -
 src/frontend/Cargo.toml                     |   1 +
 src/frontend/src/instance.rs                |   2 +-
 src/operator/src/error.rs                   |  16 +-
 src/operator/src/statement.rs               |   2 +-
 src/operator/src/statement/copy_table_to.rs |   3 +-
 src/operator/src/statement/ddl.rs           |  24 +--
 src/operator/src/statement/tql.rs           |   2 +-
 src/pipeline/src/manager/table.rs           |   7 +-
 src/query/src/datafusion.rs                 | 154 ++++++++++----------
 src/query/src/error.rs                      |   9 +-
 src/query/src/plan.rs                       |  95 +-----------
 src/query/src/planner.rs                    |   8 +-
 src/query/src/query_engine.rs               |   2 +-
 src/query/src/range_select/plan_rewrite.rs  |   5 +-
 src/query/src/tests/query_engine_test.rs    |  23 ++-
 src/script/src/table.rs                     |   5 +-
 src/servers/Cargo.toml                      |   1 +
 src/servers/src/error.rs                    |  18 +--
 src/servers/src/http.rs                     |   2 +-
 src/servers/src/interceptor.rs              |   2 +-
 src/servers/src/lib.rs                      |   2 +-
 src/servers/src/mysql/handler.rs            |  36 +++--
 src/servers/src/postgres/handler.rs         |  14 +-
 src/servers/src/postgres/types.rs           |   9 +-
 src/servers/src/prom_store.rs               |   4 +-
 src/servers/src/query_handler/sql.rs        |   2 +-
 src/servers/tests/http/influxdb_test.rs     |   2 +-
 src/servers/tests/http/opentsdb_test.rs     |   2 +-
 src/servers/tests/http/prom_store_test.rs   |   2 +-
 src/servers/tests/mod.rs                    |   2 +-
 tests-integration/src/grpc.rs               |   3 +-
 tests-integration/src/instance.rs           |   6 +-
 38 files changed, 195 insertions(+), 294 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 88c7f7f080..55e8ca8e4b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1014,7 +1014,7 @@ dependencies = [
  "bitflags 2.5.0",
  "cexpr",
  "clang-sys",
- "itertools 0.10.5",
+ "itertools 0.12.1",
  "lazy_static",
  "lazycell",
  "log",
@@ -4019,6 +4019,7 @@ dependencies = [
  "common-test-util",
  "common-time",
  "common-version",
+ "datafusion-expr",
  "datanode",
  "datatypes",
  "futures",
@@ -4836,7 +4837,7 @@ dependencies = [
  "httpdate",
  "itoa",
  "pin-project-lite",
- "socket2 0.4.10",
+ "socket2 0.5.7",
  "tokio",
  "tower-service",
  "tracing",
@@ -8439,7 +8440,7 @@ checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
 dependencies = [
  "bytes",
  "heck 0.5.0",
- "itertools 0.10.5",
+ "itertools 0.12.1",
  "log",
  "multimap",
  "once_cell",
@@ -8491,7 +8492,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1"
 dependencies = [
  "anyhow",
- "itertools 0.10.5",
+ "itertools 0.12.1",
  "proc-macro2",
  "quote",
  "syn 2.0.66",
@@ -8651,7 +8652,7 @@ dependencies = [
  "indoc",
  "libc",
  "memoffset 0.9.1",
- "parking_lot 0.11.2",
+ "parking_lot 0.12.3",
  "portable-atomic",
  "pyo3-build-config",
  "pyo3-ffi",
@@ -10480,6 +10481,7 @@ dependencies = [
  "dashmap",
  "datafusion",
  "datafusion-common",
+ "datafusion-expr",
  "datatypes",
  "derive_builder 0.12.0",
  "futures",
diff --git a/src/cmd/src/cli/repl.rs b/src/cmd/src/cli/repl.rs
index 6d7e211d7c..3fc2184d37 100644
--- a/src/cmd/src/cli/repl.rs
+++ b/src/cmd/src/cli/repl.rs
@@ -35,7 +35,6 @@ use either::Either;
 use meta_client::client::MetaClientBuilder;
 use query::datafusion::DatafusionQueryEngine;
 use query::parser::QueryLanguageParser;
-use query::plan::LogicalPlan;
 use query::query_engine::{DefaultSerializer, QueryEngineState};
 use query::QueryEngine;
 use rustyline::error::ReadlineError;
@@ -179,7 +178,7 @@ impl Repl {
                 .await
                 .context(PlanStatementSnafu)?;
 
-            let LogicalPlan::DfPlan(plan) = query_engine
+            let plan = query_engine
                 .optimize(&query_engine.engine_context(query_ctx), &plan)
                 .context(PlanStatementSnafu)?;
 
diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs
index 56068a38c3..54e1cdbafd 100644
--- a/src/datanode/src/region_server.rs
+++ b/src/datanode/src/region_server.rs
@@ -843,7 +843,7 @@ impl RegionServerInner {
 
         let result = self
             .query_engine
-            .execute(request.plan.into(), query_ctx)
+            .execute(request.plan, query_ctx)
             .await
             .context(ExecuteLogicalPlanSnafu)?;
 
diff --git a/src/datanode/src/tests.rs b/src/datanode/src/tests.rs
index 87285eabfa..89be76511d 100644
--- a/src/datanode/src/tests.rs
+++ b/src/datanode/src/tests.rs
@@ -24,8 +24,8 @@ use common_function::scalars::aggregate::AggregateFunctionMetaRef;
 use common_query::prelude::ScalarUdf;
 use common_query::Output;
 use common_runtime::Runtime;
+use datafusion_expr::LogicalPlan;
 use query::dataframe::DataFrame;
-use query::plan::LogicalPlan;
 use query::planner::LogicalPlanner;
 use query::query_engine::{DescribeResult, QueryEngineState};
 use query::{QueryEngine, QueryEngineContext};
diff --git a/src/flow/src/df_optimizer.rs b/src/flow/src/df_optimizer.rs
index d5368d5189..d73bf91bfb 100644
--- a/src/flow/src/df_optimizer.rs
+++ b/src/flow/src/df_optimizer.rs
@@ -41,7 +41,6 @@ use datafusion_expr::{
     BinaryExpr, Expr, Operator, Projection, ScalarUDFImpl, Signature, TypeSignature, Volatility,
 };
 use query::parser::QueryLanguageParser;
-use query::plan::LogicalPlan;
 use query::query_engine::DefaultSerializer;
 use query::QueryEngine;
 use snafu::ResultExt;
@@ -111,7 +110,6 @@ pub async fn sql_to_flow_plan(
         .await
         .map_err(BoxedError::new)
         .context(ExternalSnafu)?;
-    let LogicalPlan::DfPlan(plan) = plan;
 
     let opted_plan = apply_df_optimizer(plan).await?;
 
diff --git a/src/flow/src/transform.rs b/src/flow/src/transform.rs
index f6dff58856..6d4de97417 100644
--- a/src/flow/src/transform.rs
+++ b/src/flow/src/transform.rs
@@ -163,7 +163,6 @@ mod test {
     use itertools::Itertools;
     use prost::Message;
     use query::parser::QueryLanguageParser;
-    use query::plan::LogicalPlan;
     use query::query_engine::DefaultSerializer;
     use query::QueryEngine;
     use session::context::QueryContext;
@@ -274,7 +273,6 @@ mod test {
             .plan(stmt, QueryContext::arc())
             .await
             .unwrap();
-        let LogicalPlan::DfPlan(plan) = plan;
         let plan = apply_df_optimizer(plan).await.unwrap();
 
         // encode then decode so to rely on the impl of conversion from logical plan to substrait plan
@@ -297,7 +295,6 @@ mod test {
             .plan(stmt, QueryContext::arc())
             .await
             .unwrap();
-        let LogicalPlan::DfPlan(plan) = plan;
         let plan = apply_df_optimizer(plan).await;
 
         assert!(plan.is_err());
diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml
index 362d147d13..555a201280 100644
--- a/src/frontend/Cargo.toml
+++ b/src/frontend/Cargo.toml
@@ -37,6 +37,7 @@ common-runtime.workspace = true
 common-telemetry.workspace = true
 common-time.workspace = true
 common-version.workspace = true
+datafusion-expr.workspace = true
 datanode.workspace = true
 humantime-serde.workspace = true
 lazy_static.workspace = true
diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs
index 2c5544c51a..dcc56f7581 100644
--- a/src/frontend/src/instance.rs
+++ b/src/frontend/src/instance.rs
@@ -40,6 +40,7 @@ use common_procedure::options::ProcedureConfig;
 use common_procedure::ProcedureManagerRef;
 use common_query::Output;
 use common_telemetry::{debug, error, tracing};
+use datafusion_expr::LogicalPlan;
 use log_store::raft_engine::RaftEngineBackend;
 use operator::delete::DeleterRef;
 use operator::insert::InserterRef;
@@ -48,7 +49,6 @@ use pipeline::pipeline_operator::PipelineOperator;
 use prometheus::HistogramTimer;
 use query::metrics::OnDone;
 use query::parser::{PromQuery, QueryLanguageParser, QueryStatement};
-use query::plan::LogicalPlan;
 use query::query_engine::options::{validate_catalog_and_schema, QueryOptions};
 use query::query_engine::DescribeResult;
 use query::QueryEngineRef;
diff --git a/src/operator/src/error.rs b/src/operator/src/error.rs
index 02a880a903..1931814c8b 100644
--- a/src/operator/src/error.rs
+++ b/src/operator/src/error.rs
@@ -133,13 +133,6 @@ pub enum Error {
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to get schema from logical plan"))]
-    GetSchema {
-        #[snafu(implicit)]
-        location: Location,
-        source: query::error::Error,
-    },
-
     #[snafu(display("Column datatype error"))]
     ColumnDataType {
         #[snafu(implicit)]
@@ -184,6 +177,13 @@ pub enum Error {
         source: datatypes::error::Error,
     },
 
+    #[snafu(display("Failed to convert datafusion schema"))]
+    ConvertSchema {
+        source: datatypes::error::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Failed to convert expr to struct"))]
     InvalidExpr {
         #[snafu(implicit)]
@@ -795,6 +795,7 @@ impl ErrorExt for Error {
             | Error::PrepareFileTable { .. }
             | Error::InferFileTableSchema { .. }
             | Error::SchemaIncompatible { .. }
+            | Error::ConvertSchema { .. }
             | Error::UnsupportedRegionRequest { .. }
             | Error::InvalidTableName { .. }
             | Error::InvalidViewName { .. }
@@ -872,7 +873,6 @@ impl ErrorExt for Error {
             | Error::FindNewColumnsOnInsertion { source, .. } => source.status_code(),
 
             Error::ExecuteStatement { source, .. }
-            | Error::GetSchema { source, .. }
             | Error::ExtractTableNames { source, .. }
             | Error::PlanStatement { source, .. }
             | Error::ParseQuery { source, .. }
diff --git a/src/operator/src/statement.rs b/src/operator/src/statement.rs
index 35e6752d08..4dc43e0d92 100644
--- a/src/operator/src/statement.rs
+++ b/src/operator/src/statement.rs
@@ -42,9 +42,9 @@ use common_query::Output;
 use common_telemetry::tracing;
 use common_time::range::TimestampRange;
 use common_time::Timestamp;
+use datafusion_expr::LogicalPlan;
 use partition::manager::{PartitionRuleManager, PartitionRuleManagerRef};
 use query::parser::QueryStatement;
-use query::plan::LogicalPlan;
 use query::QueryEngineRef;
 use session::context::QueryContextRef;
 use session::table_name::table_idents_to_full_name;
diff --git a/src/operator/src/statement/copy_table_to.rs b/src/operator/src/statement/copy_table_to.rs
index 8a90d10955..0e53d79ac6 100644
--- a/src/operator/src/statement/copy_table_to.rs
+++ b/src/operator/src/statement/copy_table_to.rs
@@ -29,7 +29,6 @@ use datafusion::datasource::DefaultTableSource;
 use datafusion_common::TableReference as DfTableReference;
 use datafusion_expr::LogicalPlanBuilder;
 use object_store::ObjectStore;
-use query::plan::LogicalPlan;
 use session::context::QueryContextRef;
 use snafu::{OptionExt, ResultExt};
 use table::requests::CopyTableRequest;
@@ -133,7 +132,7 @@ impl StatementExecutor {
 
         let output = self
             .query_engine
-            .execute(LogicalPlan::DfPlan(plan), query_ctx)
+            .execute(plan, query_ctx)
             .await
             .context(ExecLogicalPlanSnafu)?;
         let stream = match output.data {
diff --git a/src/operator/src/statement/ddl.rs b/src/operator/src/statement/ddl.rs
index aa1a070875..e9b186bd63 100644
--- a/src/operator/src/statement/ddl.rs
+++ b/src/operator/src/statement/ddl.rs
@@ -37,7 +37,7 @@ use common_query::Output;
 use common_telemetry::{debug, info, tracing};
 use common_time::Timezone;
 use datatypes::prelude::ConcreteDataType;
-use datatypes::schema::RawSchema;
+use datatypes::schema::{RawSchema, Schema};
 use datatypes::value::Value;
 use lazy_static::lazy_static;
 use partition::expr::{Operand, PartitionExpr, RestrictedOp};
@@ -69,11 +69,11 @@ use table::TableRef;
 use super::StatementExecutor;
 use crate::error::{
     self, AlterExprToRequestSnafu, CatalogSnafu, ColumnDataTypeSnafu, ColumnNotFoundSnafu,
-    CreateLogicalTablesSnafu, CreateTableInfoSnafu, DeserializePartitionSnafu, EmptyDdlExprSnafu,
-    ExtractTableNamesSnafu, FlowNotFoundSnafu, InvalidPartitionRuleSnafu, InvalidPartitionSnafu,
-    InvalidTableNameSnafu, InvalidViewNameSnafu, InvalidViewStmtSnafu, ParseSqlValueSnafu, Result,
-    SchemaInUseSnafu, SchemaNotFoundSnafu, SchemaReadOnlySnafu, SubstraitCodecSnafu,
-    TableAlreadyExistsSnafu, TableMetadataManagerSnafu, TableNotFoundSnafu,
+    ConvertSchemaSnafu, CreateLogicalTablesSnafu, CreateTableInfoSnafu, DeserializePartitionSnafu,
+    EmptyDdlExprSnafu, ExtractTableNamesSnafu, FlowNotFoundSnafu, InvalidPartitionRuleSnafu,
+    InvalidPartitionSnafu, InvalidTableNameSnafu, InvalidViewNameSnafu, InvalidViewStmtSnafu,
+    ParseSqlValueSnafu, Result, SchemaInUseSnafu, SchemaNotFoundSnafu, SchemaReadOnlySnafu,
+    SubstraitCodecSnafu, TableAlreadyExistsSnafu, TableMetadataManagerSnafu, TableNotFoundSnafu,
     UnrecognizedTableOptionSnafu, ViewAlreadyExistsSnafu,
 };
 use crate::expr_factory;
@@ -406,9 +406,12 @@ impl StatementExecutor {
 
         // Save the columns in plan, it may changed when the schemas of tables in plan
         // are altered.
-        let plan_columns: Vec<_> = logical_plan
+        let schema: Schema = logical_plan
             .schema()
-            .context(error::GetSchemaSnafu)?
+            .clone()
+            .try_into()
+            .context(ConvertSchemaSnafu)?;
+        let plan_columns: Vec<_> = schema
             .column_schemas()
             .iter()
             .map(|c| c.name.clone())
@@ -434,9 +437,8 @@ impl StatementExecutor {
 
         // Extract the table names from the original plan
         // and rewrite them as fully qualified names.
-        let (table_names, plan) =
-            extract_and_rewrite_full_table_names(logical_plan.unwrap_df_plan(), ctx.clone())
-                .context(ExtractTableNamesSnafu)?;
+        let (table_names, plan) = extract_and_rewrite_full_table_names(logical_plan, ctx.clone())
+            .context(ExtractTableNamesSnafu)?;
 
         let table_names = table_names.into_iter().map(|t| t.into()).collect();
 
diff --git a/src/operator/src/statement/tql.rs b/src/operator/src/statement/tql.rs
index 72b2db641b..008aba0d78 100644
--- a/src/operator/src/statement/tql.rs
+++ b/src/operator/src/statement/tql.rs
@@ -16,11 +16,11 @@ use std::collections::HashMap;
 
 use common_query::Output;
 use common_telemetry::tracing;
+use datafusion_expr::LogicalPlan;
 use query::parser::{
     PromQuery, QueryLanguageParser, ANALYZE_NODE_NAME, ANALYZE_VERBOSE_NODE_NAME,
     DEFAULT_LOOKBACK_STRING, EXPLAIN_NODE_NAME, EXPLAIN_VERBOSE_NODE_NAME,
 };
-use query::plan::LogicalPlan;
 use session::context::QueryContextRef;
 use snafu::ResultExt;
 use sql::statements::tql::Tql;
diff --git a/src/pipeline/src/manager/table.rs b/src/pipeline/src/manager/table.rs
index 3c69f59f2a..8989412c3e 100644
--- a/src/pipeline/src/manager/table.rs
+++ b/src/pipeline/src/manager/table.rs
@@ -26,7 +26,7 @@ use common_telemetry::{debug, info};
 use common_time::timestamp::{TimeUnit, Timestamp};
 use datafusion::logical_expr::col;
 use datafusion_common::{TableReference, ToDFSchema};
-use datafusion_expr::{DmlStatement, LogicalPlan as DfLogicalPlan};
+use datafusion_expr::{DmlStatement, LogicalPlan};
 use datatypes::prelude::ScalarVector;
 use datatypes::timestamp::TimestampNanosecond;
 use datatypes::vectors::{StringVector, TimestampNanosecondVector, Vector};
@@ -34,7 +34,6 @@ use moka::sync::Cache;
 use operator::insert::InserterRef;
 use operator::statement::StatementExecutorRef;
 use query::dataframe::DataFrame;
-use query::plan::LogicalPlan;
 use query::QueryEngineRef;
 use session::context::{QueryContextBuilder, QueryContextRef};
 use snafu::{ensure, OptionExt, ResultExt};
@@ -373,7 +372,7 @@ impl PipelineTable {
             Arc::new(dataframe.into_parts().1),
         );
 
-        let plan = LogicalPlan::DfPlan(DfLogicalPlan::Dml(stmt));
+        let plan = LogicalPlan::Dml(stmt);
 
         // 4. execute dml stmt
         let output = self
@@ -427,7 +426,7 @@ impl PipelineTable {
             .limit(0, Some(1))
             .context(BuildDfLogicalPlanSnafu)?;
 
-        let plan = LogicalPlan::DfPlan(dataframe.into_parts().1);
+        let plan = dataframe.into_parts().1;
 
         let table_info = self.table.table_info();
 
diff --git a/src/query/src/datafusion.rs b/src/query/src/datafusion.rs
index 6ed5844de0..888ebbba83 100644
--- a/src/query/src/datafusion.rs
+++ b/src/query/src/datafusion.rs
@@ -36,7 +36,7 @@ use datafusion::physical_plan::analyze::AnalyzeExec;
 use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion_common::ResolvedTableReference;
-use datafusion_expr::{DmlStatement, LogicalPlan as DfLogicalPlan, WriteOp};
+use datafusion_expr::{DmlStatement, LogicalPlan as DfLogicalPlan, LogicalPlan, WriteOp};
 use datatypes::prelude::VectorRef;
 use datatypes::schema::Schema;
 use futures_util::StreamExt;
@@ -50,14 +50,13 @@ use crate::dataframe::DataFrame;
 pub use crate::datafusion::planner::DfContextProviderAdapter;
 use crate::dist_plan::MergeScanLogicalPlan;
 use crate::error::{
-    CatalogSnafu, CreateRecordBatchSnafu, DataFusionSnafu, MissingTableMutationHandlerSnafu,
-    MissingTimestampColumnSnafu, QueryExecutionSnafu, Result, TableMutationSnafu,
-    TableNotFoundSnafu, TableReadOnlySnafu, UnsupportedExprSnafu,
+    CatalogSnafu, ConvertSchemaSnafu, CreateRecordBatchSnafu, DataFusionSnafu,
+    MissingTableMutationHandlerSnafu, MissingTimestampColumnSnafu, QueryExecutionSnafu, Result,
+    TableMutationSnafu, TableNotFoundSnafu, TableReadOnlySnafu, UnsupportedExprSnafu,
 };
 use crate::executor::QueryExecutor;
 use crate::metrics::{OnDone, QUERY_STAGE_ELAPSED};
 use crate::physical_wrapper::PhysicalPlanWrapperRef;
-use crate::plan::LogicalPlan;
 use crate::planner::{DfLogicalPlanner, LogicalPlanner};
 use crate::query_engine::{DescribeResult, QueryEngineContext, QueryEngineState};
 use crate::{metrics, QueryEngine};
@@ -119,7 +118,7 @@ impl DatafusionQueryEngine {
         let table = self.find_table(&table_name, &query_ctx).await?;
 
         let output = self
-            .exec_query_plan(LogicalPlan::DfPlan((*dml.input).clone()), query_ctx.clone())
+            .exec_query_plan((*dml.input).clone(), query_ctx.clone())
             .await?;
         let mut stream = match output.data {
             OutputData::RecordBatches(batches) => batches.as_stream(),
@@ -265,52 +264,48 @@ impl DatafusionQueryEngine {
         logical_plan: &LogicalPlan,
     ) -> Result<Arc<dyn ExecutionPlan>> {
         let _timer = metrics::CREATE_PHYSICAL_ELAPSED.start_timer();
-        match logical_plan {
-            LogicalPlan::DfPlan(df_plan) => {
-                let state = ctx.state();
+        let state = ctx.state();
 
-                // special handle EXPLAIN plan
-                if matches!(df_plan, DfLogicalPlan::Explain(_)) {
-                    return state
-                        .create_physical_plan(df_plan)
-                        .await
-                        .context(error::DatafusionSnafu)
-                        .map_err(BoxedError::new)
-                        .context(QueryExecutionSnafu);
-                }
-
-                // analyze first
-                let analyzed_plan = state
-                    .analyzer()
-                    .execute_and_check(df_plan.clone(), state.config_options(), |_, _| {})
-                    .context(error::DatafusionSnafu)
-                    .map_err(BoxedError::new)
-                    .context(QueryExecutionSnafu)?;
-                // skip optimize for MergeScan
-                let optimized_plan = if let DfLogicalPlan::Extension(ext) = &analyzed_plan
-                    && ext.node.name() == MergeScanLogicalPlan::name()
-                {
-                    analyzed_plan.clone()
-                } else {
-                    state
-                        .optimizer()
-                        .optimize(analyzed_plan, state, |_, _| {})
-                        .context(error::DatafusionSnafu)
-                        .map_err(BoxedError::new)
-                        .context(QueryExecutionSnafu)?
-                };
-
-                let physical_plan = state
-                    .query_planner()
-                    .create_physical_plan(&optimized_plan, state)
-                    .await
-                    .context(error::DatafusionSnafu)
-                    .map_err(BoxedError::new)
-                    .context(QueryExecutionSnafu)?;
-
-                Ok(physical_plan)
-            }
+        // special handle EXPLAIN plan
+        if matches!(logical_plan, DfLogicalPlan::Explain(_)) {
+            return state
+                .create_physical_plan(logical_plan)
+                .await
+                .context(error::DatafusionSnafu)
+                .map_err(BoxedError::new)
+                .context(QueryExecutionSnafu);
         }
+
+        // analyze first
+        let analyzed_plan = state
+            .analyzer()
+            .execute_and_check(logical_plan.clone(), state.config_options(), |_, _| {})
+            .context(error::DatafusionSnafu)
+            .map_err(BoxedError::new)
+            .context(QueryExecutionSnafu)?;
+        // skip optimize for MergeScan
+        let optimized_plan = if let DfLogicalPlan::Extension(ext) = &analyzed_plan
+            && ext.node.name() == MergeScanLogicalPlan::name()
+        {
+            analyzed_plan.clone()
+        } else {
+            state
+                .optimizer()
+                .optimize(analyzed_plan, state, |_, _| {})
+                .context(error::DatafusionSnafu)
+                .map_err(BoxedError::new)
+                .context(QueryExecutionSnafu)?
+        };
+
+        let physical_plan = state
+            .query_planner()
+            .create_physical_plan(&optimized_plan, state)
+            .await
+            .context(error::DatafusionSnafu)
+            .map_err(BoxedError::new)
+            .context(QueryExecutionSnafu)?;
+
+        Ok(physical_plan)
     }
 
     #[tracing::instrument(skip_all)]
@@ -320,28 +315,25 @@ impl DatafusionQueryEngine {
         plan: &LogicalPlan,
     ) -> Result<LogicalPlan> {
         let _timer = metrics::OPTIMIZE_LOGICAL_ELAPSED.start_timer();
-        match plan {
-            LogicalPlan::DfPlan(df_plan) => {
-                // Optimized by extension rules
-                let optimized_plan = self
-                    .state
-                    .optimize_by_extension_rules(df_plan.clone(), context)
-                    .context(error::DatafusionSnafu)
-                    .map_err(BoxedError::new)
-                    .context(QueryExecutionSnafu)?;
 
-                // Optimized by datafusion optimizer
-                let optimized_plan = self
-                    .state
-                    .session_state()
-                    .optimize(&optimized_plan)
-                    .context(error::DatafusionSnafu)
-                    .map_err(BoxedError::new)
-                    .context(QueryExecutionSnafu)?;
+        // Optimized by extension rules
+        let optimized_plan = self
+            .state
+            .optimize_by_extension_rules(plan.clone(), context)
+            .context(error::DatafusionSnafu)
+            .map_err(BoxedError::new)
+            .context(QueryExecutionSnafu)?;
 
-                Ok(LogicalPlan::DfPlan(optimized_plan))
-            }
-        }
+        // Optimized by datafusion optimizer
+        let optimized_plan = self
+            .state
+            .session_state()
+            .optimize(&optimized_plan)
+            .context(error::DatafusionSnafu)
+            .map_err(BoxedError::new)
+            .context(QueryExecutionSnafu)?;
+
+        Ok(optimized_plan)
     }
 
     #[tracing::instrument(skip_all)]
@@ -399,15 +391,25 @@ impl QueryEngine for DatafusionQueryEngine {
     ) -> Result<DescribeResult> {
         let ctx = self.engine_context(query_ctx);
         if let Ok(optimised_plan) = self.optimize(&ctx, &plan) {
+            let schema = optimised_plan
+                .schema()
+                .clone()
+                .try_into()
+                .context(ConvertSchemaSnafu)?;
             Ok(DescribeResult {
-                schema: optimised_plan.schema()?,
+                schema,
                 logical_plan: optimised_plan,
             })
         } else {
             // Table's like those in information_schema cannot be optimized when
             // it contains parameters. So we fallback to original plans.
+            let schema = plan
+                .schema()
+                .clone()
+                .try_into()
+                .context(ConvertSchemaSnafu)?;
             Ok(DescribeResult {
-                schema: plan.schema()?,
+                schema,
                 logical_plan: plan,
             })
         }
@@ -415,9 +417,7 @@ impl QueryEngine for DatafusionQueryEngine {
 
     async fn execute(&self, plan: LogicalPlan, query_ctx: QueryContextRef) -> Result<Output> {
         match plan {
-            LogicalPlan::DfPlan(DfLogicalPlan::Dml(dml)) => {
-                self.exec_dml_statement(dml, query_ctx).await
-            }
+            LogicalPlan::Dml(dml) => self.exec_dml_statement(dml, query_ctx).await,
             _ => self.exec_query_plan(plan, query_ctx).await,
         }
     }
@@ -577,10 +577,10 @@ mod tests {
         // TODO(sunng87): do not rely on to_string for compare
         assert_eq!(
             format!("{plan:?}"),
-            r#"DfPlan(Limit: skip=0, fetch=20
+            r#"Limit: skip=0, fetch=20
   Projection: SUM(numbers.number)
     Aggregate: groupBy=[[]], aggr=[[SUM(numbers.number)]]
-      TableScan: numbers)"#
+      TableScan: numbers"#
         );
     }
 
diff --git a/src/query/src/error.rs b/src/query/src/error.rs
index 4eca47e175..7e246d11c3 100644
--- a/src/query/src/error.rs
+++ b/src/query/src/error.rs
@@ -108,13 +108,6 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to convert Datafusion schema"))]
-    ConvertDatafusionSchema {
-        source: datatypes::error::Error,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
     #[snafu(display("Failed to parse timestamp `{}`", raw))]
     ParseTimestamp {
         raw: String,
@@ -228,6 +221,7 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
     #[snafu(display("Unknown table type, downcast failed"))]
     UnknownTable {
         #[snafu(implicit)]
@@ -354,7 +348,6 @@ impl ErrorExt for Error {
 
             QueryAccessDenied { .. } => StatusCode::AccessDenied,
             Catalog { source, .. } => source.status_code(),
-            ConvertDatafusionSchema { source, .. } => source.status_code(),
             CreateRecordBatch { source, .. } => source.status_code(),
             QueryExecution { source, .. } | QueryPlan { source, .. } => source.status_code(),
             PlanSql { error, .. } => {
diff --git a/src/query/src/plan.rs b/src/query/src/plan.rs
index ea9dae3770..710e92129d 100644
--- a/src/query/src/plan.rs
+++ b/src/query/src/plan.rs
@@ -12,108 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{HashMap, HashSet};
-use std::fmt::{Debug, Display};
+use std::collections::HashSet;
 
-use common_query::prelude::ScalarValue;
 use datafusion::datasource::DefaultTableSource;
 use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRewriter};
-use datafusion_common::{ParamValues, TableReference};
+use datafusion_common::TableReference;
 use datafusion_expr::LogicalPlan as DfLogicalPlan;
-use datatypes::data_type::ConcreteDataType;
-use datatypes::schema::Schema;
 use session::context::QueryContextRef;
 use snafu::ResultExt;
 pub use table::metadata::TableType;
 use table::table::adapter::DfTableProviderAdapter;
 use table::table_name::TableName;
 
-use crate::error::{ConvertDatafusionSchemaSnafu, DataFusionSnafu, Result};
-
-/// A LogicalPlan represents the different types of relational
-/// operators (such as Projection, Filter, etc) and can be created by
-/// the SQL query planner.
-///
-/// A LogicalPlan represents transforming an input relation (table) to
-/// an output relation (table) with a (potentially) different
-/// schema. A plan represents a dataflow tree where data flows
-/// from leaves up to the root to produce the query result.
-#[derive(Clone, Debug)]
-pub enum LogicalPlan {
-    DfPlan(DfLogicalPlan),
-}
-
-impl LogicalPlan {
-    /// Get the schema for this logical plan
-    pub fn schema(&self) -> Result<Schema> {
-        match self {
-            Self::DfPlan(plan) => {
-                let df_schema = plan.schema();
-                df_schema
-                    .clone()
-                    .try_into()
-                    .context(ConvertDatafusionSchemaSnafu)
-            }
-        }
-    }
-
-    /// Return a `format`able structure that produces a single line
-    /// per node. For example:
-    ///
-    /// ```text
-    /// Projection: employee.id
-    ///    Filter: employee.state Eq Utf8(\"CO\")\
-    ///       CsvScan: employee projection=Some([0, 3])
-    /// ```
-    pub fn display_indent(&self) -> impl Display + '_ {
-        let LogicalPlan::DfPlan(plan) = self;
-        plan.display_indent()
-    }
-
-    /// Walk the logical plan, find any `PlaceHolder` tokens,
-    /// and return a map of their IDs and ConcreteDataTypes
-    pub fn get_param_types(&self) -> Result<HashMap<String, Option<ConcreteDataType>>> {
-        let LogicalPlan::DfPlan(plan) = self;
-        let types = plan.get_parameter_types().context(DataFusionSnafu)?;
-
-        Ok(types
-            .into_iter()
-            .map(|(k, v)| (k, v.map(|v| ConcreteDataType::from_arrow_type(&v))))
-            .collect())
-    }
-
-    /// Return a logical plan with all placeholders/params (e.g $1 $2,
-    /// ...) replaced with corresponding values provided in the
-    /// params_values
-    pub fn replace_params_with_values(&self, values: &[ScalarValue]) -> Result<LogicalPlan> {
-        let LogicalPlan::DfPlan(plan) = self;
-
-        plan.clone()
-            .replace_params_with_values(&ParamValues::List(values.to_vec()))
-            .context(DataFusionSnafu)
-            .map(LogicalPlan::DfPlan)
-    }
-
-    /// Unwrap the logical plan into a DataFusion logical plan
-    pub fn unwrap_df_plan(self) -> DfLogicalPlan {
-        match self {
-            LogicalPlan::DfPlan(plan) => plan,
-        }
-    }
-
-    /// Returns the DataFusion logical plan reference
-    pub fn df_plan(&self) -> &DfLogicalPlan {
-        match self {
-            LogicalPlan::DfPlan(plan) => plan,
-        }
-    }
-}
-
-impl From<DfLogicalPlan> for LogicalPlan {
-    fn from(plan: DfLogicalPlan) -> Self {
-        Self::DfPlan(plan)
-    }
-}
+use crate::error::{DataFusionSnafu, Result};
 
 struct TableNamesExtractAndRewriter {
     pub(crate) table_names: HashSet<TableName>,
diff --git a/src/query/src/planner.rs b/src/query/src/planner.rs
index 4c09860335..0f4f74133a 100644
--- a/src/query/src/planner.rs
+++ b/src/query/src/planner.rs
@@ -22,7 +22,7 @@ use common_telemetry::tracing;
 use datafusion::common::DFSchema;
 use datafusion::execution::context::SessionState;
 use datafusion::sql::planner::PlannerContext;
-use datafusion_expr::Expr as DfExpr;
+use datafusion_expr::{Expr as DfExpr, LogicalPlan};
 use datafusion_sql::planner::{ParserOptions, SqlToRel};
 use promql_parser::parser::EvalStmt;
 use session::context::QueryContextRef;
@@ -32,7 +32,6 @@ use sql::statements::statement::Statement;
 
 use crate::error::{DataFusionSnafu, PlanSqlSnafu, QueryPlanSnafu, Result, SqlSnafu};
 use crate::parser::QueryStatement;
-use crate::plan::LogicalPlan;
 use crate::promql::planner::PromPlanner;
 use crate::query_engine::{DefaultPlanDecoder, QueryEngineState};
 use crate::range_select::plan_rewrite::RangePlanRewriter;
@@ -109,7 +108,7 @@ impl DfLogicalPlanner {
             .optimize_by_extension_rules(plan, &context)
             .context(DataFusionSnafu)?;
 
-        Ok(LogicalPlan::DfPlan(plan))
+        Ok(plan)
     }
 
     /// Generate a relational expression from a SQL expression
@@ -160,7 +159,6 @@ impl DfLogicalPlanner {
         );
         PromPlanner::stmt_to_plan(table_provider, stmt, &self.session_state)
             .await
-            .map(LogicalPlan::DfPlan)
             .map_err(BoxedError::new)
             .context(QueryPlanSnafu)
     }
@@ -168,7 +166,7 @@ impl DfLogicalPlanner {
     #[tracing::instrument(skip_all)]
     fn optimize_logical_plan(&self, plan: LogicalPlan) -> Result<LogicalPlan> {
         self.engine_state
-            .optimize_logical_plan(plan.unwrap_df_plan())
+            .optimize_logical_plan(plan)
             .context(DataFusionSnafu)
             .map(Into::into)
     }
diff --git a/src/query/src/query_engine.rs b/src/query/src/query_engine.rs
index e2d3e01c91..61f6e1a8f0 100644
--- a/src/query/src/query_engine.rs
+++ b/src/query/src/query_engine.rs
@@ -30,6 +30,7 @@ use common_function::handlers::{
 use common_function::scalars::aggregate::AggregateFunctionMetaRef;
 use common_query::prelude::ScalarUdf;
 use common_query::Output;
+use datafusion_expr::LogicalPlan;
 use datatypes::schema::Schema;
 pub use default_serializer::{DefaultPlanDecoder, DefaultSerializer};
 use session::context::QueryContextRef;
@@ -38,7 +39,6 @@ use table::TableRef;
 use crate::dataframe::DataFrame;
 use crate::datafusion::DatafusionQueryEngine;
 use crate::error::Result;
-use crate::plan::LogicalPlan;
 use crate::planner::LogicalPlanner;
 pub use crate::query_engine::context::QueryEngineContext;
 pub use crate::query_engine::state::QueryEngineState;
diff --git a/src/query/src/range_select/plan_rewrite.rs b/src/query/src/range_select/plan_rewrite.rs
index 9eb16b359e..43b0928539 100644
--- a/src/query/src/range_select/plan_rewrite.rs
+++ b/src/query/src/range_select/plan_rewrite.rs
@@ -556,7 +556,6 @@ mod test {
 
     use super::*;
     use crate::parser::QueryLanguageParser;
-    use crate::plan::LogicalPlan as GreptimeLogicalPlan;
     use crate::{QueryEngineFactory, QueryEngineRef};
 
     async fn create_test_engine() -> QueryEngineRef {
@@ -611,14 +610,14 @@ mod test {
         QueryEngineFactory::new(catalog_list, None, None, None, None, false).query_engine()
     }
 
-    async fn do_query(sql: &str) -> Result<crate::plan::LogicalPlan> {
+    async fn do_query(sql: &str) -> Result<LogicalPlan> {
         let stmt = QueryLanguageParser::parse_sql(sql, &QueryContext::arc()).unwrap();
         let engine = create_test_engine().await;
         engine.planner().plan(stmt, QueryContext::arc()).await
     }
 
     async fn query_plan_compare(sql: &str, expected: String) {
-        let GreptimeLogicalPlan::DfPlan(plan) = do_query(sql).await.unwrap();
+        let plan = do_query(sql).await.unwrap();
         assert_eq!(plan.display_indent_schema().to_string(), expected);
     }
 
diff --git a/src/query/src/tests/query_engine_test.rs b/src/query/src/tests/query_engine_test.rs
index 6359940d4f..687346dbcb 100644
--- a/src/query/src/tests/query_engine_test.rs
+++ b/src/query/src/tests/query_engine_test.rs
@@ -35,7 +35,6 @@ use table::test_util::MemTable;
 
 use crate::error::{QueryExecutionSnafu, Result};
 use crate::parser::QueryLanguageParser;
-use crate::plan::LogicalPlan;
 use crate::query_engine::options::QueryOptions;
 use crate::query_engine::QueryEngineFactory;
 use crate::tests::exec_selection;
@@ -64,18 +63,16 @@ async fn test_datafusion_query_engine() -> Result<()> {
 
     let limit = 10;
     let table_provider = Arc::new(DfTableProviderAdapter::new(table.clone()));
-    let plan = LogicalPlan::DfPlan(
-        LogicalPlanBuilder::scan(
-            "numbers",
-            Arc::new(DefaultTableSource { table_provider }),
-            None,
-        )
-        .unwrap()
-        .limit(0, Some(limit))
-        .unwrap()
-        .build()
-        .unwrap(),
-    );
+    let plan = LogicalPlanBuilder::scan(
+        "numbers",
+        Arc::new(DefaultTableSource { table_provider }),
+        None,
+    )
+    .unwrap()
+    .limit(0, Some(limit))
+    .unwrap()
+    .build()
+    .unwrap();
 
     let output = engine.execute(plan, QueryContext::arc()).await?;
 
diff --git a/src/script/src/table.rs b/src/script/src/table.rs
index bbc04e6f64..cc20cd4f84 100644
--- a/src/script/src/table.rs
+++ b/src/script/src/table.rs
@@ -33,7 +33,6 @@ use datafusion_common::TableReference;
 use datafusion_expr::LogicalPlanBuilder;
 use datatypes::prelude::ScalarVector;
 use datatypes::vectors::{StringVector, Vector};
-use query::plan::LogicalPlan;
 use query::QueryEngineRef;
 use servers::query_handler::grpc::GrpcQueryHandlerRef;
 use session::context::{QueryContextBuilder, QueryContextRef};
@@ -224,7 +223,7 @@ impl<E: ErrorExt + Send + Sync + 'static> ScriptsTable<E> {
 
         let output = self
             .query_engine
-            .execute(LogicalPlan::DfPlan(plan), query_ctx(&table_info))
+            .execute(plan, query_ctx(&table_info))
             .await
             .context(ExecuteInternalStatementSnafu)?;
         let stream = match output.data {
@@ -279,7 +278,7 @@ impl<E: ErrorExt + Send + Sync + 'static> ScriptsTable<E> {
             .context(BuildDfLogicalPlanSnafu)?;
 
         let output = query_engine
-            .execute(LogicalPlan::DfPlan(plan), query_ctx(&table_info))
+            .execute(plan, query_ctx(&table_info))
             .await
             .context(ExecuteInternalStatementSnafu)?;
         let stream = match output.data {
diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml
index d088961e73..54665b8c68 100644
--- a/src/servers/Cargo.toml
+++ b/src/servers/Cargo.toml
@@ -46,6 +46,7 @@ common-version = { workspace = true, features = ["codec"] }
 dashmap.workspace = true
 datafusion.workspace = true
 datafusion-common.workspace = true
+datafusion-expr.workspace = true
 datatypes.workspace = true
 derive_builder.workspace = true
 futures = "0.3"
diff --git a/src/servers/src/error.rs b/src/servers/src/error.rs
index 43bb458bd5..0fde3b527c 100644
--- a/src/servers/src/error.rs
+++ b/src/servers/src/error.rs
@@ -398,13 +398,6 @@ pub enum Error {
         source: query::error::Error,
     },
 
-    #[snafu(display("Failed to get param types"))]
-    GetPreparedStmtParams {
-        source: query::error::Error,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
     #[snafu(display("{}", reason))]
     UnexpectedResult {
         reason: String,
@@ -452,13 +445,6 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to replace params with values in prepared statement"))]
-    ReplacePreparedStmtParams {
-        source: query::error::Error,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
     #[snafu(display("Failed to convert scalar value"))]
     ConvertScalarValue {
         source: datatypes::error::Error,
@@ -635,9 +621,7 @@ impl ErrorExt for Error {
 
             InvalidUtf8Value { .. } => StatusCode::InvalidArguments,
 
-            ReplacePreparedStmtParams { source, .. }
-            | GetPreparedStmtParams { source, .. }
-            | ParsePromQL { source, .. } => source.status_code(),
+            ParsePromQL { source, .. } => source.status_code(),
             Other { source, .. } => source.status_code(),
 
             UnexpectedResult { .. } => StatusCode::Unexpected,
diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs
index 18388998e7..a2b72b548b 100644
--- a/src/servers/src/http.rs
+++ b/src/servers/src/http.rs
@@ -969,11 +969,11 @@ mod test {
     use axum::routing::get;
     use common_query::Output;
     use common_recordbatch::RecordBatches;
+    use datafusion_expr::LogicalPlan;
     use datatypes::prelude::*;
     use datatypes::schema::{ColumnSchema, Schema};
     use datatypes::vectors::{StringVector, UInt32Vector};
     use query::parser::PromQuery;
-    use query::plan::LogicalPlan;
     use query::query_engine::DescribeResult;
     use session::context::QueryContextRef;
     use tokio::sync::mpsc;
diff --git a/src/servers/src/interceptor.rs b/src/servers/src/interceptor.rs
index d3478a56ea..76ed4728e0 100644
--- a/src/servers/src/interceptor.rs
+++ b/src/servers/src/interceptor.rs
@@ -21,8 +21,8 @@ use api::v1::RowInsertRequests;
 use async_trait::async_trait;
 use common_error::ext::ErrorExt;
 use common_query::Output;
+use datafusion_expr::LogicalPlan;
 use query::parser::PromQuery;
-use query::plan::LogicalPlan;
 use serde_json::Value;
 use session::context::QueryContextRef;
 use sql::statements::statement::Statement;
diff --git a/src/servers/src/lib.rs b/src/servers/src/lib.rs
index ff1af967fa..ce6857c6d2 100644
--- a/src/servers/src/lib.rs
+++ b/src/servers/src/lib.rs
@@ -18,8 +18,8 @@
 #![feature(let_chains)]
 #![feature(if_let_guard)]
 
+use datafusion_expr::LogicalPlan;
 use datatypes::schema::Schema;
-use query::plan::LogicalPlan;
 
 pub mod addrs;
 pub mod configurator;
diff --git a/src/servers/src/mysql/handler.rs b/src/servers/src/mysql/handler.rs
index 8e7b4630eb..3c0ac36f4c 100644
--- a/src/servers/src/mysql/handler.rs
+++ b/src/servers/src/mysql/handler.rs
@@ -25,6 +25,8 @@ use common_catalog::parse_optional_catalog_and_schema_from_db_string;
 use common_error::ext::ErrorExt;
 use common_query::Output;
 use common_telemetry::{debug, error, tracing, warn};
+use datafusion_common::ParamValues;
+use datafusion_expr::LogicalPlan;
 use datatypes::prelude::ConcreteDataType;
 use itertools::Itertools;
 use opensrv_mysql::{
@@ -32,7 +34,6 @@ use opensrv_mysql::{
     StatementMetaWriter, ValueInner,
 };
 use parking_lot::RwLock;
-use query::plan::LogicalPlan;
 use query::query_engine::DescribeResult;
 use rand::RngCore;
 use session::context::{Channel, QueryContextRef};
@@ -43,7 +44,7 @@ use sql::parser::{ParseOptions, ParserContext};
 use sql::statements::statement::Statement;
 use tokio::io::AsyncWrite;
 
-use crate::error::{self, InvalidPrepareStatementSnafu, Result};
+use crate::error::{self, DataFrameSnafu, InvalidPrepareStatementSnafu, Result};
 use crate::metrics::METRIC_AUTH_FAILURE;
 use crate::mysql::helper::{
     self, format_placeholder, replace_placeholders, transform_placeholders,
@@ -175,8 +176,11 @@ impl MysqlInstanceShim {
         let params = if let Some(plan) = &plan {
             prepared_params(
                 &plan
-                    .get_param_types()
-                    .context(error::GetPreparedStmtParamsSnafu)?,
+                    .get_parameter_types()
+                    .context(DataFrameSnafu)?
+                    .into_iter()
+                    .map(|(k, v)| (k, v.map(|v| ConcreteDataType::from_arrow_type(&v))))
+                    .collect(),
             )?
         } else {
             dummy_params(param_num)?
@@ -323,8 +327,11 @@ impl<W: AsyncWrite + Send + Sync + Unpin> AsyncMysqlShim<W> for MysqlInstanceShi
         let outputs = match sql_plan.plan {
             Some(plan) => {
                 let param_types = plan
-                    .get_param_types()
-                    .context(error::GetPreparedStmtParamsSnafu)?;
+                    .get_parameter_types()
+                    .context(DataFrameSnafu)?
+                    .into_iter()
+                    .map(|(k, v)| (k, v.map(|v| ConcreteDataType::from_arrow_type(&v))))
+                    .collect::<HashMap<_, _>>();
 
                 if params.len() != param_types.len() {
                     return error::InternalSnafu {
@@ -436,8 +443,11 @@ impl<W: AsyncWrite + Send + Sync + Unpin> AsyncMysqlShim<W> for MysqlInstanceShi
                     let outputs = match sql_plan.plan {
                         Some(plan) => {
                             let param_types = plan
-                                .get_param_types()
-                                .context(error::GetPreparedStmtParamsSnafu)?;
+                                .get_parameter_types()
+                                .context(DataFrameSnafu)?
+                                .into_iter()
+                                .map(|(k, v)| (k, v.map(|v| ConcreteDataType::from_arrow_type(&v))))
+                                .collect::<HashMap<_, _>>();
 
                             if params.len() != param_types.len() {
                                 writer
@@ -618,8 +628,9 @@ fn replace_params_with_values(
         }
     }
 
-    plan.replace_params_with_values(&values)
-        .context(error::ReplacePreparedStmtParamsSnafu)
+    plan.clone()
+        .replace_params_with_values(&ParamValues::List(values.clone()))
+        .context(DataFrameSnafu)
 }
 
 fn replace_params_with_exprs(
@@ -645,8 +656,9 @@ fn replace_params_with_exprs(
         }
     }
 
-    plan.replace_params_with_values(&values)
-        .context(error::ReplacePreparedStmtParamsSnafu)
+    plan.clone()
+        .replace_params_with_values(&ParamValues::List(values.clone()))
+        .context(DataFrameSnafu)
 }
 
 async fn validate_query(query: &str) -> Result<Statement> {
diff --git a/src/servers/src/postgres/handler.rs b/src/servers/src/postgres/handler.rs
index 53d907d814..e10a45ddec 100644
--- a/src/servers/src/postgres/handler.rs
+++ b/src/servers/src/postgres/handler.rs
@@ -20,6 +20,8 @@ use common_query::{Output, OutputData};
 use common_recordbatch::error::Result as RecordBatchResult;
 use common_recordbatch::RecordBatch;
 use common_telemetry::{debug, error, tracing};
+use datafusion_common::ParamValues;
+use datatypes::prelude::ConcreteDataType;
 use datatypes::schema::SchemaRef;
 use futures::{future, stream, Stream, StreamExt};
 use pgwire::api::portal::{Format, Portal};
@@ -272,7 +274,10 @@ impl ExtendedQueryHandler for PostgresServerHandler {
 
         let output = if let Some(plan) = &sql_plan.plan {
             let plan = plan
-                .replace_params_with_values(parameters_to_scalar_values(plan, portal)?.as_ref())
+                .clone()
+                .replace_params_with_values(&ParamValues::List(parameters_to_scalar_values(
+                    plan, portal,
+                )?))
                 .map_err(|e| PgWireError::ApiError(Box::new(e)))?;
             self.query_handler
                 .do_exec_plan(plan, query_ctx.clone())
@@ -306,8 +311,11 @@ impl ExtendedQueryHandler for PostgresServerHandler {
         let sql_plan = &stmt.statement;
         let (param_types, sql_plan, format) = if let Some(plan) = &sql_plan.plan {
             let param_types = plan
-                .get_param_types()
-                .map_err(|e| PgWireError::ApiError(Box::new(e)))?;
+                .get_parameter_types()
+                .map_err(|e| PgWireError::ApiError(Box::new(e)))?
+                .into_iter()
+                .map(|(k, v)| (k, v.map(|v| ConcreteDataType::from_arrow_type(&v))))
+                .collect();
 
             let types = param_types_to_pg_types(&param_types)
                 .map_err(|e| PgWireError::ApiError(Box::new(e)))?;
diff --git a/src/servers/src/postgres/types.rs b/src/servers/src/postgres/types.rs
index 9f9d94905e..85b626cb1a 100644
--- a/src/servers/src/postgres/types.rs
+++ b/src/servers/src/postgres/types.rs
@@ -23,6 +23,7 @@ use std::ops::Deref;
 use chrono::{NaiveDate, NaiveDateTime};
 use common_time::Interval;
 use datafusion_common::ScalarValue;
+use datafusion_expr::LogicalPlan;
 use datatypes::prelude::{ConcreteDataType, Value};
 use datatypes::schema::Schema;
 use datatypes::types::TimestampType;
@@ -30,7 +31,6 @@ use pgwire::api::portal::{Format, Portal};
 use pgwire::api::results::{DataRowEncoder, FieldInfo};
 use pgwire::api::Type;
 use pgwire::error::{PgWireError, PgWireResult};
-use query::plan::LogicalPlan;
 use session::context::QueryContextRef;
 use session::session_config::PGByteaOutputValue;
 
@@ -276,8 +276,11 @@ pub(super) fn parameters_to_scalar_values(
 
     let client_param_types = &portal.statement.parameter_types;
     let param_types = plan
-        .get_param_types()
-        .map_err(|e| PgWireError::ApiError(Box::new(e)))?;
+        .get_parameter_types()
+        .map_err(|e| PgWireError::ApiError(Box::new(e)))?
+        .into_iter()
+        .map(|(k, v)| (k, v.map(|v| ConcreteDataType::from_arrow_type(&v))))
+        .collect::<HashMap<_, _>>();
 
     for idx in 0..param_count {
         let server_type = param_types
diff --git a/src/servers/src/prom_store.rs b/src/servers/src/prom_store.rs
index d548a843b5..34d332e81d 100644
--- a/src/servers/src/prom_store.rs
+++ b/src/servers/src/prom_store.rs
@@ -28,10 +28,10 @@ use common_telemetry::tracing;
 use common_time::timestamp::TimeUnit;
 use datafusion::prelude::{col, lit, regexp_match, Expr};
 use datafusion_common::ScalarValue;
+use datafusion_expr::LogicalPlan;
 use datatypes::prelude::{ConcreteDataType, Value};
 use openmetrics_parser::{MetricsExposition, PrometheusType, PrometheusValue};
 use query::dataframe::DataFrame;
-use query::plan::LogicalPlan;
 use snafu::{ensure, OptionExt, ResultExt};
 use snap::raw::{Decoder, Encoder};
 
@@ -123,7 +123,7 @@ pub fn query_to_plan(dataframe: DataFrame, q: &Query) -> Result<LogicalPlan> {
         .filter(conditions)
         .context(error::DataFrameSnafu)?;
 
-    Ok(LogicalPlan::DfPlan(dataframe.into_parts().1))
+    Ok(dataframe.into_parts().1)
 }
 
 #[inline]
diff --git a/src/servers/src/query_handler/sql.rs b/src/servers/src/query_handler/sql.rs
index 9b9148af88..eef443bbec 100644
--- a/src/servers/src/query_handler/sql.rs
+++ b/src/servers/src/query_handler/sql.rs
@@ -17,8 +17,8 @@ use std::sync::Arc;
 use async_trait::async_trait;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_query::Output;
+use datafusion_expr::LogicalPlan;
 use query::parser::PromQuery;
-use query::plan::LogicalPlan;
 use session::context::QueryContextRef;
 use snafu::ResultExt;
 use sql::statements::statement::Statement;
diff --git a/src/servers/tests/http/influxdb_test.rs b/src/servers/tests/http/influxdb_test.rs
index 3e6e743b5c..a0ef088050 100644
--- a/src/servers/tests/http/influxdb_test.rs
+++ b/src/servers/tests/http/influxdb_test.rs
@@ -21,8 +21,8 @@ use auth::tests::{DatabaseAuthInfo, MockUserProvider};
 use axum::{http, Router};
 use common_query::Output;
 use common_test_util::ports;
+use datafusion_expr::LogicalPlan;
 use query::parser::PromQuery;
-use query::plan::LogicalPlan;
 use query::query_engine::DescribeResult;
 use servers::error::{Error, Result};
 use servers::http::header::constants::GREPTIME_DB_HEADER_NAME;
diff --git a/src/servers/tests/http/opentsdb_test.rs b/src/servers/tests/http/opentsdb_test.rs
index 1a719aa93d..635060261a 100644
--- a/src/servers/tests/http/opentsdb_test.rs
+++ b/src/servers/tests/http/opentsdb_test.rs
@@ -19,8 +19,8 @@ use async_trait::async_trait;
 use axum::Router;
 use common_query::Output;
 use common_test_util::ports;
+use datafusion_expr::LogicalPlan;
 use query::parser::PromQuery;
-use query::plan::LogicalPlan;
 use query::query_engine::DescribeResult;
 use servers::error::{self, Result};
 use servers::http::test_helpers::TestClient;
diff --git a/src/servers/tests/http/prom_store_test.rs b/src/servers/tests/http/prom_store_test.rs
index fd2d01304e..bb800e5b7c 100644
--- a/src/servers/tests/http/prom_store_test.rs
+++ b/src/servers/tests/http/prom_store_test.rs
@@ -23,9 +23,9 @@ use async_trait::async_trait;
 use axum::Router;
 use common_query::Output;
 use common_test_util::ports;
+use datafusion_expr::LogicalPlan;
 use prost::Message;
 use query::parser::PromQuery;
-use query::plan::LogicalPlan;
 use query::query_engine::DescribeResult;
 use servers::error::{Error, Result};
 use servers::http::header::{CONTENT_ENCODING_SNAPPY, CONTENT_TYPE_PROTOBUF};
diff --git a/src/servers/tests/mod.rs b/src/servers/tests/mod.rs
index daf9382ec7..ca098546aa 100644
--- a/src/servers/tests/mod.rs
+++ b/src/servers/tests/mod.rs
@@ -21,8 +21,8 @@ use async_trait::async_trait;
 use catalog::memory::MemoryCatalogManager;
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_query::Output;
+use datafusion_expr::LogicalPlan;
 use query::parser::{PromQuery, QueryLanguageParser, QueryStatement};
-use query::plan::LogicalPlan;
 use query::query_engine::DescribeResult;
 use query::{QueryEngineFactory, QueryEngineRef};
 use script::engine::{CompileContext, EvalContext, Script, ScriptEngine};
diff --git a/tests-integration/src/grpc.rs b/tests-integration/src/grpc.rs
index fa88de07de..a4ef632b33 100644
--- a/tests-integration/src/grpc.rs
+++ b/tests-integration/src/grpc.rs
@@ -33,7 +33,6 @@ mod test {
     use common_recordbatch::RecordBatches;
     use frontend::instance::Instance;
     use query::parser::QueryLanguageParser;
-    use query::plan::LogicalPlan;
     use query::query_engine::DefaultSerializer;
     use servers::query_handler::grpc::GrpcQueryHandler;
     use session::context::QueryContext;
@@ -540,7 +539,7 @@ CREATE TABLE {table_name} (
             &QueryContext::arc(),
         )
         .unwrap();
-        let LogicalPlan::DfPlan(plan) = instance
+        let plan = instance
             .frontend()
             .statement_executor()
             .plan(stmt, QueryContext::arc())
diff --git a/tests-integration/src/instance.rs b/tests-integration/src/instance.rs
index b3f966c811..096be44e0c 100644
--- a/tests-integration/src/instance.rs
+++ b/tests-integration/src/instance.rs
@@ -28,10 +28,10 @@ mod tests {
     use common_query::Output;
     use common_recordbatch::RecordBatches;
     use common_telemetry::debug;
+    use datafusion_expr::LogicalPlan;
     use frontend::error::{self, Error, Result};
     use frontend::instance::Instance;
     use query::parser::QueryLanguageParser;
-    use query::plan::LogicalPlan;
     use query::query_engine::DefaultSerializer;
     use servers::interceptor::{SqlQueryInterceptor, SqlQueryInterceptorRef};
     use servers::query_handler::sql::SqlQueryHandler;
@@ -233,7 +233,7 @@ mod tests {
             &QueryContext::arc(),
         )
         .unwrap();
-        let LogicalPlan::DfPlan(plan) = instance
+        let plan = instance
             .frontend()
             .statement_executor()
             .plan(stmt, QueryContext::arc())
@@ -317,7 +317,7 @@ mod tests {
             fn pre_execute(
                 &self,
                 _statement: &Statement,
-                _plan: Option<&query::plan::LogicalPlan>,
+                _plan: Option<&LogicalPlan>,
                 _query_ctx: QueryContextRef,
             ) -> Result<()> {
                 let _ = self.c.fetch_add(1, std::sync::atomic::Ordering::Relaxed);

From f02410c39bd0edce6e4786b2f6b0c6dfd7d03be4 Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Fri, 20 Sep 2024 08:35:37 +0800
Subject: [PATCH 023/128] fix: disable field pruning in last non null mode
 (#4740)

* fix: don't prune fields in last non null mode

* test: add sqlness test for field pruning

* test: add flush

* refine implementation

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
Co-authored-by: Ruihang Xia <waynestxia@gmail.com>
---
 src/mito2/src/read/scan_region.rs             | 39 ++++++++++-
 .../common/select/prune_field.result          | 69 +++++++++++++++++++
 .../standalone/common/select/prune_field.sql  | 24 +++++++
 3 files changed, 131 insertions(+), 1 deletion(-)
 create mode 100644 tests/cases/standalone/common/select/prune_field.result
 create mode 100644 tests/cases/standalone/common/select/prune_field.sql

diff --git a/src/mito2/src/read/scan_region.rs b/src/mito2/src/read/scan_region.rs
index 40f18c393b..d28562c554 100644
--- a/src/mito2/src/read/scan_region.rs
+++ b/src/mito2/src/read/scan_region.rs
@@ -14,6 +14,7 @@
 
 //! Scans a region according to the scan request.
 
+use std::collections::HashSet;
 use std::fmt;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
@@ -24,6 +25,7 @@ use common_telemetry::{debug, error, tracing, warn};
 use common_time::range::TimestampRange;
 use common_time::Timestamp;
 use datafusion::physical_plan::DisplayFormatType;
+use datafusion_expr::utils::expr_to_columns;
 use smallvec::SmallVec;
 use store_api::region_engine::{PartitionRange, RegionScannerRef};
 use store_api::storage::{ScanRequest, TimeSeriesRowSelector};
@@ -295,6 +297,9 @@ impl ScanRegion {
             self.version.options.append_mode,
         );
 
+        // Remove field filters for LastNonNull mode after logging the request.
+        self.maybe_remove_field_filters();
+
         let inverted_index_applier = self.build_invereted_index_applier();
         let fulltext_index_applier = self.build_fulltext_index_applier();
         let predicate = Predicate::new(self.request.filters.clone());
@@ -321,7 +326,7 @@ impl ScanRegion {
         Ok(input)
     }
 
-    /// Build time range predicate from filters.
+    /// Build time range predicate from filters, also remove time filters from request.
     fn build_time_range_predicate(&mut self) -> TimestampRange {
         let time_index = self.version.metadata.time_index_column();
         let unit = time_index
@@ -337,6 +342,38 @@ impl ScanRegion {
         )
     }
 
+    /// Remove field filters if the merge mode is [MergeMode::LastNonNull].
+    fn maybe_remove_field_filters(&mut self) {
+        if self.version.options.merge_mode() != MergeMode::LastNonNull {
+            return;
+        }
+
+        // TODO(yingwen): We can ignore field filters only when there are multiple sources in the same time window.
+        let field_columns = self
+            .version
+            .metadata
+            .field_columns()
+            .map(|col| &col.column_schema.name)
+            .collect::<HashSet<_>>();
+        // Columns in the expr.
+        let mut columns = HashSet::new();
+
+        self.request.filters.retain(|expr| {
+            columns.clear();
+            // `expr_to_columns` won't return error.
+            if expr_to_columns(expr, &mut columns).is_err() {
+                return false;
+            }
+            for column in &columns {
+                if field_columns.contains(&column.name) {
+                    // This expr uses the field column.
+                    return false;
+                }
+            }
+            true
+        });
+    }
+
     /// Use the latest schema to build the inveretd index applier.
     fn build_invereted_index_applier(&self) -> Option<InvertedIndexApplierRef> {
         if self.ignore_inverted_index {
diff --git a/tests/cases/standalone/common/select/prune_field.result b/tests/cases/standalone/common/select/prune_field.result
new file mode 100644
index 0000000000..d309d384be
--- /dev/null
+++ b/tests/cases/standalone/common/select/prune_field.result
@@ -0,0 +1,69 @@
+CREATE TABLE IF NOT EXISTS prune_field (
+  ts TIMESTAMP TIME INDEX,
+  tag UInt16,
+  a UInt8,
+  b UInt8,
+PRIMARY KEY (tag)) ENGINE = mito WITH('merge_mode'='last_non_null');
+
+Affected Rows: 0
+
+insert into prune_field(ts, tag, a, b) values(0, 1, 1, null);
+
+Affected Rows: 1
+
+admin flush_table('prune_field');
+
++----------------------------------+
+| ADMIN flush_table('prune_field') |
++----------------------------------+
+| 0                                |
++----------------------------------+
+
+insert into prune_field(ts, tag, a, b) values(0, 1, null, 1);
+
+Affected Rows: 1
+
+admin flush_table('prune_field');
+
++----------------------------------+
+| ADMIN flush_table('prune_field') |
++----------------------------------+
+| 0                                |
++----------------------------------+
+
+select * from prune_field where a = 1;
+
++---------------------+-----+---+---+
+| ts                  | tag | a | b |
++---------------------+-----+---+---+
+| 1970-01-01T00:00:00 | 1   | 1 | 1 |
++---------------------+-----+---+---+
+
+select * from prune_field where b = 1;
+
++---------------------+-----+---+---+
+| ts                  | tag | a | b |
++---------------------+-----+---+---+
+| 1970-01-01T00:00:00 | 1   | 1 | 1 |
++---------------------+-----+---+---+
+
+select * from prune_field;
+
++---------------------+-----+---+---+
+| ts                  | tag | a | b |
++---------------------+-----+---+---+
+| 1970-01-01T00:00:00 | 1   | 1 | 1 |
++---------------------+-----+---+---+
+
+select * from prune_field where a = 1 and b = 1;
+
++---------------------+-----+---+---+
+| ts                  | tag | a | b |
++---------------------+-----+---+---+
+| 1970-01-01T00:00:00 | 1   | 1 | 1 |
++---------------------+-----+---+---+
+
+drop table prune_field;
+
+Affected Rows: 0
+
diff --git a/tests/cases/standalone/common/select/prune_field.sql b/tests/cases/standalone/common/select/prune_field.sql
new file mode 100644
index 0000000000..6488175508
--- /dev/null
+++ b/tests/cases/standalone/common/select/prune_field.sql
@@ -0,0 +1,24 @@
+CREATE TABLE IF NOT EXISTS prune_field (
+  ts TIMESTAMP TIME INDEX,
+  tag UInt16,
+  a UInt8,
+  b UInt8,
+PRIMARY KEY (tag)) ENGINE = mito WITH('merge_mode'='last_non_null');
+
+insert into prune_field(ts, tag, a, b) values(0, 1, 1, null);
+
+admin flush_table('prune_field');
+
+insert into prune_field(ts, tag, a, b) values(0, 1, null, 1);
+
+admin flush_table('prune_field');
+
+select * from prune_field where a = 1;
+
+select * from prune_field where b = 1;
+
+select * from prune_field;
+
+select * from prune_field where a = 1 and b = 1;
+
+drop table prune_field;

From c4e52ebf918d4ae7f758c3336c07e6d60b022618 Mon Sep 17 00:00:00 2001
From: discord9 <55937128+discord9@users.noreply.github.com>
Date: Fri, 20 Sep 2024 10:34:45 +0800
Subject: [PATCH 024/128] feat: use new image for gcc-10 (#4748)

feat: use new image
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 697a31aed6..da9e8da5c0 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,7 @@ CARGO_BUILD_OPTS := --locked
 IMAGE_REGISTRY ?= docker.io
 IMAGE_NAMESPACE ?= greptime
 IMAGE_TAG ?= latest
-DEV_BUILDER_IMAGE_TAG ?= 2024-06-06-b4b105ad-20240827021230
+DEV_BUILDER_IMAGE_TAG ?= 2024-06-06-1acda74c-20240919113454
 BUILDX_MULTI_PLATFORM_BUILD ?= false
 BUILDX_BUILDER_NAME ?= gtbuilder
 BASE_IMAGE ?= ubuntu

From e12ffbeb2f807f83e1e15c6a921880d2ba2bdb4d Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Fri, 20 Sep 2024 10:55:31 +0800
Subject: [PATCH 025/128] feat: flush other workers if still need flush (#4746)

---
 src/mito2/src/engine/flush_test.rs | 69 ++++++++++++++++++++++++++++++
 src/mito2/src/engine/listener.rs   | 10 ++++-
 src/mito2/src/worker.rs            |  5 +++
 3 files changed, 83 insertions(+), 1 deletion(-)

diff --git a/src/mito2/src/engine/flush_test.rs b/src/mito2/src/engine/flush_test.rs
index aac02db91e..c134def6aa 100644
--- a/src/mito2/src/engine/flush_test.rs
+++ b/src/mito2/src/engine/flush_test.rs
@@ -402,3 +402,72 @@ async fn test_auto_flush_engine() {
 +-------+---------+---------------------+";
     assert_eq!(expected, batches.pretty_print().unwrap());
 }
+
+#[tokio::test]
+async fn test_flush_workers() {
+    let mut env = TestEnv::new();
+    let write_buffer_manager = Arc::new(MockWriteBufferManager::default());
+    let listener = Arc::new(FlushListener::default());
+    let engine = env
+        .create_engine_with(
+            MitoConfig {
+                num_workers: 2,
+                ..Default::default()
+            },
+            Some(write_buffer_manager.clone()),
+            Some(listener.clone()),
+        )
+        .await;
+
+    let region_id0 = RegionId::new(1, 0);
+    let region_id1 = RegionId::new(1, 1);
+    let request = CreateRequestBuilder::new().region_dir("r0").build();
+    let column_schemas = rows_schema(&request);
+    engine
+        .handle_request(region_id0, RegionRequest::Create(request))
+        .await
+        .unwrap();
+    let request = CreateRequestBuilder::new().region_dir("r1").build();
+    engine
+        .handle_request(region_id1, RegionRequest::Create(request.clone()))
+        .await
+        .unwrap();
+
+    // Prepares rows for flush.
+    let rows = Rows {
+        schema: column_schemas.clone(),
+        rows: build_rows_for_key("a", 0, 2, 0),
+    };
+    put_rows(&engine, region_id0, rows.clone()).await;
+    put_rows(&engine, region_id1, rows).await;
+
+    write_buffer_manager.set_should_flush(true);
+
+    // Writes to the mutable memtable and triggers flush for region 0.
+    let rows = Rows {
+        schema: column_schemas.clone(),
+        rows: build_rows_for_key("b", 0, 2, 0),
+    };
+    put_rows(&engine, region_id0, rows).await;
+
+    // Waits until flush is finished.
+    while listener.success_count() < 2 {
+        listener.wait().await;
+    }
+
+    // Scans region 1.
+    let request = ScanRequest::default();
+    let scanner = engine.scanner(region_id1, request).unwrap();
+    assert_eq!(0, scanner.num_memtables());
+    assert_eq!(1, scanner.num_files());
+    let stream = scanner.scan().await.unwrap();
+    let batches = RecordBatches::try_collect(stream).await.unwrap();
+    let expected = "\
++-------+---------+---------------------+
+| tag_0 | field_0 | ts                  |
++-------+---------+---------------------+
+| a     | 0.0     | 1970-01-01T00:00:00 |
+| a     | 1.0     | 1970-01-01T00:00:01 |
++-------+---------+---------------------+";
+    assert_eq!(expected, batches.pretty_print().unwrap());
+}
diff --git a/src/mito2/src/engine/listener.rs b/src/mito2/src/engine/listener.rs
index a79cb6eafd..83679f96e4 100644
--- a/src/mito2/src/engine/listener.rs
+++ b/src/mito2/src/engine/listener.rs
@@ -14,6 +14,7 @@
 
 //! Engine event listener for tests.
 
+use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -77,6 +78,7 @@ pub type EventListenerRef = Arc<dyn EventListener>;
 #[derive(Default)]
 pub struct FlushListener {
     notify: Notify,
+    success_count: AtomicUsize,
 }
 
 impl FlushListener {
@@ -84,6 +86,11 @@ impl FlushListener {
     pub async fn wait(&self) {
         self.notify.notified().await;
     }
+
+    /// Returns the success count.
+    pub fn success_count(&self) -> usize {
+        self.success_count.load(Ordering::Relaxed)
+    }
 }
 
 #[async_trait]
@@ -91,7 +98,8 @@ impl EventListener for FlushListener {
     fn on_flush_success(&self, region_id: RegionId) {
         info!("Region {} flush successfully", region_id);
 
-        self.notify.notify_one()
+        self.success_count.fetch_add(1, Ordering::Relaxed);
+        self.notify.notify_one();
     }
 }
 
diff --git a/src/mito2/src/worker.rs b/src/mito2/src/worker.rs
index 0f872e24e4..c2fbc80982 100644
--- a/src/mito2/src/worker.rs
+++ b/src/mito2/src/worker.rs
@@ -670,6 +670,11 @@ impl<S: LogStore> RegionWorkerLoop<S> {
                         // The channel is disconnected.
                         break;
                     } else {
+                        // Also flush this worker if other workers trigger flush as this worker may have
+                        // a large memtable to flush. We may not have chance to flush that memtable if we
+                        // never write to this worker. So only flushing other workers may not release enough
+                        // memory.
+                        self.maybe_flush_worker();
                         // A flush job is finished, handles stalled requests.
                         self.handle_stalled_requests().await;
                         continue;

From 75c6fad1a3d409229ea88bb9e17ebff56d297d04 Mon Sep 17 00:00:00 2001
From: Ning Sun <sunng@protonmail.com>
Date: Fri, 20 Sep 2024 12:19:50 +0800
Subject: [PATCH 026/128] feat: add more h3 scalar functions (#4707)

* feat: add more h3 scalar functions

* chore: comment up
---
 Cargo.lock                                    |  22 +
 src/common/function/Cargo.toml                |   1 +
 src/common/function/src/scalars/geo.rs        |  16 +-
 src/common/function/src/scalars/geo/h3.rs     | 718 +++++++++++++++++-
 .../standalone/common/function/geo.result     | 174 +++--
 .../cases/standalone/common/function/geo.sql  |  41 +-
 6 files changed, 857 insertions(+), 115 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 55e8ca8e4b..e7f180eb17 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1997,6 +1997,7 @@ dependencies = [
  "common-version",
  "datafusion",
  "datatypes",
+ "derive_more",
  "geohash",
  "h3o",
  "jsonb",
@@ -3413,6 +3414,27 @@ dependencies = [
  "syn 2.0.66",
 ]
 
+[[package]]
+name = "derive_more"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05"
+dependencies = [
+ "derive_more-impl",
+]
+
+[[package]]
+name = "derive_more-impl"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.66",
+ "unicode-xid",
+]
+
 [[package]]
 name = "diff"
 version = "0.1.13"
diff --git a/src/common/function/Cargo.toml b/src/common/function/Cargo.toml
index b2e9c5a98b..e9a3dd6b55 100644
--- a/src/common/function/Cargo.toml
+++ b/src/common/function/Cargo.toml
@@ -27,6 +27,7 @@ common-time.workspace = true
 common-version.workspace = true
 datafusion.workspace = true
 datatypes.workspace = true
+derive_more = { version = "1", default-features = false, features = ["display"] }
 geohash = { version = "0.13", optional = true }
 h3o = { version = "0.6", optional = true }
 jsonb.workspace = true
diff --git a/src/common/function/src/scalars/geo.rs b/src/common/function/src/scalars/geo.rs
index 4b126f20f0..9e415a3fdd 100644
--- a/src/common/function/src/scalars/geo.rs
+++ b/src/common/function/src/scalars/geo.rs
@@ -17,7 +17,6 @@ mod geohash;
 mod h3;
 
 use geohash::GeohashFunction;
-use h3::H3Function;
 
 use crate::function_registry::FunctionRegistry;
 
@@ -25,7 +24,20 @@ pub(crate) struct GeoFunctions;
 
 impl GeoFunctions {
     pub fn register(registry: &FunctionRegistry) {
+        // geohash
         registry.register(Arc::new(GeohashFunction));
-        registry.register(Arc::new(H3Function));
+        // h3 family
+        registry.register(Arc::new(h3::H3LatLngToCell));
+        registry.register(Arc::new(h3::H3LatLngToCellString));
+        registry.register(Arc::new(h3::H3CellBase));
+        registry.register(Arc::new(h3::H3CellCenterChild));
+        registry.register(Arc::new(h3::H3CellCenterLat));
+        registry.register(Arc::new(h3::H3CellCenterLng));
+        registry.register(Arc::new(h3::H3CellIsPentagon));
+        registry.register(Arc::new(h3::H3CellParent));
+        registry.register(Arc::new(h3::H3CellResolution));
+        registry.register(Arc::new(h3::H3CellToString));
+        registry.register(Arc::new(h3::H3IsNeighbour));
+        registry.register(Arc::new(h3::H3StringToCell));
     }
 }
diff --git a/src/common/function/src/scalars/geo/h3.rs b/src/common/function/src/scalars/geo/h3.rs
index 26ec246997..672fbfd714 100644
--- a/src/common/function/src/scalars/geo/h3.rs
+++ b/src/common/function/src/scalars/geo/h3.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::fmt;
+use std::str::FromStr;
 
 use common_error::ext::{BoxedError, PlainError};
 use common_error::status_code::StatusCode;
@@ -22,23 +22,118 @@ use datafusion::logical_expr::Volatility;
 use datatypes::prelude::ConcreteDataType;
 use datatypes::scalars::ScalarVectorBuilder;
 use datatypes::value::Value;
-use datatypes::vectors::{MutableVector, StringVectorBuilder, VectorRef};
-use h3o::{LatLng, Resolution};
+use datatypes::vectors::{
+    BooleanVectorBuilder, Float64VectorBuilder, MutableVector, StringVectorBuilder,
+    UInt64VectorBuilder, UInt8VectorBuilder, VectorRef,
+};
+use derive_more::Display;
+use h3o::{CellIndex, LatLng, Resolution};
 use snafu::{ensure, ResultExt};
 
 use crate::function::{Function, FunctionContext};
 
-/// Function that returns [h3] encoding string for a given geospatial coordinate.
+/// Function that returns [h3] encoding cellid for a given geospatial coordinate.
 ///
 /// [h3]: https://h3geo.org/
-#[derive(Clone, Debug, Default)]
-pub struct H3Function;
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3LatLngToCell;
 
-const NAME: &str = "h3";
-
-impl Function for H3Function {
+impl Function for H3LatLngToCell {
     fn name(&self) -> &str {
-        NAME
+        "h3_latlng_to_cell"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::uint64_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        let mut signatures = Vec::new();
+        for coord_type in &[
+            ConcreteDataType::float32_datatype(),
+            ConcreteDataType::float64_datatype(),
+        ] {
+            for resolution_type in &[
+                ConcreteDataType::int8_datatype(),
+                ConcreteDataType::int16_datatype(),
+                ConcreteDataType::int32_datatype(),
+                ConcreteDataType::int64_datatype(),
+                ConcreteDataType::uint8_datatype(),
+                ConcreteDataType::uint16_datatype(),
+                ConcreteDataType::uint32_datatype(),
+                ConcreteDataType::uint64_datatype(),
+            ] {
+                signatures.push(TypeSignature::Exact(vec![
+                    // latitude
+                    coord_type.clone(),
+                    // longitude
+                    coord_type.clone(),
+                    // resolution
+                    resolution_type.clone(),
+                ]));
+            }
+        }
+        Signature::one_of(signatures, Volatility::Stable)
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 3,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 3, provided : {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let lat_vec = &columns[0];
+        let lon_vec = &columns[1];
+        let resolution_vec = &columns[2];
+
+        let size = lat_vec.len();
+        let mut results = UInt64VectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let lat = lat_vec.get(i).as_f64_lossy();
+            let lon = lon_vec.get(i).as_f64_lossy();
+            let r = value_to_resolution(resolution_vec.get(i))?;
+
+            let result = match (lat, lon) {
+                (Some(lat), Some(lon)) => {
+                    let coord = LatLng::new(lat, lon)
+                        .map_err(|e| {
+                            BoxedError::new(PlainError::new(
+                                format!("H3 error: {}", e),
+                                StatusCode::EngineExecuteQuery,
+                            ))
+                        })
+                        .context(error::ExecuteSnafu)?;
+                    let encoded: u64 = coord.to_cell(r).into();
+                    Some(encoded)
+                }
+                _ => None,
+            };
+
+            results.push(result);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that returns [h3] encoding cellid in string form for a given
+/// geospatial coordinate.
+///
+/// [h3]: https://h3geo.org/
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3LatLngToCellString;
+
+impl Function for H3LatLngToCellString {
+    fn name(&self) -> &str {
+        "h3_latlng_to_cell_string"
     }
 
     fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
@@ -95,17 +190,7 @@ impl Function for H3Function {
         for i in 0..size {
             let lat = lat_vec.get(i).as_f64_lossy();
             let lon = lon_vec.get(i).as_f64_lossy();
-            let r = match resolution_vec.get(i) {
-                Value::Int8(v) => v as u8,
-                Value::Int16(v) => v as u8,
-                Value::Int32(v) => v as u8,
-                Value::Int64(v) => v as u8,
-                Value::UInt8(v) => v,
-                Value::UInt16(v) => v as u8,
-                Value::UInt32(v) => v as u8,
-                Value::UInt64(v) => v as u8,
-                _ => unreachable!(),
-            };
+            let r = value_to_resolution(resolution_vec.get(i))?;
 
             let result = match (lat, lon) {
                 (Some(lat), Some(lon)) => {
@@ -117,14 +202,6 @@ impl Function for H3Function {
                             ))
                         })
                         .context(error::ExecuteSnafu)?;
-                    let r = Resolution::try_from(r)
-                        .map_err(|e| {
-                            BoxedError::new(PlainError::new(
-                                format!("H3 error: {}", e),
-                                StatusCode::EngineExecuteQuery,
-                            ))
-                        })
-                        .context(error::ExecuteSnafu)?;
                     let encoded = coord.to_cell(r).to_string();
                     Some(encoded)
                 }
@@ -138,8 +215,585 @@ impl Function for H3Function {
     }
 }
 
-impl fmt::Display for H3Function {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", NAME)
+/// Function that converts cell id to its string form
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3CellToString;
+
+impl Function for H3CellToString {
+    fn name(&self) -> &str {
+        "h3_cell_to_string"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::string_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 1,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 1, provided : {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let cell_vec = &columns[0];
+        let size = cell_vec.len();
+        let mut results = StringVectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell_id_string = cell_from_value(cell_vec.get(i))?.map(|c| c.to_string());
+
+            results.push(cell_id_string.as_deref());
+        }
+
+        Ok(results.to_vector())
     }
 }
+
+/// Function that converts cell string id to uint64 number
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3StringToCell;
+
+impl Function for H3StringToCell {
+    fn name(&self) -> &str {
+        "h3_string_to_cell"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::uint64_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::new(
+            TypeSignature::Exact(vec![ConcreteDataType::string_datatype()]),
+            Volatility::Stable,
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 1,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 1, provided : {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let string_vec = &columns[0];
+        let size = string_vec.len();
+        let mut results = UInt64VectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell = string_vec.get(i);
+
+            let cell_id = match cell {
+                Value::String(v) => Some(
+                    CellIndex::from_str(v.as_utf8())
+                        .map_err(|e| {
+                            BoxedError::new(PlainError::new(
+                                format!("H3 error: {}", e),
+                                StatusCode::EngineExecuteQuery,
+                            ))
+                        })
+                        .context(error::ExecuteSnafu)?
+                        .into(),
+                ),
+                _ => None,
+            };
+
+            results.push(cell_id);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that returns centroid latitude of given cell id
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3CellCenterLat;
+
+impl Function for H3CellCenterLat {
+    fn name(&self) -> &str {
+        "h3_cell_center_lat"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::float64_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 1,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 1, provided : {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let cell_vec = &columns[0];
+        let size = cell_vec.len();
+        let mut results = Float64VectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i))?;
+            let lat = cell.map(|cell| LatLng::from(cell).lat());
+
+            results.push(lat);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that returns centroid longitude of given cell id
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3CellCenterLng;
+
+impl Function for H3CellCenterLng {
+    fn name(&self) -> &str {
+        "h3_cell_center_lng"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::float64_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 1,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 1, provided : {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let cell_vec = &columns[0];
+        let size = cell_vec.len();
+        let mut results = Float64VectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i))?;
+            let lat = cell.map(|cell| LatLng::from(cell).lng());
+
+            results.push(lat);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that returns resolution of given cell id
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3CellResolution;
+
+impl Function for H3CellResolution {
+    fn name(&self) -> &str {
+        "h3_cell_resolution"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::uint8_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 1,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 1, provided : {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let cell_vec = &columns[0];
+        let size = cell_vec.len();
+        let mut results = UInt8VectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i))?;
+            let res = cell.map(|cell| cell.resolution().into());
+
+            results.push(res);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that returns base cell of given cell id
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3CellBase;
+
+impl Function for H3CellBase {
+    fn name(&self) -> &str {
+        "h3_cell_base"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::uint8_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 1,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 1, provided : {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let cell_vec = &columns[0];
+        let size = cell_vec.len();
+        let mut results = UInt8VectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i))?;
+            let res = cell.map(|cell| cell.base_cell().into());
+
+            results.push(res);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that check if given cell id is a pentagon
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3CellIsPentagon;
+
+impl Function for H3CellIsPentagon {
+    fn name(&self) -> &str {
+        "h3_cell_is_pentagon"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::boolean_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 1,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 1, provided : {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let cell_vec = &columns[0];
+        let size = cell_vec.len();
+        let mut results = BooleanVectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i))?;
+            let res = cell.map(|cell| cell.is_pentagon());
+
+            results.push(res);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that returns center child cell of given cell id
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3CellCenterChild;
+
+impl Function for H3CellCenterChild {
+    fn name(&self) -> &str {
+        "h3_cell_center_child"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::uint64_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell_and_resolution()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 2, provided : {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let cell_vec = &columns[0];
+        let res_vec = &columns[1];
+        let size = cell_vec.len();
+        let mut results = UInt64VectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i))?;
+            let res = value_to_resolution(res_vec.get(i))?;
+            let result = cell
+                .and_then(|cell| cell.center_child(res))
+                .map(|c| c.into());
+
+            results.push(result);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that returns parent cell of given cell id and resolution
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3CellParent;
+
+impl Function for H3CellParent {
+    fn name(&self) -> &str {
+        "h3_cell_parent"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::uint64_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell_and_resolution()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 2, provided : {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let cell_vec = &columns[0];
+        let res_vec = &columns[1];
+        let size = cell_vec.len();
+        let mut results = UInt64VectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i))?;
+            let res = value_to_resolution(res_vec.get(i))?;
+            let result = cell.and_then(|cell| cell.parent(res)).map(|c| c.into());
+
+            results.push(result);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that checks if two cells are neighbour
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3IsNeighbour;
+
+impl Function for H3IsNeighbour {
+    fn name(&self) -> &str {
+        "h3_is_neighbour"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::boolean_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_double_cell()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 2, provided : {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let cell_vec = &columns[0];
+        let cell2_vec = &columns[1];
+        let size = cell_vec.len();
+        let mut results = BooleanVectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let result = match (
+                cell_from_value(cell_vec.get(i))?,
+                cell_from_value(cell2_vec.get(i))?,
+            ) {
+                (Some(cell_this), Some(cell_that)) => {
+                    let is_neighbour = cell_this
+                        .is_neighbor_with(cell_that)
+                        .map_err(|e| {
+                            BoxedError::new(PlainError::new(
+                                format!("H3 error: {}", e),
+                                StatusCode::EngineExecuteQuery,
+                            ))
+                        })
+                        .context(error::ExecuteSnafu)?;
+                    Some(is_neighbour)
+                }
+                _ => None,
+            };
+
+            results.push(result);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+fn value_to_resolution(v: Value) -> Result<Resolution> {
+    let r = match v {
+        Value::Int8(v) => v as u8,
+        Value::Int16(v) => v as u8,
+        Value::Int32(v) => v as u8,
+        Value::Int64(v) => v as u8,
+        Value::UInt8(v) => v,
+        Value::UInt16(v) => v as u8,
+        Value::UInt32(v) => v as u8,
+        Value::UInt64(v) => v as u8,
+        _ => unreachable!(),
+    };
+    Resolution::try_from(r)
+        .map_err(|e| {
+            BoxedError::new(PlainError::new(
+                format!("H3 error: {}", e),
+                StatusCode::EngineExecuteQuery,
+            ))
+        })
+        .context(error::ExecuteSnafu)
+}
+
+fn signature_of_cell() -> Signature {
+    let mut signatures = Vec::new();
+    for cell_type in &[
+        ConcreteDataType::uint64_datatype(),
+        ConcreteDataType::int64_datatype(),
+    ] {
+        signatures.push(TypeSignature::Exact(vec![cell_type.clone()]));
+    }
+
+    Signature::one_of(signatures, Volatility::Stable)
+}
+
+fn signature_of_double_cell() -> Signature {
+    let mut signatures = Vec::new();
+    let cell_types = &[
+        ConcreteDataType::uint64_datatype(),
+        ConcreteDataType::int64_datatype(),
+    ];
+    for cell_type in cell_types {
+        for cell_type2 in cell_types {
+            signatures.push(TypeSignature::Exact(vec![
+                cell_type.clone(),
+                cell_type2.clone(),
+            ]));
+        }
+    }
+
+    Signature::one_of(signatures, Volatility::Stable)
+}
+
+fn signature_of_cell_and_resolution() -> Signature {
+    let mut signatures = Vec::new();
+    for cell_type in &[
+        ConcreteDataType::uint64_datatype(),
+        ConcreteDataType::int64_datatype(),
+    ] {
+        for resolution_type in &[
+            ConcreteDataType::int8_datatype(),
+            ConcreteDataType::int16_datatype(),
+            ConcreteDataType::int32_datatype(),
+            ConcreteDataType::int64_datatype(),
+            ConcreteDataType::uint8_datatype(),
+            ConcreteDataType::uint16_datatype(),
+            ConcreteDataType::uint32_datatype(),
+            ConcreteDataType::uint64_datatype(),
+        ] {
+            signatures.push(TypeSignature::Exact(vec![
+                cell_type.clone(),
+                resolution_type.clone(),
+            ]));
+        }
+    }
+    Signature::one_of(signatures, Volatility::Stable)
+}
+
+fn cell_from_value(v: Value) -> Result<Option<CellIndex>> {
+    let cell = match v {
+        Value::Int64(v) => Some(
+            CellIndex::try_from(v as u64)
+                .map_err(|e| {
+                    BoxedError::new(PlainError::new(
+                        format!("H3 error: {}", e),
+                        StatusCode::EngineExecuteQuery,
+                    ))
+                })
+                .context(error::ExecuteSnafu)?,
+        ),
+        Value::UInt64(v) => Some(
+            CellIndex::try_from(v)
+                .map_err(|e| {
+                    BoxedError::new(PlainError::new(
+                        format!("H3 error: {}", e),
+                        StatusCode::EngineExecuteQuery,
+                    ))
+                })
+                .context(error::ExecuteSnafu)?,
+        ),
+        _ => None,
+    };
+    Ok(cell)
+}
diff --git a/tests/cases/standalone/common/function/geo.result b/tests/cases/standalone/common/function/geo.result
index 6d44c3ac04..3a63b5890b 100644
--- a/tests/cases/standalone/common/function/geo.result
+++ b/tests/cases/standalone/common/function/geo.result
@@ -1,98 +1,136 @@
-SELECT h3(37.76938, -122.3889, 0);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 0), h3_latlng_to_cell_string(37.76938, -122.3889, 0);;
 
-+---------------------------------------------------+
-| h3(Float64(37.76938),Float64(-122.3889),Int64(0)) |
-+---------------------------------------------------+
-| 8029fffffffffff                                   |
-+---------------------------------------------------+
++------------------------------------------------------------------+-------------------------------------------------------------------------+
+| h3_latlng_to_cell(Float64(37.76938),Float64(-122.3889),Int64(0)) | h3_latlng_to_cell_string(Float64(37.76938),Float64(-122.3889),Int64(0)) |
++------------------------------------------------------------------+-------------------------------------------------------------------------+
+| 577199624117288959                                               | 8029fffffffffff                                                         |
++------------------------------------------------------------------+-------------------------------------------------------------------------+
 
-SELECT h3(37.76938, -122.3889, 1);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 1), h3_latlng_to_cell_string(37.76938, -122.3889, 1);
 
-+---------------------------------------------------+
-| h3(Float64(37.76938),Float64(-122.3889),Int64(1)) |
-+---------------------------------------------------+
-| 81283ffffffffff                                   |
-+---------------------------------------------------+
++------------------------------------------------------------------+-------------------------------------------------------------------------+
+| h3_latlng_to_cell(Float64(37.76938),Float64(-122.3889),Int64(1)) | h3_latlng_to_cell_string(Float64(37.76938),Float64(-122.3889),Int64(1)) |
++------------------------------------------------------------------+-------------------------------------------------------------------------+
+| 581672437419081727                                               | 81283ffffffffff                                                         |
++------------------------------------------------------------------+-------------------------------------------------------------------------+
 
-SELECT h3(37.76938, -122.3889, 8);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8), h3_latlng_to_cell_string(37.76938, -122.3889, 8);
 
-+---------------------------------------------------+
-| h3(Float64(37.76938),Float64(-122.3889),Int64(8)) |
-+---------------------------------------------------+
-| 88283082e7fffff                                   |
-+---------------------------------------------------+
++------------------------------------------------------------------+-------------------------------------------------------------------------+
+| h3_latlng_to_cell(Float64(37.76938),Float64(-122.3889),Int64(8)) | h3_latlng_to_cell_string(Float64(37.76938),Float64(-122.3889),Int64(8)) |
++------------------------------------------------------------------+-------------------------------------------------------------------------+
+| 613196570438926335                                               | 88283082e7fffff                                                         |
++------------------------------------------------------------------+-------------------------------------------------------------------------+
 
-SELECT h3(37.76938, -122.3889, 100);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 100), h3_latlng_to_cell_string(37.76938, -122.3889, 100);
 
 Error: 3001(EngineExecuteQuery), H3 error: invalid resolution (got Some(100)): out of range
 
-SELECT h3(37.76938, -122.3889, -1);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, -1), h3_latlng_to_cell_string(37.76938, -122.3889, -1);
 
 Error: 3001(EngineExecuteQuery), H3 error: invalid resolution (got Some(255)): out of range
 
-SELECT h3(37.76938, -122.3889, 8::Int8);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::Int8), h3_latlng_to_cell_string(37.76938, -122.3889, 8::Int8);
 
-+---------------------------------------------------+
-| h3(Float64(37.76938),Float64(-122.3889),Int64(8)) |
-+---------------------------------------------------+
-| 88283082e7fffff                                   |
-+---------------------------------------------------+
++------------------------------------------------------------------+-------------------------------------------------------------------------+
+| h3_latlng_to_cell(Float64(37.76938),Float64(-122.3889),Int64(8)) | h3_latlng_to_cell_string(Float64(37.76938),Float64(-122.3889),Int64(8)) |
++------------------------------------------------------------------+-------------------------------------------------------------------------+
+| 613196570438926335                                               | 88283082e7fffff                                                         |
++------------------------------------------------------------------+-------------------------------------------------------------------------+
 
-SELECT h3(37.76938, -122.3889, 8::Int16);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::Int16), h3_latlng_to_cell_string(37.76938, -122.3889, 8::Int16);
 
-+-----------------------------------------------------------------------------+
-| h3(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("Int16"))) |
-+-----------------------------------------------------------------------------+
-| 88283082e7fffff                                                             |
-+-----------------------------------------------------------------------------+
++--------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
+| h3_latlng_to_cell(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("Int16"))) | h3_latlng_to_cell_string(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("Int16"))) |
++--------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
+| 613196570438926335                                                                         | 88283082e7fffff                                                                                   |
++--------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
 
-SELECT h3(37.76938, -122.3889, 8::Int32);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::Int32), h3_latlng_to_cell_string(37.76938, -122.3889, 8::Int32);
 
-+-----------------------------------------------------------------------------+
-| h3(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("Int32"))) |
-+-----------------------------------------------------------------------------+
-| 88283082e7fffff                                                             |
-+-----------------------------------------------------------------------------+
++--------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
+| h3_latlng_to_cell(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("Int32"))) | h3_latlng_to_cell_string(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("Int32"))) |
++--------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
+| 613196570438926335                                                                         | 88283082e7fffff                                                                                   |
++--------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
 
-SELECT h3(37.76938, -122.3889, 8::Int64);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::Int64), h3_latlng_to_cell_string(37.76938, -122.3889, 8::Int64);
 
-+-----------------------------------------------------------------------------+
-| h3(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("Int64"))) |
-+-----------------------------------------------------------------------------+
-| 88283082e7fffff                                                             |
-+-----------------------------------------------------------------------------+
++--------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
+| h3_latlng_to_cell(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("Int64"))) | h3_latlng_to_cell_string(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("Int64"))) |
++--------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
+| 613196570438926335                                                                         | 88283082e7fffff                                                                                   |
++--------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
 
-SELECT h3(37.76938, -122.3889, 8::UInt8);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt8), h3_latlng_to_cell_string(37.76938, -122.3889, 8::UInt8);
 
-+-----------------------------------------------------------------------------+
-| h3(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("UInt8"))) |
-+-----------------------------------------------------------------------------+
-| 88283082e7fffff                                                             |
-+-----------------------------------------------------------------------------+
++--------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
+| h3_latlng_to_cell(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("UInt8"))) | h3_latlng_to_cell_string(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("UInt8"))) |
++--------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
+| 613196570438926335                                                                         | 88283082e7fffff                                                                                   |
++--------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
 
-SELECT h3(37.76938, -122.3889, 8::UInt16);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt16), h3_latlng_to_cell_string(37.76938, -122.3889, 8::UInt8);
 
-+------------------------------------------------------------------------------+
-| h3(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("UInt16"))) |
-+------------------------------------------------------------------------------+
-| 88283082e7fffff                                                              |
-+------------------------------------------------------------------------------+
++---------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
+| h3_latlng_to_cell(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("UInt16"))) | h3_latlng_to_cell_string(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("UInt8"))) |
++---------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
+| 613196570438926335                                                                          | 88283082e7fffff                                                                                   |
++---------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------+
 
-SELECT h3(37.76938, -122.3889, 8::UInt32);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt32), h3_latlng_to_cell_string(37.76938, -122.3889, 8::UInt32);
 
-+------------------------------------------------------------------------------+
-| h3(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("UInt32"))) |
-+------------------------------------------------------------------------------+
-| 88283082e7fffff                                                              |
-+------------------------------------------------------------------------------+
++---------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+
+| h3_latlng_to_cell(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("UInt32"))) | h3_latlng_to_cell_string(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("UInt32"))) |
++---------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+
+| 613196570438926335                                                                          | 88283082e7fffff                                                                                    |
++---------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+
 
-SELECT h3(37.76938, -122.3889, 8::UInt64);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64), h3_latlng_to_cell_string(37.76938, -122.3889, 8::UInt64);
 
-+------------------------------------------------------------------------------+
-| h3(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("UInt64"))) |
-+------------------------------------------------------------------------------+
-| 88283082e7fffff                                                              |
-+------------------------------------------------------------------------------+
++---------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+
+| h3_latlng_to_cell(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("UInt64"))) | h3_latlng_to_cell_string(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(8),Utf8("UInt64"))) |
++---------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+
+| 613196570438926335                                                                          | 88283082e7fffff                                                                                    |
++---------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------+
+
+SELECT h3_cell_to_string(h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64)) AS cell_str, h3_string_to_cell(h3_latlng_to_cell_string(37.76938, -122.3889, 8::UInt64)) AS cell_index;
+
++-----------------+--------------------+
+| cell_str        | cell_index         |
++-----------------+--------------------+
+| 88283082e7fffff | 613196570438926335 |
++-----------------+--------------------+
+
+SELECT h3_cell_center_lat(h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64)) AS cell_center_lat, h3_cell_center_lng(h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64)) AS cell_center_lng;
+
++-------------------+---------------------+
+| cell_center_lat   | cell_center_lng     |
++-------------------+---------------------+
+| 37.77246152245501 | -122.39010997087324 |
++-------------------+---------------------+
+
+SELECT
+    h3_cell_resolution(cell) AS resolution,
+    h3_cell_base(cell) AS base,
+    h3_cell_is_pentagon(cell) AS pentagon,
+    h3_cell_parent(cell, 6::UInt64) AS parent,
+FROM (SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64) AS cell);
+
++------------+------+----------+--------------------+
+| resolution | base | pentagon | parent             |
++------------+------+----------+--------------------+
+| 8          | 20   | false    | 604189371209351167 |
++------------+------+----------+--------------------+
+
+SELECT h3_is_neighbour(cell1, cell2)
+FROM (SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64) AS cell1, h3_latlng_to_cell(36.76938, -122.3889, 8::UInt64) AS cell2);
+
++------------------------------+
+| h3_is_neighbour(cell1,cell2) |
++------------------------------+
+| false                        |
++------------------------------+
 
 SELECT geohash(37.76938, -122.3889, 9);
 
diff --git a/tests/cases/standalone/common/function/geo.sql b/tests/cases/standalone/common/function/geo.sql
index 8f6f70f4a4..3a0e668acc 100644
--- a/tests/cases/standalone/common/function/geo.sql
+++ b/tests/cases/standalone/common/function/geo.sql
@@ -1,28 +1,43 @@
-SELECT h3(37.76938, -122.3889, 0);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 0), h3_latlng_to_cell_string(37.76938, -122.3889, 0);;
 
-SELECT h3(37.76938, -122.3889, 1);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 1), h3_latlng_to_cell_string(37.76938, -122.3889, 1);
 
-SELECT h3(37.76938, -122.3889, 8);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8), h3_latlng_to_cell_string(37.76938, -122.3889, 8);
 
-SELECT h3(37.76938, -122.3889, 100);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 100), h3_latlng_to_cell_string(37.76938, -122.3889, 100);
 
-SELECT h3(37.76938, -122.3889, -1);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, -1), h3_latlng_to_cell_string(37.76938, -122.3889, -1);
 
-SELECT h3(37.76938, -122.3889, 8::Int8);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::Int8), h3_latlng_to_cell_string(37.76938, -122.3889, 8::Int8);
 
-SELECT h3(37.76938, -122.3889, 8::Int16);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::Int16), h3_latlng_to_cell_string(37.76938, -122.3889, 8::Int16);
 
-SELECT h3(37.76938, -122.3889, 8::Int32);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::Int32), h3_latlng_to_cell_string(37.76938, -122.3889, 8::Int32);
 
-SELECT h3(37.76938, -122.3889, 8::Int64);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::Int64), h3_latlng_to_cell_string(37.76938, -122.3889, 8::Int64);
 
-SELECT h3(37.76938, -122.3889, 8::UInt8);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt8), h3_latlng_to_cell_string(37.76938, -122.3889, 8::UInt8);
 
-SELECT h3(37.76938, -122.3889, 8::UInt16);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt16), h3_latlng_to_cell_string(37.76938, -122.3889, 8::UInt8);
 
-SELECT h3(37.76938, -122.3889, 8::UInt32);
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt32), h3_latlng_to_cell_string(37.76938, -122.3889, 8::UInt32);
+
+SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64), h3_latlng_to_cell_string(37.76938, -122.3889, 8::UInt64);
+
+SELECT h3_cell_to_string(h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64)) AS cell_str, h3_string_to_cell(h3_latlng_to_cell_string(37.76938, -122.3889, 8::UInt64)) AS cell_index;
+
+SELECT h3_cell_center_lat(h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64)) AS cell_center_lat, h3_cell_center_lng(h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64)) AS cell_center_lng;
+
+SELECT
+    h3_cell_resolution(cell) AS resolution,
+    h3_cell_base(cell) AS base,
+    h3_cell_is_pentagon(cell) AS pentagon,
+    h3_cell_parent(cell, 6::UInt64) AS parent,
+FROM (SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64) AS cell);
+
+SELECT h3_is_neighbour(cell1, cell2)
+FROM (SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64) AS cell1, h3_latlng_to_cell(36.76938, -122.3889, 8::UInt64) AS cell2);
 
-SELECT h3(37.76938, -122.3889, 8::UInt64);
 
 SELECT geohash(37.76938, -122.3889, 9);
 

From 0c9b8eb0d2ff6dcb6c6c5f78aae3a3336f2090d7 Mon Sep 17 00:00:00 2001
From: taobo <realtaobo@outlook.com>
Date: Fri, 20 Sep 2024 14:07:53 +0800
Subject: [PATCH 027/128] feat: improve observability for procedure (#4675)

* feat: improve observability for procedure

* fix: test error

* test: add sqlness test for information_schema.procedure_info

* fix: sqlness test error

* fix: cr comment

* chore: update proto version

* fix: apply cr comment

* update version

* fix: cr comment

* optimize procedure type output format

* upgrade dep version

* fix: clippy error

* fix: `procedure` borrowed error

* fix: optimize code
---
 Cargo.lock                                    |   4 +-
 Cargo.toml                                    |   2 +-
 src/catalog/Cargo.toml                        |   1 +
 src/catalog/src/error.rs                      |  35 +-
 src/catalog/src/kvbackend/manager.rs          |   8 +
 .../src/system_schema/information_schema.rs   |  11 +-
 .../information_schema/procedure_info.rs      | 310 ++++++++++++++++++
 .../information_schema/table_names.rs         |   1 +
 src/catalog/src/system_schema/utils.rs        |  15 +
 src/catalog/src/table_source.rs               |   1 +
 src/cmd/src/cli/repl.rs                       |   1 +
 src/cmd/src/flownode.rs                       |   1 +
 src/cmd/src/frontend.rs                       |   1 +
 src/cmd/src/standalone.rs                     |   1 +
 src/common/catalog/src/consts.rs              |   2 +
 src/common/meta/src/ddl.rs                    |   3 +
 src/common/meta/src/ddl_manager.rs            |  10 +
 src/common/meta/src/rpc/procedure.rs          |  39 ++-
 src/common/procedure/Cargo.toml               |   1 +
 src/common/procedure/src/lib.rs               |   4 +-
 src/common/procedure/src/local.rs             |  48 ++-
 src/common/procedure/src/local/runner.rs      |   7 +-
 src/common/procedure/src/procedure.rs         |  44 +++
 src/meta-client/src/client.rs                 |  12 +-
 src/meta-client/src/client/procedure.rs       |  29 +-
 src/meta-srv/src/service/procedure.rs         |  18 +-
 tests-integration/src/cluster.rs              |   1 +
 tests-integration/src/standalone.rs           |   1 +
 .../information_schema/procedure_info.result  |  40 +++
 .../information_schema/procedure_info.sql     |  20 ++
 .../common/show/show_databases_tables.result  |   3 +
 .../common/system/information_schema.result   |   7 +
 .../standalone/common/view/create.result      |   1 +
 33 files changed, 662 insertions(+), 20 deletions(-)
 create mode 100644 src/catalog/src/system_schema/information_schema/procedure_info.rs
 create mode 100644 tests/cases/standalone/common/information_schema/procedure_info.result
 create mode 100644 tests/cases/standalone/common/information_schema/procedure_info.sql

diff --git a/Cargo.lock b/Cargo.lock
index e7f180eb17..2183cf593d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1379,6 +1379,7 @@ dependencies = [
  "common-error",
  "common-macro",
  "common-meta",
+ "common-procedure",
  "common-query",
  "common-recordbatch",
  "common-runtime",
@@ -2179,6 +2180,7 @@ dependencies = [
  "common-runtime",
  "common-telemetry",
  "common-test-util",
+ "common-time",
  "futures",
  "futures-util",
  "humantime-serde",
@@ -4394,7 +4396,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 [[package]]
 name = "greptime-proto"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=973f49cde88a582fb65755cc572ebcf6fb93ccf7#973f49cde88a582fb65755cc572ebcf6fb93ccf7"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=796ce9b003c6689e853825f649e03543c81ede99#796ce9b003c6689e853825f649e03543c81ede99"
 dependencies = [
  "prost 0.12.6",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index 25cd5b5dd2..df70b98332 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -120,7 +120,7 @@ etcd-client = { version = "0.13" }
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "973f49cde88a582fb65755cc572ebcf6fb93ccf7" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "796ce9b003c6689e853825f649e03543c81ede99" }
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"
diff --git a/src/catalog/Cargo.toml b/src/catalog/Cargo.toml
index cfea5e02c6..a5ad92e891 100644
--- a/src/catalog/Cargo.toml
+++ b/src/catalog/Cargo.toml
@@ -22,6 +22,7 @@ common-config.workspace = true
 common-error.workspace = true
 common-macro.workspace = true
 common-meta.workspace = true
+common-procedure.workspace = true
 common-query.workspace = true
 common-recordbatch.workspace = true
 common-runtime.workspace = true
diff --git a/src/catalog/src/error.rs b/src/catalog/src/error.rs
index d44a5b7683..14748a4695 100644
--- a/src/catalog/src/error.rs
+++ b/src/catalog/src/error.rs
@@ -82,6 +82,33 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Failed to get procedure client in {mode} mode"))]
+    GetProcedureClient {
+        mode: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Failed to list procedures"))]
+    ListProcedures {
+        #[snafu(implicit)]
+        location: Location,
+        source: BoxedError,
+    },
+
+    #[snafu(display("Procedure id not found"))]
+    ProcedureIdNotFound {
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("convert proto data error"))]
+    ConvertProtoData {
+        #[snafu(implicit)]
+        location: Location,
+        source: BoxedError,
+    },
+
     #[snafu(display("Failed to re-compile script due to internal error"))]
     CompileScriptInternal {
         #[snafu(implicit)]
@@ -266,7 +293,9 @@ impl ErrorExt for Error {
             | Error::FindRegionRoutes { .. }
             | Error::CacheNotFound { .. }
             | Error::CastManager { .. }
-            | Error::Json { .. } => StatusCode::Unexpected,
+            | Error::Json { .. }
+            | Error::GetProcedureClient { .. }
+            | Error::ProcedureIdNotFound { .. } => StatusCode::Unexpected,
 
             Error::ViewPlanColumnsChanged { .. } => StatusCode::InvalidArguments,
 
@@ -283,7 +312,9 @@ impl ErrorExt for Error {
             | Error::ListNodes { source, .. }
             | Error::ListSchemas { source, .. }
             | Error::ListTables { source, .. }
-            | Error::ListFlows { source, .. } => source.status_code(),
+            | Error::ListFlows { source, .. }
+            | Error::ListProcedures { source, .. }
+            | Error::ConvertProtoData { source, .. } => source.status_code(),
 
             Error::CreateTable { source, .. } => source.status_code(),
 
diff --git a/src/catalog/src/kvbackend/manager.rs b/src/catalog/src/kvbackend/manager.rs
index 1559022514..2495a95255 100644
--- a/src/catalog/src/kvbackend/manager.rs
+++ b/src/catalog/src/kvbackend/manager.rs
@@ -31,6 +31,7 @@ use common_meta::key::table_info::TableInfoValue;
 use common_meta::key::table_name::TableNameKey;
 use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
 use common_meta::kv_backend::KvBackendRef;
+use common_procedure::ProcedureManagerRef;
 use futures_util::stream::BoxStream;
 use futures_util::{StreamExt, TryStreamExt};
 use meta_client::client::MetaClient;
@@ -69,6 +70,7 @@ pub struct KvBackendCatalogManager {
     /// A sub-CatalogManager that handles system tables
     system_catalog: SystemCatalog,
     cache_registry: LayeredCacheRegistryRef,
+    procedure_manager: Option<ProcedureManagerRef>,
 }
 
 const CATALOG_CACHE_MAX_CAPACITY: u64 = 128;
@@ -79,6 +81,7 @@ impl KvBackendCatalogManager {
         meta_client: Option<Arc<MetaClient>>,
         backend: KvBackendRef,
         cache_registry: LayeredCacheRegistryRef,
+        procedure_manager: Option<ProcedureManagerRef>,
     ) -> Arc<Self> {
         Arc::new_cyclic(|me| Self {
             mode,
@@ -106,6 +109,7 @@ impl KvBackendCatalogManager {
                 backend,
             },
             cache_registry,
+            procedure_manager,
         })
     }
 
@@ -132,6 +136,10 @@ impl KvBackendCatalogManager {
     pub fn table_metadata_manager_ref(&self) -> &TableMetadataManagerRef {
         &self.table_metadata_manager
     }
+
+    pub fn procedure_manager(&self) -> Option<ProcedureManagerRef> {
+        self.procedure_manager.clone()
+    }
 }
 
 #[async_trait::async_trait]
diff --git a/src/catalog/src/system_schema/information_schema.rs b/src/catalog/src/system_schema/information_schema.rs
index 93dfaa75b5..83f2ff4926 100644
--- a/src/catalog/src/system_schema/information_schema.rs
+++ b/src/catalog/src/system_schema/information_schema.rs
@@ -18,6 +18,7 @@ pub mod flows;
 mod information_memory_table;
 pub mod key_column_usage;
 mod partitions;
+mod procedure_info;
 mod region_peers;
 mod runtime_metrics;
 pub mod schemata;
@@ -188,6 +189,11 @@ impl SystemSchemaProviderInner for InformationSchemaProvider {
                 self.catalog_name.clone(),
                 self.flow_metadata_manager.clone(),
             )) as _),
+            PROCEDURE_INFO => Some(
+                Arc::new(procedure_info::InformationSchemaProcedureInfo::new(
+                    self.catalog_manager.clone(),
+                )) as _,
+            ),
             _ => None,
         }
     }
@@ -250,7 +256,10 @@ impl InformationSchemaProvider {
             self.build_table(TABLE_CONSTRAINTS).unwrap(),
         );
         tables.insert(FLOWS.to_string(), self.build_table(FLOWS).unwrap());
-
+        tables.insert(
+            PROCEDURE_INFO.to_string(),
+            self.build_table(PROCEDURE_INFO).unwrap(),
+        );
         // Add memory tables
         for name in MEMORY_TABLES.iter() {
             tables.insert((*name).to_string(), self.build_table(name).expect(name));
diff --git a/src/catalog/src/system_schema/information_schema/procedure_info.rs b/src/catalog/src/system_schema/information_schema/procedure_info.rs
new file mode 100644
index 0000000000..56c36c2210
--- /dev/null
+++ b/src/catalog/src/system_schema/information_schema/procedure_info.rs
@@ -0,0 +1,310 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::{Arc, Weak};
+
+use api::v1::meta::{ProcedureMeta, ProcedureStatus};
+use arrow_schema::SchemaRef as ArrowSchemaRef;
+use common_catalog::consts::INFORMATION_SCHEMA_PROCEDURE_INFO_TABLE_ID;
+use common_config::Mode;
+use common_error::ext::BoxedError;
+use common_meta::ddl::{ExecutorContext, ProcedureExecutor};
+use common_meta::rpc::procedure;
+use common_procedure::{ProcedureInfo, ProcedureState};
+use common_recordbatch::adapter::RecordBatchStreamAdapter;
+use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
+use common_time::timestamp::Timestamp;
+use datafusion::execution::TaskContext;
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
+use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
+use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
+use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder, VectorRef};
+use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
+use datatypes::timestamp::TimestampMillisecond;
+use datatypes::value::Value;
+use datatypes::vectors::{StringVectorBuilder, TimestampMillisecondVectorBuilder};
+use snafu::ResultExt;
+use store_api::storage::{ScanRequest, TableId};
+
+use super::PROCEDURE_INFO;
+use crate::error::{
+    ConvertProtoDataSnafu, CreateRecordBatchSnafu, GetProcedureClientSnafu, InternalSnafu,
+    ListProceduresSnafu, ProcedureIdNotFoundSnafu, Result,
+};
+use crate::system_schema::information_schema::{InformationTable, Predicates};
+use crate::system_schema::utils;
+use crate::CatalogManager;
+
+const PROCEDURE_ID: &str = "procedure_id";
+const PROCEDURE_TYPE: &str = "procedure_type";
+const START_TIME: &str = "start_time";
+const END_TIME: &str = "end_time";
+const STATUS: &str = "status";
+const LOCK_KEYS: &str = "lock_keys";
+
+const INIT_CAPACITY: usize = 42;
+
+/// The `PROCEDURE_INFO` table provides information about the current procedure information of the cluster.
+///
+/// - `procedure_id`: the unique identifier of the procedure.
+/// - `procedure_name`: the name of the procedure.
+/// - `start_time`: the starting execution time of the procedure.
+/// - `end_time`: the ending execution time of the procedure.
+/// - `status`: the status of the procedure.
+/// - `lock_keys`: the lock keys of the procedure.
+///
+pub(super) struct InformationSchemaProcedureInfo {
+    schema: SchemaRef,
+    catalog_manager: Weak<dyn CatalogManager>,
+}
+
+impl InformationSchemaProcedureInfo {
+    pub(super) fn new(catalog_manager: Weak<dyn CatalogManager>) -> Self {
+        Self {
+            schema: Self::schema(),
+            catalog_manager,
+        }
+    }
+
+    pub(crate) fn schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![
+            ColumnSchema::new(PROCEDURE_ID, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(PROCEDURE_TYPE, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(
+                START_TIME,
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                true,
+            ),
+            ColumnSchema::new(
+                END_TIME,
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                true,
+            ),
+            ColumnSchema::new(STATUS, ConcreteDataType::string_datatype(), false),
+            ColumnSchema::new(LOCK_KEYS, ConcreteDataType::string_datatype(), true),
+        ]))
+    }
+
+    fn builder(&self) -> InformationSchemaProcedureInfoBuilder {
+        InformationSchemaProcedureInfoBuilder::new(
+            self.schema.clone(),
+            self.catalog_manager.clone(),
+        )
+    }
+}
+
+impl InformationTable for InformationSchemaProcedureInfo {
+    fn table_id(&self) -> TableId {
+        INFORMATION_SCHEMA_PROCEDURE_INFO_TABLE_ID
+    }
+
+    fn table_name(&self) -> &'static str {
+        PROCEDURE_INFO
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn to_stream(&self, request: ScanRequest) -> Result<SendableRecordBatchStream> {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        let stream = Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_procedure_info(Some(request))
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ));
+        Ok(Box::pin(
+            RecordBatchStreamAdapter::try_new(stream)
+                .map_err(BoxedError::new)
+                .context(InternalSnafu)?,
+        ))
+    }
+}
+
+struct InformationSchemaProcedureInfoBuilder {
+    schema: SchemaRef,
+    catalog_manager: Weak<dyn CatalogManager>,
+
+    procedure_ids: StringVectorBuilder,
+    procedure_types: StringVectorBuilder,
+    start_times: TimestampMillisecondVectorBuilder,
+    end_times: TimestampMillisecondVectorBuilder,
+    statuses: StringVectorBuilder,
+    lock_keys: StringVectorBuilder,
+}
+
+impl InformationSchemaProcedureInfoBuilder {
+    fn new(schema: SchemaRef, catalog_manager: Weak<dyn CatalogManager>) -> Self {
+        Self {
+            schema,
+            catalog_manager,
+            procedure_ids: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+            procedure_types: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+            start_times: TimestampMillisecondVectorBuilder::with_capacity(INIT_CAPACITY),
+            end_times: TimestampMillisecondVectorBuilder::with_capacity(INIT_CAPACITY),
+            statuses: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+            lock_keys: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+        }
+    }
+
+    /// Construct the `information_schema.procedure_info` virtual table
+    async fn make_procedure_info(&mut self, request: Option<ScanRequest>) -> Result<RecordBatch> {
+        let predicates = Predicates::from_scan_request(&request);
+        let mode = utils::running_mode(&self.catalog_manager)?.unwrap_or(Mode::Standalone);
+        match mode {
+            Mode::Standalone => {
+                if let Some(procedure_manager) = utils::procedure_manager(&self.catalog_manager)? {
+                    let procedures = procedure_manager
+                        .list_procedures()
+                        .await
+                        .map_err(BoxedError::new)
+                        .context(ListProceduresSnafu)?;
+                    for procedure in procedures {
+                        self.add_procedure(
+                            &predicates,
+                            procedure.state.as_str_name().to_string(),
+                            procedure,
+                        );
+                    }
+                } else {
+                    return GetProcedureClientSnafu { mode: "standalone" }.fail();
+                }
+            }
+            Mode::Distributed => {
+                if let Some(meta_client) = utils::meta_client(&self.catalog_manager)? {
+                    let procedures = meta_client
+                        .list_procedures(&ExecutorContext::default())
+                        .await
+                        .map_err(BoxedError::new)
+                        .context(ListProceduresSnafu)?;
+                    for procedure in procedures.procedures {
+                        self.add_procedure_info(&predicates, procedure)?;
+                    }
+                } else {
+                    return GetProcedureClientSnafu {
+                        mode: "distributed",
+                    }
+                    .fail();
+                }
+            }
+        };
+
+        self.finish()
+    }
+
+    fn add_procedure(
+        &mut self,
+        predicates: &Predicates,
+        status: String,
+        procedure_info: ProcedureInfo,
+    ) {
+        let ProcedureInfo {
+            id,
+            type_name,
+            start_time_ms,
+            end_time_ms,
+            lock_keys,
+            ..
+        } = procedure_info;
+        let pid = id.to_string();
+        let start_time = TimestampMillisecond(Timestamp::new_millisecond(start_time_ms));
+        let end_time = TimestampMillisecond(Timestamp::new_millisecond(end_time_ms));
+        let lock_keys = lock_keys.join(",");
+
+        let row = [
+            (PROCEDURE_ID, &Value::from(pid.clone())),
+            (PROCEDURE_TYPE, &Value::from(type_name.clone())),
+            (START_TIME, &Value::from(start_time)),
+            (END_TIME, &Value::from(end_time)),
+            (STATUS, &Value::from(status.clone())),
+            (LOCK_KEYS, &Value::from(lock_keys.clone())),
+        ];
+        if !predicates.eval(&row) {
+            return;
+        }
+        self.procedure_ids.push(Some(&pid));
+        self.procedure_types.push(Some(&type_name));
+        self.start_times.push(Some(start_time));
+        self.end_times.push(Some(end_time));
+        self.statuses.push(Some(&status));
+        self.lock_keys.push(Some(&lock_keys));
+    }
+
+    fn add_procedure_info(
+        &mut self,
+        predicates: &Predicates,
+        procedure: ProcedureMeta,
+    ) -> Result<()> {
+        let pid = match procedure.id {
+            Some(pid) => pid,
+            None => return ProcedureIdNotFoundSnafu {}.fail(),
+        };
+        let pid = procedure::pb_pid_to_pid(&pid)
+            .map_err(BoxedError::new)
+            .context(ConvertProtoDataSnafu)?;
+        let status = ProcedureStatus::try_from(procedure.status)
+            .map(|v| v.as_str_name())
+            .unwrap_or("Unknown")
+            .to_string();
+        let procedure_info = ProcedureInfo {
+            id: pid,
+            type_name: procedure.type_name,
+            start_time_ms: procedure.start_time_ms,
+            end_time_ms: procedure.end_time_ms,
+            state: ProcedureState::Running,
+            lock_keys: procedure.lock_keys,
+        };
+        self.add_procedure(predicates, status, procedure_info);
+        Ok(())
+    }
+
+    fn finish(&mut self) -> Result<RecordBatch> {
+        let columns: Vec<VectorRef> = vec![
+            Arc::new(self.procedure_ids.finish()),
+            Arc::new(self.procedure_types.finish()),
+            Arc::new(self.start_times.finish()),
+            Arc::new(self.end_times.finish()),
+            Arc::new(self.statuses.finish()),
+            Arc::new(self.lock_keys.finish()),
+        ];
+        RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu)
+    }
+}
+
+impl DfPartitionStream for InformationSchemaProcedureInfo {
+    fn schema(&self) -> &ArrowSchemaRef {
+        self.schema.arrow_schema()
+    }
+
+    fn execute(&self, _: Arc<TaskContext>) -> DfSendableRecordBatchStream {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_procedure_info(None)
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ))
+    }
+}
diff --git a/src/catalog/src/system_schema/information_schema/table_names.rs b/src/catalog/src/system_schema/information_schema/table_names.rs
index c2c9eeff24..a62f4ddb40 100644
--- a/src/catalog/src/system_schema/information_schema/table_names.rs
+++ b/src/catalog/src/system_schema/information_schema/table_names.rs
@@ -45,3 +45,4 @@ pub const TABLE_CONSTRAINTS: &str = "table_constraints";
 pub const CLUSTER_INFO: &str = "cluster_info";
 pub const VIEWS: &str = "views";
 pub const FLOWS: &str = "flows";
+pub const PROCEDURE_INFO: &str = "procedure_info";
diff --git a/src/catalog/src/system_schema/utils.rs b/src/catalog/src/system_schema/utils.rs
index 2a2e89516a..b9786bc260 100644
--- a/src/catalog/src/system_schema/utils.rs
+++ b/src/catalog/src/system_schema/utils.rs
@@ -18,6 +18,7 @@ use std::sync::{Arc, Weak};
 
 use common_config::Mode;
 use common_meta::key::TableMetadataManagerRef;
+use common_procedure::ProcedureManagerRef;
 use meta_client::client::MetaClient;
 use snafu::OptionExt;
 
@@ -68,3 +69,17 @@ pub fn table_meta_manager(
         .downcast_ref::<KvBackendCatalogManager>()
         .map(|manager| manager.table_metadata_manager_ref().clone()))
 }
+
+/// Try to get the `[ProcedureManagerRef]` from `[CatalogManager]` weak reference.
+pub fn procedure_manager(
+    catalog_manager: &Weak<dyn CatalogManager>,
+) -> Result<Option<ProcedureManagerRef>> {
+    let catalog_manager = catalog_manager
+        .upgrade()
+        .context(UpgradeWeakCatalogManagerRefSnafu)?;
+
+    Ok(catalog_manager
+        .as_any()
+        .downcast_ref::<KvBackendCatalogManager>()
+        .and_then(|manager| manager.procedure_manager()))
+}
diff --git a/src/catalog/src/table_source.rs b/src/catalog/src/table_source.rs
index d6d81fa134..09d3d9d2fd 100644
--- a/src/catalog/src/table_source.rs
+++ b/src/catalog/src/table_source.rs
@@ -327,6 +327,7 @@ mod tests {
             None,
             backend.clone(),
             layered_cache_registry,
+            None,
         );
         let table_metadata_manager = TableMetadataManager::new(backend);
         let mut view_info = common_meta::key::test_utils::new_test_table_info(1024, vec![]);
diff --git a/src/cmd/src/cli/repl.rs b/src/cmd/src/cli/repl.rs
index 3fc2184d37..b55b1c44d6 100644
--- a/src/cmd/src/cli/repl.rs
+++ b/src/cmd/src/cli/repl.rs
@@ -280,6 +280,7 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
         Some(meta_client.clone()),
         cached_meta_backend.clone(),
         layered_cache_registry,
+        None,
     );
     let plugins: Plugins = Default::default();
     let state = Arc::new(QueryEngineState::new(
diff --git a/src/cmd/src/flownode.rs b/src/cmd/src/flownode.rs
index e950d9bb43..78dfc90607 100644
--- a/src/cmd/src/flownode.rs
+++ b/src/cmd/src/flownode.rs
@@ -274,6 +274,7 @@ impl StartCommand {
             Some(meta_client.clone()),
             cached_meta_backend.clone(),
             layered_cache_registry.clone(),
+            None,
         );
 
         let table_metadata_manager =
diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs
index 6d8fd97070..320dc49c19 100644
--- a/src/cmd/src/frontend.rs
+++ b/src/cmd/src/frontend.rs
@@ -320,6 +320,7 @@ impl StartCommand {
             Some(meta_client.clone()),
             cached_meta_backend.clone(),
             layered_cache_registry.clone(),
+            None,
         );
 
         let executor = HandlerGroupExecutor::new(vec![
diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
index c8083c5f80..80b38ebaa7 100644
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -482,6 +482,7 @@ impl StartCommand {
             None,
             kv_backend.clone(),
             layered_cache_registry.clone(),
+            Some(procedure_manager.clone()),
         );
 
         let table_metadata_manager =
diff --git a/src/common/catalog/src/consts.rs b/src/common/catalog/src/consts.rs
index 6020a4bef8..2a8e2fc0e4 100644
--- a/src/common/catalog/src/consts.rs
+++ b/src/common/catalog/src/consts.rs
@@ -98,6 +98,8 @@ pub const INFORMATION_SCHEMA_CLUSTER_INFO_TABLE_ID: u32 = 31;
 pub const INFORMATION_SCHEMA_VIEW_TABLE_ID: u32 = 32;
 /// id for information_schema.FLOWS
 pub const INFORMATION_SCHEMA_FLOW_TABLE_ID: u32 = 33;
+/// id for information_schema.procedure_info
+pub const INFORMATION_SCHEMA_PROCEDURE_INFO_TABLE_ID: u32 = 34;
 /// ----- End of information_schema tables -----
 
 /// ----- Begin of pg_catalog tables -----
diff --git a/src/common/meta/src/ddl.rs b/src/common/meta/src/ddl.rs
index 7186997906..11654f04d6 100644
--- a/src/common/meta/src/ddl.rs
+++ b/src/common/meta/src/ddl.rs
@@ -15,6 +15,7 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
+use api::v1::meta::ProcedureDetailResponse;
 use common_telemetry::tracing_context::W3cTrace;
 use store_api::storage::{RegionId, RegionNumber, TableId};
 
@@ -82,6 +83,8 @@ pub trait ProcedureExecutor: Send + Sync {
         ctx: &ExecutorContext,
         pid: &str,
     ) -> Result<ProcedureStateResponse>;
+
+    async fn list_procedures(&self, ctx: &ExecutorContext) -> Result<ProcedureDetailResponse>;
 }
 
 pub type ProcedureExecutorRef = Arc<dyn ProcedureExecutor>;
diff --git a/src/common/meta/src/ddl_manager.rs b/src/common/meta/src/ddl_manager.rs
index 152a4631e2..1ee148406d 100644
--- a/src/common/meta/src/ddl_manager.rs
+++ b/src/common/meta/src/ddl_manager.rs
@@ -14,6 +14,7 @@
 
 use std::sync::Arc;
 
+use api::v1::meta::ProcedureDetailResponse;
 use common_procedure::{
     watcher, BoxedProcedureLoader, Output, ProcedureId, ProcedureManagerRef, ProcedureWithId,
 };
@@ -825,6 +826,15 @@ impl ProcedureExecutor for DdlManager {
 
         Ok(procedure::procedure_state_to_pb_response(&state))
     }
+
+    async fn list_procedures(&self, _ctx: &ExecutorContext) -> Result<ProcedureDetailResponse> {
+        let metas = self
+            .procedure_manager
+            .list_procedures()
+            .await
+            .context(QueryProcedureSnafu)?;
+        Ok(procedure::procedure_details_to_pb_response(metas))
+    }
 }
 
 #[cfg(test)]
diff --git a/src/common/meta/src/rpc/procedure.rs b/src/common/meta/src/rpc/procedure.rs
index 6dabe899a8..2c2a69b5b6 100644
--- a/src/common/meta/src/rpc/procedure.rs
+++ b/src/common/meta/src/rpc/procedure.rs
@@ -16,10 +16,11 @@ use std::time::Duration;
 
 pub use api::v1::meta::{MigrateRegionResponse, ProcedureStateResponse};
 use api::v1::meta::{
-    ProcedureId as PbProcedureId, ProcedureStateResponse as PbProcedureStateResponse,
+    ProcedureDetailResponse as PbProcedureDetailResponse, ProcedureId as PbProcedureId,
+    ProcedureMeta as PbProcedureMeta, ProcedureStateResponse as PbProcedureStateResponse,
     ProcedureStatus as PbProcedureStatus,
 };
-use common_procedure::{ProcedureId, ProcedureState};
+use common_procedure::{ProcedureId, ProcedureInfo, ProcedureState};
 use snafu::ResultExt;
 
 use crate::error::{ParseProcedureIdSnafu, Result};
@@ -49,9 +50,9 @@ pub fn pid_to_pb_pid(pid: ProcedureId) -> PbProcedureId {
     }
 }
 
-/// Cast the common [`ProcedureState`] to pb [`ProcedureStateResponse`].
-pub fn procedure_state_to_pb_response(state: &ProcedureState) -> PbProcedureStateResponse {
-    let (status, error) = match state {
+/// Cast the [`ProcedureState`] to protobuf [`PbProcedureStatus`].
+pub fn procedure_state_to_pb_state(state: &ProcedureState) -> (PbProcedureStatus, String) {
+    match state {
         ProcedureState::Running => (PbProcedureStatus::Running, String::default()),
         ProcedureState::Done { .. } => (PbProcedureStatus::Done, String::default()),
         ProcedureState::Retrying { error } => (PbProcedureStatus::Retrying, error.to_string()),
@@ -62,8 +63,12 @@ pub fn procedure_state_to_pb_response(state: &ProcedureState) -> PbProcedureStat
         ProcedureState::RollingBack { error } => {
             (PbProcedureStatus::RollingBack, error.to_string())
         }
-    };
+    }
+}
 
+/// Cast the common [`ProcedureState`] to pb [`ProcedureStateResponse`].
+pub fn procedure_state_to_pb_response(state: &ProcedureState) -> PbProcedureStateResponse {
+    let (status, error) = procedure_state_to_pb_state(state);
     PbProcedureStateResponse {
         status: status.into(),
         error,
@@ -71,6 +76,28 @@ pub fn procedure_state_to_pb_response(state: &ProcedureState) -> PbProcedureStat
     }
 }
 
+pub fn procedure_details_to_pb_response(metas: Vec<ProcedureInfo>) -> PbProcedureDetailResponse {
+    let procedures = metas
+        .into_iter()
+        .map(|meta| {
+            let (status, error) = procedure_state_to_pb_state(&meta.state);
+            PbProcedureMeta {
+                id: Some(pid_to_pb_pid(meta.id)),
+                type_name: meta.type_name.to_string(),
+                status: status.into(),
+                start_time_ms: meta.start_time_ms,
+                end_time_ms: meta.end_time_ms,
+                lock_keys: meta.lock_keys,
+                error,
+            }
+        })
+        .collect();
+    PbProcedureDetailResponse {
+        procedures,
+        ..Default::default()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::sync::Arc;
diff --git a/src/common/procedure/Cargo.toml b/src/common/procedure/Cargo.toml
index b56aa46a91..1d8c6736e3 100644
--- a/src/common/procedure/Cargo.toml
+++ b/src/common/procedure/Cargo.toml
@@ -19,6 +19,7 @@ common-error.workspace = true
 common-macro.workspace = true
 common-runtime.workspace = true
 common-telemetry.workspace = true
+common-time.workspace = true
 futures.workspace = true
 humantime-serde.workspace = true
 object-store.workspace = true
diff --git a/src/common/procedure/src/lib.rs b/src/common/procedure/src/lib.rs
index ece2ce4189..269ad2c529 100644
--- a/src/common/procedure/src/lib.rs
+++ b/src/common/procedure/src/lib.rs
@@ -26,7 +26,7 @@ pub mod watcher;
 pub use crate::error::{Error, Result};
 pub use crate::procedure::{
     BoxedProcedure, BoxedProcedureLoader, Context, ContextProvider, LockKey, Output, ParseIdError,
-    Procedure, ProcedureId, ProcedureManager, ProcedureManagerRef, ProcedureState, ProcedureWithId,
-    Status, StringKey,
+    Procedure, ProcedureId, ProcedureInfo, ProcedureManager, ProcedureManagerRef, ProcedureState,
+    ProcedureWithId, Status, StringKey,
 };
 pub use crate::watcher::Watcher;
diff --git a/src/common/procedure/src/local.rs b/src/common/procedure/src/local.rs
index 63fc06270a..da456fb37a 100644
--- a/src/common/procedure/src/local.rs
+++ b/src/common/procedure/src/local.rs
@@ -16,7 +16,7 @@ mod runner;
 mod rwlock;
 
 use std::collections::{HashMap, VecDeque};
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicI64, Ordering};
 use std::sync::{Arc, Mutex, RwLock};
 use std::time::{Duration, Instant};
 
@@ -35,7 +35,7 @@ use crate::error::{
     StartRemoveOutdatedMetaTaskSnafu, StopRemoveOutdatedMetaTaskSnafu,
 };
 use crate::local::runner::Runner;
-use crate::procedure::{BoxedProcedureLoader, InitProcedureState};
+use crate::procedure::{BoxedProcedureLoader, InitProcedureState, ProcedureInfo};
 use crate::store::{ProcedureMessage, ProcedureMessages, ProcedureStore, StateStoreRef};
 use crate::{
     BoxedProcedure, ContextProvider, LockKey, ProcedureId, ProcedureManager, ProcedureState,
@@ -57,6 +57,8 @@ const META_TTL: Duration = Duration::from_secs(60 * 10);
 pub(crate) struct ProcedureMeta {
     /// Id of this procedure.
     id: ProcedureId,
+    /// Type name of this procedure.
+    type_name: String,
     /// Parent procedure id.
     parent_id: Option<ProcedureId>,
     /// Notify to wait for subprocedures.
@@ -69,6 +71,10 @@ pub(crate) struct ProcedureMeta {
     state_receiver: Receiver<ProcedureState>,
     /// Id of child procedures.
     children: Mutex<Vec<ProcedureId>>,
+    /// Start execution time of this procedure.
+    start_time_ms: AtomicI64,
+    /// End execution time of this procedure.
+    end_time_ms: AtomicI64,
 }
 
 impl ProcedureMeta {
@@ -77,6 +83,7 @@ impl ProcedureMeta {
         procedure_state: ProcedureState,
         parent_id: Option<ProcedureId>,
         lock_key: LockKey,
+        type_name: &str,
     ) -> ProcedureMeta {
         let (state_sender, state_receiver) = watch::channel(procedure_state);
         ProcedureMeta {
@@ -87,6 +94,9 @@ impl ProcedureMeta {
             state_sender,
             state_receiver,
             children: Mutex::new(Vec::new()),
+            start_time_ms: AtomicI64::new(0),
+            end_time_ms: AtomicI64::new(0),
+            type_name: type_name.to_string(),
         }
     }
 
@@ -117,6 +127,18 @@ impl ProcedureMeta {
     fn num_children(&self) -> usize {
         self.children.lock().unwrap().len()
     }
+
+    /// update the start time of the procedure.
+    fn set_start_time_ms(&self) {
+        self.start_time_ms
+            .store(common_time::util::current_time_millis(), Ordering::Relaxed);
+    }
+
+    /// update the end time of the procedure.
+    fn set_end_time_ms(&self) {
+        self.end_time_ms
+            .store(common_time::util::current_time_millis(), Ordering::Relaxed);
+    }
 }
 
 /// Reference counted pointer to [ProcedureMeta].
@@ -210,6 +232,22 @@ impl ManagerContext {
         procedures.get(&procedure_id).map(|meta| meta.state())
     }
 
+    /// Returns the [ProcedureMeta] of all procedures.
+    fn list_procedure(&self) -> Vec<ProcedureInfo> {
+        let procedures = self.procedures.read().unwrap();
+        procedures
+            .values()
+            .map(|meta| ProcedureInfo {
+                id: meta.id,
+                type_name: meta.type_name.clone(),
+                start_time_ms: meta.start_time_ms.load(Ordering::Relaxed),
+                end_time_ms: meta.end_time_ms.load(Ordering::Relaxed),
+                state: meta.state(),
+                lock_keys: meta.lock_key.get_keys(),
+            })
+            .collect()
+    }
+
     /// Returns the [Watcher] of specific `procedure_id`.
     fn watcher(&self, procedure_id: ProcedureId) -> Option<Watcher> {
         let procedures = self.procedures.read().unwrap();
@@ -438,6 +476,7 @@ impl LocalManager {
             procedure_state,
             None,
             procedure.lock_key(),
+            procedure.type_name(),
         ));
         let runner = Runner {
             meta: meta.clone(),
@@ -641,6 +680,10 @@ impl ProcedureManager for LocalManager {
     fn procedure_watcher(&self, procedure_id: ProcedureId) -> Option<Watcher> {
         self.manager_ctx.watcher(procedure_id)
     }
+
+    async fn list_procedures(&self) -> Result<Vec<ProcedureInfo>> {
+        Ok(self.manager_ctx.list_procedure())
+    }
 }
 
 struct RemoveOutdatedMetaFunction {
@@ -675,6 +718,7 @@ pub(crate) mod test_util {
             ProcedureState::Running,
             None,
             LockKey::default(),
+            "ProcedureAdapter",
         )
     }
 
diff --git a/src/common/procedure/src/local/runner.rs b/src/common/procedure/src/local/runner.rs
index 2f38ae135d..c2d15001fb 100644
--- a/src/common/procedure/src/local/runner.rs
+++ b/src/common/procedure/src/local/runner.rs
@@ -27,7 +27,9 @@ use crate::error::{self, ProcedurePanicSnafu, Result, RollbackTimesExceededSnafu
 use crate::local::{ManagerContext, ProcedureMeta, ProcedureMetaRef};
 use crate::procedure::{Output, StringKey};
 use crate::store::{ProcedureMessage, ProcedureStore};
-use crate::{BoxedProcedure, Context, Error, ProcedureId, ProcedureState, ProcedureWithId, Status};
+use crate::{
+    BoxedProcedure, Context, Error, Procedure, ProcedureId, ProcedureState, ProcedureWithId, Status,
+};
 
 /// A guard to cleanup procedure state.
 struct ProcedureGuard {
@@ -129,7 +131,9 @@ impl Runner {
 
         // Execute the procedure. We need to release the lock whenever the execution
         // is successful or fail.
+        self.meta.set_start_time_ms();
         self.execute_procedure_in_loop().await;
+        self.meta.set_end_time_ms();
 
         // We can't remove the metadata of the procedure now as users and its parent might
         // need to query its state.
@@ -368,6 +372,7 @@ impl Runner {
             procedure_state,
             Some(self.meta.id),
             procedure.lock_key(),
+            procedure.type_name(),
         ));
         let runner = Runner {
             meta: meta.clone(),
diff --git a/src/common/procedure/src/procedure.rs b/src/common/procedure/src/procedure.rs
index 6c694315e9..ddf5dc74a3 100644
--- a/src/common/procedure/src/procedure.rs
+++ b/src/common/procedure/src/procedure.rs
@@ -159,6 +159,14 @@ impl<T: Procedure + ?Sized> Procedure for Box<T> {
         (**self).execute(ctx).await
     }
 
+    async fn rollback(&mut self, ctx: &Context) -> Result<()> {
+        (**self).rollback(ctx).await
+    }
+
+    fn rollback_supported(&self) -> bool {
+        (**self).rollback_supported()
+    }
+
     fn dump(&self) -> Result<String> {
         (**self).dump()
     }
@@ -227,6 +235,11 @@ impl LockKey {
     pub fn keys_to_lock(&self) -> impl Iterator<Item = &StringKey> {
         self.0.iter()
     }
+
+    /// Returns the keys to lock.
+    pub fn get_keys(&self) -> Vec<String> {
+        self.0.iter().map(|key| format!("{:?}", key)).collect()
+    }
 }
 
 /// Boxed [Procedure].
@@ -374,6 +387,18 @@ impl ProcedureState {
             _ => None,
         }
     }
+
+    /// Return the string values of the enum field names.
+    pub fn as_str_name(&self) -> &str {
+        match self {
+            ProcedureState::Running => "Running",
+            ProcedureState::Done { .. } => "Done",
+            ProcedureState::Retrying { .. } => "Retrying",
+            ProcedureState::Failed { .. } => "Failed",
+            ProcedureState::PrepareRollback { .. } => "PrepareRollback",
+            ProcedureState::RollingBack { .. } => "RollingBack",
+        }
+    }
 }
 
 /// The initial procedure state.
@@ -412,11 +437,30 @@ pub trait ProcedureManager: Send + Sync + 'static {
 
     /// Returns a [Watcher] to watch [ProcedureState] of specific procedure.
     fn procedure_watcher(&self, procedure_id: ProcedureId) -> Option<Watcher>;
+
+    /// Returns the details of the procedure.
+    async fn list_procedures(&self) -> Result<Vec<ProcedureInfo>>;
 }
 
 /// Ref-counted pointer to the [ProcedureManager].
 pub type ProcedureManagerRef = Arc<dyn ProcedureManager>;
 
+#[derive(Debug, Clone)]
+pub struct ProcedureInfo {
+    /// Id of this procedure.
+    pub id: ProcedureId,
+    /// Type name of this procedure.
+    pub type_name: String,
+    /// Start execution time of this procedure.
+    pub start_time_ms: i64,
+    /// End execution time of this procedure.
+    pub end_time_ms: i64,
+    /// status of this procedure.
+    pub state: ProcedureState,
+    /// Lock keys of this procedure.
+    pub lock_keys: Vec<String>,
+}
+
 #[cfg(test)]
 mod tests {
     use common_error::mock::MockError;
diff --git a/src/meta-client/src/client.rs b/src/meta-client/src/client.rs
index 7c2a6ed923..0a9df7293f 100644
--- a/src/meta-client/src/client.rs
+++ b/src/meta-client/src/client.rs
@@ -22,7 +22,7 @@ mod cluster;
 mod store;
 mod util;
 
-use api::v1::meta::Role;
+use api::v1::meta::{ProcedureDetailResponse, Role};
 use cluster::Client as ClusterClient;
 use common_error::ext::BoxedError;
 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
@@ -259,6 +259,16 @@ impl ProcedureExecutor for MetaClient {
             .map_err(BoxedError::new)
             .context(meta_error::ExternalSnafu)
     }
+
+    async fn list_procedures(&self, _ctx: &ExecutorContext) -> MetaResult<ProcedureDetailResponse> {
+        self.procedure_client()
+            .map_err(BoxedError::new)
+            .context(meta_error::ExternalSnafu)?
+            .list_procedures()
+            .await
+            .map_err(BoxedError::new)
+            .context(meta_error::ExternalSnafu)
+    }
 }
 
 #[async_trait::async_trait]
diff --git a/src/meta-client/src/client/procedure.rs b/src/meta-client/src/client/procedure.rs
index 32049dbabd..f45cfb7879 100644
--- a/src/meta-client/src/client/procedure.rs
+++ b/src/meta-client/src/client/procedure.rs
@@ -18,8 +18,9 @@ use std::time::Duration;
 
 use api::v1::meta::procedure_service_client::ProcedureServiceClient;
 use api::v1::meta::{
-    DdlTaskRequest, DdlTaskResponse, MigrateRegionRequest, MigrateRegionResponse, ProcedureId,
-    ProcedureStateResponse, QueryProcedureRequest, ResponseHeader, Role,
+    DdlTaskRequest, DdlTaskResponse, MigrateRegionRequest, MigrateRegionResponse,
+    ProcedureDetailRequest, ProcedureDetailResponse, ProcedureId, ProcedureStateResponse,
+    QueryProcedureRequest, ResponseHeader, Role,
 };
 use common_grpc::channel_manager::ChannelManager;
 use common_telemetry::tracing_context::TracingContext;
@@ -89,6 +90,11 @@ impl Client {
             .migrate_region(region_id, from_peer, to_peer, replay_timeout)
             .await
     }
+
+    pub async fn list_procedures(&self) -> Result<ProcedureDetailResponse> {
+        let inner = self.inner.read().await;
+        inner.list_procedures().await
+    }
 }
 
 #[derive(Debug)]
@@ -279,4 +285,23 @@ impl Inner {
         )
         .await
     }
+
+    async fn list_procedures(&self) -> Result<ProcedureDetailResponse> {
+        let mut req = ProcedureDetailRequest::default();
+        req.set_header(
+            self.id,
+            self.role,
+            TracingContext::from_current_span().to_w3c(),
+        );
+
+        self.with_retry(
+            "list procedure",
+            move |mut client| {
+                let req = req.clone();
+                async move { client.details(req).await.map(|res| res.into_inner()) }
+            },
+            |resp: &ProcedureDetailResponse| &resp.header,
+        )
+        .await
+    }
 }
diff --git a/src/meta-srv/src/service/procedure.rs b/src/meta-srv/src/service/procedure.rs
index 05e6d98fa4..d19d0902ae 100644
--- a/src/meta-srv/src/service/procedure.rs
+++ b/src/meta-srv/src/service/procedure.rs
@@ -18,7 +18,7 @@ use std::time::Duration;
 use api::v1::meta::{
     procedure_service_server, DdlTaskRequest as PbDdlTaskRequest,
     DdlTaskResponse as PbDdlTaskResponse, MigrateRegionRequest, MigrateRegionResponse,
-    ProcedureStateResponse, QueryProcedureRequest,
+    ProcedureDetailRequest, ProcedureDetailResponse, ProcedureStateResponse, QueryProcedureRequest,
 };
 use common_meta::ddl::ExecutorContext;
 use common_meta::rpc::ddl::{DdlTask, SubmitDdlTaskRequest};
@@ -146,4 +146,20 @@ impl procedure_service_server::ProcedureService for Metasrv {
 
         Ok(Response::new(resp))
     }
+
+    async fn details(
+        &self,
+        request: Request<ProcedureDetailRequest>,
+    ) -> GrpcResult<ProcedureDetailResponse> {
+        let ProcedureDetailRequest { header } = request.into_inner();
+        let _header = header.context(error::MissingRequestHeaderSnafu)?;
+        let metas = self
+            .procedure_manager()
+            .list_procedures()
+            .await
+            .context(error::QueryProcedureSnafu)?;
+        Ok(Response::new(procedure::procedure_details_to_pb_response(
+            metas,
+        )))
+    }
 }
diff --git a/tests-integration/src/cluster.rs b/tests-integration/src/cluster.rs
index 19cb36a5b9..ce2803996a 100644
--- a/tests-integration/src/cluster.rs
+++ b/tests-integration/src/cluster.rs
@@ -370,6 +370,7 @@ impl GreptimeDbClusterBuilder {
             Some(meta_client.clone()),
             cached_meta_backend.clone(),
             cache_registry.clone(),
+            None,
         );
 
         let handlers_executor = HandlerGroupExecutor::new(vec![
diff --git a/tests-integration/src/standalone.rs b/tests-integration/src/standalone.rs
index 9f7188568f..fa6e8f2a9e 100644
--- a/tests-integration/src/standalone.rs
+++ b/tests-integration/src/standalone.rs
@@ -149,6 +149,7 @@ impl GreptimeDbStandaloneBuilder {
             None,
             kv_backend.clone(),
             cache_registry.clone(),
+            Some(procedure_manager.clone()),
         );
 
         let flow_builder = FlownodeBuilder::new(
diff --git a/tests/cases/standalone/common/information_schema/procedure_info.result b/tests/cases/standalone/common/information_schema/procedure_info.result
new file mode 100644
index 0000000000..965e02b8ba
--- /dev/null
+++ b/tests/cases/standalone/common/information_schema/procedure_info.result
@@ -0,0 +1,40 @@
+--- test information_schema.procedure_info ----
+USE public;
+
+Affected Rows: 0
+
+CREATE TABLE procedure_info_for_sql_test1(
+  ts TIMESTAMP TIME INDEX,
+  temperature DOUBLE DEFAULT 10,
+) engine=mito with('append_mode'='true');
+
+Affected Rows: 0
+
+CREATE TABLE procedure_info_for_sql_test2(
+  ts TIMESTAMP TIME INDEX,
+  temperature DOUBLE DEFAULT 10,
+) engine=mito with('append_mode'='true');
+
+Affected Rows: 0
+
+use INFORMATION_SCHEMA;
+
+Affected Rows: 0
+
+select procedure_type from procedure_info where lock_keys like '%procedure_info_for_sql_test%';
+
++--------------------------------+
+| procedure_type                 |
++--------------------------------+
+| metasrv-procedure::CreateTable |
+| metasrv-procedure::CreateTable |
++--------------------------------+
+
+use public;
+
+Affected Rows: 0
+
+DROP TABLE procedure_info_for_sql_test1, procedure_info_for_sql_test2;
+
+Affected Rows: 0
+
diff --git a/tests/cases/standalone/common/information_schema/procedure_info.sql b/tests/cases/standalone/common/information_schema/procedure_info.sql
new file mode 100644
index 0000000000..763dde12ef
--- /dev/null
+++ b/tests/cases/standalone/common/information_schema/procedure_info.sql
@@ -0,0 +1,20 @@
+--- test information_schema.procedure_info ----
+USE public;
+
+CREATE TABLE procedure_info_for_sql_test1(
+  ts TIMESTAMP TIME INDEX,
+  temperature DOUBLE DEFAULT 10,
+) engine=mito with('append_mode'='true');
+
+CREATE TABLE procedure_info_for_sql_test2(
+  ts TIMESTAMP TIME INDEX,
+  temperature DOUBLE DEFAULT 10,
+) engine=mito with('append_mode'='true');
+
+use INFORMATION_SCHEMA;
+
+select procedure_type from procedure_info where lock_keys like '%procedure_info_for_sql_test%';
+
+use public;
+
+DROP TABLE procedure_info_for_sql_test1, procedure_info_for_sql_test2;
diff --git a/tests/cases/standalone/common/show/show_databases_tables.result b/tests/cases/standalone/common/show/show_databases_tables.result
index fa50fb2aab..c92a5b33c8 100644
--- a/tests/cases/standalone/common/show/show_databases_tables.result
+++ b/tests/cases/standalone/common/show/show_databases_tables.result
@@ -45,6 +45,7 @@ SHOW TABLES;
 | optimizer_trace                       |
 | parameters                            |
 | partitions                            |
+| procedure_info                        |
 | profiling                             |
 | referential_constraints               |
 | region_peers                          |
@@ -91,6 +92,7 @@ SHOW FULL TABLES;
 | optimizer_trace                       | LOCAL TEMPORARY |
 | parameters                            | LOCAL TEMPORARY |
 | partitions                            | LOCAL TEMPORARY |
+| procedure_info                        | LOCAL TEMPORARY |
 | profiling                             | LOCAL TEMPORARY |
 | referential_constraints               | LOCAL TEMPORARY |
 | region_peers                          | LOCAL TEMPORARY |
@@ -131,6 +133,7 @@ SHOW TABLE STATUS;
 |optimizer_trace||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0|||
 |parameters||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0|||
 |partitions||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0|||
+|procedure_info||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0|||
 |profiling||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0|||
 |referential_constraints||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0|||
 |region_peers||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0|||
diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result
index cbb0c12b6f..03b65cf390 100644
--- a/tests/cases/standalone/common/system/information_schema.result
+++ b/tests/cases/standalone/common/system/information_schema.result
@@ -32,6 +32,7 @@ order by table_schema, table_name;
 |greptime|information_schema|optimizer_trace|LOCALTEMPORARY|17|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y|
 |greptime|information_schema|parameters|LOCALTEMPORARY|18|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y|
 |greptime|information_schema|partitions|LOCALTEMPORARY|28|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y|
+|greptime|information_schema|procedure_info|LOCALTEMPORARY|34|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y|
 |greptime|information_schema|profiling|LOCALTEMPORARY|19|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y|
 |greptime|information_schema|referential_constraints|LOCALTEMPORARY|20|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y|
 |greptime|information_schema|region_peers|LOCALTEMPORARY|29|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y|
@@ -256,6 +257,12 @@ select * from information_schema.columns order by table_schema, table_name, colu
 | greptime      | information_schema | partitions                            | table_schema                      | 2                | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | No          | string          |                |        |
 | greptime      | information_schema | partitions                            | tablespace_name                   | 25               | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | Yes         | string          |                |        |
 | greptime      | information_schema | partitions                            | update_time                       | 20               |                          |                        |                   |               | 3                  |                    |                |            |       | select,insert |                       | DateTime             | datetime        | FIELD         |                | Yes         | datetime        |                |        |
+| greptime      | information_schema | procedure_info                        | end_time                          | 4                |                          |                        |                   |               | 3                  |                    |                |            |       | select,insert |                       | TimestampMillisecond | timestamp(3)    | FIELD         |                | Yes         | timestamp(3)    |                |        |
+| greptime      | information_schema | procedure_info                        | lock_keys                         | 6                | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | Yes         | string          |                |        |
+| greptime      | information_schema | procedure_info                        | procedure_id                      | 1                | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | No          | string          |                |        |
+| greptime      | information_schema | procedure_info                        | procedure_type                    | 2                | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | No          | string          |                |        |
+| greptime      | information_schema | procedure_info                        | start_time                        | 3                |                          |                        |                   |               | 3                  |                    |                |            |       | select,insert |                       | TimestampMillisecond | timestamp(3)    | FIELD         |                | Yes         | timestamp(3)    |                |        |
+| greptime      | information_schema | procedure_info                        | status                            | 5                | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | No          | string          |                |        |
 | greptime      | information_schema | profiling                             | block_ops_in                      | 9                |                          |                        | 19                | 0             |                    |                    |                |            |       | select,insert |                       | Int64                | bigint          | FIELD         |                | No          | bigint          |                |        |
 | greptime      | information_schema | profiling                             | block_ops_out                     | 10               |                          |                        | 19                | 0             |                    |                    |                |            |       | select,insert |                       | Int64                | bigint          | FIELD         |                | No          | bigint          |                |        |
 | greptime      | information_schema | profiling                             | context_involuntary               | 8                |                          |                        | 19                | 0             |                    |                    |                |            |       | select,insert |                       | Int64                | bigint          | FIELD         |                | No          | bigint          |                |        |
diff --git a/tests/cases/standalone/common/view/create.result b/tests/cases/standalone/common/view/create.result
index 4cf8084cd3..4fd2f59356 100644
--- a/tests/cases/standalone/common/view/create.result
+++ b/tests/cases/standalone/common/view/create.result
@@ -106,6 +106,7 @@ SELECT * FROM INFORMATION_SCHEMA.TABLES ORDER BY TABLE_NAME, TABLE_TYPE;
 |greptime|information_schema|optimizer_trace|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y|
 |greptime|information_schema|parameters|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y|
 |greptime|information_schema|partitions|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y|
+|greptime|information_schema|procedure_info|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y|
 |greptime|information_schema|profiling|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y|
 |greptime|information_schema|referential_constraints|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y|
 |greptime|information_schema|region_peers|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y|

From 163cea81c26ded94b0780840eabbaae022c3b6d4 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Fri, 20 Sep 2024 17:27:20 +0900
Subject: [PATCH 028/128] feat: migrate local WAL regions (#4715)

* feat: allow to flush region before migrating

* fix: fix unit tests

* feat: allow to set `flush_timeout`

* feat: skip to replay memtable

* fix: fix unit tests

* test: add more tests

* refactor: simplify timeout logical

* test: add unit tests

* test: add unit tests

* chore: update comments

* fix: fix unit tests

* fix: fmt and clippy

* feat: change default timeout to 30s

* fix: throw `ExceededDeadline` error

* test: add tests for `downgrade_region_with_retry`

* chore: apply suggestions from CR

* chore: apply suggestions from CR

* chore: apply suggestions from CR

* chore: update proto to `3633474`

* refactor: refactor `upgrade_region_with_retry`

* chore: apply suggestions from CR
---
 Cargo.lock                                    |   2 +-
 Cargo.toml                                    |   2 +-
 .../function/src/table/migrate_region.rs      |  21 +-
 src/common/meta/src/instruction.rs            |  13 +-
 src/common/meta/src/rpc/procedure.rs          |   2 +-
 src/datanode/src/heartbeat/handler.rs         |  24 +-
 .../src/heartbeat/handler/downgrade_region.rs | 401 +++++++++++++++++-
 .../src/heartbeat/handler/upgrade_region.rs   |  57 +--
 src/datanode/src/tests.rs                     |  15 +-
 src/meta-client/src/client.rs                 |   2 +-
 src/meta-client/src/client/procedure.rs       |  10 +-
 src/meta-srv/src/error.rs                     |  10 +-
 .../src/procedure/region_migration.rs         |  28 +-
 .../downgrade_leader_region.rs                | 178 +++++---
 .../src/procedure/region_migration/manager.rs |  24 +-
 .../procedure/region_migration/test_util.rs   |   2 +-
 .../upgrade_candidate_region.rs               | 124 ++++--
 src/meta-srv/src/region/supervisor.rs         |   2 +-
 src/meta-srv/src/service/procedure.rs         |   5 +-
 src/mito2/src/engine/catchup_test.rs          | 120 +++++-
 src/mito2/src/test_util.rs                    |   6 +
 src/mito2/src/worker/handle_catchup.rs        |  67 +--
 src/store-api/src/logstore/provider.rs        |   5 +
 23 files changed, 848 insertions(+), 272 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2183cf593d..000d6316ec 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4396,7 +4396,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 [[package]]
 name = "greptime-proto"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=796ce9b003c6689e853825f649e03543c81ede99#796ce9b003c6689e853825f649e03543c81ede99"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=36334744c7020734dcb4a6b8d24d52ae7ed53fe1#36334744c7020734dcb4a6b8d24d52ae7ed53fe1"
 dependencies = [
  "prost 0.12.6",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index df70b98332..f0b93ac653 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -120,7 +120,7 @@ etcd-client = { version = "0.13" }
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "796ce9b003c6689e853825f649e03543c81ede99" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "36334744c7020734dcb4a6b8d24d52ae7ed53fe1" }
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"
diff --git a/src/common/function/src/table/migrate_region.rs b/src/common/function/src/table/migrate_region.rs
index b46231eb45..0a487973d3 100644
--- a/src/common/function/src/table/migrate_region.rs
+++ b/src/common/function/src/table/migrate_region.rs
@@ -25,13 +25,13 @@ use session::context::QueryContextRef;
 use crate::handlers::ProcedureServiceHandlerRef;
 use crate::helper::cast_u64;
 
-const DEFAULT_REPLAY_TIMEOUT_SECS: u64 = 10;
+const DEFAULT_TIMEOUT_SECS: u64 = 30;
 
 /// A function to migrate a region from source peer to target peer.
 /// Returns the submitted procedure id if success. Only available in cluster mode.
 ///
-/// - `migrate_region(region_id, from_peer, to_peer)`, with default replay WAL timeout(10 seconds).
-/// - `migrate_region(region_id, from_peer, to_peer, timeout(secs))`
+/// - `migrate_region(region_id, from_peer, to_peer)`, with timeout(30 seconds).
+/// - `migrate_region(region_id, from_peer, to_peer, timeout(secs))`.
 ///
 /// The parameters:
 /// - `region_id`:  the region id
@@ -48,18 +48,13 @@ pub(crate) async fn migrate_region(
     _ctx: &QueryContextRef,
     params: &[ValueRef<'_>],
 ) -> Result<Value> {
-    let (region_id, from_peer, to_peer, replay_timeout) = match params.len() {
+    let (region_id, from_peer, to_peer, timeout) = match params.len() {
         3 => {
             let region_id = cast_u64(&params[0])?;
             let from_peer = cast_u64(&params[1])?;
             let to_peer = cast_u64(&params[2])?;
 
-            (
-                region_id,
-                from_peer,
-                to_peer,
-                Some(DEFAULT_REPLAY_TIMEOUT_SECS),
-            )
+            (region_id, from_peer, to_peer, Some(DEFAULT_TIMEOUT_SECS))
         }
 
         4 => {
@@ -82,14 +77,14 @@ pub(crate) async fn migrate_region(
         }
     };
 
-    match (region_id, from_peer, to_peer, replay_timeout) {
-        (Some(region_id), Some(from_peer), Some(to_peer), Some(replay_timeout)) => {
+    match (region_id, from_peer, to_peer, timeout) {
+        (Some(region_id), Some(from_peer), Some(to_peer), Some(timeout)) => {
             let pid = procedure_service_handler
                 .migrate_region(MigrateRegionRequest {
                     region_id,
                     from_peer,
                     to_peer,
-                    replay_timeout: Duration::from_secs(replay_timeout),
+                    timeout: Duration::from_secs(timeout),
                 })
                 .await?;
 
diff --git a/src/common/meta/src/instruction.rs b/src/common/meta/src/instruction.rs
index d620cc3449..61e2811e72 100644
--- a/src/common/meta/src/instruction.rs
+++ b/src/common/meta/src/instruction.rs
@@ -132,11 +132,20 @@ impl OpenRegion {
 pub struct DowngradeRegion {
     /// The [RegionId].
     pub region_id: RegionId,
+    /// The timeout of waiting for flush the region.
+    ///
+    /// `None` stands for don't flush before downgrading the region.
+    #[serde(default)]
+    pub flush_timeout: Option<Duration>,
 }
 
 impl Display for DowngradeRegion {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        write!(f, "DowngradeRegion(region_id={})", self.region_id)
+        write!(
+            f,
+            "DowngradeRegion(region_id={}, flush_timeout={:?})",
+            self.region_id, self.flush_timeout,
+        )
     }
 }
 
@@ -152,7 +161,7 @@ pub struct UpgradeRegion {
     /// `None` stands for no wait,
     /// it's helpful to verify whether the leader region is ready.
     #[serde(with = "humantime_serde")]
-    pub wait_for_replay_timeout: Option<Duration>,
+    pub replay_timeout: Option<Duration>,
     /// The hint for replaying memtable.
     #[serde(default)]
     pub location_id: Option<u64>,
diff --git a/src/common/meta/src/rpc/procedure.rs b/src/common/meta/src/rpc/procedure.rs
index 2c2a69b5b6..2e25a4aa5d 100644
--- a/src/common/meta/src/rpc/procedure.rs
+++ b/src/common/meta/src/rpc/procedure.rs
@@ -31,7 +31,7 @@ pub struct MigrateRegionRequest {
     pub region_id: u64,
     pub from_peer: u64,
     pub to_peer: u64,
-    pub replay_timeout: Duration,
+    pub timeout: Duration,
 }
 
 /// Cast the protobuf [`ProcedureId`] to common [`ProcedureId`].
diff --git a/src/datanode/src/heartbeat/handler.rs b/src/datanode/src/heartbeat/handler.rs
index 573b94cb11..d23615eb13 100644
--- a/src/datanode/src/heartbeat/handler.rs
+++ b/src/datanode/src/heartbeat/handler.rs
@@ -37,6 +37,7 @@ use crate::region_server::RegionServer;
 pub struct RegionHeartbeatResponseHandler {
     region_server: RegionServer,
     catchup_tasks: TaskTracker<()>,
+    downgrade_tasks: TaskTracker<()>,
 }
 
 /// Handler of the instruction.
@@ -47,12 +48,22 @@ pub type InstructionHandler =
 pub struct HandlerContext {
     region_server: RegionServer,
     catchup_tasks: TaskTracker<()>,
+    downgrade_tasks: TaskTracker<()>,
 }
 
 impl HandlerContext {
     fn region_ident_to_region_id(region_ident: &RegionIdent) -> RegionId {
         RegionId::new(region_ident.table_id, region_ident.region_number)
     }
+
+    #[cfg(test)]
+    pub fn new_for_test(region_server: RegionServer) -> Self {
+        Self {
+            region_server,
+            catchup_tasks: TaskTracker::new(),
+            downgrade_tasks: TaskTracker::new(),
+        }
+    }
 }
 
 impl RegionHeartbeatResponseHandler {
@@ -61,6 +72,7 @@ impl RegionHeartbeatResponseHandler {
         Self {
             region_server,
             catchup_tasks: TaskTracker::new(),
+            downgrade_tasks: TaskTracker::new(),
         }
     }
 
@@ -107,11 +119,13 @@ impl HeartbeatResponseHandler for RegionHeartbeatResponseHandler {
         let mailbox = ctx.mailbox.clone();
         let region_server = self.region_server.clone();
         let catchup_tasks = self.catchup_tasks.clone();
+        let downgrade_tasks = self.downgrade_tasks.clone();
         let handler = Self::build_handler(instruction)?;
         let _handle = common_runtime::spawn_global(async move {
             let reply = handler(HandlerContext {
                 region_server,
                 catchup_tasks,
+                downgrade_tasks,
             })
             .await;
 
@@ -129,6 +143,7 @@ mod tests {
     use std::assert_matches::assert_matches;
     use std::collections::HashMap;
     use std::sync::Arc;
+    use std::time::Duration;
 
     use common_meta::heartbeat::mailbox::{
         HeartbeatMailbox, IncomingMessage, MailboxRef, MessageMeta,
@@ -197,6 +212,7 @@ mod tests {
         // Downgrade region
         let instruction = Instruction::DowngradeRegion(DowngradeRegion {
             region_id: RegionId::new(2048, 1),
+            flush_timeout: Some(Duration::from_secs(1)),
         });
         assert!(heartbeat_handler
             .is_acceptable(&heartbeat_env.create_handler_ctx((meta.clone(), instruction))));
@@ -205,7 +221,7 @@ mod tests {
         let instruction = Instruction::UpgradeRegion(UpgradeRegion {
             region_id,
             last_entry_id: None,
-            wait_for_replay_timeout: None,
+            replay_timeout: None,
             location_id: None,
         });
         assert!(
@@ -392,7 +408,10 @@ mod tests {
         // Should be ok, if we try to downgrade it twice.
         for _ in 0..2 {
             let meta = MessageMeta::new_test(1, "test", "dn-1", "me-0");
-            let instruction = Instruction::DowngradeRegion(DowngradeRegion { region_id });
+            let instruction = Instruction::DowngradeRegion(DowngradeRegion {
+                region_id,
+                flush_timeout: Some(Duration::from_secs(1)),
+            });
 
             let mut ctx = heartbeat_env.create_handler_ctx((meta, instruction));
             let control = heartbeat_handler.handle(&mut ctx).await.unwrap();
@@ -413,6 +432,7 @@ mod tests {
         let meta = MessageMeta::new_test(1, "test", "dn-1", "me-0");
         let instruction = Instruction::DowngradeRegion(DowngradeRegion {
             region_id: RegionId::new(2048, 1),
+            flush_timeout: Some(Duration::from_secs(1)),
         });
         let mut ctx = heartbeat_env.create_handler_ctx((meta, instruction));
         let control = heartbeat_handler.handle(&mut ctx).await.unwrap();
diff --git a/src/datanode/src/heartbeat/handler/downgrade_region.rs b/src/datanode/src/heartbeat/handler/downgrade_region.rs
index 4dccdc26aa..ac11792803 100644
--- a/src/datanode/src/heartbeat/handler/downgrade_region.rs
+++ b/src/datanode/src/heartbeat/handler/downgrade_region.rs
@@ -13,38 +13,399 @@
 // limitations under the License.
 
 use common_meta::instruction::{DowngradeRegion, DowngradeRegionReply, InstructionReply};
+use common_telemetry::tracing::info;
+use common_telemetry::warn;
 use futures_util::future::BoxFuture;
 use store_api::region_engine::SetReadonlyResponse;
+use store_api::region_request::{RegionFlushRequest, RegionRequest};
+use store_api::storage::RegionId;
 
 use crate::heartbeat::handler::HandlerContext;
+use crate::heartbeat::task_tracker::WaitResult;
 
 impl HandlerContext {
+    async fn set_readonly_gracefully(&self, region_id: RegionId) -> InstructionReply {
+        match self.region_server.set_readonly_gracefully(region_id).await {
+            Ok(SetReadonlyResponse::Success { last_entry_id }) => {
+                InstructionReply::DowngradeRegion(DowngradeRegionReply {
+                    last_entry_id,
+                    exists: true,
+                    error: None,
+                })
+            }
+            Ok(SetReadonlyResponse::NotFound) => {
+                InstructionReply::DowngradeRegion(DowngradeRegionReply {
+                    last_entry_id: None,
+                    exists: false,
+                    error: None,
+                })
+            }
+            Err(err) => InstructionReply::DowngradeRegion(DowngradeRegionReply {
+                last_entry_id: None,
+                exists: true,
+                error: Some(format!("{err:?}")),
+            }),
+        }
+    }
+
     pub(crate) fn handle_downgrade_region_instruction(
         self,
-        DowngradeRegion { region_id }: DowngradeRegion,
+        DowngradeRegion {
+            region_id,
+            flush_timeout,
+        }: DowngradeRegion,
     ) -> BoxFuture<'static, InstructionReply> {
         Box::pin(async move {
-            match self.region_server.set_readonly_gracefully(region_id).await {
-                Ok(SetReadonlyResponse::Success { last_entry_id }) => {
-                    InstructionReply::DowngradeRegion(DowngradeRegionReply {
-                        last_entry_id,
-                        exists: true,
-                        error: None,
-                    })
-                }
-                Ok(SetReadonlyResponse::NotFound) => {
-                    InstructionReply::DowngradeRegion(DowngradeRegionReply {
-                        last_entry_id: None,
-                        exists: false,
-                        error: None,
-                    })
-                }
-                Err(err) => InstructionReply::DowngradeRegion(DowngradeRegionReply {
+            let Some(writable) = self.region_server.is_writable(region_id) else {
+                return InstructionReply::DowngradeRegion(DowngradeRegionReply {
                     last_entry_id: None,
-                    exists: true,
-                    error: Some(format!("{err:?}")),
-                }),
+                    exists: false,
+                    error: None,
+                });
+            };
+
+            // Ignores flush request
+            if !writable {
+                return self.set_readonly_gracefully(region_id).await;
+            }
+
+            let region_server_moved = self.region_server.clone();
+            if let Some(flush_timeout) = flush_timeout {
+                let register_result = self
+                    .downgrade_tasks
+                    .try_register(
+                        region_id,
+                        Box::pin(async move {
+                            info!("Flush region: {region_id} before downgrading region");
+                            region_server_moved
+                                .handle_request(
+                                    region_id,
+                                    RegionRequest::Flush(RegionFlushRequest {
+                                        row_group_size: None,
+                                    }),
+                                )
+                                .await?;
+
+                            Ok(())
+                        }),
+                    )
+                    .await;
+
+                if register_result.is_busy() {
+                    warn!("Another flush task is running for the region: {region_id}");
+                }
+
+                let mut watcher = register_result.into_watcher();
+                let result = self.catchup_tasks.wait(&mut watcher, flush_timeout).await;
+
+                match result {
+                    WaitResult::Timeout => {
+                        InstructionReply::DowngradeRegion(DowngradeRegionReply {
+                            last_entry_id: None,
+                            exists: true,
+                            error: Some(format!(
+                                "Flush region: {region_id} before downgrading region is timeout"
+                            )),
+                        })
+                    }
+                    WaitResult::Finish(Ok(_)) => self.set_readonly_gracefully(region_id).await,
+                    WaitResult::Finish(Err(err)) => {
+                        InstructionReply::DowngradeRegion(DowngradeRegionReply {
+                            last_entry_id: None,
+                            exists: true,
+                            error: Some(format!("{err:?}")),
+                        })
+                    }
+                }
+            } else {
+                self.set_readonly_gracefully(region_id).await
             }
         })
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::assert_matches::assert_matches;
+    use std::time::Duration;
+
+    use common_meta::instruction::{DowngradeRegion, InstructionReply};
+    use mito2::engine::MITO_ENGINE_NAME;
+    use store_api::region_engine::{RegionRole, SetReadonlyResponse};
+    use store_api::region_request::RegionRequest;
+    use store_api::storage::RegionId;
+    use tokio::time::Instant;
+
+    use crate::error;
+    use crate::heartbeat::handler::HandlerContext;
+    use crate::tests::{mock_region_server, MockRegionEngine};
+
+    #[tokio::test]
+    async fn test_region_not_exist() {
+        let mut mock_region_server = mock_region_server();
+        let (mock_engine, _) = MockRegionEngine::new(MITO_ENGINE_NAME);
+        mock_region_server.register_engine(mock_engine);
+        let handler_context = HandlerContext::new_for_test(mock_region_server);
+        let region_id = RegionId::new(1024, 1);
+        let waits = vec![None, Some(Duration::from_millis(100u64))];
+
+        for flush_timeout in waits {
+            let reply = handler_context
+                .clone()
+                .handle_downgrade_region_instruction(DowngradeRegion {
+                    region_id,
+                    flush_timeout,
+                })
+                .await;
+            assert_matches!(reply, InstructionReply::DowngradeRegion(_));
+
+            if let InstructionReply::DowngradeRegion(reply) = reply {
+                assert!(!reply.exists);
+                assert!(reply.error.is_none());
+                assert!(reply.last_entry_id.is_none());
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn test_region_readonly() {
+        let mock_region_server = mock_region_server();
+        let region_id = RegionId::new(1024, 1);
+        let (mock_engine, _) =
+            MockRegionEngine::with_custom_apply_fn(MITO_ENGINE_NAME, |region_engine| {
+                region_engine.mock_role = Some(Some(RegionRole::Follower));
+                region_engine.handle_request_mock_fn = Some(Box::new(|_, req| {
+                    if let RegionRequest::Flush(_) = req {
+                        // Should be unreachable.
+                        unreachable!();
+                    };
+
+                    Ok(0)
+                }));
+                region_engine.handle_set_readonly_gracefully_mock_fn =
+                    Some(Box::new(|_| Ok(SetReadonlyResponse::success(Some(1024)))))
+            });
+        mock_region_server.register_test_region(region_id, mock_engine);
+        let handler_context = HandlerContext::new_for_test(mock_region_server);
+
+        let waits = vec![None, Some(Duration::from_millis(100u64))];
+        for flush_timeout in waits {
+            let reply = handler_context
+                .clone()
+                .handle_downgrade_region_instruction(DowngradeRegion {
+                    region_id,
+                    flush_timeout,
+                })
+                .await;
+            assert_matches!(reply, InstructionReply::DowngradeRegion(_));
+
+            if let InstructionReply::DowngradeRegion(reply) = reply {
+                assert!(reply.exists);
+                assert!(reply.error.is_none());
+                assert_eq!(reply.last_entry_id.unwrap(), 1024);
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn test_region_flush_timeout() {
+        let mock_region_server = mock_region_server();
+        let region_id = RegionId::new(1024, 1);
+        let (mock_engine, _) =
+            MockRegionEngine::with_custom_apply_fn(MITO_ENGINE_NAME, |region_engine| {
+                region_engine.mock_role = Some(Some(RegionRole::Leader));
+                region_engine.handle_request_delay = Some(Duration::from_secs(100));
+                region_engine.handle_set_readonly_gracefully_mock_fn =
+                    Some(Box::new(|_| Ok(SetReadonlyResponse::success(Some(1024)))))
+            });
+        mock_region_server.register_test_region(region_id, mock_engine);
+        let handler_context = HandlerContext::new_for_test(mock_region_server);
+
+        let flush_timeout = Duration::from_millis(100);
+        let reply = handler_context
+            .clone()
+            .handle_downgrade_region_instruction(DowngradeRegion {
+                region_id,
+                flush_timeout: Some(flush_timeout),
+            })
+            .await;
+        assert_matches!(reply, InstructionReply::DowngradeRegion(_));
+
+        if let InstructionReply::DowngradeRegion(reply) = reply {
+            assert!(reply.exists);
+            assert!(reply.error.unwrap().contains("timeout"));
+            assert!(reply.last_entry_id.is_none());
+        }
+    }
+
+    #[tokio::test]
+    async fn test_region_flush_timeout_and_retry() {
+        let mock_region_server = mock_region_server();
+        let region_id = RegionId::new(1024, 1);
+        let (mock_engine, _) =
+            MockRegionEngine::with_custom_apply_fn(MITO_ENGINE_NAME, |region_engine| {
+                region_engine.mock_role = Some(Some(RegionRole::Leader));
+                region_engine.handle_request_delay = Some(Duration::from_millis(300));
+                region_engine.handle_set_readonly_gracefully_mock_fn =
+                    Some(Box::new(|_| Ok(SetReadonlyResponse::success(Some(1024)))))
+            });
+        mock_region_server.register_test_region(region_id, mock_engine);
+        let handler_context = HandlerContext::new_for_test(mock_region_server);
+
+        let waits = vec![
+            Some(Duration::from_millis(100u64)),
+            Some(Duration::from_millis(100u64)),
+        ];
+
+        for flush_timeout in waits {
+            let reply = handler_context
+                .clone()
+                .handle_downgrade_region_instruction(DowngradeRegion {
+                    region_id,
+                    flush_timeout,
+                })
+                .await;
+            assert_matches!(reply, InstructionReply::DowngradeRegion(_));
+            if let InstructionReply::DowngradeRegion(reply) = reply {
+                assert!(reply.exists);
+                assert!(reply.error.unwrap().contains("timeout"));
+                assert!(reply.last_entry_id.is_none());
+            }
+        }
+        let timer = Instant::now();
+        let reply = handler_context
+            .handle_downgrade_region_instruction(DowngradeRegion {
+                region_id,
+                flush_timeout: Some(Duration::from_millis(500)),
+            })
+            .await;
+        assert_matches!(reply, InstructionReply::DowngradeRegion(_));
+        // Must less than 300 ms.
+        assert!(timer.elapsed().as_millis() < 300);
+
+        if let InstructionReply::DowngradeRegion(reply) = reply {
+            assert!(reply.exists);
+            assert!(reply.error.is_none());
+            assert_eq!(reply.last_entry_id.unwrap(), 1024);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_region_flush_timeout_and_retry_error() {
+        let mock_region_server = mock_region_server();
+        let region_id = RegionId::new(1024, 1);
+        let (mock_engine, _) =
+            MockRegionEngine::with_custom_apply_fn(MITO_ENGINE_NAME, |region_engine| {
+                region_engine.mock_role = Some(Some(RegionRole::Leader));
+                region_engine.handle_request_delay = Some(Duration::from_millis(300));
+                region_engine.handle_request_mock_fn = Some(Box::new(|_, _| {
+                    error::UnexpectedSnafu {
+                        violated: "mock flush failed",
+                    }
+                    .fail()
+                }));
+                region_engine.handle_set_readonly_gracefully_mock_fn =
+                    Some(Box::new(|_| Ok(SetReadonlyResponse::success(Some(1024)))))
+            });
+        mock_region_server.register_test_region(region_id, mock_engine);
+        let handler_context = HandlerContext::new_for_test(mock_region_server);
+
+        let waits = vec![
+            Some(Duration::from_millis(100u64)),
+            Some(Duration::from_millis(100u64)),
+        ];
+
+        for flush_timeout in waits {
+            let reply = handler_context
+                .clone()
+                .handle_downgrade_region_instruction(DowngradeRegion {
+                    region_id,
+                    flush_timeout,
+                })
+                .await;
+            assert_matches!(reply, InstructionReply::DowngradeRegion(_));
+            if let InstructionReply::DowngradeRegion(reply) = reply {
+                assert!(reply.exists);
+                assert!(reply.error.unwrap().contains("timeout"));
+                assert!(reply.last_entry_id.is_none());
+            }
+        }
+        let timer = Instant::now();
+        let reply = handler_context
+            .handle_downgrade_region_instruction(DowngradeRegion {
+                region_id,
+                flush_timeout: Some(Duration::from_millis(500)),
+            })
+            .await;
+        assert_matches!(reply, InstructionReply::DowngradeRegion(_));
+        // Must less than 300 ms.
+        assert!(timer.elapsed().as_millis() < 300);
+
+        if let InstructionReply::DowngradeRegion(reply) = reply {
+            assert!(reply.exists);
+            assert!(reply.error.unwrap().contains("flush failed"));
+            assert!(reply.last_entry_id.is_none());
+        }
+    }
+
+    #[tokio::test]
+    async fn test_set_region_readonly_not_found() {
+        let mock_region_server = mock_region_server();
+        let region_id = RegionId::new(1024, 1);
+        let (mock_engine, _) =
+            MockRegionEngine::with_custom_apply_fn(MITO_ENGINE_NAME, |region_engine| {
+                region_engine.mock_role = Some(Some(RegionRole::Leader));
+                region_engine.handle_set_readonly_gracefully_mock_fn =
+                    Some(Box::new(|_| Ok(SetReadonlyResponse::NotFound)));
+            });
+        mock_region_server.register_test_region(region_id, mock_engine);
+        let handler_context = HandlerContext::new_for_test(mock_region_server);
+        let reply = handler_context
+            .clone()
+            .handle_downgrade_region_instruction(DowngradeRegion {
+                region_id,
+                flush_timeout: None,
+            })
+            .await;
+        assert_matches!(reply, InstructionReply::DowngradeRegion(_));
+        if let InstructionReply::DowngradeRegion(reply) = reply {
+            assert!(!reply.exists);
+            assert!(reply.error.is_none());
+            assert!(reply.last_entry_id.is_none());
+        }
+    }
+
+    #[tokio::test]
+    async fn test_set_region_readonly_error() {
+        let mock_region_server = mock_region_server();
+        let region_id = RegionId::new(1024, 1);
+        let (mock_engine, _) =
+            MockRegionEngine::with_custom_apply_fn(MITO_ENGINE_NAME, |region_engine| {
+                region_engine.mock_role = Some(Some(RegionRole::Leader));
+                region_engine.handle_set_readonly_gracefully_mock_fn = Some(Box::new(|_| {
+                    error::UnexpectedSnafu {
+                        violated: "Failed to set region to readonly",
+                    }
+                    .fail()
+                }));
+            });
+        mock_region_server.register_test_region(region_id, mock_engine);
+        let handler_context = HandlerContext::new_for_test(mock_region_server);
+        let reply = handler_context
+            .clone()
+            .handle_downgrade_region_instruction(DowngradeRegion {
+                region_id,
+                flush_timeout: None,
+            })
+            .await;
+        assert_matches!(reply, InstructionReply::DowngradeRegion(_));
+        if let InstructionReply::DowngradeRegion(reply) = reply {
+            assert!(reply.exists);
+            assert!(reply
+                .error
+                .unwrap()
+                .contains("Failed to set region to readonly"));
+            assert!(reply.last_entry_id.is_none());
+        }
+    }
+}
diff --git a/src/datanode/src/heartbeat/handler/upgrade_region.rs b/src/datanode/src/heartbeat/handler/upgrade_region.rs
index 6143731663..0d1ef0476c 100644
--- a/src/datanode/src/heartbeat/handler/upgrade_region.rs
+++ b/src/datanode/src/heartbeat/handler/upgrade_region.rs
@@ -26,7 +26,7 @@ impl HandlerContext {
         UpgradeRegion {
             region_id,
             last_entry_id,
-            wait_for_replay_timeout,
+            replay_timeout,
             location_id,
         }: UpgradeRegion,
     ) -> BoxFuture<'static, InstructionReply> {
@@ -78,7 +78,7 @@ impl HandlerContext {
             }
 
             // Returns immediately
-            let Some(wait_for_replay_timeout) = wait_for_replay_timeout else {
+            let Some(replay_timeout) = replay_timeout else {
                 return InstructionReply::UpgradeRegion(UpgradeRegionReply {
                     ready: false,
                     exists: true,
@@ -88,10 +88,7 @@ impl HandlerContext {
 
             // We don't care that it returns a newly registered or running task.
             let mut watcher = register_result.into_watcher();
-            let result = self
-                .catchup_tasks
-                .wait(&mut watcher, wait_for_replay_timeout)
-                .await;
+            let result = self.catchup_tasks.wait(&mut watcher, replay_timeout).await;
 
             match result {
                 WaitResult::Timeout => InstructionReply::UpgradeRegion(UpgradeRegionReply {
@@ -129,7 +126,6 @@ mod tests {
 
     use crate::error;
     use crate::heartbeat::handler::HandlerContext;
-    use crate::heartbeat::task_tracker::TaskTracker;
     use crate::tests::{mock_region_server, MockRegionEngine};
 
     #[tokio::test]
@@ -138,21 +134,18 @@ mod tests {
         let (mock_engine, _) = MockRegionEngine::new(MITO_ENGINE_NAME);
         mock_region_server.register_engine(mock_engine);
 
-        let handler_context = HandlerContext {
-            region_server: mock_region_server,
-            catchup_tasks: TaskTracker::new(),
-        };
+        let handler_context = HandlerContext::new_for_test(mock_region_server);
 
         let region_id = RegionId::new(1024, 1);
         let waits = vec![None, Some(Duration::from_millis(100u64))];
 
-        for wait_for_replay_timeout in waits {
+        for replay_timeout in waits {
             let reply = handler_context
                 .clone()
                 .handle_upgrade_region_instruction(UpgradeRegion {
                     region_id,
                     last_entry_id: None,
-                    wait_for_replay_timeout,
+                    replay_timeout,
                     location_id: None,
                 })
                 .await;
@@ -180,20 +173,17 @@ mod tests {
             });
         mock_region_server.register_test_region(region_id, mock_engine);
 
-        let handler_context = HandlerContext {
-            region_server: mock_region_server,
-            catchup_tasks: TaskTracker::new(),
-        };
+        let handler_context = HandlerContext::new_for_test(mock_region_server);
 
         let waits = vec![None, Some(Duration::from_millis(100u64))];
 
-        for wait_for_replay_timeout in waits {
+        for replay_timeout in waits {
             let reply = handler_context
                 .clone()
                 .handle_upgrade_region_instruction(UpgradeRegion {
                     region_id,
                     last_entry_id: None,
-                    wait_for_replay_timeout,
+                    replay_timeout,
                     location_id: None,
                 })
                 .await;
@@ -222,20 +212,17 @@ mod tests {
             });
         mock_region_server.register_test_region(region_id, mock_engine);
 
-        let handler_context = HandlerContext {
-            region_server: mock_region_server,
-            catchup_tasks: TaskTracker::new(),
-        };
+        let handler_context = HandlerContext::new_for_test(mock_region_server);
 
         let waits = vec![None, Some(Duration::from_millis(100u64))];
 
-        for wait_for_replay_timeout in waits {
+        for replay_timeout in waits {
             let reply = handler_context
                 .clone()
                 .handle_upgrade_region_instruction(UpgradeRegion {
                     region_id,
                     last_entry_id: None,
-                    wait_for_replay_timeout,
+                    replay_timeout,
                     location_id: None,
                 })
                 .await;
@@ -269,17 +256,14 @@ mod tests {
             Some(Duration::from_millis(100u64)),
         ];
 
-        let handler_context = HandlerContext {
-            region_server: mock_region_server,
-            catchup_tasks: TaskTracker::new(),
-        };
+        let handler_context = HandlerContext::new_for_test(mock_region_server);
 
-        for wait_for_replay_timeout in waits {
+        for replay_timeout in waits {
             let reply = handler_context
                 .clone()
                 .handle_upgrade_region_instruction(UpgradeRegion {
                     region_id,
-                    wait_for_replay_timeout,
+                    replay_timeout,
                     last_entry_id: None,
                     location_id: None,
                 })
@@ -298,7 +282,7 @@ mod tests {
             .handle_upgrade_region_instruction(UpgradeRegion {
                 region_id,
                 last_entry_id: None,
-                wait_for_replay_timeout: Some(Duration::from_millis(500)),
+                replay_timeout: Some(Duration::from_millis(500)),
                 location_id: None,
             })
             .await;
@@ -333,17 +317,14 @@ mod tests {
             });
         mock_region_server.register_test_region(region_id, mock_engine);
 
-        let handler_context = HandlerContext {
-            region_server: mock_region_server,
-            catchup_tasks: TaskTracker::new(),
-        };
+        let handler_context = HandlerContext::new_for_test(mock_region_server);
 
         let reply = handler_context
             .clone()
             .handle_upgrade_region_instruction(UpgradeRegion {
                 region_id,
                 last_entry_id: None,
-                wait_for_replay_timeout: None,
+                replay_timeout: None,
                 location_id: None,
             })
             .await;
@@ -361,7 +342,7 @@ mod tests {
             .handle_upgrade_region_instruction(UpgradeRegion {
                 region_id,
                 last_entry_id: None,
-                wait_for_replay_timeout: Some(Duration::from_millis(200)),
+                replay_timeout: Some(Duration::from_millis(200)),
                 location_id: None,
             })
             .await;
diff --git a/src/datanode/src/tests.rs b/src/datanode/src/tests.rs
index 89be76511d..8966dc4932 100644
--- a/src/datanode/src/tests.rs
+++ b/src/datanode/src/tests.rs
@@ -103,10 +103,14 @@ pub fn mock_region_server() -> RegionServer {
 pub type MockRequestHandler =
     Box<dyn Fn(RegionId, RegionRequest) -> Result<AffectedRows, Error> + Send + Sync>;
 
+pub type MockSetReadonlyGracefullyHandler =
+    Box<dyn Fn(RegionId) -> Result<SetReadonlyResponse, Error> + Send + Sync>;
+
 pub struct MockRegionEngine {
     sender: Sender<(RegionId, RegionRequest)>,
     pub(crate) handle_request_delay: Option<Duration>,
     pub(crate) handle_request_mock_fn: Option<MockRequestHandler>,
+    pub(crate) handle_set_readonly_gracefully_mock_fn: Option<MockSetReadonlyGracefullyHandler>,
     pub(crate) mock_role: Option<Option<RegionRole>>,
     engine: String,
 }
@@ -120,6 +124,7 @@ impl MockRegionEngine {
                 handle_request_delay: None,
                 sender: tx,
                 handle_request_mock_fn: None,
+                handle_set_readonly_gracefully_mock_fn: None,
                 mock_role: None,
                 engine: engine.to_string(),
             }),
@@ -138,6 +143,7 @@ impl MockRegionEngine {
                 handle_request_delay: None,
                 sender: tx,
                 handle_request_mock_fn: Some(mock_fn),
+                handle_set_readonly_gracefully_mock_fn: None,
                 mock_role: None,
                 engine: engine.to_string(),
             }),
@@ -157,6 +163,7 @@ impl MockRegionEngine {
             handle_request_delay: None,
             sender: tx,
             handle_request_mock_fn: None,
+            handle_set_readonly_gracefully_mock_fn: None,
             mock_role: None,
             engine: engine.to_string(),
         };
@@ -217,9 +224,13 @@ impl RegionEngine for MockRegionEngine {
 
     async fn set_readonly_gracefully(
         &self,
-        _region_id: RegionId,
+        region_id: RegionId,
     ) -> Result<SetReadonlyResponse, BoxedError> {
-        unimplemented!()
+        if let Some(mock_fn) = &self.handle_set_readonly_gracefully_mock_fn {
+            return mock_fn(region_id).map_err(BoxedError::new);
+        };
+
+        unreachable!()
     }
 
     fn role(&self, _region_id: RegionId) -> Option<RegionRole> {
diff --git a/src/meta-client/src/client.rs b/src/meta-client/src/client.rs
index 0a9df7293f..d5f7c6aaaf 100644
--- a/src/meta-client/src/client.rs
+++ b/src/meta-client/src/client.rs
@@ -483,7 +483,7 @@ impl MetaClient {
                 request.region_id,
                 request.from_peer,
                 request.to_peer,
-                request.replay_timeout,
+                request.timeout,
             )
             .await
     }
diff --git a/src/meta-client/src/client/procedure.rs b/src/meta-client/src/client/procedure.rs
index f45cfb7879..eccfe90dff 100644
--- a/src/meta-client/src/client/procedure.rs
+++ b/src/meta-client/src/client/procedure.rs
@@ -77,17 +77,17 @@ impl Client {
     /// - `region_id`:  the migrated region id
     /// - `from_peer`:  the source datanode id
     /// - `to_peer`:  the target datanode id
-    /// - `replay_timeout`: replay WAL timeout after migration.
+    /// - `timeout`: timeout for downgrading region and upgrading region operations
     pub async fn migrate_region(
         &self,
         region_id: u64,
         from_peer: u64,
         to_peer: u64,
-        replay_timeout: Duration,
+        timeout: Duration,
     ) -> Result<MigrateRegionResponse> {
         let inner = self.inner.read().await;
         inner
-            .migrate_region(region_id, from_peer, to_peer, replay_timeout)
+            .migrate_region(region_id, from_peer, to_peer, timeout)
             .await
     }
 
@@ -216,13 +216,13 @@ impl Inner {
         region_id: u64,
         from_peer: u64,
         to_peer: u64,
-        replay_timeout: Duration,
+        timeout: Duration,
     ) -> Result<MigrateRegionResponse> {
         let mut req = MigrateRegionRequest {
             region_id,
             from_peer,
             to_peer,
-            replay_timeout_secs: replay_timeout.as_secs() as u32,
+            timeout_secs: timeout.as_secs() as u32,
             ..Default::default()
         };
 
diff --git a/src/meta-srv/src/error.rs b/src/meta-srv/src/error.rs
index d0a58d688e..728f326871 100644
--- a/src/meta-srv/src/error.rs
+++ b/src/meta-srv/src/error.rs
@@ -32,6 +32,13 @@ use crate::pubsub::Message;
 #[snafu(visibility(pub))]
 #[stack_trace_debug]
 pub enum Error {
+    #[snafu(display("Exceeded deadline, operation: {}", operation))]
+    ExceededDeadline {
+        #[snafu(implicit)]
+        location: Location,
+        operation: String,
+    },
+
     #[snafu(display("The target peer is unavailable temporally: {}", peer_id))]
     PeerUnavailable {
         #[snafu(implicit)]
@@ -783,7 +790,8 @@ impl ErrorExt for Error {
             | Error::Join { .. }
             | Error::WeightArray { .. }
             | Error::NotSetWeightArray { .. }
-            | Error::PeerUnavailable { .. } => StatusCode::Internal,
+            | Error::PeerUnavailable { .. }
+            | Error::ExceededDeadline { .. } => StatusCode::Internal,
 
             Error::Unsupported { .. } => StatusCode::Unsupported,
 
diff --git a/src/meta-srv/src/procedure/region_migration.rs b/src/meta-srv/src/procedure/region_migration.rs
index e56afdbf11..9b5bdfc006 100644
--- a/src/meta-srv/src/procedure/region_migration.rs
+++ b/src/meta-srv/src/procedure/region_migration.rs
@@ -73,13 +73,13 @@ pub struct PersistentContext {
     to_peer: Peer,
     /// The [RegionId] of migration region.
     region_id: RegionId,
-    /// The timeout of waiting for a candidate to replay the WAL.
-    #[serde(with = "humantime_serde", default = "default_replay_timeout")]
-    replay_timeout: Duration,
+    /// The timeout for downgrading leader region and upgrading candidate region operations.
+    #[serde(with = "humantime_serde", default = "default_timeout")]
+    timeout: Duration,
 }
 
-fn default_replay_timeout() -> Duration {
-    Duration::from_secs(1)
+fn default_timeout() -> Duration {
+    Duration::from_secs(10)
 }
 
 impl PersistentContext {
@@ -123,6 +123,8 @@ pub struct VolatileContext {
     leader_region_lease_deadline: Option<Instant>,
     /// The last_entry_id of leader region.
     leader_region_last_entry_id: Option<u64>,
+    /// Elapsed time of downgrading region and upgrading region.
+    operations_elapsed: Duration,
 }
 
 impl VolatileContext {
@@ -211,6 +213,18 @@ pub struct Context {
 }
 
 impl Context {
+    /// Returns the next operation's timeout.
+    pub fn next_operation_timeout(&self) -> Option<Duration> {
+        self.persistent_ctx
+            .timeout
+            .checked_sub(self.volatile_ctx.operations_elapsed)
+    }
+
+    /// Updates operations elapsed.
+    pub fn update_operations_elapsed(&mut self, instant: Instant) {
+        self.volatile_ctx.operations_elapsed += instant.elapsed();
+    }
+
     /// Returns address of meta server.
     pub fn server_addr(&self) -> &str {
         &self.server_addr
@@ -441,7 +455,7 @@ impl RegionMigrationProcedure {
             region_id: persistent_ctx.region_id,
             from_peer: persistent_ctx.from_peer.clone(),
             to_peer: persistent_ctx.to_peer.clone(),
-            replay_timeout: persistent_ctx.replay_timeout,
+            timeout: persistent_ctx.timeout,
         });
         let context = context_factory.new_context(persistent_ctx);
 
@@ -537,7 +551,7 @@ mod tests {
         let procedure = RegionMigrationProcedure::new(persistent_context, context, None);
 
         let serialized = procedure.dump().unwrap();
-        let expected = r#"{"persistent_ctx":{"catalog":"greptime","schema":"public","cluster_id":0,"from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_id":4398046511105,"replay_timeout":"1s"},"state":{"region_migration_state":"RegionMigrationStart"}}"#;
+        let expected = r#"{"persistent_ctx":{"catalog":"greptime","schema":"public","cluster_id":0,"from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_id":4398046511105,"timeout":"10s"},"state":{"region_migration_state":"RegionMigrationStart"}}"#;
         assert_eq!(expected, serialized);
     }
 
diff --git a/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs b/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs
index 17c5762906..836ca4c532 100644
--- a/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs
+++ b/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs
@@ -16,24 +16,23 @@ use std::any::Any;
 use std::time::Duration;
 
 use api::v1::meta::MailboxMessage;
-use common_meta::distributed_time_constants::{MAILBOX_RTT_SECS, REGION_LEASE_SECS};
+use common_meta::distributed_time_constants::REGION_LEASE_SECS;
 use common_meta::instruction::{
     DowngradeRegion, DowngradeRegionReply, Instruction, InstructionReply,
 };
 use common_procedure::Status;
-use common_telemetry::{info, warn};
+use common_telemetry::{error, info, warn};
 use serde::{Deserialize, Serialize};
-use snafu::ResultExt;
-use tokio::time::sleep;
+use snafu::{OptionExt, ResultExt};
+use tokio::time::{sleep, Instant};
 
+use super::update_metadata::UpdateMetadata;
 use super::upgrade_candidate_region::UpgradeCandidateRegion;
 use crate::error::{self, Result};
 use crate::handler::HeartbeatMailbox;
 use crate::procedure::region_migration::{Context, State};
 use crate::service::mailbox::Channel;
 
-const DOWNGRADE_LEADER_REGION_TIMEOUT: Duration = Duration::from_secs(MAILBOX_RTT_SECS);
-
 #[derive(Debug, Serialize, Deserialize)]
 pub struct DowngradeLeaderRegion {
     // The optimistic retry times.
@@ -55,25 +54,32 @@ impl Default for DowngradeLeaderRegion {
 #[typetag::serde]
 impl State for DowngradeLeaderRegion {
     async fn next(&mut self, ctx: &mut Context) -> Result<(Box<dyn State>, Status)> {
-        let replay_timeout = ctx.persistent_ctx.replay_timeout;
         // Ensures the `leader_region_lease_deadline` must exist after recovering.
         ctx.volatile_ctx
             .set_leader_region_lease_deadline(Duration::from_secs(REGION_LEASE_SECS));
-        self.downgrade_region_with_retry(ctx).await;
 
-        if let Some(deadline) = ctx.volatile_ctx.leader_region_lease_deadline.as_ref() {
-            info!(
-                "Running into the downgrade leader slow path, sleep until {:?}",
-                deadline
-            );
-            tokio::time::sleep_until(*deadline).await;
+        match self.downgrade_region_with_retry(ctx).await {
+            Ok(_) => {
+                // Do nothing
+            }
+            Err(error::Error::ExceededDeadline { .. }) => {
+                // Rollbacks the metadata if procedure is timeout
+                return Ok((Box::new(UpdateMetadata::Rollback), Status::executing(false)));
+            }
+            Err(err) => {
+                error!(err; "Occurs non-retryable error");
+                if let Some(deadline) = ctx.volatile_ctx.leader_region_lease_deadline.as_ref() {
+                    info!(
+                        "Running into the downgrade leader slow path, sleep until {:?}",
+                        deadline
+                    );
+                    tokio::time::sleep_until(*deadline).await;
+                }
+            }
         }
 
         Ok((
-            Box::new(UpgradeCandidateRegion {
-                replay_timeout,
-                ..Default::default()
-            }),
+            Box::new(UpgradeCandidateRegion::default()),
             Status::executing(false),
         ))
     }
@@ -85,10 +91,17 @@ impl State for DowngradeLeaderRegion {
 
 impl DowngradeLeaderRegion {
     /// Builds downgrade region instruction.
-    fn build_downgrade_region_instruction(&self, ctx: &Context) -> Instruction {
+    fn build_downgrade_region_instruction(
+        &self,
+        ctx: &Context,
+        flush_timeout: Duration,
+    ) -> Instruction {
         let pc = &ctx.persistent_ctx;
         let region_id = pc.region_id;
-        Instruction::DowngradeRegion(DowngradeRegion { region_id })
+        Instruction::DowngradeRegion(DowngradeRegion {
+            region_id,
+            flush_timeout: Some(flush_timeout),
+        })
     }
 
     /// Tries to downgrade a leader region.
@@ -102,32 +115,32 @@ impl DowngradeLeaderRegion {
     /// - [PushMessage](error::Error::PushMessage), The receiver is dropped.
     /// - [MailboxReceiver](error::Error::MailboxReceiver), The sender is dropped without sending (impossible).
     /// - [UnexpectedInstructionReply](error::Error::UnexpectedInstructionReply).
+    /// - [ExceededDeadline](error::Error::ExceededDeadline)
     /// - Invalid JSON.
-    async fn downgrade_region(
-        &self,
-        ctx: &mut Context,
-        downgrade_instruction: &Instruction,
-    ) -> Result<()> {
+    async fn downgrade_region(&self, ctx: &mut Context) -> Result<()> {
         let pc = &ctx.persistent_ctx;
         let region_id = pc.region_id;
         let leader = &pc.from_peer;
+        let operation_timeout =
+            ctx.next_operation_timeout()
+                .context(error::ExceededDeadlineSnafu {
+                    operation: "Downgrade region",
+                })?;
+        let downgrade_instruction = self.build_downgrade_region_instruction(ctx, operation_timeout);
 
         let msg = MailboxMessage::json_message(
             &format!("Downgrade leader region: {}", region_id),
             &format!("Meta@{}", ctx.server_addr()),
             &format!("Datanode-{}@{}", leader.id, leader.addr),
             common_time::util::current_time_millis(),
-            downgrade_instruction,
+            &downgrade_instruction,
         )
         .with_context(|_| error::SerializeToJsonSnafu {
             input: downgrade_instruction.to_string(),
         })?;
 
         let ch = Channel::Datanode(leader.id);
-        let receiver = ctx
-            .mailbox
-            .send(&ch, msg, DOWNGRADE_LEADER_REGION_TIMEOUT)
-            .await?;
+        let receiver = ctx.mailbox.send(&ch, msg, operation_timeout).await?;
 
         match receiver.await? {
             Ok(msg) => {
@@ -191,26 +204,36 @@ impl DowngradeLeaderRegion {
     ///
     /// Slow path:
     /// - Waits for the lease of the leader region expired.
-    async fn downgrade_region_with_retry(&self, ctx: &mut Context) {
-        let instruction = self.build_downgrade_region_instruction(ctx);
-
+    ///
+    /// Abort:
+    /// - ExceededDeadline
+    async fn downgrade_region_with_retry(&self, ctx: &mut Context) -> Result<()> {
         let mut retry = 0;
 
         loop {
-            if let Err(err) = self.downgrade_region(ctx, &instruction).await {
+            let timer = Instant::now();
+            if let Err(err) = self.downgrade_region(ctx).await {
+                ctx.update_operations_elapsed(timer);
                 retry += 1;
-                if err.is_retryable() && retry < self.optimistic_retry {
-                    warn!("Failed to downgrade region, error: {err:?}, retry later");
+                // Throws the error immediately if the procedure exceeded the deadline.
+                if matches!(err, error::Error::ExceededDeadline { .. }) {
+                    return Err(err);
+                } else if err.is_retryable() && retry < self.optimistic_retry {
+                    error!("Failed to downgrade region, error: {err:?}, retry later");
                     sleep(self.retry_initial_interval).await;
                 } else {
+                    error!("Failed to downgrade region, error: {err:?}");
                     break;
                 }
             } else {
+                ctx.update_operations_elapsed(timer);
                 // Resets the deadline.
                 ctx.volatile_ctx.reset_leader_region_lease_deadline();
                 break;
             }
         }
+
+        Ok(())
     }
 }
 
@@ -237,7 +260,7 @@ mod tests {
             to_peer: Peer::empty(2),
             region_id: RegionId::new(1024, 1),
             cluster_id: 0,
-            replay_timeout: Duration::from_millis(1000),
+            timeout: Duration::from_millis(1000),
         }
     }
 
@@ -248,11 +271,7 @@ mod tests {
         let env = TestingEnv::new();
         let mut ctx = env.context_factory().new_context(persistent_context);
 
-        let instruction = &state.build_downgrade_region_instruction(&ctx);
-        let err = state
-            .downgrade_region(&mut ctx, instruction)
-            .await
-            .unwrap_err();
+        let err = state.downgrade_region(&mut ctx).await.unwrap_err();
 
         assert_matches!(err, Error::PusherNotFound { .. });
         assert!(!err.is_retryable());
@@ -276,16 +295,33 @@ mod tests {
 
         drop(rx);
 
-        let instruction = &state.build_downgrade_region_instruction(&ctx);
-        let err = state
-            .downgrade_region(&mut ctx, instruction)
-            .await
-            .unwrap_err();
+        let err = state.downgrade_region(&mut ctx).await.unwrap_err();
 
         assert_matches!(err, Error::PushMessage { .. });
         assert!(!err.is_retryable());
     }
 
+    #[tokio::test]
+    async fn test_procedure_exceeded_deadline() {
+        let state = DowngradeLeaderRegion::default();
+        let persistent_context = new_persistent_context();
+        let env = TestingEnv::new();
+        let mut ctx = env.context_factory().new_context(persistent_context);
+        ctx.volatile_ctx.operations_elapsed = ctx.persistent_ctx.timeout + Duration::from_secs(1);
+
+        let err = state.downgrade_region(&mut ctx).await.unwrap_err();
+
+        assert_matches!(err, Error::ExceededDeadline { .. });
+        assert!(!err.is_retryable());
+
+        let err = state
+            .downgrade_region_with_retry(&mut ctx)
+            .await
+            .unwrap_err();
+        assert_matches!(err, Error::ExceededDeadline { .. });
+        assert!(!err.is_retryable());
+    }
+
     #[tokio::test]
     async fn test_unexpected_instruction_reply() {
         let state = DowngradeLeaderRegion::default();
@@ -306,11 +342,7 @@ mod tests {
         // Sends an incorrect reply.
         send_mock_reply(mailbox, rx, |id| Ok(new_close_region_reply(id)));
 
-        let instruction = &state.build_downgrade_region_instruction(&ctx);
-        let err = state
-            .downgrade_region(&mut ctx, instruction)
-            .await
-            .unwrap_err();
+        let err = state.downgrade_region(&mut ctx).await.unwrap_err();
 
         assert_matches!(err, Error::UnexpectedInstructionReply { .. });
         assert!(!err.is_retryable());
@@ -337,11 +369,7 @@ mod tests {
             Err(error::MailboxTimeoutSnafu { id }.build())
         });
 
-        let instruction = &state.build_downgrade_region_instruction(&ctx);
-        let err = state
-            .downgrade_region(&mut ctx, instruction)
-            .await
-            .unwrap_err();
+        let err = state.downgrade_region(&mut ctx).await.unwrap_err();
 
         assert_matches!(err, Error::RetryLater { .. });
         assert!(err.is_retryable());
@@ -373,11 +401,7 @@ mod tests {
             ))
         });
 
-        let instruction = &state.build_downgrade_region_instruction(&ctx);
-        let err = state
-            .downgrade_region(&mut ctx, instruction)
-            .await
-            .unwrap_err();
+        let err = state.downgrade_region(&mut ctx).await.unwrap_err();
 
         assert_matches!(err, Error::RetryLater { .. });
         assert!(err.is_retryable());
@@ -425,7 +449,7 @@ mod tests {
                 .unwrap();
         });
 
-        state.downgrade_region_with_retry(&mut ctx).await;
+        state.downgrade_region_with_retry(&mut ctx).await.unwrap();
         assert_eq!(ctx.volatile_ctx.leader_region_last_entry_id, Some(1));
         assert!(ctx.volatile_ctx.leader_region_lease_deadline.is_none());
     }
@@ -467,7 +491,7 @@ mod tests {
         ctx.volatile_ctx
             .set_leader_region_lease_deadline(Duration::from_secs(5));
         let expected_deadline = ctx.volatile_ctx.leader_region_lease_deadline.unwrap();
-        state.downgrade_region_with_retry(&mut ctx).await;
+        state.downgrade_region_with_retry(&mut ctx).await.unwrap();
         assert_eq!(ctx.volatile_ctx.leader_region_last_entry_id, None);
         // Should remain no change.
         assert_eq!(
@@ -509,4 +533,30 @@ mod tests {
             .downcast_ref::<UpgradeCandidateRegion>()
             .unwrap();
     }
+
+    #[tokio::test]
+    async fn test_downgrade_region_procedure_exceeded_deadline() {
+        let mut state = Box::<UpgradeCandidateRegion>::default();
+        state.retry_initial_interval = Duration::from_millis(100);
+        let persistent_context = new_persistent_context();
+        let to_peer_id = persistent_context.to_peer.id;
+
+        let mut env = TestingEnv::new();
+        let mut ctx = env.context_factory().new_context(persistent_context);
+        let mailbox_ctx = env.mailbox_context();
+        let mailbox = mailbox_ctx.mailbox().clone();
+        ctx.volatile_ctx.operations_elapsed = ctx.persistent_ctx.timeout + Duration::from_secs(1);
+
+        let (tx, rx) = tokio::sync::mpsc::channel(1);
+        mailbox_ctx
+            .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
+            .await;
+
+        send_mock_reply(mailbox, rx, |id| {
+            Ok(new_downgrade_region_reply(id, None, true, None))
+        });
+        let (next, _) = state.next(&mut ctx).await.unwrap();
+        let update_metadata = next.as_any().downcast_ref::<UpdateMetadata>().unwrap();
+        assert_matches!(update_metadata, UpdateMetadata::Rollback);
+    }
 }
diff --git a/src/meta-srv/src/procedure/region_migration/manager.rs b/src/meta-srv/src/procedure/region_migration/manager.rs
index 73aa4371f4..bb3eff80c0 100644
--- a/src/meta-srv/src/procedure/region_migration/manager.rs
+++ b/src/meta-srv/src/procedure/region_migration/manager.rs
@@ -104,7 +104,7 @@ pub struct RegionMigrationProcedureTask {
     pub(crate) region_id: RegionId,
     pub(crate) from_peer: Peer,
     pub(crate) to_peer: Peer,
-    pub(crate) replay_timeout: Duration,
+    pub(crate) timeout: Duration,
 }
 
 impl RegionMigrationProcedureTask {
@@ -113,14 +113,14 @@ impl RegionMigrationProcedureTask {
         region_id: RegionId,
         from_peer: Peer,
         to_peer: Peer,
-        replay_timeout: Duration,
+        timeout: Duration,
     ) -> Self {
         Self {
             cluster_id,
             region_id,
             from_peer,
             to_peer,
-            replay_timeout,
+            timeout,
         }
     }
 }
@@ -328,7 +328,7 @@ impl RegionMigrationManager {
             region_id,
             from_peer,
             to_peer,
-            replay_timeout,
+            timeout,
         } = task.clone();
         let procedure = RegionMigrationProcedure::new(
             PersistentContext {
@@ -338,7 +338,7 @@ impl RegionMigrationManager {
                 region_id,
                 from_peer,
                 to_peer,
-                replay_timeout,
+                timeout,
             },
             self.context_factory.clone(),
             Some(guard),
@@ -390,7 +390,7 @@ mod test {
             region_id,
             from_peer: Peer::empty(2),
             to_peer: Peer::empty(1),
-            replay_timeout: Duration::from_millis(1000),
+            timeout: Duration::from_millis(1000),
         };
         // Inserts one
         manager
@@ -415,7 +415,7 @@ mod test {
             region_id,
             from_peer: Peer::empty(1),
             to_peer: Peer::empty(1),
-            replay_timeout: Duration::from_millis(1000),
+            timeout: Duration::from_millis(1000),
         };
 
         let err = manager.submit_procedure(task).await.unwrap_err();
@@ -433,7 +433,7 @@ mod test {
             region_id,
             from_peer: Peer::empty(1),
             to_peer: Peer::empty(2),
-            replay_timeout: Duration::from_millis(1000),
+            timeout: Duration::from_millis(1000),
         };
 
         let err = manager.submit_procedure(task).await.unwrap_err();
@@ -451,7 +451,7 @@ mod test {
             region_id,
             from_peer: Peer::empty(1),
             to_peer: Peer::empty(2),
-            replay_timeout: Duration::from_millis(1000),
+            timeout: Duration::from_millis(1000),
         };
 
         let table_info = new_test_table_info(1024, vec![1]).into();
@@ -479,7 +479,7 @@ mod test {
             region_id,
             from_peer: Peer::empty(1),
             to_peer: Peer::empty(2),
-            replay_timeout: Duration::from_millis(1000),
+            timeout: Duration::from_millis(1000),
         };
 
         let table_info = new_test_table_info(1024, vec![1]).into();
@@ -511,7 +511,7 @@ mod test {
             region_id,
             from_peer: Peer::empty(1),
             to_peer: Peer::empty(2),
-            replay_timeout: Duration::from_millis(1000),
+            timeout: Duration::from_millis(1000),
         };
 
         let table_info = new_test_table_info(1024, vec![1]).into();
@@ -538,7 +538,7 @@ mod test {
             region_id,
             from_peer: Peer::empty(1),
             to_peer: Peer::empty(2),
-            replay_timeout: Duration::from_millis(1000),
+            timeout: Duration::from_millis(1000),
         };
 
         let err = manager
diff --git a/src/meta-srv/src/procedure/region_migration/test_util.rs b/src/meta-srv/src/procedure/region_migration/test_util.rs
index edfb89515f..65e33ab3d9 100644
--- a/src/meta-srv/src/procedure/region_migration/test_util.rs
+++ b/src/meta-srv/src/procedure/region_migration/test_util.rs
@@ -316,7 +316,7 @@ pub fn new_persistent_context(from: u64, to: u64, region_id: RegionId) -> Persis
         to_peer: Peer::empty(to),
         region_id,
         cluster_id: 0,
-        replay_timeout: Duration::from_millis(1000),
+        timeout: Duration::from_secs(10),
     }
 }
 
diff --git a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs
index 88caf5f08d..49100e92f3 100644
--- a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs
+++ b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs
@@ -16,13 +16,12 @@ use std::any::Any;
 use std::time::Duration;
 
 use api::v1::meta::MailboxMessage;
-use common_meta::distributed_time_constants::MAILBOX_RTT_SECS;
 use common_meta::instruction::{Instruction, InstructionReply, UpgradeRegion, UpgradeRegionReply};
 use common_procedure::Status;
-use common_telemetry::warn;
+use common_telemetry::error;
 use serde::{Deserialize, Serialize};
-use snafu::{ensure, ResultExt};
-use tokio::time::sleep;
+use snafu::{ensure, OptionExt, ResultExt};
+use tokio::time::{sleep, Instant};
 
 use super::update_metadata::UpdateMetadata;
 use crate::error::{self, Result};
@@ -36,8 +35,6 @@ pub struct UpgradeCandidateRegion {
     pub(crate) optimistic_retry: usize,
     // The retry initial interval.
     pub(crate) retry_initial_interval: Duration,
-    // The replay timeout of a instruction.
-    pub(crate) replay_timeout: Duration,
     // If it's true it requires the candidate region MUST replay the WAL to the latest entry id.
     // Otherwise, it will rollback to the old leader region.
     pub(crate) require_ready: bool,
@@ -48,7 +45,6 @@ impl Default for UpgradeCandidateRegion {
         Self {
             optimistic_retry: 3,
             retry_initial_interval: Duration::from_millis(500),
-            replay_timeout: Duration::from_millis(1000),
             require_ready: true,
         }
     }
@@ -71,17 +67,12 @@ impl State for UpgradeCandidateRegion {
 }
 
 impl UpgradeCandidateRegion {
-    const UPGRADE_CANDIDATE_REGION_RTT: Duration = Duration::from_secs(MAILBOX_RTT_SECS);
-
-    /// Returns the timeout of the upgrade candidate region.
-    ///
-    /// Equals `replay_timeout` + RTT
-    fn send_upgrade_candidate_region_timeout(&self) -> Duration {
-        self.replay_timeout + UpgradeCandidateRegion::UPGRADE_CANDIDATE_REGION_RTT
-    }
-
     /// Builds upgrade region instruction.
-    fn build_upgrade_region_instruction(&self, ctx: &Context) -> Instruction {
+    fn build_upgrade_region_instruction(
+        &self,
+        ctx: &Context,
+        replay_timeout: Duration,
+    ) -> Instruction {
         let pc = &ctx.persistent_ctx;
         let region_id = pc.region_id;
         let last_entry_id = ctx.volatile_ctx.leader_region_last_entry_id;
@@ -89,7 +80,7 @@ impl UpgradeCandidateRegion {
         Instruction::UpgradeRegion(UpgradeRegion {
             region_id,
             last_entry_id,
-            wait_for_replay_timeout: Some(self.replay_timeout),
+            replay_timeout: Some(replay_timeout),
             location_id: Some(ctx.persistent_ctx.from_peer.id),
         })
     }
@@ -106,28 +97,32 @@ impl UpgradeCandidateRegion {
     /// - [PushMessage](error::Error::PushMessage), The receiver is dropped.
     /// - [MailboxReceiver](error::Error::MailboxReceiver), The sender is dropped without sending (impossible).
     /// - [UnexpectedInstructionReply](error::Error::UnexpectedInstructionReply) (impossible).
+    /// - [ExceededDeadline](error::Error::ExceededDeadline)
     /// - Invalid JSON (impossible).
-    async fn upgrade_region(&self, ctx: &Context, upgrade_instruction: &Instruction) -> Result<()> {
+    async fn upgrade_region(&self, ctx: &Context) -> Result<()> {
         let pc = &ctx.persistent_ctx;
         let region_id = pc.region_id;
         let candidate = &pc.to_peer;
+        let operation_timeout =
+            ctx.next_operation_timeout()
+                .context(error::ExceededDeadlineSnafu {
+                    operation: "Upgrade region",
+                })?;
+        let upgrade_instruction = self.build_upgrade_region_instruction(ctx, operation_timeout);
 
         let msg = MailboxMessage::json_message(
             &format!("Upgrade candidate region: {}", region_id),
             &format!("Meta@{}", ctx.server_addr()),
             &format!("Datanode-{}@{}", candidate.id, candidate.addr),
             common_time::util::current_time_millis(),
-            upgrade_instruction,
+            &upgrade_instruction,
         )
         .with_context(|_| error::SerializeToJsonSnafu {
             input: upgrade_instruction.to_string(),
         })?;
 
         let ch = Channel::Datanode(candidate.id);
-        let receiver = ctx
-            .mailbox
-            .send(&ch, msg, self.send_upgrade_candidate_region_timeout())
-            .await?;
+        let receiver = ctx.mailbox.send(&ch, msg, operation_timeout).await?;
 
         match receiver.await? {
             Ok(msg) => {
@@ -192,22 +187,27 @@ impl UpgradeCandidateRegion {
     /// Upgrades a candidate region.
     ///
     /// Returns true if the candidate region is upgraded successfully.
-    async fn upgrade_region_with_retry(&self, ctx: &Context) -> bool {
-        let upgrade_instruction = self.build_upgrade_region_instruction(ctx);
-
+    async fn upgrade_region_with_retry(&self, ctx: &mut Context) -> bool {
         let mut retry = 0;
         let mut upgraded = false;
 
         loop {
-            if let Err(err) = self.upgrade_region(ctx, &upgrade_instruction).await {
+            let timer = Instant::now();
+            if let Err(err) = self.upgrade_region(ctx).await {
                 retry += 1;
-                if err.is_retryable() && retry < self.optimistic_retry {
-                    warn!("Failed to upgrade region, error: {err:?}, retry later");
+                ctx.update_operations_elapsed(timer);
+                if matches!(err, error::Error::ExceededDeadline { .. }) {
+                    error!("Failed to upgrade region, exceeded deadline");
+                    break;
+                } else if err.is_retryable() && retry < self.optimistic_retry {
+                    error!("Failed to upgrade region, error: {err:?}, retry later");
                     sleep(self.retry_initial_interval).await;
                 } else {
+                    error!("Failed to upgrade region, error: {err:?}");
                     break;
                 }
             } else {
+                ctx.update_operations_elapsed(timer);
                 upgraded = true;
                 break;
             }
@@ -239,7 +239,7 @@ mod tests {
             to_peer: Peer::empty(2),
             region_id: RegionId::new(1024, 1),
             cluster_id: 0,
-            replay_timeout: Duration::from_millis(1000),
+            timeout: Duration::from_millis(1000),
         }
     }
 
@@ -250,8 +250,7 @@ mod tests {
         let env = TestingEnv::new();
         let ctx = env.context_factory().new_context(persistent_context);
 
-        let instruction = &state.build_upgrade_region_instruction(&ctx);
-        let err = state.upgrade_region(&ctx, instruction).await.unwrap_err();
+        let err = state.upgrade_region(&ctx).await.unwrap_err();
 
         assert_matches!(err, Error::PusherNotFound { .. });
         assert!(!err.is_retryable());
@@ -275,13 +274,26 @@ mod tests {
 
         drop(rx);
 
-        let instruction = &state.build_upgrade_region_instruction(&ctx);
-        let err = state.upgrade_region(&ctx, instruction).await.unwrap_err();
+        let err = state.upgrade_region(&ctx).await.unwrap_err();
 
         assert_matches!(err, Error::PushMessage { .. });
         assert!(!err.is_retryable());
     }
 
+    #[tokio::test]
+    async fn test_procedure_exceeded_deadline() {
+        let state = UpgradeCandidateRegion::default();
+        let persistent_context = new_persistent_context();
+        let env = TestingEnv::new();
+        let mut ctx = env.context_factory().new_context(persistent_context);
+        ctx.volatile_ctx.operations_elapsed = ctx.persistent_ctx.timeout + Duration::from_secs(1);
+
+        let err = state.upgrade_region(&ctx).await.unwrap_err();
+
+        assert_matches!(err, Error::ExceededDeadline { .. });
+        assert!(!err.is_retryable());
+    }
+
     #[tokio::test]
     async fn test_unexpected_instruction_reply() {
         let state = UpgradeCandidateRegion::default();
@@ -301,8 +313,7 @@ mod tests {
 
         send_mock_reply(mailbox, rx, |id| Ok(new_close_region_reply(id)));
 
-        let instruction = &state.build_upgrade_region_instruction(&ctx);
-        let err = state.upgrade_region(&ctx, instruction).await.unwrap_err();
+        let err = state.upgrade_region(&ctx).await.unwrap_err();
         assert_matches!(err, Error::UnexpectedInstructionReply { .. });
         assert!(!err.is_retryable());
     }
@@ -334,8 +345,7 @@ mod tests {
             ))
         });
 
-        let instruction = &state.build_upgrade_region_instruction(&ctx);
-        let err = state.upgrade_region(&ctx, instruction).await.unwrap_err();
+        let err = state.upgrade_region(&ctx).await.unwrap_err();
 
         assert_matches!(err, Error::RetryLater { .. });
         assert!(err.is_retryable());
@@ -363,8 +373,7 @@ mod tests {
             Ok(new_upgrade_region_reply(id, true, false, None))
         });
 
-        let instruction = &state.build_upgrade_region_instruction(&ctx);
-        let err = state.upgrade_region(&ctx, instruction).await.unwrap_err();
+        let err = state.upgrade_region(&ctx).await.unwrap_err();
 
         assert_matches!(err, Error::Unexpected { .. });
         assert!(!err.is_retryable());
@@ -396,8 +405,7 @@ mod tests {
             Ok(new_upgrade_region_reply(id, false, true, None))
         });
 
-        let instruction = &state.build_upgrade_region_instruction(&ctx);
-        let err = state.upgrade_region(&ctx, instruction).await.unwrap_err();
+        let err = state.upgrade_region(&ctx).await.unwrap_err();
 
         assert_matches!(err, Error::RetryLater { .. });
         assert!(err.is_retryable());
@@ -417,8 +425,7 @@ mod tests {
             Ok(new_upgrade_region_reply(id, false, true, None))
         });
 
-        let instruction = &state.build_upgrade_region_instruction(&ctx);
-        state.upgrade_region(&ctx, instruction).await.unwrap();
+        state.upgrade_region(&ctx).await.unwrap();
     }
 
     #[tokio::test]
@@ -537,4 +544,31 @@ mod tests {
         let update_metadata = next.as_any().downcast_ref::<UpdateMetadata>().unwrap();
         assert_matches!(update_metadata, UpdateMetadata::Rollback);
     }
+
+    #[tokio::test]
+    async fn test_upgrade_region_procedure_exceeded_deadline() {
+        let mut state = Box::<UpgradeCandidateRegion>::default();
+        state.retry_initial_interval = Duration::from_millis(100);
+        let persistent_context = new_persistent_context();
+        let to_peer_id = persistent_context.to_peer.id;
+
+        let mut env = TestingEnv::new();
+        let mut ctx = env.context_factory().new_context(persistent_context);
+        let mailbox_ctx = env.mailbox_context();
+        let mailbox = mailbox_ctx.mailbox().clone();
+        ctx.volatile_ctx.operations_elapsed = ctx.persistent_ctx.timeout + Duration::from_secs(1);
+
+        let (tx, rx) = tokio::sync::mpsc::channel(1);
+        mailbox_ctx
+            .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
+            .await;
+
+        send_mock_reply(mailbox, rx, |id| {
+            Ok(new_upgrade_region_reply(id, false, true, None))
+        });
+
+        let (next, _) = state.next(&mut ctx).await.unwrap();
+        let update_metadata = next.as_any().downcast_ref::<UpdateMetadata>().unwrap();
+        assert_matches!(update_metadata, UpdateMetadata::Rollback);
+    }
 }
diff --git a/src/meta-srv/src/region/supervisor.rs b/src/meta-srv/src/region/supervisor.rs
index 8c7dff9b3d..79b305bb80 100644
--- a/src/meta-srv/src/region/supervisor.rs
+++ b/src/meta-srv/src/region/supervisor.rs
@@ -402,7 +402,7 @@ impl RegionSupervisor {
             region_id,
             from_peer,
             to_peer,
-            replay_timeout: Duration::from_secs(60),
+            timeout: Duration::from_secs(60),
         };
 
         if let Err(err) = self.region_migration_manager.submit_procedure(task).await {
diff --git a/src/meta-srv/src/service/procedure.rs b/src/meta-srv/src/service/procedure.rs
index d19d0902ae..e20bb2c4db 100644
--- a/src/meta-srv/src/service/procedure.rs
+++ b/src/meta-srv/src/service/procedure.rs
@@ -111,8 +111,7 @@ impl procedure_service_server::ProcedureService for Metasrv {
             region_id,
             from_peer,
             to_peer,
-            replay_timeout_secs,
-            ..
+            timeout_secs,
         } = request.into_inner();
 
         let header = header.context(error::MissingRequestHeaderSnafu)?;
@@ -134,7 +133,7 @@ impl procedure_service_server::ProcedureService for Metasrv {
                 region_id: region_id.into(),
                 from_peer,
                 to_peer,
-                replay_timeout: Duration::from_secs(replay_timeout_secs.into()),
+                timeout: Duration::from_secs(timeout_secs.into()),
             })
             .await?
             .map(procedure::pid_to_pb_pid);
diff --git a/src/mito2/src/engine/catchup_test.rs b/src/mito2/src/engine/catchup_test.rs
index de72bb6128..5f4dd3b15a 100644
--- a/src/mito2/src/engine/catchup_test.rs
+++ b/src/mito2/src/engine/catchup_test.rs
@@ -19,6 +19,9 @@ use api::v1::Rows;
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use common_recordbatch::RecordBatches;
+use common_wal::options::{KafkaWalOptions, WalOptions, WAL_OPTIONS_KEY};
+use rstest::rstest;
+use rstest_reuse::{self, apply};
 use store_api::region_engine::{RegionEngine, SetReadonlyResponse};
 use store_api::region_request::{RegionCatchupRequest, RegionOpenRequest, RegionRequest};
 use store_api::storage::{RegionId, ScanRequest};
@@ -26,7 +29,8 @@ use store_api::storage::{RegionId, ScanRequest};
 use crate::config::MitoConfig;
 use crate::error::{self, Error};
 use crate::test_util::{
-    build_rows, flush_region, put_rows, rows_schema, CreateRequestBuilder, TestEnv,
+    build_rows, flush_region, kafka_log_store_factory, prepare_test_for_kafka_log_store, put_rows,
+    rows_schema, single_kafka_log_store_factory, CreateRequestBuilder, LogStoreFactory, TestEnv,
 };
 use crate::wal::EntryId;
 
@@ -38,15 +42,23 @@ fn get_last_entry_id(resp: SetReadonlyResponse) -> Option<EntryId> {
     }
 }
 
-#[tokio::test]
-async fn test_catchup_with_last_entry_id() {
+#[apply(single_kafka_log_store_factory)]
+
+async fn test_catchup_with_last_entry_id(factory: Option<LogStoreFactory>) {
     common_telemetry::init_default_ut_logging();
-    let mut env = TestEnv::with_prefix("last_entry_id");
+    let Some(factory) = factory else {
+        return;
+    };
+
+    let mut env = TestEnv::with_prefix("last_entry_id").with_log_store_factory(factory.clone());
+    let topic = prepare_test_for_kafka_log_store(&factory).await;
     let leader_engine = env.create_engine(MitoConfig::default()).await;
     let follower_engine = env.create_follower_engine(MitoConfig::default()).await;
 
     let region_id = RegionId::new(1, 1);
-    let request = CreateRequestBuilder::new().build();
+    let request = CreateRequestBuilder::new()
+        .kafka_topic(topic.clone())
+        .build();
     let region_dir = request.region_dir.clone();
 
     let column_schemas = rows_schema(&request);
@@ -55,13 +67,23 @@ async fn test_catchup_with_last_entry_id() {
         .await
         .unwrap();
 
+    let mut options = HashMap::new();
+    if let Some(topic) = &topic {
+        options.insert(
+            WAL_OPTIONS_KEY.to_string(),
+            serde_json::to_string(&WalOptions::Kafka(KafkaWalOptions {
+                topic: topic.to_string(),
+            }))
+            .unwrap(),
+        );
+    };
     follower_engine
         .handle_request(
             region_id,
             RegionRequest::Open(RegionOpenRequest {
                 engine: String::new(),
                 region_dir,
-                options: HashMap::default(),
+                options,
                 skip_wal_replay: false,
             }),
         )
@@ -135,15 +157,23 @@ async fn test_catchup_with_last_entry_id() {
     assert!(resp.is_ok());
 }
 
-#[tokio::test]
-async fn test_catchup_with_incorrect_last_entry_id() {
+#[apply(single_kafka_log_store_factory)]
+async fn test_catchup_with_incorrect_last_entry_id(factory: Option<LogStoreFactory>) {
     common_telemetry::init_default_ut_logging();
-    let mut env = TestEnv::with_prefix("incorrect_last_entry_id");
+    let Some(factory) = factory else {
+        return;
+    };
+
+    let mut env =
+        TestEnv::with_prefix("incorrect_last_entry_id").with_log_store_factory(factory.clone());
+    let topic = prepare_test_for_kafka_log_store(&factory).await;
     let leader_engine = env.create_engine(MitoConfig::default()).await;
     let follower_engine = env.create_follower_engine(MitoConfig::default()).await;
 
     let region_id = RegionId::new(1, 1);
-    let request = CreateRequestBuilder::new().build();
+    let request = CreateRequestBuilder::new()
+        .kafka_topic(topic.clone())
+        .build();
     let region_dir = request.region_dir.clone();
 
     let column_schemas = rows_schema(&request);
@@ -152,13 +182,23 @@ async fn test_catchup_with_incorrect_last_entry_id() {
         .await
         .unwrap();
 
+    let mut options = HashMap::new();
+    if let Some(topic) = &topic {
+        options.insert(
+            WAL_OPTIONS_KEY.to_string(),
+            serde_json::to_string(&WalOptions::Kafka(KafkaWalOptions {
+                topic: topic.to_string(),
+            }))
+            .unwrap(),
+        );
+    };
     follower_engine
         .handle_request(
             region_id,
             RegionRequest::Open(RegionOpenRequest {
                 engine: String::new(),
                 region_dir,
-                options: HashMap::default(),
+                options,
                 skip_wal_replay: false,
             }),
         )
@@ -217,14 +257,23 @@ async fn test_catchup_with_incorrect_last_entry_id() {
     assert!(resp.is_ok());
 }
 
-#[tokio::test]
-async fn test_catchup_without_last_entry_id() {
-    let mut env = TestEnv::with_prefix("without_last_entry_id");
+#[apply(single_kafka_log_store_factory)]
+async fn test_catchup_without_last_entry_id(factory: Option<LogStoreFactory>) {
+    common_telemetry::init_default_ut_logging();
+    let Some(factory) = factory else {
+        return;
+    };
+
+    let mut env =
+        TestEnv::with_prefix("without_last_entry_id").with_log_store_factory(factory.clone());
+    let topic = prepare_test_for_kafka_log_store(&factory).await;
     let leader_engine = env.create_engine(MitoConfig::default()).await;
     let follower_engine = env.create_follower_engine(MitoConfig::default()).await;
 
     let region_id = RegionId::new(1, 1);
-    let request = CreateRequestBuilder::new().build();
+    let request = CreateRequestBuilder::new()
+        .kafka_topic(topic.clone())
+        .build();
     let region_dir = request.region_dir.clone();
 
     let column_schemas = rows_schema(&request);
@@ -233,13 +282,23 @@ async fn test_catchup_without_last_entry_id() {
         .await
         .unwrap();
 
+    let mut options = HashMap::new();
+    if let Some(topic) = &topic {
+        options.insert(
+            WAL_OPTIONS_KEY.to_string(),
+            serde_json::to_string(&WalOptions::Kafka(KafkaWalOptions {
+                topic: topic.to_string(),
+            }))
+            .unwrap(),
+        );
+    };
     follower_engine
         .handle_request(
             region_id,
             RegionRequest::Open(RegionOpenRequest {
                 engine: String::new(),
                 region_dir,
-                options: HashMap::default(),
+                options,
                 skip_wal_replay: false,
             }),
         )
@@ -299,14 +358,23 @@ async fn test_catchup_without_last_entry_id() {
     assert!(region.is_writable());
 }
 
-#[tokio::test]
-async fn test_catchup_with_manifest_update() {
-    let mut env = TestEnv::with_prefix("without_manifest_update");
+#[apply(single_kafka_log_store_factory)]
+async fn test_catchup_with_manifest_update(factory: Option<LogStoreFactory>) {
+    common_telemetry::init_default_ut_logging();
+    let Some(factory) = factory else {
+        return;
+    };
+
+    let mut env =
+        TestEnv::with_prefix("without_manifest_update").with_log_store_factory(factory.clone());
+    let topic = prepare_test_for_kafka_log_store(&factory).await;
     let leader_engine = env.create_engine(MitoConfig::default()).await;
     let follower_engine = env.create_follower_engine(MitoConfig::default()).await;
 
     let region_id = RegionId::new(1, 1);
-    let request = CreateRequestBuilder::new().build();
+    let request = CreateRequestBuilder::new()
+        .kafka_topic(topic.clone())
+        .build();
     let region_dir = request.region_dir.clone();
 
     let column_schemas = rows_schema(&request);
@@ -315,13 +383,23 @@ async fn test_catchup_with_manifest_update() {
         .await
         .unwrap();
 
+    let mut options = HashMap::new();
+    if let Some(topic) = &topic {
+        options.insert(
+            WAL_OPTIONS_KEY.to_string(),
+            serde_json::to_string(&WalOptions::Kafka(KafkaWalOptions {
+                topic: topic.to_string(),
+            }))
+            .unwrap(),
+        );
+    };
     follower_engine
         .handle_request(
             region_id,
             RegionRequest::Open(RegionOpenRequest {
                 engine: String::new(),
                 region_dir,
-                options: HashMap::default(),
+                options,
                 skip_wal_replay: false,
             }),
         )
diff --git a/src/mito2/src/test_util.rs b/src/mito2/src/test_util.rs
index da0451bc1a..cd449e53fa 100644
--- a/src/mito2/src/test_util.rs
+++ b/src/mito2/src/test_util.rs
@@ -112,6 +112,12 @@ pub(crate) fn kafka_log_store_factory() -> Option<LogStoreFactory> {
 #[tokio::test]
 pub(crate) fn multiple_log_store_factories(#[case] factory: Option<LogStoreFactory>) {}
 
+#[template]
+#[rstest]
+#[case::with_kafka(kafka_log_store_factory())]
+#[tokio::test]
+pub(crate) fn single_kafka_log_store_factory(#[case] factory: Option<LogStoreFactory>) {}
+
 #[derive(Clone)]
 pub(crate) struct RaftEngineLogStoreFactory;
 
diff --git a/src/mito2/src/worker/handle_catchup.rs b/src/mito2/src/worker/handle_catchup.rs
index bee00fae5e..505c994d36 100644
--- a/src/mito2/src/worker/handle_catchup.rs
+++ b/src/mito2/src/worker/handle_catchup.rs
@@ -17,6 +17,7 @@
 use std::sync::Arc;
 
 use common_telemetry::info;
+use common_telemetry::tracing::warn;
 use snafu::ensure;
 use store_api::logstore::LogStore;
 use store_api::region_request::{AffectedRows, RegionCatchupRequest};
@@ -72,38 +73,42 @@ impl<S: LogStore> RegionWorkerLoop<S> {
             region
         };
 
-        let flushed_entry_id = region.version_control.current().last_entry_id;
-        info!("Trying to replay memtable for region: {region_id}, flushed entry id: {flushed_entry_id}");
-        let timer = Instant::now();
-        let wal_entry_reader =
-            self.wal
-                .wal_entry_reader(&region.provider, region_id, request.location_id);
-        let on_region_opened = self.wal.on_region_opened();
-        let last_entry_id = replay_memtable(
-            &region.provider,
-            wal_entry_reader,
-            region_id,
-            flushed_entry_id,
-            &region.version_control,
-            self.config.allow_stale_entries,
-            on_region_opened,
-        )
-        .await?;
-        info!(
-            "Elapsed: {:?}, region: {region_id} catchup finished. last entry id: {last_entry_id}, expected: {:?}.",
-            timer.elapsed(),
-            request.entry_id
-        );
-        if let Some(expected_last_entry_id) = request.entry_id {
-            ensure!(
-                // The replayed last entry id may be greater than the `expected_last_entry_id`.
-                last_entry_id >= expected_last_entry_id,
-                error::UnexpectedReplaySnafu {
-                    region_id,
-                    expected_last_entry_id,
-                    replayed_last_entry_id: last_entry_id,
-                }
+        if region.provider.is_remote_wal() {
+            let flushed_entry_id = region.version_control.current().last_entry_id;
+            info!("Trying to replay memtable for region: {region_id}, flushed entry id: {flushed_entry_id}");
+            let timer = Instant::now();
+            let wal_entry_reader =
+                self.wal
+                    .wal_entry_reader(&region.provider, region_id, request.location_id);
+            let on_region_opened = self.wal.on_region_opened();
+            let last_entry_id = replay_memtable(
+                &region.provider,
+                wal_entry_reader,
+                region_id,
+                flushed_entry_id,
+                &region.version_control,
+                self.config.allow_stale_entries,
+                on_region_opened,
             )
+            .await?;
+            info!(
+                "Elapsed: {:?}, region: {region_id} catchup finished. last entry id: {last_entry_id}, expected: {:?}.",
+                timer.elapsed(),
+                request.entry_id
+            );
+            if let Some(expected_last_entry_id) = request.entry_id {
+                ensure!(
+                    // The replayed last entry id may be greater than the `expected_last_entry_id`.
+                    last_entry_id >= expected_last_entry_id,
+                    error::UnexpectedReplaySnafu {
+                        region_id,
+                        expected_last_entry_id,
+                        replayed_last_entry_id: last_entry_id,
+                    }
+                )
+            }
+        } else {
+            warn!("Skips to replay memtable for region: {}", region.region_id);
         }
 
         if request.set_writable {
diff --git a/src/store-api/src/logstore/provider.rs b/src/store-api/src/logstore/provider.rs
index 16f907f3b4..b37b2de907 100644
--- a/src/store-api/src/logstore/provider.rs
+++ b/src/store-api/src/logstore/provider.rs
@@ -84,6 +84,11 @@ impl Provider {
         Provider::Kafka(Arc::new(KafkaProvider { topic }))
     }
 
+    /// Returns true if it's remote WAL.
+    pub fn is_remote_wal(&self) -> bool {
+        matches!(self, Provider::Kafka(_))
+    }
+
     /// Returns the type name.
     pub fn type_name(&self) -> &'static str {
         match self {

From 0f992183867c145330ec5602b6f50c13a3b9840d Mon Sep 17 00:00:00 2001
From: Ning Sun <sunng@protonmail.com>
Date: Sun, 22 Sep 2024 10:39:38 +0800
Subject: [PATCH 029/128] feat: list/array/timezone support for postgres output
 (#4727)

* feat: list/array support for postgres output

* fix: implement time zone support for postgrsql

* feat: add a geohash function that returns array

* fix: typo

* fix: lint warnings

* test: add sqlness test

* refactor: check resolution range before convert value

* fix: test result for sqlness

* feat: upgrade pgwire apis
---
 Cargo.lock                                    |  42 +-
 src/common/function/src/scalars/geo.rs        |   3 +-
 .../function/src/scalars/geo/geohash.rs       | 198 +++++++-
 src/servers/Cargo.toml                        |   2 +-
 src/servers/src/postgres.rs                   |  35 +-
 src/servers/src/postgres/auth_handler.rs      |   4 +-
 src/servers/src/postgres/handler.rs           |   6 +-
 src/servers/src/postgres/server.rs            |  10 +-
 src/servers/src/postgres/types.rs             | 448 ++++++++++++++++--
 src/servers/src/postgres/types/datetime.rs    |  61 ++-
 tests-integration/tests/sql.rs                |  32 ++
 .../standalone/common/function/geo.result     |  12 +-
 .../cases/standalone/common/function/geo.sql  |   2 +
 13 files changed, 742 insertions(+), 113 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 000d6316ec..013b0a205a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3303,9 +3303,9 @@ dependencies = [
 
 [[package]]
 name = "derive-new"
-version = "0.6.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d150dea618e920167e5973d70ae6ece4385b7164e0d799fe7c122dd0a5d912ad"
+checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5760,6 +5760,29 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "lazy-regex"
+version = "3.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d8e41c97e6bc7ecb552016274b99fbb5d035e8de288c582d9b933af6677bfda"
+dependencies = [
+ "lazy-regex-proc_macros",
+ "once_cell",
+ "regex-lite",
+]
+
+[[package]]
+name = "lazy-regex-proc_macros"
+version = "3.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76e1d8b05d672c53cb9c7b920bbba8783845ae4f0b076e02a3db1d02c81b4163"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "regex",
+ "syn 2.0.66",
+]
+
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -7856,20 +7879,22 @@ dependencies = [
 
 [[package]]
 name = "pgwire"
-version = "0.22.0"
+version = "0.24.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3770f56e1e8a608c6de40011b9a00c6b669c14d121024411701b4bc3b2a5be99"
+checksum = "ed4ca46dd335b3a030d977be54dfe121b1b9fe22aa8bbd69161ac2434524fc68"
 dependencies = [
  "async-trait",
  "bytes",
  "chrono",
- "derive-new 0.6.0",
+ "derive-new 0.7.0",
  "futures",
  "hex",
+ "lazy-regex",
  "md5",
  "postgres-types",
  "rand",
  "ring 0.17.8",
+ "rust_decimal",
  "thiserror",
  "tokio",
  "tokio-rustls 0.26.0",
@@ -9075,6 +9100,12 @@ dependencies = [
  "regex-syntax 0.8.4",
 ]
 
+[[package]]
+name = "regex-lite"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a"
+
 [[package]]
 name = "regex-syntax"
 version = "0.6.29"
@@ -9550,6 +9581,7 @@ dependencies = [
  "borsh",
  "bytes",
  "num-traits",
+ "postgres-types",
  "rand",
  "rkyv",
  "serde",
diff --git a/src/common/function/src/scalars/geo.rs b/src/common/function/src/scalars/geo.rs
index 9e415a3fdd..8ad6a7aef2 100644
--- a/src/common/function/src/scalars/geo.rs
+++ b/src/common/function/src/scalars/geo.rs
@@ -16,7 +16,7 @@ use std::sync::Arc;
 mod geohash;
 mod h3;
 
-use geohash::GeohashFunction;
+use geohash::{GeohashFunction, GeohashNeighboursFunction};
 
 use crate::function_registry::FunctionRegistry;
 
@@ -26,6 +26,7 @@ impl GeoFunctions {
     pub fn register(registry: &FunctionRegistry) {
         // geohash
         registry.register(Arc::new(GeohashFunction));
+        registry.register(Arc::new(GeohashNeighboursFunction));
         // h3 family
         registry.register(Arc::new(h3::H3LatLngToCell));
         registry.register(Arc::new(h3::H3LatLngToCellString));
diff --git a/src/common/function/src/scalars/geo/geohash.rs b/src/common/function/src/scalars/geo/geohash.rs
index 2daa8223cc..d35a6a06ff 100644
--- a/src/common/function/src/scalars/geo/geohash.rs
+++ b/src/common/function/src/scalars/geo/geohash.rs
@@ -20,23 +20,69 @@ use common_query::error::{self, InvalidFuncArgsSnafu, Result};
 use common_query::prelude::{Signature, TypeSignature};
 use datafusion::logical_expr::Volatility;
 use datatypes::prelude::ConcreteDataType;
-use datatypes::scalars::ScalarVectorBuilder;
-use datatypes::value::Value;
-use datatypes::vectors::{MutableVector, StringVectorBuilder, VectorRef};
+use datatypes::scalars::{Scalar, ScalarVectorBuilder};
+use datatypes::value::{ListValue, Value};
+use datatypes::vectors::{ListVectorBuilder, MutableVector, StringVectorBuilder, VectorRef};
 use geohash::Coord;
 use snafu::{ensure, ResultExt};
 
 use crate::function::{Function, FunctionContext};
 
+macro_rules! ensure_resolution_usize {
+    ($v: ident) => {
+        if !($v > 0 && $v <= 12) {
+            Err(BoxedError::new(PlainError::new(
+                format!("Invalid geohash resolution {}, expect value: [1, 12]", $v),
+                StatusCode::EngineExecuteQuery,
+            )))
+            .context(error::ExecuteSnafu)
+        } else {
+            Ok($v as usize)
+        }
+    };
+}
+
+fn try_into_resolution(v: Value) -> Result<usize> {
+    match v {
+        Value::Int8(v) => {
+            ensure_resolution_usize!(v)
+        }
+        Value::Int16(v) => {
+            ensure_resolution_usize!(v)
+        }
+        Value::Int32(v) => {
+            ensure_resolution_usize!(v)
+        }
+        Value::Int64(v) => {
+            ensure_resolution_usize!(v)
+        }
+        Value::UInt8(v) => {
+            ensure_resolution_usize!(v)
+        }
+        Value::UInt16(v) => {
+            ensure_resolution_usize!(v)
+        }
+        Value::UInt32(v) => {
+            ensure_resolution_usize!(v)
+        }
+        Value::UInt64(v) => {
+            ensure_resolution_usize!(v)
+        }
+        _ => unreachable!(),
+    }
+}
+
 /// Function that return geohash string for a given geospatial coordinate.
 #[derive(Clone, Debug, Default)]
 pub struct GeohashFunction;
 
-const NAME: &str = "geohash";
+impl GeohashFunction {
+    const NAME: &'static str = "geohash";
+}
 
 impl Function for GeohashFunction {
     fn name(&self) -> &str {
-        NAME
+        Self::NAME
     }
 
     fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
@@ -93,17 +139,7 @@ impl Function for GeohashFunction {
         for i in 0..size {
             let lat = lat_vec.get(i).as_f64_lossy();
             let lon = lon_vec.get(i).as_f64_lossy();
-            let r = match resolution_vec.get(i) {
-                Value::Int8(v) => v as usize,
-                Value::Int16(v) => v as usize,
-                Value::Int32(v) => v as usize,
-                Value::Int64(v) => v as usize,
-                Value::UInt8(v) => v as usize,
-                Value::UInt16(v) => v as usize,
-                Value::UInt32(v) => v as usize,
-                Value::UInt64(v) => v as usize,
-                _ => unreachable!(),
-            };
+            let r = try_into_resolution(resolution_vec.get(i))?;
 
             let result = match (lat, lon) {
                 (Some(lat), Some(lon)) => {
@@ -130,6 +166,134 @@ impl Function for GeohashFunction {
 
 impl fmt::Display for GeohashFunction {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", NAME)
+        write!(f, "{}", Self::NAME)
+    }
+}
+
+/// Function that return geohash string for a given geospatial coordinate.
+#[derive(Clone, Debug, Default)]
+pub struct GeohashNeighboursFunction;
+
+impl GeohashNeighboursFunction {
+    const NAME: &'static str = "geohash_neighbours";
+}
+
+impl Function for GeohashNeighboursFunction {
+    fn name(&self) -> &str {
+        GeohashNeighboursFunction::NAME
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::list_datatype(
+            ConcreteDataType::string_datatype(),
+        ))
+    }
+
+    fn signature(&self) -> Signature {
+        let mut signatures = Vec::new();
+        for coord_type in &[
+            ConcreteDataType::float32_datatype(),
+            ConcreteDataType::float64_datatype(),
+        ] {
+            for resolution_type in &[
+                ConcreteDataType::int8_datatype(),
+                ConcreteDataType::int16_datatype(),
+                ConcreteDataType::int32_datatype(),
+                ConcreteDataType::int64_datatype(),
+                ConcreteDataType::uint8_datatype(),
+                ConcreteDataType::uint16_datatype(),
+                ConcreteDataType::uint32_datatype(),
+                ConcreteDataType::uint64_datatype(),
+            ] {
+                signatures.push(TypeSignature::Exact(vec![
+                    // latitude
+                    coord_type.clone(),
+                    // longitude
+                    coord_type.clone(),
+                    // resolution
+                    resolution_type.clone(),
+                ]));
+            }
+        }
+        Signature::one_of(signatures, Volatility::Stable)
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 3,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect 3, provided : {}",
+                    columns.len()
+                ),
+            }
+        );
+
+        let lat_vec = &columns[0];
+        let lon_vec = &columns[1];
+        let resolution_vec = &columns[2];
+
+        let size = lat_vec.len();
+        let mut results =
+            ListVectorBuilder::with_type_capacity(ConcreteDataType::string_datatype(), size);
+
+        for i in 0..size {
+            let lat = lat_vec.get(i).as_f64_lossy();
+            let lon = lon_vec.get(i).as_f64_lossy();
+            let r = try_into_resolution(resolution_vec.get(i))?;
+
+            let result = match (lat, lon) {
+                (Some(lat), Some(lon)) => {
+                    let coord = Coord { x: lon, y: lat };
+                    let encoded = geohash::encode(coord, r)
+                        .map_err(|e| {
+                            BoxedError::new(PlainError::new(
+                                format!("Geohash error: {}", e),
+                                StatusCode::EngineExecuteQuery,
+                            ))
+                        })
+                        .context(error::ExecuteSnafu)?;
+                    let neighbours = geohash::neighbors(&encoded)
+                        .map_err(|e| {
+                            BoxedError::new(PlainError::new(
+                                format!("Geohash error: {}", e),
+                                StatusCode::EngineExecuteQuery,
+                            ))
+                        })
+                        .context(error::ExecuteSnafu)?;
+                    Some(ListValue::new(
+                        vec![
+                            neighbours.n,
+                            neighbours.nw,
+                            neighbours.w,
+                            neighbours.sw,
+                            neighbours.s,
+                            neighbours.se,
+                            neighbours.e,
+                            neighbours.ne,
+                        ]
+                        .into_iter()
+                        .map(Value::from)
+                        .collect(),
+                        ConcreteDataType::string_datatype(),
+                    ))
+                }
+                _ => None,
+            };
+
+            if let Some(list_value) = result {
+                results.push(Some(list_value.as_scalar_ref()));
+            } else {
+                results.push(None);
+            }
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+impl fmt::Display for GeohashNeighboursFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", GeohashNeighboursFunction::NAME)
     }
 }
diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml
index 54665b8c68..b94fa17d44 100644
--- a/src/servers/Cargo.toml
+++ b/src/servers/Cargo.toml
@@ -71,7 +71,7 @@ openmetrics-parser = "0.4"
 opensrv-mysql = { git = "https://github.com/datafuselabs/opensrv", rev = "6bbc3b65e6b19212c4f7fc4f40c20daf6f452deb" }
 opentelemetry-proto.workspace = true
 parking_lot = "0.12"
-pgwire = { version = "0.22", default-features = false, features = ["server-api-ring"] }
+pgwire = { version = "0.24.2", default-features = false, features = ["server-api-ring"] }
 pin-project = "1.0"
 pipeline.workspace = true
 postgres-types = { version = "0.2", features = ["with-chrono-0_4", "with-serde_json-1"] }
diff --git a/src/servers/src/postgres.rs b/src/servers/src/postgres.rs
index 36f6730b43..5e8de2294e 100644
--- a/src/servers/src/postgres.rs
+++ b/src/servers/src/postgres.rs
@@ -32,7 +32,8 @@ use std::sync::Arc;
 use ::auth::UserProviderRef;
 use derive_builder::Builder;
 use pgwire::api::auth::ServerParameterProvider;
-use pgwire::api::ClientInfo;
+use pgwire::api::copy::NoopCopyHandler;
+use pgwire::api::{ClientInfo, PgWireHandlerFactory};
 pub use server::PostgresServer;
 use session::context::Channel;
 use session::Session;
@@ -68,7 +69,7 @@ impl ServerParameterProvider for GreptimeDBStartupParameters {
     }
 }
 
-pub struct PostgresServerHandler {
+pub struct PostgresServerHandlerInner {
     query_handler: ServerSqlQueryHandlerRef,
     login_verifier: PgLoginVerifier,
     force_tls: bool,
@@ -87,10 +88,35 @@ pub(crate) struct MakePostgresServerHandler {
     force_tls: bool,
 }
 
+pub(crate) struct PostgresServerHandler(Arc<PostgresServerHandlerInner>);
+
+impl PgWireHandlerFactory for PostgresServerHandler {
+    type StartupHandler = PostgresServerHandlerInner;
+    type SimpleQueryHandler = PostgresServerHandlerInner;
+    type ExtendedQueryHandler = PostgresServerHandlerInner;
+    type CopyHandler = NoopCopyHandler;
+
+    fn simple_query_handler(&self) -> Arc<Self::SimpleQueryHandler> {
+        self.0.clone()
+    }
+
+    fn extended_query_handler(&self) -> Arc<Self::ExtendedQueryHandler> {
+        self.0.clone()
+    }
+
+    fn startup_handler(&self) -> Arc<Self::StartupHandler> {
+        self.0.clone()
+    }
+
+    fn copy_handler(&self) -> Arc<Self::CopyHandler> {
+        Arc::new(NoopCopyHandler)
+    }
+}
+
 impl MakePostgresServerHandler {
     fn make(&self, addr: Option<SocketAddr>) -> PostgresServerHandler {
         let session = Arc::new(Session::new(addr, Channel::Postgres, Default::default()));
-        PostgresServerHandler {
+        let handler = PostgresServerHandlerInner {
             query_handler: self.query_handler.clone(),
             login_verifier: PgLoginVerifier::new(self.user_provider.clone()),
             force_tls: self.force_tls,
@@ -98,6 +124,7 @@ impl MakePostgresServerHandler {
 
             session: session.clone(),
             query_parser: Arc::new(DefaultQueryParser::new(self.query_handler.clone(), session)),
-        }
+        };
+        PostgresServerHandler(Arc::new(handler))
     }
 }
diff --git a/src/servers/src/postgres/auth_handler.rs b/src/servers/src/postgres/auth_handler.rs
index 83d508215d..3f33603858 100644
--- a/src/servers/src/postgres/auth_handler.rs
+++ b/src/servers/src/postgres/auth_handler.rs
@@ -29,7 +29,7 @@ use pgwire::messages::{PgWireBackendMessage, PgWireFrontendMessage};
 use session::Session;
 use snafu::IntoError;
 
-use super::PostgresServerHandler;
+use super::PostgresServerHandlerInner;
 use crate::error::{AuthSnafu, Result};
 use crate::metrics::METRIC_AUTH_FAILURE;
 use crate::postgres::types::PgErrorCode;
@@ -127,7 +127,7 @@ where
 }
 
 #[async_trait]
-impl StartupHandler for PostgresServerHandler {
+impl StartupHandler for PostgresServerHandlerInner {
     async fn on_startup<C>(
         &self,
         client: &mut C,
diff --git a/src/servers/src/postgres/handler.rs b/src/servers/src/postgres/handler.rs
index e10a45ddec..158e2cab4d 100644
--- a/src/servers/src/postgres/handler.rs
+++ b/src/servers/src/postgres/handler.rs
@@ -39,13 +39,13 @@ use sql::dialect::PostgreSqlDialect;
 use sql::parser::{ParseOptions, ParserContext};
 
 use super::types::*;
-use super::{fixtures, PostgresServerHandler};
+use super::{fixtures, PostgresServerHandlerInner};
 use crate::error::Result;
 use crate::query_handler::sql::ServerSqlQueryHandlerRef;
 use crate::SqlPlan;
 
 #[async_trait]
-impl SimpleQueryHandler for PostgresServerHandler {
+impl SimpleQueryHandler for PostgresServerHandlerInner {
     #[tracing::instrument(skip_all, fields(protocol = "postgres"))]
     async fn do_query<'a, C>(
         &self,
@@ -237,7 +237,7 @@ impl QueryParser for DefaultQueryParser {
 }
 
 #[async_trait]
-impl ExtendedQueryHandler for PostgresServerHandler {
+impl ExtendedQueryHandler for PostgresServerHandlerInner {
     type Statement = SqlPlan;
     type QueryParser = DefaultQueryParser;
 
diff --git a/src/servers/src/postgres/server.rs b/src/servers/src/postgres/server.rs
index cca9c43181..e904845547 100644
--- a/src/servers/src/postgres/server.rs
+++ b/src/servers/src/postgres/server.rs
@@ -94,14 +94,8 @@ impl PostgresServer {
                         let _handle = io_runtime.spawn(async move {
                             crate::metrics::METRIC_POSTGRES_CONNECTIONS.inc();
                             let pg_handler = Arc::new(handler_maker.make(addr));
-                            let r = process_socket(
-                                io_stream,
-                                tls_acceptor.clone(),
-                                pg_handler.clone(),
-                                pg_handler.clone(),
-                                pg_handler,
-                            )
-                            .await;
+                            let r =
+                                process_socket(io_stream, tls_acceptor.clone(), pg_handler).await;
                             crate::metrics::METRIC_POSTGRES_CONNECTIONS.dec();
                             r
                         });
diff --git a/src/servers/src/postgres/types.rs b/src/servers/src/postgres/types.rs
index 85b626cb1a..2e4a805ef0 100644
--- a/src/servers/src/postgres/types.rs
+++ b/src/servers/src/postgres/types.rs
@@ -20,13 +20,14 @@ mod interval;
 use std::collections::HashMap;
 use std::ops::Deref;
 
-use chrono::{NaiveDate, NaiveDateTime};
+use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
 use common_time::Interval;
 use datafusion_common::ScalarValue;
 use datafusion_expr::LogicalPlan;
 use datatypes::prelude::{ConcreteDataType, Value};
 use datatypes::schema::Schema;
 use datatypes::types::TimestampType;
+use datatypes::value::ListValue;
 use pgwire::api::portal::{Format, Portal};
 use pgwire::api::results::{DataRowEncoder, FieldInfo};
 use pgwire::api::Type;
@@ -58,6 +59,317 @@ pub(super) fn schema_to_pg(origin: &Schema, field_formats: &Format) -> Result<Ve
         .collect::<Result<Vec<FieldInfo>>>()
 }
 
+fn encode_array(
+    query_ctx: &QueryContextRef,
+    value_list: &ListValue,
+    builder: &mut DataRowEncoder,
+) -> PgWireResult<()> {
+    match value_list.datatype() {
+        &ConcreteDataType::Boolean(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::Boolean(v) => Ok(Some(*v)),
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!("Invalid list item type, find {v:?}, expected bool",),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<bool>>>>()?;
+            builder.encode_field(&array)
+        }
+        &ConcreteDataType::Int8(_) | &ConcreteDataType::UInt8(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::Int8(v) => Ok(Some(*v)),
+                    Value::UInt8(v) => Ok(Some(*v as i8)),
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!(
+                            "Invalid list item type, find {v:?}, expected int8 or uint8",
+                        ),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<i8>>>>()?;
+            builder.encode_field(&array)
+        }
+        &ConcreteDataType::Int16(_) | &ConcreteDataType::UInt16(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::Int16(v) => Ok(Some(*v)),
+                    Value::UInt16(v) => Ok(Some(*v as i16)),
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!(
+                            "Invalid list item type, find {v:?}, expected int16 or uint16",
+                        ),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<i16>>>>()?;
+            builder.encode_field(&array)
+        }
+        &ConcreteDataType::Int32(_) | &ConcreteDataType::UInt32(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::Int32(v) => Ok(Some(*v)),
+                    Value::UInt32(v) => Ok(Some(*v as i32)),
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!(
+                            "Invalid list item type, find {v:?}, expected int32 or uint32",
+                        ),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<i32>>>>()?;
+            builder.encode_field(&array)
+        }
+        &ConcreteDataType::Int64(_) | &ConcreteDataType::UInt64(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::Int64(v) => Ok(Some(*v)),
+                    Value::UInt64(v) => Ok(Some(*v as i64)),
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!(
+                            "Invalid list item type, find {v:?}, expected int64 or uint64",
+                        ),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<i64>>>>()?;
+            builder.encode_field(&array)
+        }
+        &ConcreteDataType::Float32(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::Float32(v) => Ok(Some(v.0)),
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!("Invalid list item type, find {v:?}, expected float32",),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<f32>>>>()?;
+            builder.encode_field(&array)
+        }
+        &ConcreteDataType::Float64(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::Float64(v) => Ok(Some(v.0)),
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!("Invalid list item type, find {v:?}, expected float64",),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<f64>>>>()?;
+            builder.encode_field(&array)
+        }
+        &ConcreteDataType::Binary(_) => {
+            let bytea_output = query_ctx.configuration_parameter().postgres_bytea_output();
+
+            match *bytea_output {
+                PGByteaOutputValue::ESCAPE => {
+                    let array = value_list
+                        .items()
+                        .iter()
+                        .map(|v| match v {
+                            Value::Null => Ok(None),
+                            Value::Binary(v) => Ok(Some(EscapeOutputBytea(v.deref()))),
+
+                            _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                                err_msg: format!(
+                                    "Invalid list item type, find {v:?}, expected binary",
+                                ),
+                            }))),
+                        })
+                        .collect::<PgWireResult<Vec<Option<EscapeOutputBytea>>>>()?;
+                    builder.encode_field(&array)
+                }
+                PGByteaOutputValue::HEX => {
+                    let array = value_list
+                        .items()
+                        .iter()
+                        .map(|v| match v {
+                            Value::Null => Ok(None),
+                            Value::Binary(v) => Ok(Some(HexOutputBytea(v.deref()))),
+
+                            _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                                err_msg: format!(
+                                    "Invalid list item type, find {v:?}, expected binary",
+                                ),
+                            }))),
+                        })
+                        .collect::<PgWireResult<Vec<Option<HexOutputBytea>>>>()?;
+                    builder.encode_field(&array)
+                }
+            }
+        }
+        &ConcreteDataType::String(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::String(v) => Ok(Some(v.as_utf8())),
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!("Invalid list item type, find {v:?}, expected string",),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<&str>>>>()?;
+            builder.encode_field(&array)
+        }
+        &ConcreteDataType::Date(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::Date(v) => {
+                        if let Some(date) = v.to_chrono_date() {
+                            let (style, order) =
+                                *query_ctx.configuration_parameter().pg_datetime_style();
+                            Ok(Some(StylingDate(date, style, order)))
+                        } else {
+                            Err(PgWireError::ApiError(Box::new(Error::Internal {
+                                err_msg: format!("Failed to convert date to postgres type {v:?}",),
+                            })))
+                        }
+                    }
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!("Invalid list item type, find {v:?}, expected date",),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<StylingDate>>>>()?;
+            builder.encode_field(&array)
+        }
+        &ConcreteDataType::DateTime(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::DateTime(v) => {
+                        if let Some(datetime) =
+                            v.to_chrono_datetime_with_timezone(Some(&query_ctx.timezone()))
+                        {
+                            let (style, order) =
+                                *query_ctx.configuration_parameter().pg_datetime_style();
+                            Ok(Some(StylingDateTime(datetime, style, order)))
+                        } else {
+                            Err(PgWireError::ApiError(Box::new(Error::Internal {
+                                err_msg: format!("Failed to convert date to postgres type {v:?}",),
+                            })))
+                        }
+                    }
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!("Invalid list item type, find {v:?}, expected date",),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<StylingDateTime>>>>()?;
+            builder.encode_field(&array)
+        }
+        &ConcreteDataType::Timestamp(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::Timestamp(v) => {
+                        if let Some(datetime) =
+                            v.to_chrono_datetime_with_timezone(Some(&query_ctx.timezone()))
+                        {
+                            let (style, order) =
+                                *query_ctx.configuration_parameter().pg_datetime_style();
+                            Ok(Some(StylingDateTime(datetime, style, order)))
+                        } else {
+                            Err(PgWireError::ApiError(Box::new(Error::Internal {
+                                err_msg: format!("Failed to convert date to postgres type {v:?}",),
+                            })))
+                        }
+                    }
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!("Invalid list item type, find {v:?}, expected timestamp",),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<StylingDateTime>>>>()?;
+            builder.encode_field(&array)
+        }
+        &ConcreteDataType::Time(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::Time(v) => Ok(v.to_chrono_time()),
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!("Invalid list item type, find {v:?}, expected time",),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<NaiveTime>>>>()?;
+            builder.encode_field(&array)
+        }
+        &ConcreteDataType::Interval(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::Interval(v) => Ok(Some(PgInterval::from(*v))),
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!("Invalid list item type, find {v:?}, expected interval",),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<PgInterval>>>>()?;
+            builder.encode_field(&array)
+        }
+        &ConcreteDataType::Decimal128(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::Decimal128(v) => Ok(Some(v.to_string())),
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!("Invalid list item type, find {v:?}, expected decimal",),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<String>>>>()?;
+            builder.encode_field(&array)
+        }
+        &ConcreteDataType::Json(_) => {
+            let array = value_list
+                .items()
+                .iter()
+                .map(|v| match v {
+                    Value::Null => Ok(None),
+                    Value::Binary(v) => Ok(Some(jsonb::to_string(v))),
+                    _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+                        err_msg: format!("Invalid list item type, find {v:?}, expected json",),
+                    }))),
+                })
+                .collect::<PgWireResult<Vec<Option<String>>>>()?;
+            builder.encode_field(&array)
+        }
+        _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
+            err_msg: format!(
+                "cannot write array type {:?} in postgres protocol: unimplemented",
+                value_list.datatype()
+            ),
+        }))),
+    }
+}
+
 pub(super) fn encode_value(
     query_ctx: &QueryContextRef,
     value: &Value,
@@ -93,7 +405,7 @@ pub(super) fn encode_value(
         Value::Date(v) => {
             if let Some(date) = v.to_chrono_date() {
                 let (style, order) = *query_ctx.configuration_parameter().pg_datetime_style();
-                builder.encode_field(&StylingDate(&date, style, order))
+                builder.encode_field(&StylingDate(date, style, order))
             } else {
                 Err(PgWireError::ApiError(Box::new(Error::Internal {
                     err_msg: format!("Failed to convert date to postgres type {v:?}",),
@@ -101,9 +413,10 @@ pub(super) fn encode_value(
             }
         }
         Value::DateTime(v) => {
-            if let Some(datetime) = v.to_chrono_datetime() {
+            if let Some(datetime) = v.to_chrono_datetime_with_timezone(Some(&query_ctx.timezone()))
+            {
                 let (style, order) = *query_ctx.configuration_parameter().pg_datetime_style();
-                builder.encode_field(&StylingDateTime(&datetime, style, order))
+                builder.encode_field(&StylingDateTime(datetime, style, order))
             } else {
                 Err(PgWireError::ApiError(Box::new(Error::Internal {
                     err_msg: format!("Failed to convert date to postgres type {v:?}",),
@@ -111,9 +424,10 @@ pub(super) fn encode_value(
             }
         }
         Value::Timestamp(v) => {
-            if let Some(datetime) = v.to_chrono_datetime() {
+            if let Some(datetime) = v.to_chrono_datetime_with_timezone(Some(&query_ctx.timezone()))
+            {
                 let (style, order) = *query_ctx.configuration_parameter().pg_datetime_style();
-                builder.encode_field(&StylingDateTime(&datetime, style, order))
+                builder.encode_field(&StylingDateTime(datetime, style, order))
             } else {
                 Err(PgWireError::ApiError(Box::new(Error::Internal {
                     err_msg: format!("Failed to convert date to postgres type {v:?}",),
@@ -131,14 +445,13 @@ pub(super) fn encode_value(
         }
         Value::Interval(v) => builder.encode_field(&PgInterval::from(*v)),
         Value::Decimal128(v) => builder.encode_field(&v.to_string()),
-        Value::List(_) | Value::Duration(_) => {
-            Err(PgWireError::ApiError(Box::new(Error::Internal {
-                err_msg: format!(
-                    "cannot write value {:?} in postgres protocol: unimplemented",
-                    &value
-                ),
-            })))
-        }
+        Value::List(values) => encode_array(query_ctx, values, builder),
+        Value::Duration(_) => Err(PgWireError::ApiError(Box::new(Error::Internal {
+            err_msg: format!(
+                "cannot write value {:?} in postgres protocol: unimplemented",
+                &value
+            ),
+        }))),
     }
 }
 
@@ -155,19 +468,45 @@ pub(super) fn type_gt_to_pg(origin: &ConcreteDataType) -> Result<Type> {
         &ConcreteDataType::Binary(_) => Ok(Type::BYTEA),
         &ConcreteDataType::String(_) => Ok(Type::VARCHAR),
         &ConcreteDataType::Date(_) => Ok(Type::DATE),
-        &ConcreteDataType::DateTime(_) => Ok(Type::TIMESTAMP),
-        &ConcreteDataType::Timestamp(_) => Ok(Type::TIMESTAMP),
+        &ConcreteDataType::DateTime(_) | &ConcreteDataType::Timestamp(_) => Ok(Type::TIMESTAMP),
         &ConcreteDataType::Time(_) => Ok(Type::TIME),
         &ConcreteDataType::Interval(_) => Ok(Type::INTERVAL),
         &ConcreteDataType::Decimal128(_) => Ok(Type::NUMERIC),
         &ConcreteDataType::Json(_) => Ok(Type::JSON),
-        &ConcreteDataType::Duration(_)
-        | &ConcreteDataType::List(_)
-        | &ConcreteDataType::Dictionary(_) => server_error::UnsupportedDataTypeSnafu {
-            data_type: origin,
-            reason: "not implemented",
+        ConcreteDataType::List(list) => match list.item_type() {
+            &ConcreteDataType::Null(_) => Ok(Type::UNKNOWN),
+            &ConcreteDataType::Boolean(_) => Ok(Type::BOOL_ARRAY),
+            &ConcreteDataType::Int8(_) | &ConcreteDataType::UInt8(_) => Ok(Type::CHAR_ARRAY),
+            &ConcreteDataType::Int16(_) | &ConcreteDataType::UInt16(_) => Ok(Type::INT2_ARRAY),
+            &ConcreteDataType::Int32(_) | &ConcreteDataType::UInt32(_) => Ok(Type::INT4_ARRAY),
+            &ConcreteDataType::Int64(_) | &ConcreteDataType::UInt64(_) => Ok(Type::INT8_ARRAY),
+            &ConcreteDataType::Float32(_) => Ok(Type::FLOAT4_ARRAY),
+            &ConcreteDataType::Float64(_) => Ok(Type::FLOAT8_ARRAY),
+            &ConcreteDataType::Binary(_) => Ok(Type::BYTEA_ARRAY),
+            &ConcreteDataType::String(_) => Ok(Type::VARCHAR_ARRAY),
+            &ConcreteDataType::Date(_) => Ok(Type::DATE_ARRAY),
+            &ConcreteDataType::DateTime(_) | &ConcreteDataType::Timestamp(_) => {
+                Ok(Type::TIMESTAMP_ARRAY)
+            }
+            &ConcreteDataType::Time(_) => Ok(Type::TIME_ARRAY),
+            &ConcreteDataType::Interval(_) => Ok(Type::INTERVAL_ARRAY),
+            &ConcreteDataType::Decimal128(_) => Ok(Type::NUMERIC_ARRAY),
+            &ConcreteDataType::Json(_) => Ok(Type::JSON_ARRAY),
+            &ConcreteDataType::Duration(_)
+            | &ConcreteDataType::Dictionary(_)
+            | &ConcreteDataType::List(_) => server_error::UnsupportedDataTypeSnafu {
+                data_type: origin,
+                reason: "not implemented",
+            }
+            .fail(),
+        },
+        &ConcreteDataType::Duration(_) | &ConcreteDataType::Dictionary(_) => {
+            server_error::UnsupportedDataTypeSnafu {
+                data_type: origin,
+                reason: "not implemented",
+            }
+            .fail()
         }
-        .fail(),
     }
 }
 
@@ -621,6 +960,7 @@ mod test {
 
     use common_time::interval::IntervalUnit;
     use common_time::timestamp::TimeUnit;
+    use common_time::Timestamp;
     use datatypes::schema::{ColumnSchema, Schema};
     use datatypes::value::ListValue;
     use pgwire::api::results::{FieldFormat, FieldInfo};
@@ -816,6 +1156,34 @@ mod test {
                 Type::INTERVAL,
                 FieldFormat::Text,
             ),
+            FieldInfo::new(
+                "int_list".into(),
+                None,
+                None,
+                Type::INT8_ARRAY,
+                FieldFormat::Text,
+            ),
+            FieldInfo::new(
+                "float_list".into(),
+                None,
+                None,
+                Type::FLOAT8_ARRAY,
+                FieldFormat::Text,
+            ),
+            FieldInfo::new(
+                "string_list".into(),
+                None,
+                None,
+                Type::VARCHAR_ARRAY,
+                FieldFormat::Text,
+            ),
+            FieldInfo::new(
+                "timestamp_list".into(),
+                None,
+                None,
+                Type::TIMESTAMP_ARRAY,
+                FieldFormat::Text,
+            ),
         ];
 
         let datatypes = vec![
@@ -846,6 +1214,10 @@ mod test {
             ConcreteDataType::datetime_datatype(),
             ConcreteDataType::timestamp_datatype(TimeUnit::Second),
             ConcreteDataType::interval_datatype(IntervalUnit::YearMonth),
+            ConcreteDataType::list_datatype(ConcreteDataType::int64_datatype()),
+            ConcreteDataType::list_datatype(ConcreteDataType::float64_datatype()),
+            ConcreteDataType::list_datatype(ConcreteDataType::string_datatype()),
+            ConcreteDataType::list_datatype(ConcreteDataType::timestamp_second_datatype()),
         ];
         let values = vec![
             Value::Null,
@@ -875,6 +1247,22 @@ mod test {
             Value::DateTime(1000001i64.into()),
             Value::Timestamp(1000001i64.into()),
             Value::Interval(1000001i128.into()),
+            Value::List(ListValue::new(
+                vec![Value::Int64(1i64)],
+                ConcreteDataType::int64_datatype(),
+            )),
+            Value::List(ListValue::new(
+                vec![Value::Float64(1.0f64.into())],
+                ConcreteDataType::float64_datatype(),
+            )),
+            Value::List(ListValue::new(
+                vec![Value::String("tom".into())],
+                ConcreteDataType::string_datatype(),
+            )),
+            Value::List(ListValue::new(
+                vec![Value::Timestamp(Timestamp::new(1i64, TimeUnit::Second))],
+                ConcreteDataType::timestamp_second_datatype(),
+            )),
         ];
         let query_context = QueryContextBuilder::default()
             .configuration_parameter(Default::default())
@@ -884,22 +1272,6 @@ mod test {
         for (value, datatype) in values.iter().zip(datatypes) {
             encode_value(&query_context, value, &mut builder, &datatype).unwrap();
         }
-
-        let err = encode_value(
-            &query_context,
-            &Value::List(ListValue::new(vec![], ConcreteDataType::int16_datatype())),
-            &mut builder,
-            &ConcreteDataType::list_datatype(ConcreteDataType::int16_datatype()),
-        )
-        .unwrap_err();
-        match err {
-            PgWireError::ApiError(e) => {
-                assert!(format!("{e}").contains("Internal error:"));
-            }
-            _ => {
-                unreachable!()
-            }
-        }
     }
 
     #[test]
diff --git a/src/servers/src/postgres/types/datetime.rs b/src/servers/src/postgres/types/datetime.rs
index fc324c047b..700f6dc2b5 100644
--- a/src/servers/src/postgres/types/datetime.rs
+++ b/src/servers/src/postgres/types/datetime.rs
@@ -19,10 +19,10 @@ use postgres_types::{IsNull, ToSql, Type};
 use session::session_config::{PGDateOrder, PGDateTimeStyle};
 
 #[derive(Debug)]
-pub struct StylingDate<'a>(pub &'a NaiveDate, pub PGDateTimeStyle, pub PGDateOrder);
+pub struct StylingDate(pub NaiveDate, pub PGDateTimeStyle, pub PGDateOrder);
 
 #[derive(Debug)]
-pub struct StylingDateTime<'a>(pub &'a NaiveDateTime, pub PGDateTimeStyle, pub PGDateOrder);
+pub struct StylingDateTime(pub NaiveDateTime, pub PGDateTimeStyle, pub PGDateOrder);
 
 fn date_format_string(style: PGDateTimeStyle, order: PGDateOrder) -> &'static str {
     match style {
@@ -53,7 +53,7 @@ fn datetime_format_string(style: PGDateTimeStyle, order: PGDateOrder) -> &'stati
         },
     }
 }
-impl ToSqlText for StylingDate<'_> {
+impl ToSqlText for StylingDate {
     fn to_sql_text(
         &self,
         ty: &Type,
@@ -78,7 +78,7 @@ impl ToSqlText for StylingDate<'_> {
     }
 }
 
-impl ToSqlText for StylingDateTime<'_> {
+impl ToSqlText for StylingDateTime {
     fn to_sql_text(
         &self,
         ty: &Type,
@@ -112,7 +112,7 @@ impl ToSqlText for StylingDateTime<'_> {
 
 macro_rules! delegate_to_sql {
     ($delegator:ident, $delegatee:ident) => {
-        impl ToSql for $delegator<'_> {
+        impl ToSql for $delegator {
             fn to_sql(
                 &self,
                 ty: &Type,
@@ -148,7 +148,7 @@ mod tests {
         let naive_date = NaiveDate::from_ymd_opt(1997, 12, 17).unwrap();
 
         {
-            let styling_date = StylingDate(&naive_date, PGDateTimeStyle::ISO, PGDateOrder::MDY);
+            let styling_date = StylingDate(naive_date, PGDateTimeStyle::ISO, PGDateOrder::MDY);
             let expected = "1997-12-17";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_date.to_sql_text(&Type::DATE, &mut out).unwrap();
@@ -157,7 +157,7 @@ mod tests {
         }
 
         {
-            let styling_date = StylingDate(&naive_date, PGDateTimeStyle::ISO, PGDateOrder::YMD);
+            let styling_date = StylingDate(naive_date, PGDateTimeStyle::ISO, PGDateOrder::YMD);
             let expected = "1997-12-17";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_date.to_sql_text(&Type::DATE, &mut out).unwrap();
@@ -166,7 +166,7 @@ mod tests {
         }
 
         {
-            let styling_date = StylingDate(&naive_date, PGDateTimeStyle::ISO, PGDateOrder::DMY);
+            let styling_date = StylingDate(naive_date, PGDateTimeStyle::ISO, PGDateOrder::DMY);
             let expected = "1997-12-17";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_date.to_sql_text(&Type::DATE, &mut out).unwrap();
@@ -175,7 +175,7 @@ mod tests {
         }
 
         {
-            let styling_date = StylingDate(&naive_date, PGDateTimeStyle::German, PGDateOrder::MDY);
+            let styling_date = StylingDate(naive_date, PGDateTimeStyle::German, PGDateOrder::MDY);
             let expected = "17.12.1997";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_date.to_sql_text(&Type::DATE, &mut out).unwrap();
@@ -184,7 +184,7 @@ mod tests {
         }
 
         {
-            let styling_date = StylingDate(&naive_date, PGDateTimeStyle::German, PGDateOrder::YMD);
+            let styling_date = StylingDate(naive_date, PGDateTimeStyle::German, PGDateOrder::YMD);
             let expected = "17.12.1997";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_date.to_sql_text(&Type::DATE, &mut out).unwrap();
@@ -193,7 +193,7 @@ mod tests {
         }
 
         {
-            let styling_date = StylingDate(&naive_date, PGDateTimeStyle::German, PGDateOrder::DMY);
+            let styling_date = StylingDate(naive_date, PGDateTimeStyle::German, PGDateOrder::DMY);
             let expected = "17.12.1997";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_date.to_sql_text(&Type::DATE, &mut out).unwrap();
@@ -202,8 +202,7 @@ mod tests {
         }
 
         {
-            let styling_date =
-                StylingDate(&naive_date, PGDateTimeStyle::Postgres, PGDateOrder::MDY);
+            let styling_date = StylingDate(naive_date, PGDateTimeStyle::Postgres, PGDateOrder::MDY);
             let expected = "12-17-1997";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_date.to_sql_text(&Type::DATE, &mut out).unwrap();
@@ -212,8 +211,7 @@ mod tests {
         }
 
         {
-            let styling_date =
-                StylingDate(&naive_date, PGDateTimeStyle::Postgres, PGDateOrder::YMD);
+            let styling_date = StylingDate(naive_date, PGDateTimeStyle::Postgres, PGDateOrder::YMD);
             let expected = "12-17-1997";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_date.to_sql_text(&Type::DATE, &mut out).unwrap();
@@ -222,8 +220,7 @@ mod tests {
         }
 
         {
-            let styling_date =
-                StylingDate(&naive_date, PGDateTimeStyle::Postgres, PGDateOrder::DMY);
+            let styling_date = StylingDate(naive_date, PGDateTimeStyle::Postgres, PGDateOrder::DMY);
             let expected = "17-12-1997";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_date.to_sql_text(&Type::DATE, &mut out).unwrap();
@@ -232,7 +229,7 @@ mod tests {
         }
 
         {
-            let styling_date = StylingDate(&naive_date, PGDateTimeStyle::SQL, PGDateOrder::MDY);
+            let styling_date = StylingDate(naive_date, PGDateTimeStyle::SQL, PGDateOrder::MDY);
             let expected = "12/17/1997";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_date.to_sql_text(&Type::DATE, &mut out).unwrap();
@@ -241,7 +238,7 @@ mod tests {
         }
 
         {
-            let styling_date = StylingDate(&naive_date, PGDateTimeStyle::SQL, PGDateOrder::YMD);
+            let styling_date = StylingDate(naive_date, PGDateTimeStyle::SQL, PGDateOrder::YMD);
             let expected = "12/17/1997";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_date.to_sql_text(&Type::DATE, &mut out).unwrap();
@@ -250,7 +247,7 @@ mod tests {
         }
 
         {
-            let styling_date = StylingDate(&naive_date, PGDateTimeStyle::SQL, PGDateOrder::DMY);
+            let styling_date = StylingDate(naive_date, PGDateTimeStyle::SQL, PGDateOrder::DMY);
             let expected = "17/12/1997";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_date.to_sql_text(&Type::DATE, &mut out).unwrap();
@@ -266,7 +263,7 @@ mod tests {
                 .unwrap();
 
         {
-            let styling_datetime = StylingDateTime(&input, PGDateTimeStyle::ISO, PGDateOrder::MDY);
+            let styling_datetime = StylingDateTime(input, PGDateTimeStyle::ISO, PGDateOrder::MDY);
             let expected = "2021-09-01 12:34:56.789012";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_datetime
@@ -277,7 +274,7 @@ mod tests {
         }
 
         {
-            let styling_datetime = StylingDateTime(&input, PGDateTimeStyle::ISO, PGDateOrder::YMD);
+            let styling_datetime = StylingDateTime(input, PGDateTimeStyle::ISO, PGDateOrder::YMD);
             let expected = "2021-09-01 12:34:56.789012";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_datetime
@@ -288,7 +285,7 @@ mod tests {
         }
 
         {
-            let styling_datetime = StylingDateTime(&input, PGDateTimeStyle::ISO, PGDateOrder::DMY);
+            let styling_datetime = StylingDateTime(input, PGDateTimeStyle::ISO, PGDateOrder::DMY);
             let expected = "2021-09-01 12:34:56.789012";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_datetime
@@ -300,7 +297,7 @@ mod tests {
 
         {
             let styling_datetime =
-                StylingDateTime(&input, PGDateTimeStyle::German, PGDateOrder::MDY);
+                StylingDateTime(input, PGDateTimeStyle::German, PGDateOrder::MDY);
             let expected = "01.09.2021 12:34:56.789012";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_datetime
@@ -312,7 +309,7 @@ mod tests {
 
         {
             let styling_datetime =
-                StylingDateTime(&input, PGDateTimeStyle::German, PGDateOrder::YMD);
+                StylingDateTime(input, PGDateTimeStyle::German, PGDateOrder::YMD);
             let expected = "01.09.2021 12:34:56.789012";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_datetime
@@ -324,7 +321,7 @@ mod tests {
 
         {
             let styling_datetime =
-                StylingDateTime(&input, PGDateTimeStyle::German, PGDateOrder::DMY);
+                StylingDateTime(input, PGDateTimeStyle::German, PGDateOrder::DMY);
             let expected = "01.09.2021 12:34:56.789012";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_datetime
@@ -336,7 +333,7 @@ mod tests {
 
         {
             let styling_datetime =
-                StylingDateTime(&input, PGDateTimeStyle::Postgres, PGDateOrder::MDY);
+                StylingDateTime(input, PGDateTimeStyle::Postgres, PGDateOrder::MDY);
             let expected = "Wed Sep 01 12:34:56.789012 2021";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_datetime
@@ -348,7 +345,7 @@ mod tests {
 
         {
             let styling_datetime =
-                StylingDateTime(&input, PGDateTimeStyle::Postgres, PGDateOrder::YMD);
+                StylingDateTime(input, PGDateTimeStyle::Postgres, PGDateOrder::YMD);
             let expected = "Wed Sep 01 12:34:56.789012 2021";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_datetime
@@ -360,7 +357,7 @@ mod tests {
 
         {
             let styling_datetime =
-                StylingDateTime(&input, PGDateTimeStyle::Postgres, PGDateOrder::DMY);
+                StylingDateTime(input, PGDateTimeStyle::Postgres, PGDateOrder::DMY);
             let expected = "Wed 01 Sep 12:34:56.789012 2021";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_datetime
@@ -371,7 +368,7 @@ mod tests {
         }
 
         {
-            let styling_datetime = StylingDateTime(&input, PGDateTimeStyle::SQL, PGDateOrder::MDY);
+            let styling_datetime = StylingDateTime(input, PGDateTimeStyle::SQL, PGDateOrder::MDY);
             let expected = "09/01/2021 12:34:56.789012";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_datetime
@@ -382,7 +379,7 @@ mod tests {
         }
 
         {
-            let styling_datetime = StylingDateTime(&input, PGDateTimeStyle::SQL, PGDateOrder::YMD);
+            let styling_datetime = StylingDateTime(input, PGDateTimeStyle::SQL, PGDateOrder::YMD);
             let expected = "09/01/2021 12:34:56.789012";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_datetime
@@ -393,7 +390,7 @@ mod tests {
         }
 
         {
-            let styling_datetime = StylingDateTime(&input, PGDateTimeStyle::SQL, PGDateOrder::DMY);
+            let styling_datetime = StylingDateTime(input, PGDateTimeStyle::SQL, PGDateOrder::DMY);
             let expected = "01/09/2021 12:34:56.789012";
             let mut out = bytes::BytesMut::new();
             let is_null = styling_datetime
diff --git a/tests-integration/tests/sql.rs b/tests-integration/tests/sql.rs
index 1e87c54e5f..19acc37ea6 100644
--- a/tests-integration/tests/sql.rs
+++ b/tests-integration/tests/sql.rs
@@ -69,6 +69,7 @@ macro_rules! sql_tests {
                 test_postgres_bytea,
                 test_postgres_datestyle,
                 test_postgres_parameter_inference,
+                test_postgres_array_types,
                 test_mysql_prepare_stmt_insert_timestamp,
             );
         )*
@@ -1111,3 +1112,34 @@ pub async fn test_mysql_prepare_stmt_insert_timestamp(store_type: StorageType) {
     let _ = server.shutdown().await;
     guard.remove_all().await;
 }
+
+pub async fn test_postgres_array_types(store_type: StorageType) {
+    let (addr, mut guard, fe_pg_server) = setup_pg_server(store_type, "sql_inference").await;
+
+    let (client, connection) = tokio_postgres::connect(&format!("postgres://{addr}/public"), NoTls)
+        .await
+        .unwrap();
+
+    let (tx, rx) = tokio::sync::oneshot::channel();
+    tokio::spawn(async move {
+        connection.await.unwrap();
+        tx.send(()).unwrap();
+    });
+
+    let rows = client
+        .query(
+            "SELECT arrow_cast(1, 'List(Int8)'), arrow_cast('tom', 'List(Utf8)'), arrow_cast(3.14, 'List(Float32)'), arrow_cast('2023-01-02T12:53:02', 'List(Timestamp(Millisecond, None))')",
+            &[],
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(1, rows.len());
+
+    // Shutdown the client.
+    drop(client);
+    rx.await.unwrap();
+
+    let _ = fe_pg_server.shutdown().await;
+    guard.remove_all().await;
+}
diff --git a/tests/cases/standalone/common/function/geo.result b/tests/cases/standalone/common/function/geo.result
index 3a63b5890b..8954447650 100644
--- a/tests/cases/standalone/common/function/geo.result
+++ b/tests/cases/standalone/common/function/geo.result
@@ -158,11 +158,11 @@ SELECT geohash(37.76938, -122.3889, 11);
 
 SELECT geohash(37.76938, -122.3889, 100);
 
-Error: 3001(EngineExecuteQuery), Geohash error: Invalid length specified: 100. Accepted values are between 1 and 12, inclusive
+Error: 3001(EngineExecuteQuery), Invalid geohash resolution 100, expect value: [1, 12]
 
 SELECT geohash(37.76938, -122.3889, -1);
 
-Error: 3001(EngineExecuteQuery), Geohash error: Invalid length specified: 18446744073709551615. Accepted values are between 1 and 12, inclusive
+Error: 3001(EngineExecuteQuery), Invalid geohash resolution -1, expect value: [1, 12]
 
 SELECT geohash(37.76938, -122.3889, 11::Int8);
 
@@ -228,3 +228,11 @@ SELECT geohash(37.76938, -122.3889, 11::UInt64);
 | 9q8yygxneft                                                                        |
 +------------------------------------------------------------------------------------+
 
+SELECT geohash_neighbours(37.76938, -122.3889, 11);
+
++----------------------------------------------------------------------------------------------------------+
+| geohash_neighbours(Float64(37.76938),Float64(-122.3889),Int64(11))                                       |
++----------------------------------------------------------------------------------------------------------+
+| [9q8yygxnefv, 9q8yygxnefu, 9q8yygxnefs, 9q8yygxnefk, 9q8yygxnefm, 9q8yygxnefq, 9q8yygxnefw, 9q8yygxnefy] |
++----------------------------------------------------------------------------------------------------------+
+
diff --git a/tests/cases/standalone/common/function/geo.sql b/tests/cases/standalone/common/function/geo.sql
index 3a0e668acc..be2b3947bb 100644
--- a/tests/cases/standalone/common/function/geo.sql
+++ b/tests/cases/standalone/common/function/geo.sql
@@ -64,3 +64,5 @@ SELECT geohash(37.76938, -122.3889, 11::UInt16);
 SELECT geohash(37.76938, -122.3889, 11::UInt32);
 
 SELECT geohash(37.76938, -122.3889, 11::UInt64);
+
+SELECT geohash_neighbours(37.76938, -122.3889, 11);

From 2feddca1cb550a3fd896d116a67c5821897ac7ab Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Mon, 23 Sep 2024 16:35:06 +0800
Subject: [PATCH 030/128] feat: include order by to commutativity rule set
 (#4753)

* feat: include order by to commutativity rule set

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* tune sqlness replace interceptor

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 src/query/src/dist_plan/analyzer.rs           | 101 ++++++++++++++----
 src/query/src/dist_plan/commutativity.rs      |   2 +-
 .../standalone/common/order/order_by.result   |  99 +++++++++++++++++
 .../standalone/common/order/order_by.sql      |  41 +++++++
 4 files changed, 224 insertions(+), 19 deletions(-)

diff --git a/src/query/src/dist_plan/analyzer.rs b/src/query/src/dist_plan/analyzer.rs
index bbb3e5ddd9..c182eee1cd 100644
--- a/src/query/src/dist_plan/analyzer.rs
+++ b/src/query/src/dist_plan/analyzer.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashSet;
 use std::sync::Arc;
 
 use datafusion::datasource::DefaultTableSource;
@@ -19,7 +20,8 @@ use datafusion::error::Result as DfResult;
 use datafusion_common::config::ConfigOptions;
 use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRewriter};
 use datafusion_expr::expr::{Exists, InSubquery};
-use datafusion_expr::{col, Expr, LogicalPlan, LogicalPlanBuilder, Subquery};
+use datafusion_expr::utils::expr_to_columns;
+use datafusion_expr::{col as col_fn, Expr, LogicalPlan, LogicalPlanBuilder, Subquery};
 use datafusion_optimizer::analyzer::AnalyzerRule;
 use datafusion_optimizer::simplify_expressions::SimplifyExpressions;
 use datafusion_optimizer::{OptimizerContext, OptimizerRule};
@@ -104,7 +106,7 @@ impl DistPlannerAnalyzer {
             let project_exprs = output_schema
                 .fields()
                 .iter()
-                .map(|f| col(f.name()))
+                .map(|f| col_fn(f.name()))
                 .collect::<Vec<_>>();
             rewrote_subquery = LogicalPlanBuilder::from(rewrote_subquery)
                 .project(project_exprs)?
@@ -137,6 +139,7 @@ struct PlanRewriter {
     status: RewriterStatus,
     /// Partition columns of the table in current pass
     partition_cols: Option<Vec<String>>,
+    column_requirements: HashSet<String>,
 }
 
 impl PlanRewriter {
@@ -162,6 +165,7 @@ impl PlanRewriter {
             Commutativity::Commutative => {}
             Commutativity::PartialCommutative => {
                 if let Some(plan) = partial_commutative_transformer(plan) {
+                    self.update_column_requirements(&plan);
                     self.stage.push(plan)
                 }
             }
@@ -169,6 +173,7 @@ impl PlanRewriter {
                 if let Some(transformer) = transformer
                     && let Some(plan) = transformer(plan)
                 {
+                    self.update_column_requirements(&plan);
                     self.stage.push(plan)
                 }
             }
@@ -176,6 +181,7 @@ impl PlanRewriter {
                 if let Some(transformer) = transformer
                     && let Some(plan) = transformer(plan)
                 {
+                    self.update_column_requirements(&plan);
                     self.stage.push(plan)
                 }
             }
@@ -189,6 +195,18 @@ impl PlanRewriter {
         false
     }
 
+    fn update_column_requirements(&mut self, plan: &LogicalPlan) {
+        let mut container = HashSet::new();
+        for expr in plan.expressions() {
+            // this method won't fail
+            let _ = expr_to_columns(&expr, &mut container);
+        }
+
+        for col in container {
+            self.column_requirements.insert(col.flat_name());
+        }
+    }
+
     fn is_expanded(&self) -> bool {
         self.status == RewriterStatus::Expanded
     }
@@ -238,6 +256,67 @@ impl PlanRewriter {
         self.level -= 1;
         self.stack.pop();
     }
+
+    fn expand(&mut self, mut on_node: LogicalPlan) -> DfResult<LogicalPlan> {
+        let mut rewriter = EnforceDistRequirementRewriter {
+            column_requirements: std::mem::take(&mut self.column_requirements),
+        };
+        on_node = on_node.rewrite(&mut rewriter)?.data;
+
+        // add merge scan as the new root
+        let mut node = MergeScanLogicalPlan::new(on_node, false).into_logical_plan();
+        // expand stages
+        for new_stage in self.stage.drain(..) {
+            node = new_stage.with_new_exprs(new_stage.expressions(), vec![node.clone()])?
+        }
+        self.set_expanded();
+
+        Ok(node)
+    }
+}
+
+/// Implementation of the [`TreeNodeRewriter`] trait which is responsible for rewriting
+/// logical plans to enforce various requirement for distributed query.
+///
+/// Requirements enforced by this rewriter:
+/// - Enforce column requirements for `LogicalPlan::Projection` nodes. Makes sure the
+///   required columns are available in the sub plan.
+struct EnforceDistRequirementRewriter {
+    column_requirements: HashSet<String>,
+}
+
+impl TreeNodeRewriter for EnforceDistRequirementRewriter {
+    type Node = LogicalPlan;
+
+    fn f_down(&mut self, node: Self::Node) -> DfResult<Transformed<Self::Node>> {
+        if let LogicalPlan::Projection(ref projection) = node {
+            let mut column_requirements = std::mem::take(&mut self.column_requirements);
+            if column_requirements.is_empty() {
+                return Ok(Transformed::no(node));
+            }
+
+            for expr in &projection.expr {
+                column_requirements.remove(&expr.name_for_alias()?);
+            }
+            if column_requirements.is_empty() {
+                return Ok(Transformed::no(node));
+            }
+
+            let mut new_exprs = projection.expr.clone();
+            for col in &column_requirements {
+                new_exprs.push(col_fn(col));
+            }
+            let new_node =
+                node.with_new_exprs(new_exprs, node.inputs().into_iter().cloned().collect())?;
+            return Ok(Transformed::yes(new_node));
+        }
+
+        Ok(Transformed::no(node))
+    }
+
+    fn f_up(&mut self, node: Self::Node) -> DfResult<Transformed<Self::Node>> {
+        Ok(Transformed::no(node))
+    }
 }
 
 impl TreeNodeRewriter for PlanRewriter {
@@ -274,14 +353,7 @@ impl TreeNodeRewriter for PlanRewriter {
         self.maybe_set_partitions(&node);
 
         let Some(parent) = self.get_parent() else {
-            // add merge scan as the new root
-            let mut node = MergeScanLogicalPlan::new(node, false).into_logical_plan();
-            // expand stages
-            for new_stage in self.stage.drain(..) {
-                node = new_stage.with_new_exprs(node.expressions(), vec![node.clone()])?
-            }
-            self.set_expanded();
-
+            let node = self.expand(node)?;
             self.pop_stack();
             return Ok(Transformed::yes(node));
         };
@@ -289,14 +361,7 @@ impl TreeNodeRewriter for PlanRewriter {
         // TODO(ruihang): avoid this clone
         if self.should_expand(&parent.clone()) {
             // TODO(ruihang): does this work for nodes with multiple children?;
-            // replace the current node with expanded one
-            let mut node = MergeScanLogicalPlan::new(node, false).into_logical_plan();
-            // expand stages
-            for new_stage in self.stage.drain(..) {
-                node = new_stage.with_new_exprs(node.expressions(), vec![node.clone()])?
-            }
-            self.set_expanded();
-
+            let node = self.expand(node)?;
             self.pop_stack();
             return Ok(Transformed::yes(node));
         }
diff --git a/src/query/src/dist_plan/commutativity.rs b/src/query/src/dist_plan/commutativity.rs
index 6da9d4bf92..c6b3ca2c62 100644
--- a/src/query/src/dist_plan/commutativity.rs
+++ b/src/query/src/dist_plan/commutativity.rs
@@ -69,7 +69,7 @@ impl Categorizer {
 
                 // sort plan needs to consider column priority
                 // We can implement a merge-sort on partial ordered data
-                Commutativity::Unimplemented
+                Commutativity::PartialCommutative
             }
             LogicalPlan::Join(_) => Commutativity::NonCommutative,
             LogicalPlan::CrossJoin(_) => Commutativity::NonCommutative,
diff --git a/tests/cases/standalone/common/order/order_by.result b/tests/cases/standalone/common/order/order_by.result
index 2e8014c6a2..f6385537c3 100644
--- a/tests/cases/standalone/common/order/order_by.result
+++ b/tests/cases/standalone/common/order/order_by.result
@@ -223,3 +223,102 @@ DROP TABLE test;
 
 Affected Rows: 0
 
+-- ORDER BY for partition table
+CREATE TABLE IF NOT EXISTS `t` (
+  `tag` STRING NULL,
+  `ts` TIMESTAMP(3) NOT NULL,
+  `num` BIGINT NULL,
+  TIME INDEX (`ts`),
+  PRIMARY KEY (`tag`)
+)
+PARTITION ON COLUMNS (`tag`) (
+  tag <= 'z',
+  tag > 'z'
+);
+
+Affected Rows: 0
+
+INSERT INTO t (tag, ts, num) VALUES
+    ('abc', 0, 1),
+    ('abc', 3000, 2),
+    ('abc', 6000, 3),
+    ('abc', 9000, 4),
+    ('abc', 12000, 5),
+    ('zzz', 3000, 6),
+    ('zzz', 6000, 7),
+    ('zzz', 9000, 8),
+    ('zzz', 0, 9),
+    ('zzz', 3000, 10);
+
+Affected Rows: 10
+
+select * from t where num > 3 order by ts desc limit 2;
+
++-----+---------------------+-----+
+| tag | ts                  | num |
++-----+---------------------+-----+
+| abc | 1970-01-01T00:00:12 | 5   |
+| abc | 1970-01-01T00:00:09 | 4   |
++-----+---------------------+-----+
+
+select tag from t where num > 6 order by ts desc limit 2;
+
++-----+---------------------+
+| tag | ts                  |
++-----+---------------------+
+| zzz | 1970-01-01T00:00:09 |
+| zzz | 1970-01-01T00:00:06 |
++-----+---------------------+
+
+select tag from t where num > 6 order by ts;
+
++-----+---------------------+
+| tag | ts                  |
++-----+---------------------+
+| zzz | 1970-01-01T00:00:00 |
+| zzz | 1970-01-01T00:00:03 |
+| zzz | 1970-01-01T00:00:06 |
+| zzz | 1970-01-01T00:00:09 |
++-----+---------------------+
+
+-- SQLNESS REPLACE (-+) -
+-- SQLNESS REPLACE (\s\s+) _
+-- SQLNESS REPLACE (peers.*) REDACTED
+-- SQLNESS REPLACE (metrics.*) REDACTED
+-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
+-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
+explain analyze select tag from t where num > 6 order by ts desc limit 2;
+
++-+-+-+
+| stage | node | plan_|
++-+-+-+
+| 0_| 0_|_GlobalLimitExec: skip=0, fetch=2 REDACTED
+|_|_|_SortPreservingMergeExec: [ts@1 DESC] REDACTED
+|_|_|_SortExec: TopK(fetch=2), expr=[ts@1 DESC], preserve_partitioning=[true] REDACTED
+|_|_|_MergeScanExec: REDACTED
+|_|_|_|
+| 1_| 0_|_GlobalLimitExec: skip=0, fetch=2 REDACTED
+|_|_|_SortPreservingMergeExec: [ts@1 DESC] REDACTED
+|_|_|_SortExec: TopK(fetch=2), expr=[ts@1 DESC], preserve_partitioning=[true] REDACTED
+|_|_|_ProjectionExec: expr=[tag@0 as tag, ts@1 as ts] REDACTED
+|_|_|_CoalesceBatchesExec: target_batch_size=8192 REDACTED
+|_|_|_FilterExec: num@2 > 6 REDACTED
+|_|_|_RepartitionExec: partitioning=REDACTED
+|_|_|_SeqScan: region=REDACTED, partition_count=1 (1 memtable ranges, 0 file 0 ranges) REDACTED
+|_|_|_|
+| 1_| 1_|_GlobalLimitExec: skip=0, fetch=2 REDACTED
+|_|_|_SortPreservingMergeExec: [ts@1 DESC] REDACTED
+|_|_|_SortExec: TopK(fetch=2), expr=[ts@1 DESC], preserve_partitioning=[true] REDACTED
+|_|_|_ProjectionExec: expr=[tag@0 as tag, ts@1 as ts] REDACTED
+|_|_|_CoalesceBatchesExec: target_batch_size=8192 REDACTED
+|_|_|_FilterExec: num@2 > 6 REDACTED
+|_|_|_RepartitionExec: partitioning=REDACTED
+|_|_|_SeqScan: region=REDACTED, partition_count=1 (1 memtable ranges, 0 file 0 ranges) REDACTED
+|_|_|_|
+|_|_| Total rows: 2_|
++-+-+-+
+
+drop table t;
+
+Affected Rows: 0
+
diff --git a/tests/cases/standalone/common/order/order_by.sql b/tests/cases/standalone/common/order/order_by.sql
index 38ec572978..6e140e5c04 100644
--- a/tests/cases/standalone/common/order/order_by.sql
+++ b/tests/cases/standalone/common/order/order_by.sql
@@ -56,3 +56,44 @@ SELECT a-10 AS k FROM test UNION SELECT a-10 AS l FROM test ORDER BY a-10;
 SELECT a-10 AS k FROM test UNION SELECT a-11 AS l FROM test ORDER BY a-11;
 
 DROP TABLE test;
+
+-- ORDER BY for partition table
+CREATE TABLE IF NOT EXISTS `t` (
+  `tag` STRING NULL,
+  `ts` TIMESTAMP(3) NOT NULL,
+  `num` BIGINT NULL,
+  TIME INDEX (`ts`),
+  PRIMARY KEY (`tag`)
+)
+PARTITION ON COLUMNS (`tag`) (
+  tag <= 'z',
+  tag > 'z'
+);
+
+INSERT INTO t (tag, ts, num) VALUES
+    ('abc', 0, 1),
+    ('abc', 3000, 2),
+    ('abc', 6000, 3),
+    ('abc', 9000, 4),
+    ('abc', 12000, 5),
+    ('zzz', 3000, 6),
+    ('zzz', 6000, 7),
+    ('zzz', 9000, 8),
+    ('zzz', 0, 9),
+    ('zzz', 3000, 10);
+
+select * from t where num > 3 order by ts desc limit 2;
+
+select tag from t where num > 6 order by ts desc limit 2;
+
+select tag from t where num > 6 order by ts;
+
+-- SQLNESS REPLACE (-+) -
+-- SQLNESS REPLACE (\s\s+) _
+-- SQLNESS REPLACE (peers.*) REDACTED
+-- SQLNESS REPLACE (metrics.*) REDACTED
+-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
+-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
+explain analyze select tag from t where num > 6 order by ts desc limit 2;
+
+drop table t;

From 5c64f0ce0950e653ac2238c61b37b32a489810c7 Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Mon, 23 Sep 2024 18:55:20 +0800
Subject: [PATCH 031/128] refactor!: simplify NativeType trait and remove
 percentile UDAF (#4758)

* refactor!: simplify NativeType trait and remove percentile UDAF

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* remove NativeType

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* recover a mis-deleted case

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 src/common/function/src/scalars/aggregate.rs  |   3 -
 .../src/scalars/aggregate/percentile.rs       | 436 ------------------
 src/datatypes/src/types.rs                    |   2 +-
 src/datatypes/src/types/primitive_type.rs     |  28 +-
 src/query/src/tests.rs                        |   1 -
 src/query/src/tests/percentile_test.rs        |  97 ----
 .../src/python/rspython/builtins/test.rs      |   2 +-
 .../python/rspython/builtins/testcases.ron    |  19 -
 8 files changed, 5 insertions(+), 583 deletions(-)
 delete mode 100644 src/common/function/src/scalars/aggregate/percentile.rs
 delete mode 100644 src/query/src/tests/percentile_test.rs

diff --git a/src/common/function/src/scalars/aggregate.rs b/src/common/function/src/scalars/aggregate.rs
index 7ed4530983..c6875a680c 100644
--- a/src/common/function/src/scalars/aggregate.rs
+++ b/src/common/function/src/scalars/aggregate.rs
@@ -16,7 +16,6 @@ mod argmax;
 mod argmin;
 mod diff;
 mod mean;
-mod percentile;
 mod polyval;
 mod scipy_stats_norm_cdf;
 mod scipy_stats_norm_pdf;
@@ -28,7 +27,6 @@ pub use argmin::ArgminAccumulatorCreator;
 use common_query::logical_plan::AggregateFunctionCreatorRef;
 pub use diff::DiffAccumulatorCreator;
 pub use mean::MeanAccumulatorCreator;
-pub use percentile::PercentileAccumulatorCreator;
 pub use polyval::PolyvalAccumulatorCreator;
 pub use scipy_stats_norm_cdf::ScipyStatsNormCdfAccumulatorCreator;
 pub use scipy_stats_norm_pdf::ScipyStatsNormPdfAccumulatorCreator;
@@ -91,7 +89,6 @@ impl AggregateFunctions {
         register_aggr_func!("polyval", 2, PolyvalAccumulatorCreator);
         register_aggr_func!("argmax", 1, ArgmaxAccumulatorCreator);
         register_aggr_func!("argmin", 1, ArgminAccumulatorCreator);
-        register_aggr_func!("percentile", 2, PercentileAccumulatorCreator);
         register_aggr_func!("scipystatsnormcdf", 2, ScipyStatsNormCdfAccumulatorCreator);
         register_aggr_func!("scipystatsnormpdf", 2, ScipyStatsNormPdfAccumulatorCreator);
     }
diff --git a/src/common/function/src/scalars/aggregate/percentile.rs b/src/common/function/src/scalars/aggregate/percentile.rs
deleted file mode 100644
index 2d3e5482fe..0000000000
--- a/src/common/function/src/scalars/aggregate/percentile.rs
+++ /dev/null
@@ -1,436 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::cmp::Reverse;
-use std::collections::BinaryHeap;
-use std::sync::Arc;
-
-use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
-use common_query::error::{
-    self, BadAccumulatorImplSnafu, CreateAccumulatorSnafu, DowncastVectorSnafu,
-    FromScalarValueSnafu, InvalidInputColSnafu, Result,
-};
-use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
-use common_query::prelude::*;
-use datatypes::prelude::*;
-use datatypes::types::OrdPrimitive;
-use datatypes::value::{ListValue, OrderedFloat};
-use datatypes::vectors::{ConstantVector, Float64Vector, Helper, ListVector};
-use datatypes::with_match_primitive_type_id;
-use num::NumCast;
-use snafu::{ensure, OptionExt, ResultExt};
-
-// https://numpy.org/doc/stable/reference/generated/numpy.percentile.html?highlight=percentile#numpy.percentile
-// if the p is 50,then the Percentile become median
-// we use two heap great and not_greater
-// the not_greater push the value that smaller than P-value
-// the greater push the value that bigger than P-value
-// just like the percentile in numpy:
-// Given a vector V of length N, the q-th percentile of V is the value q/100 of the way from the minimum to the maximum in a sorted copy of V.
-// The values and distances of the two nearest neighbors as well as the method parameter will determine the percentile
-// if the normalized ranking does not match the location of q exactly.
-// This function is the same as the median if q=50, the same as the minimum if q=0 and the same as the maximum if q=100.
-// This optional method parameter specifies the method to use when the desired quantile lies between two data points i < j.
-// If g is the fractional part of the index surrounded by i and alpha and beta are correction constants modifying i and j.
-//              i+g = (q-alpha)/(n-alpha-beta+1)
-// Below, 'q' is the quantile value, 'n' is the sample size and alpha and beta are constants. The following formula gives an interpolation "i + g" of where the quantile would be in the sorted sample.
-// With 'i' being the floor and 'g' the fractional part of the result.
-// the default method is linear where
-// alpha = 1
-// beta = 1
-#[derive(Debug, Default)]
-pub struct Percentile<T>
-where
-    T: WrapperType,
-{
-    greater: BinaryHeap<Reverse<OrdPrimitive<T>>>,
-    not_greater: BinaryHeap<OrdPrimitive<T>>,
-    n: u64,
-    p: Option<f64>,
-}
-
-impl<T> Percentile<T>
-where
-    T: WrapperType,
-{
-    fn push(&mut self, value: T) {
-        let value = OrdPrimitive::<T>(value);
-
-        self.n += 1;
-        if self.not_greater.is_empty() {
-            self.not_greater.push(value);
-            return;
-        }
-        // to keep the not_greater length == floor+1
-        // so to ensure the peek of the not_greater is array[floor]
-        // and the peek of the greater is array[floor+1]
-        let p = self.p.unwrap_or(0.0_f64);
-        let floor = (((self.n - 1) as f64) * p / (100_f64)).floor();
-        if value <= *self.not_greater.peek().unwrap() {
-            self.not_greater.push(value);
-            if self.not_greater.len() > (floor + 1.0) as usize {
-                self.greater.push(Reverse(self.not_greater.pop().unwrap()));
-            }
-        } else {
-            self.greater.push(Reverse(value));
-            if self.not_greater.len() < (floor + 1.0) as usize {
-                self.not_greater.push(self.greater.pop().unwrap().0);
-            }
-        }
-    }
-}
-
-impl<T> Accumulator for Percentile<T>
-where
-    T: WrapperType,
-{
-    fn state(&self) -> Result<Vec<Value>> {
-        let nums = self
-            .greater
-            .iter()
-            .map(|x| &x.0)
-            .chain(self.not_greater.iter())
-            .map(|&n| n.into())
-            .collect::<Vec<Value>>();
-        Ok(vec![
-            Value::List(ListValue::new(nums, T::LogicalType::build_data_type())),
-            self.p.into(),
-        ])
-    }
-
-    fn update_batch(&mut self, values: &[VectorRef]) -> Result<()> {
-        if values.is_empty() {
-            return Ok(());
-        }
-        ensure!(values.len() == 2, InvalidInputStateSnafu);
-        ensure!(values[0].len() == values[1].len(), InvalidInputStateSnafu);
-
-        if values[0].len() == 0 {
-            return Ok(());
-        }
-
-        // This is a unary accumulator, so only one column is provided.
-        let column = &values[0];
-        let mut len = 1;
-        let column: &<T as Scalar>::VectorType = if column.is_const() {
-            len = column.len();
-            let column: &ConstantVector = unsafe { Helper::static_cast(column) };
-            unsafe { Helper::static_cast(column.inner()) }
-        } else {
-            unsafe { Helper::static_cast(column) }
-        };
-
-        let x = &values[1];
-        let x = Helper::check_get_scalar::<f64>(x).context(error::InvalidInputTypeSnafu {
-            err_msg: "expecting \"POLYVAL\" function's second argument to be float64",
-        })?;
-        // `get(0)` is safe because we have checked `values[1].len() == values[0].len() != 0`
-        let first = x.get(0);
-        ensure!(!first.is_null(), InvalidInputColSnafu);
-
-        for i in 1..x.len() {
-            ensure!(first == x.get(i), InvalidInputColSnafu);
-        }
-
-        let first = match first {
-            Value::Float64(OrderedFloat(v)) => v,
-            // unreachable because we have checked `first` is not null and is i64 above
-            _ => unreachable!(),
-        };
-        if let Some(p) = self.p {
-            ensure!(p == first, InvalidInputColSnafu);
-        } else {
-            self.p = Some(first);
-        };
-
-        (0..len).for_each(|_| {
-            for v in column.iter_data().flatten() {
-                self.push(v);
-            }
-        });
-        Ok(())
-    }
-
-    fn merge_batch(&mut self, states: &[VectorRef]) -> Result<()> {
-        if states.is_empty() {
-            return Ok(());
-        }
-
-        ensure!(
-            states.len() == 2,
-            BadAccumulatorImplSnafu {
-                err_msg: "expect 2 states in `merge_batch`"
-            }
-        );
-
-        let p = &states[1];
-        let p = p
-            .as_any()
-            .downcast_ref::<Float64Vector>()
-            .with_context(|| DowncastVectorSnafu {
-                err_msg: format!(
-                    "expect float64vector, got vector type {}",
-                    p.vector_type_name()
-                ),
-            })?;
-        let p = p.get(0);
-        if p.is_null() {
-            return Ok(());
-        }
-        let p = match p {
-            Value::Float64(OrderedFloat(p)) => p,
-            _ => unreachable!(),
-        };
-        self.p = Some(p);
-
-        let values = &states[0];
-        let values = values
-            .as_any()
-            .downcast_ref::<ListVector>()
-            .with_context(|| DowncastVectorSnafu {
-                err_msg: format!(
-                    "expect ListVector, got vector type {}",
-                    values.vector_type_name()
-                ),
-            })?;
-        for value in values.values_iter() {
-            if let Some(value) = value.context(FromScalarValueSnafu)? {
-                let column: &<T as Scalar>::VectorType = unsafe { Helper::static_cast(&value) };
-                for v in column.iter_data().flatten() {
-                    self.push(v);
-                }
-            }
-        }
-        Ok(())
-    }
-
-    fn evaluate(&self) -> Result<Value> {
-        if self.not_greater.is_empty() {
-            assert!(
-                self.greater.is_empty(),
-                "not expected in two-heap percentile algorithm, there must be a bug when implementing it"
-            );
-        }
-        let not_greater = self.not_greater.peek();
-        if not_greater.is_none() {
-            return Ok(Value::Null);
-        }
-        let not_greater = (*self.not_greater.peek().unwrap()).as_primitive();
-        let percentile = if self.greater.is_empty() {
-            NumCast::from(not_greater).unwrap()
-        } else {
-            let greater = self.greater.peek().unwrap();
-            let p = if let Some(p) = self.p {
-                p
-            } else {
-                return Ok(Value::Null);
-            };
-            let fract = (((self.n - 1) as f64) * p / 100_f64).fract();
-            let not_greater_v: f64 = NumCast::from(not_greater).unwrap();
-            let greater_v: f64 = NumCast::from(greater.0.as_primitive()).unwrap();
-            not_greater_v * (1.0 - fract) + greater_v * fract
-        };
-        Ok(Value::from(percentile))
-    }
-}
-
-#[as_aggr_func_creator]
-#[derive(Debug, Default, AggrFuncTypeStore)]
-pub struct PercentileAccumulatorCreator {}
-
-impl AggregateFunctionCreator for PercentileAccumulatorCreator {
-    fn creator(&self) -> AccumulatorCreatorFunction {
-        let creator: AccumulatorCreatorFunction = Arc::new(move |types: &[ConcreteDataType]| {
-            let input_type = &types[0];
-            with_match_primitive_type_id!(
-                input_type.logical_type_id(),
-                |$S| {
-                    Ok(Box::new(Percentile::<<$S as LogicalPrimitiveType>::Wrapper>::default()))
-                },
-                {
-                    let err_msg = format!(
-                        "\"PERCENTILE\" aggregate function not support data type {:?}",
-                        input_type.logical_type_id(),
-                    );
-                    CreateAccumulatorSnafu { err_msg }.fail()?
-                }
-            )
-        });
-        creator
-    }
-
-    fn output_type(&self) -> Result<ConcreteDataType> {
-        let input_types = self.input_types()?;
-        ensure!(input_types.len() == 2, InvalidInputStateSnafu);
-        // unwrap is safe because we have checked input_types len must equals 1
-        Ok(ConcreteDataType::float64_datatype())
-    }
-
-    fn state_types(&self) -> Result<Vec<ConcreteDataType>> {
-        let input_types = self.input_types()?;
-        ensure!(input_types.len() == 2, InvalidInputStateSnafu);
-        Ok(vec![
-            ConcreteDataType::list_datatype(input_types.into_iter().next().unwrap()),
-            ConcreteDataType::float64_datatype(),
-        ])
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use datatypes::vectors::{Float64Vector, Int32Vector};
-
-    use super::*;
-    #[test]
-    fn test_update_batch() {
-        // test update empty batch, expect not updating anything
-        let mut percentile = Percentile::<i32>::default();
-        percentile.update_batch(&[]).unwrap();
-        assert!(percentile.not_greater.is_empty());
-        assert!(percentile.greater.is_empty());
-        assert_eq!(Value::Null, percentile.evaluate().unwrap());
-
-        // test update one not-null value
-        let mut percentile = Percentile::<i32>::default();
-        let v: Vec<VectorRef> = vec![
-            Arc::new(Int32Vector::from(vec![Some(42)])),
-            Arc::new(Float64Vector::from(vec![Some(100.0_f64)])),
-        ];
-        percentile.update_batch(&v).unwrap();
-        assert_eq!(Value::from(42.0_f64), percentile.evaluate().unwrap());
-
-        // test update one null value
-        let mut percentile = Percentile::<i32>::default();
-        let v: Vec<VectorRef> = vec![
-            Arc::new(Int32Vector::from(vec![Option::<i32>::None])),
-            Arc::new(Float64Vector::from(vec![Some(100.0_f64)])),
-        ];
-        percentile.update_batch(&v).unwrap();
-        assert_eq!(Value::Null, percentile.evaluate().unwrap());
-
-        // test update no null-value batch
-        let mut percentile = Percentile::<i32>::default();
-        let v: Vec<VectorRef> = vec![
-            Arc::new(Int32Vector::from(vec![Some(-1i32), Some(1), Some(2)])),
-            Arc::new(Float64Vector::from(vec![
-                Some(100.0_f64),
-                Some(100.0_f64),
-                Some(100.0_f64),
-            ])),
-        ];
-        percentile.update_batch(&v).unwrap();
-        assert_eq!(Value::from(2_f64), percentile.evaluate().unwrap());
-
-        // test update null-value batch
-        let mut percentile = Percentile::<i32>::default();
-        let v: Vec<VectorRef> = vec![
-            Arc::new(Int32Vector::from(vec![Some(-2i32), None, Some(3), Some(4)])),
-            Arc::new(Float64Vector::from(vec![
-                Some(100.0_f64),
-                Some(100.0_f64),
-                Some(100.0_f64),
-                Some(100.0_f64),
-            ])),
-        ];
-        percentile.update_batch(&v).unwrap();
-        assert_eq!(Value::from(4_f64), percentile.evaluate().unwrap());
-
-        // test update with constant vector
-        let mut percentile = Percentile::<i32>::default();
-        let v: Vec<VectorRef> = vec![
-            Arc::new(ConstantVector::new(
-                Arc::new(Int32Vector::from_vec(vec![4])),
-                2,
-            )),
-            Arc::new(Float64Vector::from(vec![Some(100.0_f64), Some(100.0_f64)])),
-        ];
-        percentile.update_batch(&v).unwrap();
-        assert_eq!(Value::from(4_f64), percentile.evaluate().unwrap());
-
-        // test left border
-        let mut percentile = Percentile::<i32>::default();
-        let v: Vec<VectorRef> = vec![
-            Arc::new(Int32Vector::from(vec![Some(-1i32), Some(1), Some(2)])),
-            Arc::new(Float64Vector::from(vec![
-                Some(0.0_f64),
-                Some(0.0_f64),
-                Some(0.0_f64),
-            ])),
-        ];
-        percentile.update_batch(&v).unwrap();
-        assert_eq!(Value::from(-1.0_f64), percentile.evaluate().unwrap());
-
-        // test medium
-        let mut percentile = Percentile::<i32>::default();
-        let v: Vec<VectorRef> = vec![
-            Arc::new(Int32Vector::from(vec![Some(-1i32), Some(1), Some(2)])),
-            Arc::new(Float64Vector::from(vec![
-                Some(50.0_f64),
-                Some(50.0_f64),
-                Some(50.0_f64),
-            ])),
-        ];
-        percentile.update_batch(&v).unwrap();
-        assert_eq!(Value::from(1.0_f64), percentile.evaluate().unwrap());
-
-        // test right border
-        let mut percentile = Percentile::<i32>::default();
-        let v: Vec<VectorRef> = vec![
-            Arc::new(Int32Vector::from(vec![Some(-1i32), Some(1), Some(2)])),
-            Arc::new(Float64Vector::from(vec![
-                Some(100.0_f64),
-                Some(100.0_f64),
-                Some(100.0_f64),
-            ])),
-        ];
-        percentile.update_batch(&v).unwrap();
-        assert_eq!(Value::from(2.0_f64), percentile.evaluate().unwrap());
-
-        // the following is the result of numpy.percentile
-        // numpy.percentile
-        // a = np.array([[10,7,4]])
-        // np.percentile(a,40)
-        // >> 6.400000000000
-        let mut percentile = Percentile::<i32>::default();
-        let v: Vec<VectorRef> = vec![
-            Arc::new(Int32Vector::from(vec![Some(10i32), Some(7), Some(4)])),
-            Arc::new(Float64Vector::from(vec![
-                Some(40.0_f64),
-                Some(40.0_f64),
-                Some(40.0_f64),
-            ])),
-        ];
-        percentile.update_batch(&v).unwrap();
-        assert_eq!(Value::from(6.400000000_f64), percentile.evaluate().unwrap());
-
-        // the following is the result of numpy.percentile
-        // a = np.array([[10,7,4]])
-        // np.percentile(a,95)
-        // >> 9.7000000000000011
-        let mut percentile = Percentile::<i32>::default();
-        let v: Vec<VectorRef> = vec![
-            Arc::new(Int32Vector::from(vec![Some(10i32), Some(7), Some(4)])),
-            Arc::new(Float64Vector::from(vec![
-                Some(95.0_f64),
-                Some(95.0_f64),
-                Some(95.0_f64),
-            ])),
-        ];
-        percentile.update_batch(&v).unwrap();
-        assert_eq!(
-            Value::from(9.700_000_000_000_001_f64),
-            percentile.evaluate().unwrap()
-        );
-    }
-}
diff --git a/src/datatypes/src/types.rs b/src/datatypes/src/types.rs
index 0bedd2965c..a0e6d501a6 100644
--- a/src/datatypes/src/types.rs
+++ b/src/datatypes/src/types.rs
@@ -48,7 +48,7 @@ pub use list_type::ListType;
 pub use null_type::NullType;
 pub use primitive_type::{
     Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, LogicalPrimitiveType,
-    NativeType, OrdPrimitive, UInt16Type, UInt32Type, UInt64Type, UInt8Type, WrapperType,
+    OrdPrimitive, UInt16Type, UInt32Type, UInt64Type, UInt8Type, WrapperType,
 };
 pub use string_type::StringType;
 pub use time_type::{
diff --git a/src/datatypes/src/types/primitive_type.rs b/src/datatypes/src/types/primitive_type.rs
index 800be3d11b..cae71976fd 100644
--- a/src/datatypes/src/types/primitive_type.rs
+++ b/src/datatypes/src/types/primitive_type.rs
@@ -18,7 +18,6 @@ use std::fmt;
 use arrow::datatypes::{ArrowNativeType, ArrowPrimitiveType, DataType as ArrowDataType};
 use common_time::interval::IntervalUnit;
 use common_time::{Date, DateTime};
-use num::NumCast;
 use serde::{Deserialize, Serialize};
 use snafu::OptionExt;
 
@@ -31,27 +30,6 @@ use crate::types::{DateTimeType, DateType};
 use crate::value::{Value, ValueRef};
 use crate::vectors::{MutableVector, PrimitiveVector, PrimitiveVectorBuilder, Vector};
 
-/// Data types that can be used as arrow's native type.
-pub trait NativeType: ArrowNativeType + NumCast {}
-
-macro_rules! impl_native_type {
-    ($Type: ident) => {
-        impl NativeType for $Type {}
-    };
-}
-
-impl_native_type!(u8);
-impl_native_type!(u16);
-impl_native_type!(u32);
-impl_native_type!(u64);
-impl_native_type!(i8);
-impl_native_type!(i16);
-impl_native_type!(i32);
-impl_native_type!(i64);
-impl_native_type!(i128);
-impl_native_type!(f32);
-impl_native_type!(f64);
-
 /// Represents the wrapper type that wraps a native type using the `newtype pattern`,
 /// such as [Date](`common_time::Date`) is a wrapper type for the underlying native
 /// type `i32`.
@@ -70,7 +48,7 @@ pub trait WrapperType:
     /// Logical primitive type that this wrapper type belongs to.
     type LogicalType: LogicalPrimitiveType<Wrapper = Self, Native = Self::Native>;
     /// The underlying native type.
-    type Native: NativeType;
+    type Native: ArrowNativeType;
 
     /// Convert native type into this wrapper type.
     fn from_native(value: Self::Native) -> Self;
@@ -84,7 +62,7 @@ pub trait LogicalPrimitiveType: 'static + Sized {
     /// Arrow primitive type of this logical type.
     type ArrowPrimitive: ArrowPrimitiveType<Native = Self::Native>;
     /// Native (physical) type of this logical type.
-    type Native: NativeType;
+    type Native: ArrowNativeType;
     /// Wrapper type that the vector returns.
     type Wrapper: WrapperType<LogicalType = Self, Native = Self::Native>
         + for<'a> Scalar<VectorType = PrimitiveVector<Self>, RefType<'a> = Self::Wrapper>
@@ -107,7 +85,7 @@ pub trait LogicalPrimitiveType: 'static + Sized {
 
 /// A new type for [WrapperType], complement the `Ord` feature for it. Wrapping non ordered
 /// primitive types like `f32` and `f64` in `OrdPrimitive` can make them be used in places that
-/// require `Ord`. For example, in `Median` or `Percentile` UDAFs.
+/// require `Ord`. For example, in `Median` UDAFs.
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub struct OrdPrimitive<T: WrapperType>(pub T);
 
diff --git a/src/query/src/tests.rs b/src/query/src/tests.rs
index 3ecd69a455..1fee632073 100644
--- a/src/query/src/tests.rs
+++ b/src/query/src/tests.rs
@@ -25,7 +25,6 @@ mod argmax_test;
 mod argmin_test;
 mod mean_test;
 mod my_sum_udaf_example;
-mod percentile_test;
 mod polyval_test;
 mod query_engine_test;
 mod scipy_stats_norm_cdf_test;
diff --git a/src/query/src/tests/percentile_test.rs b/src/query/src/tests/percentile_test.rs
deleted file mode 100644
index 1c01401b89..0000000000
--- a/src/query/src/tests/percentile_test.rs
+++ /dev/null
@@ -1,97 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-
-use common_recordbatch::RecordBatch;
-use datatypes::for_all_primitive_types;
-use datatypes::prelude::*;
-use datatypes::schema::{ColumnSchema, Schema};
-use datatypes::vectors::Int32Vector;
-use function::{create_query_engine, get_numbers_from_table};
-use num_traits::AsPrimitive;
-use table::test_util::MemTable;
-
-use super::new_query_engine_with_table;
-use crate::error::Result;
-use crate::tests::{exec_selection, function};
-use crate::QueryEngine;
-
-#[tokio::test]
-async fn test_percentile_aggregator() -> Result<()> {
-    common_telemetry::init_default_ut_logging();
-    let engine = create_query_engine();
-
-    macro_rules! test_percentile {
-        ([], $( { $T:ty } ),*) => {
-            $(
-                let column_name = format!("{}_number", std::any::type_name::<$T>());
-                test_percentile_success::<$T>(&column_name, "numbers", engine.clone()).await?;
-            )*
-        }
-    }
-    for_all_primitive_types! { test_percentile }
-    Ok(())
-}
-
-#[tokio::test]
-async fn test_percentile_correctness() -> Result<()> {
-    let engine = create_correctness_engine();
-    let sql = String::from("select PERCENTILE(corr_number,88.0) as percentile from corr_numbers");
-    let record_batch = exec_selection(engine, &sql).await;
-    let column = record_batch[0].column(0);
-    let value = column.get(0);
-    assert_eq!(value, Value::from(9.280_000_000_000_001_f64));
-    Ok(())
-}
-
-async fn test_percentile_success<T>(
-    column_name: &str,
-    table_name: &str,
-    engine: Arc<dyn QueryEngine>,
-) -> Result<()>
-where
-    T: WrapperType + AsPrimitive<f64>,
-{
-    let sql = format!("select PERCENTILE({column_name},50.0) as percentile from {table_name}");
-    let result = exec_selection(engine.clone(), &sql).await;
-    let value = function::get_value_from_batches("percentile", result);
-
-    let numbers = get_numbers_from_table::<T>(column_name, table_name, engine.clone()).await;
-    let expected_value = numbers.iter().map(|&n| n.as_()).collect::<Vec<f64>>();
-
-    let expected_value: inc_stats::Percentiles<f64> = expected_value.iter().cloned().collect();
-    let expected_value = expected_value.percentile(0.5).unwrap();
-    assert_eq!(value, expected_value.into());
-    Ok(())
-}
-
-fn create_correctness_engine() -> Arc<dyn QueryEngine> {
-    // create engine
-
-    let mut column_schemas = vec![];
-    let mut columns = vec![];
-
-    let column_schema = ColumnSchema::new("corr_number", ConcreteDataType::int32_datatype(), true);
-    column_schemas.push(column_schema);
-
-    let numbers = [3_i32, 6_i32, 8_i32, 10_i32];
-
-    let column: VectorRef = Arc::new(Int32Vector::from_slice(numbers));
-    columns.push(column);
-
-    let schema = Arc::new(Schema::new(column_schemas));
-    let number_table = MemTable::table("corr_numbers", RecordBatch::new(schema, columns).unwrap());
-    new_query_engine_with_table(number_table)
-}
diff --git a/src/script/src/python/rspython/builtins/test.rs b/src/script/src/python/rspython/builtins/test.rs
index a108663ad7..3908ab9eef 100644
--- a/src/script/src/python/rspython/builtins/test.rs
+++ b/src/script/src/python/rspython/builtins/test.rs
@@ -345,7 +345,7 @@ fn run_builtin_fn_testcases() {
                     match case.expect{
                         Ok(v) => {
                             error!("\nError:\n{err_res}");
-                            panic!("Expect Ok: {v:?}, found Error");
+                            panic!("Expect Ok: {v:?}, found Error in case {}", case.script);
                         },
                         Err(err) => {
                             if !err_res.contains(&err){
diff --git a/src/script/src/python/rspython/builtins/testcases.ron b/src/script/src/python/rspython/builtins/testcases.ron
index 0b053538b4..0022d0e874 100644
--- a/src/script/src/python/rspython/builtins/testcases.ron
+++ b/src/script/src/python/rspython/builtins/testcases.ron
@@ -989,25 +989,6 @@ argmin(p)"#,
         },
         script: r#"
 from greptime import *
-percentile(x, p)"#,
-        expect: Ok((
-            ty: Float64,
-            value: Float(-0.97)
-        ))
-    ),
-    TestCase(
-        input: {
-            "x": Var(
-                ty: Float64,
-                value: FloatVec([-1.0, 2.0, 3.0])
-            ),
-            "p": Var(
-                ty: Float64,
-                value: FloatVec([0.5, 0.5, 0.5])
-            )
-        },
-        script: r#"
-from greptime import *
 scipy_stats_norm_cdf(x, p)"#,
         expect: Ok((
             ty: Float64,

From 54f6e13d13a7035f3b3a0deeac6fc294a0f66507 Mon Sep 17 00:00:00 2001
From: discord9 <55937128+discord9@users.noreply.github.com>
Date: Tue, 24 Sep 2024 11:11:12 +0800
Subject: [PATCH 032/128] fix: Release CI & make rustls use `ring` (#4750)

* feat: new dev-build

* fix: copy

* chore: rm dbg run

* fix: binstall install script

* chore: typos

* fix: update useable image

* fix: properly uninstall gcc-9

* chore: another try

* chore: print current cc version

* chore: another try to release

* fix: use gcc-10 by `update-alternatives`

* chore: update dev-build image

* remove gcc-9 again and install make/cmake

* use new image

* alias gcc-10/g++-10 to every variant

* again.....

* fix....

* again release ....

* chore: remove auto remove

* feat: rustls default to ring

* chore: update Cargo.lock

* chore: update Cargo.lock

* Apply suggestions from code review

---------

Co-authored-by: Ruihang Xia <waynestxia@gmail.com>
---
 Cargo.lock                                   | 1850 ++++++++----------
 Cargo.toml                                   |    9 +
 Makefile                                     |    2 +-
 docker/dev-builder/binstall/pull_binstall.sh |   50 +
 docker/dev-builder/centos/Dockerfile         |    4 +-
 docker/dev-builder/ubuntu/Dockerfile         |   11 +-
 6 files changed, 924 insertions(+), 1002 deletions(-)
 create mode 100755 docker/dev-builder/binstall/pull_binstall.sh

diff --git a/Cargo.lock b/Cargo.lock
index 013b0a205a..991829a758 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -10,18 +10,18 @@ checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3"
 
 [[package]]
 name = "addr2line"
-version = "0.22.0"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678"
+checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375"
 dependencies = [
  "gimli",
 ]
 
 [[package]]
-name = "adler"
-version = "1.0.2"
+name = "adler2"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
 
 [[package]]
 name = "adler32"
@@ -147,9 +147,9 @@ dependencies = [
 
 [[package]]
 name = "anstream"
-version = "0.6.14"
+version = "0.6.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b"
+checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
 dependencies = [
  "anstyle",
  "anstyle-parse",
@@ -162,33 +162,33 @@ dependencies = [
 
 [[package]]
 name = "anstyle"
-version = "1.0.7"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"
+checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.2.4"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4"
+checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
 dependencies = [
  "utf8parse",
 ]
 
 [[package]]
 name = "anstyle-query"
-version = "1.1.0"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391"
+checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
 dependencies = [
  "windows-sys 0.52.0",
 ]
 
 [[package]]
 name = "anstyle-wincon"
-version = "3.0.3"
+version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19"
+checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
 dependencies = [
  "anstyle",
  "windows-sys 0.52.0",
@@ -196,9 +196,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.86"
+version = "1.0.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
+checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6"
 
 [[package]]
 name = "anymap"
@@ -282,15 +282,15 @@ checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc"
 
 [[package]]
 name = "arrayref"
-version = "0.3.7"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545"
+checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
 
 [[package]]
 name = "arrayvec"
-version = "0.7.4"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
 
 [[package]]
 name = "arrow"
@@ -443,7 +443,7 @@ dependencies = [
  "arrow-schema",
  "flatbuffers",
  "lz4_flex 0.11.3",
- "zstd 0.13.1",
+ "zstd 0.13.2",
 ]
 
 [[package]]
@@ -459,7 +459,7 @@ dependencies = [
  "arrow-schema",
  "chrono",
  "half 2.4.1",
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "lexical-core",
  "num",
  "serde",
@@ -502,7 +502,7 @@ version = "51.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "serde",
 ]
 
@@ -607,9 +607,9 @@ dependencies = [
 
 [[package]]
 name = "async-compression"
-version = "0.4.11"
+version = "0.4.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5"
+checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa"
 dependencies = [
  "brotli 6.0.0",
  "bzip2",
@@ -620,8 +620,8 @@ dependencies = [
  "pin-project-lite",
  "tokio",
  "xz2",
- "zstd 0.13.1",
- "zstd-safe 7.1.0",
+ "zstd 0.13.2",
+ "zstd-safe 7.2.1",
 ]
 
 [[package]]
@@ -654,7 +654,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -676,7 +676,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -687,13 +687,13 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de"
 
 [[package]]
 name = "async-trait"
-version = "0.1.80"
+version = "0.1.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca"
+checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -740,9 +740,12 @@ dependencies = [
 
 [[package]]
 name = "atomic"
-version = "0.5.3"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba"
+checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994"
+dependencies = [
+ "bytemuck",
+]
 
 [[package]]
 name = "atomic-waker"
@@ -788,7 +791,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -812,25 +815,13 @@ dependencies = [
  "cc",
 ]
 
-[[package]]
-name = "aws-lc-rs"
-version = "1.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f95446d919226d587817a7d21379e6eb099b97b45110a7f272a444ca5c54070"
-dependencies = [
- "aws-lc-sys",
- "mirai-annotations",
- "paste",
- "zeroize",
-]
-
 [[package]]
 name = "aws-lc-sys"
-version = "0.21.0"
+version = "0.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5055edc4a9a1b2a917a818258cdfb86a535947feebd9981adc99667a062c6f85"
+checksum = "b3ddc4a5b231dd6958b140ff3151b6412b3f4321fab354f399eec8f14b06df62"
 dependencies = [
- "bindgen",
+ "bindgen 0.69.4",
  "cc",
  "cmake",
  "dunce",
@@ -853,7 +844,7 @@ dependencies = [
  "headers 0.3.9",
  "http 0.2.12",
  "http-body 0.4.6",
- "hyper 0.14.29",
+ "hyper 0.14.30",
  "itoa",
  "matchit",
  "memchr",
@@ -866,7 +857,7 @@ dependencies = [
  "serde_json",
  "serde_path_to_error",
  "serde_urlencoded",
- "sync_wrapper",
+ "sync_wrapper 0.1.2",
  "tokio",
  "tower",
  "tower-layer",
@@ -899,7 +890,7 @@ dependencies = [
  "heck 0.4.1",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -927,26 +918,28 @@ dependencies = [
 
 [[package]]
 name = "backon"
-version = "1.0.2"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2caf634d05fe0642d0fb1ab43497fa627088ecd93f84b2d0f2a5d7b91f7730db"
+checksum = "e4fa97bb310c33c811334143cf64c5bb2b7b3c06e453db6b095d7061eff8f113"
 dependencies = [
  "fastrand",
+ "gloo-timers",
+ "tokio",
 ]
 
 [[package]]
 name = "backtrace"
-version = "0.3.73"
+version = "0.3.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a"
+checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a"
 dependencies = [
  "addr2line",
- "cc",
  "cfg-if",
  "libc",
  "miniz_oxide",
  "object",
  "rustc-demangle",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -1011,7 +1004,7 @@ version = "0.69.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cexpr",
  "clang-sys",
  "itertools 0.12.1",
@@ -1024,10 +1017,28 @@ dependencies = [
  "regex",
  "rustc-hash 1.1.0",
  "shlex",
- "syn 2.0.66",
+ "syn 2.0.77",
  "which",
 ]
 
+[[package]]
+name = "bindgen"
+version = "0.70.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f"
+dependencies = [
+ "bitflags 2.6.0",
+ "cexpr",
+ "clang-sys",
+ "itertools 0.13.0",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash 1.1.0",
+ "shlex",
+ "syn 2.0.77",
+]
+
 [[package]]
 name = "bit-set"
 version = "0.5.3"
@@ -1051,9 +1062,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.5.0"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
+checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
 
 [[package]]
 name = "bitpacking"
@@ -1087,9 +1098,9 @@ dependencies = [
 
 [[package]]
 name = "blake3"
-version = "1.5.1"
+version = "1.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52"
+checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7"
 dependencies = [
  "arrayref",
  "arrayvec",
@@ -1146,10 +1157,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b"
 dependencies = [
  "once_cell",
- "proc-macro-crate 3.1.0",
+ "proc-macro-crate 3.2.0",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
  "syn_derive",
 ]
 
@@ -1266,9 +1277,9 @@ dependencies = [
 
 [[package]]
 name = "bytemuck"
-version = "1.16.0"
+version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78834c15cb5d5efe3452d58b1e8ba890dd62d21907f867f383358198e56ebca5"
+checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae"
 
 [[package]]
 name = "byteorder"
@@ -1278,9 +1289,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.7.1"
+version = "1.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50"
+checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3"
 dependencies = [
  "serde",
 ]
@@ -1424,9 +1435,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.1.20"
+version = "1.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45bcde016d64c21da4be18b655631e5ab6d3107607e71a73a9f53eb48aae23fb"
+checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0"
 dependencies = [
  "jobserver",
  "libc",
@@ -1493,7 +1504,7 @@ version = "0.13.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6026d8cd82ada8bbcfe337805dd1eb6afdc9e80fa4d57e977b3a36315e0c5525"
 dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "lazy_static",
  "num-traits",
  "regex",
@@ -1513,7 +1524,7 @@ dependencies = [
  "num-traits",
  "serde",
  "wasm-bindgen",
- "windows-targets 0.52.5",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -1611,7 +1622,7 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
 dependencies = [
  "glob",
  "libc",
- "libloading 0.8.3",
+ "libloading 0.8.5",
 ]
 
 [[package]]
@@ -1643,9 +1654,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.7"
+version = "4.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f"
+checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -1653,26 +1664,26 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.7"
+version = "4.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f"
+checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b"
 dependencies = [
  "anstream",
  "anstyle",
- "clap_lex 0.7.1",
+ "clap_lex 0.7.2",
  "strsim 0.11.1",
 ]
 
 [[package]]
 name = "clap_derive"
-version = "4.5.5"
+version = "4.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6"
+checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab"
 dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -1686,9 +1697,9 @@ dependencies = [
 
 [[package]]
 name = "clap_lex"
-version = "0.7.1"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70"
+checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
 
 [[package]]
 name = "client"
@@ -1743,9 +1754,9 @@ dependencies = [
 
 [[package]]
 name = "cmake"
-version = "0.1.50"
+version = "0.1.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"
+checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a"
 dependencies = [
  "cc",
 ]
@@ -1760,7 +1771,7 @@ dependencies = [
  "cache",
  "catalog",
  "chrono",
- "clap 4.5.7",
+ "clap 4.5.18",
  "client",
  "common-base",
  "common-catalog",
@@ -1815,16 +1826,16 @@ dependencies = [
  "tempfile",
  "tikv-jemallocator",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.19",
  "tonic 0.11.0",
  "tracing-appender",
 ]
 
 [[package]]
 name = "colorchoice"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
+checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
 
 [[package]]
 name = "combine"
@@ -1842,7 +1853,7 @@ version = "7.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7"
 dependencies = [
- "strum 0.26.2",
+ "strum 0.26.3",
  "strum_macros 0.26.4",
  "unicode-width",
 ]
@@ -1867,7 +1878,7 @@ dependencies = [
  "paste",
  "serde",
  "snafu 0.8.4",
- "toml 0.8.14",
+ "toml 0.8.19",
  "zeroize",
 ]
 
@@ -1902,7 +1913,7 @@ dependencies = [
  "sysinfo",
  "temp-env",
  "tempfile",
- "toml 0.8.14",
+ "toml 0.8.19",
 ]
 
 [[package]]
@@ -2027,7 +2038,7 @@ dependencies = [
  "common-telemetry",
  "common-test-util",
  "common-version",
- "hyper 0.14.29",
+ "hyper 0.14.30",
  "reqwest",
  "serde",
  "tempfile",
@@ -2090,7 +2101,7 @@ dependencies = [
  "quote",
  "snafu 0.8.4",
  "static_assertions",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -2139,7 +2150,7 @@ dependencies = [
  "futures-util",
  "hex",
  "humantime-serde",
- "hyper 0.14.29",
+ "hyper 0.14.30",
  "itertools 0.10.5",
  "lazy_static",
  "moka",
@@ -2173,7 +2184,7 @@ version = "0.9.3"
 dependencies = [
  "async-stream",
  "async-trait",
- "backon 1.0.2",
+ "backon 1.2.0",
  "common-base",
  "common-error",
  "common-macro",
@@ -2347,15 +2358,15 @@ dependencies = [
  "humantime-serde",
  "num_cpus",
  "rskafka",
- "rustls 0.23.10",
- "rustls-native-certs",
- "rustls-pemfile 2.1.2",
+ "rustls 0.23.13",
+ "rustls-native-certs 0.7.3",
+ "rustls-pemfile 2.1.3",
  "serde",
  "serde_json",
  "serde_with",
  "snafu 0.8.4",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.19",
 ]
 
 [[package]]
@@ -2475,18 +2486,18 @@ checksum = "373e9fafaa20882876db20562275ff58d50e0caa2590077fe7ce7bef90211d0d"
 
 [[package]]
 name = "const_format"
-version = "0.2.32"
+version = "0.2.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3a214c7af3d04997541b18d432afaff4c455e79e2029079647e72fc2bd27673"
+checksum = "50c655d81ff1114fb0dcdea9225ea9f0cc712a6f8d189378e82bdf62a473a64b"
 dependencies = [
  "const_format_proc_macros",
 ]
 
 [[package]]
 name = "const_format_proc_macros"
-version = "0.2.32"
+version = "0.2.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7f6ff08fd20f4f299298a28e2dfa8a8ba1036e6cd2460ac1de7b425d76f2500"
+checksum = "eff1a44b93f47b1bac19a27932f5c591e43d1ba357ee4f61526c8a25603f0eb1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2495,9 +2506,9 @@ dependencies = [
 
 [[package]]
 name = "constant_time_eq"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2"
+checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
 
 [[package]]
 name = "core-foundation"
@@ -2511,9 +2522,9 @@ dependencies = [
 
 [[package]]
 name = "core-foundation-sys"
-version = "0.8.6"
+version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
 [[package]]
 name = "core2"
@@ -2526,18 +2537,18 @@ dependencies = [
 
 [[package]]
 name = "cpp_demangle"
-version = "0.4.3"
+version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119"
+checksum = "96e58d342ad113c2b878f16d5d034c03be492ae460cdbc02b7f0f2284d310c7d"
 dependencies = [
  "cfg-if",
 ]
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.12"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504"
+checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0"
 dependencies = [
  "libc",
 ]
@@ -2612,7 +2623,7 @@ dependencies = [
  "anes",
  "cast",
  "ciborium",
- "clap 4.5.7",
+ "clap 4.5.18",
  "criterion-plot",
  "is-terminal",
  "itertools 0.10.5",
@@ -2764,12 +2775,12 @@ dependencies = [
 
 [[package]]
 name = "darling"
-version = "0.20.9"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1"
+checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
 dependencies = [
- "darling_core 0.20.9",
- "darling_macro 0.20.9",
+ "darling_core 0.20.10",
+ "darling_macro 0.20.10",
 ]
 
 [[package]]
@@ -2788,16 +2799,16 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.20.9"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120"
+checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
 dependencies = [
  "fnv",
  "ident_case",
  "proc-macro2",
  "quote",
  "strsim 0.11.1",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -2813,13 +2824,13 @@ dependencies = [
 
 [[package]]
 name = "darling_macro"
-version = "0.20.9"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178"
+checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
 dependencies = [
- "darling_core 0.20.9",
+ "darling_core 0.20.10",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -2851,7 +2862,7 @@ dependencies = [
  "arrow-array",
  "arrow-ipc",
  "arrow-schema",
- "async-compression 0.4.11",
+ "async-compression 0.4.12",
  "async-trait",
  "bytes",
  "bzip2",
@@ -2874,7 +2885,7 @@ dependencies = [
  "glob",
  "half 2.4.1",
  "hashbrown 0.14.5",
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "itertools 0.12.1",
  "log",
  "num_cpus",
@@ -2890,7 +2901,7 @@ dependencies = [
  "url",
  "uuid",
  "xz2",
- "zstd 0.13.1",
+ "zstd 0.13.2",
 ]
 
 [[package]]
@@ -2955,7 +2966,7 @@ dependencies = [
  "paste",
  "serde_json",
  "sqlparser 0.45.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "strum 0.26.2",
+ "strum 0.26.3",
  "strum_macros 0.26.4",
 ]
 
@@ -3033,7 +3044,7 @@ dependencies = [
  "datafusion-expr",
  "datafusion-physical-expr",
  "hashbrown 0.14.5",
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "itertools 0.12.1",
  "log",
  "regex-syntax 0.8.4",
@@ -3061,7 +3072,7 @@ dependencies = [
  "half 2.4.1",
  "hashbrown 0.14.5",
  "hex",
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "itertools 0.12.1",
  "log",
  "paste",
@@ -3103,7 +3114,7 @@ dependencies = [
  "futures",
  "half 2.4.1",
  "hashbrown 0.14.5",
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "itertools 0.12.1",
  "log",
  "once_cell",
@@ -3126,7 +3137,7 @@ dependencies = [
  "log",
  "regex",
  "sqlparser 0.45.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "strum 0.26.2",
+ "strum 0.26.3",
 ]
 
 [[package]]
@@ -3198,7 +3209,7 @@ dependencies = [
  "substrait 0.9.3",
  "table",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.19",
  "tonic 0.11.0",
 ]
 
@@ -3309,7 +3320,7 @@ checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -3320,7 +3331,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -3343,11 +3354,11 @@ dependencies = [
 
 [[package]]
 name = "derive_builder"
-version = "0.20.0"
+version = "0.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7"
+checksum = "cd33f37ee6a119146a1781d3356a7c26028f83d779b2e04ecd45fdc75c76877b"
 dependencies = [
- "derive_builder_macro 0.20.0",
+ "derive_builder_macro 0.20.1",
 ]
 
 [[package]]
@@ -3376,14 +3387,14 @@ dependencies = [
 
 [[package]]
 name = "derive_builder_core"
-version = "0.20.0"
+version = "0.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d"
+checksum = "7431fa049613920234f22c47fdc33e6cf3ee83067091ea4277a3f8c4587aae38"
 dependencies = [
- "darling 0.20.9",
+ "darling 0.20.10",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -3408,12 +3419,12 @@ dependencies = [
 
 [[package]]
 name = "derive_builder_macro"
-version = "0.20.0"
+version = "0.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b"
+checksum = "4abae7035bf79b9877b779505d8cf3749285b80c43941eda66604841889451dc"
 dependencies = [
- "derive_builder_core 0.20.0",
- "syn 2.0.66",
+ "derive_builder_core 0.20.1",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -3433,7 +3444,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
  "unicode-xid",
 ]
 
@@ -3517,17 +3528,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "displaydoc"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
-
 [[package]]
 name = "dlv-list"
 version = "0.3.0"
@@ -3602,7 +3602,7 @@ dependencies = [
  "serde",
  "thiserror",
  "time",
- "winnow 0.6.13",
+ "winnow 0.6.18",
 ]
 
 [[package]]
@@ -3613,9 +3613,9 @@ checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125"
 
 [[package]]
 name = "either"
-version = "1.12.0"
+version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b"
+checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
 
 [[package]]
 name = "ena"
@@ -3649,14 +3649,14 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
 
 [[package]]
 name = "enum-as-inner"
-version = "0.6.0"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a"
+checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -3668,7 +3668,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -3793,9 +3793,9 @@ checksum = "59668941c55e5c186b8b58c391629af56774ec768f73c08bbcd56f09348eb00b"
 
 [[package]]
 name = "fastrand"
-version = "2.1.0"
+version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
+checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
 
 [[package]]
 name = "fd-lock"
@@ -3840,14 +3840,14 @@ dependencies = [
 
 [[package]]
 name = "filetime"
-version = "0.2.23"
+version = "0.2.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd"
+checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall 0.4.1",
- "windows-sys 0.52.0",
+ "libredox",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -3870,9 +3870,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
 
 [[package]]
 name = "flagset"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdeb3aa5e95cf9aabc17f060cfa0ced7b83f042390760ca53bf09df9968acaa1"
+checksum = "b3ea1ec5f8307826a5b71094dd91fc04d4ae75d5709b20ad351c7fb4815c86ec"
 
 [[package]]
 name = "flatbuffers"
@@ -3886,9 +3886,9 @@ dependencies = [
 
 [[package]]
 name = "flate2"
-version = "1.0.30"
+version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae"
+checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253"
 dependencies = [
  "crc32fast",
  "libz-sys",
@@ -4072,7 +4072,7 @@ dependencies = [
  "strfmt",
  "table",
  "tokio",
- "toml 0.8.14",
+ "toml 0.8.19",
  "tonic 0.11.0",
  "tower",
  "uuid",
@@ -4080,54 +4080,58 @@ dependencies = [
 
 [[package]]
 name = "frunk"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11a351b59e12f97b4176ee78497dff72e4276fb1ceb13e19056aca7fa0206287"
+checksum = "874b6a17738fc273ec753618bac60ddaeac48cb1d7684c3e7bd472e57a28b817"
 dependencies = [
  "frunk_core",
  "frunk_derives",
  "frunk_proc_macros",
+ "serde",
 ]
 
 [[package]]
 name = "frunk_core"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af2469fab0bd07e64ccf0ad57a1438f63160c69b2e57f04a439653d68eb558d6"
+checksum = "3529a07095650187788833d585c219761114005d5976185760cf794d265b6a5c"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "frunk_derives"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e"
+checksum = "e99b8b3c28ae0e84b604c75f721c21dc77afb3706076af5e8216d15fd1deaae3"
 dependencies = [
  "frunk_proc_macro_helpers",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
 name = "frunk_proc_macro_helpers"
-version = "0.1.2"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35b54add839292b743aeda6ebedbd8b11e93404f902c56223e51b9ec18a13d2c"
+checksum = "05a956ef36c377977e512e227dcad20f68c2786ac7a54dacece3746046fea5ce"
 dependencies = [
  "frunk_core",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
 name = "frunk_proc_macros"
-version = "0.1.2"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71b85a1d4a9a6b300b41c05e8e13ef2feca03e0334127f29eca9506a7fe13a93"
+checksum = "67e86c2c9183662713fea27ea527aad20fb15fee635a71081ff91bf93df4dc51"
 dependencies = [
  "frunk_core",
  "frunk_proc_macro_helpers",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -4257,7 +4261,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -4370,9 +4374,9 @@ dependencies = [
 
 [[package]]
 name = "gimli"
-version = "0.29.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd"
+checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64"
 
 [[package]]
 name = "git2"
@@ -4380,7 +4384,7 @@ version = "0.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b903b73e45dc0c6c596f2d37eccece7c1c8bb6e4407b001096387c63d0d93724"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "libc",
  "libgit2-sys",
  "log",
@@ -4393,6 +4397,18 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 
+[[package]]
+name = "gloo-timers"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "js-sys",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "greptime-proto"
 version = "0.1.0"
@@ -4419,7 +4435,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http 0.2.12",
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "slab",
  "tokio",
  "tokio-util",
@@ -4581,6 +4597,12 @@ version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
 
+[[package]]
+name = "hermit-abi"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc"
+
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -4681,9 +4703,9 @@ dependencies = [
 
 [[package]]
 name = "http-body"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643"
+checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
 dependencies = [
  "bytes",
  "http 1.1.0",
@@ -4698,7 +4720,7 @@ dependencies = [
  "bytes",
  "futures-util",
  "http 1.1.0",
- "http-body 1.0.0",
+ "http-body 1.0.1",
  "pin-project-lite",
 ]
 
@@ -4710,9 +4732,9 @@ checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f"
 
 [[package]]
 name = "httparse"
-version = "1.9.3"
+version = "1.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0e7a4dd27b9476dc40cb050d3632d3bba3a70ddbff012285f7f8559a1e7e545"
+checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9"
 
 [[package]]
 name = "httpdate"
@@ -4732,7 +4754,7 @@ dependencies = [
  "os_info",
  "serde",
  "serde_derive",
- "toml 0.8.14",
+ "toml 0.8.19",
  "uuid",
 ]
 
@@ -4793,7 +4815,7 @@ dependencies = [
  "proc-macro-crate 1.3.1",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -4808,7 +4830,7 @@ dependencies = [
  "rust-sitter",
  "rust-sitter-tool",
  "slotmap",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -4817,7 +4839,7 @@ version = "0.6.2"
 source = "git+https://github.com/GreptimeTeam/hydroflow.git?branch=main#b072ee026f97f8537165e1fb247101e0ab2fb320"
 dependencies = [
  "auto_impl",
- "clap 4.5.7",
+ "clap 4.5.18",
  "data-encoding",
  "itertools 0.10.5",
  "prettyplease",
@@ -4827,7 +4849,7 @@ dependencies = [
  "serde",
  "serde_json",
  "slotmap",
- "syn 2.0.66",
+ "syn 2.0.77",
  "webbrowser",
 ]
 
@@ -4841,14 +4863,14 @@ dependencies = [
  "proc-macro-crate 1.3.1",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
 name = "hyper"
-version = "0.14.29"
+version = "0.14.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33"
+checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9"
 dependencies = [
  "bytes",
  "futures-channel",
@@ -4870,15 +4892,15 @@ dependencies = [
 
 [[package]]
 name = "hyper"
-version = "1.3.1"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d"
+checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05"
 dependencies = [
  "bytes",
  "futures-channel",
  "futures-util",
  "http 1.1.0",
- "http-body 1.0.0",
+ "http-body 1.0.1",
  "httparse",
  "itoa",
  "pin-project-lite",
@@ -4897,11 +4919,11 @@ dependencies = [
  "futures-util",
  "headers 0.4.0",
  "http 1.1.0",
- "hyper 1.3.1",
- "hyper-rustls 0.27.2",
+ "hyper 1.4.1",
+ "hyper-rustls",
  "hyper-util",
  "pin-project-lite",
- "rustls-native-certs",
+ "rustls-native-certs 0.7.3",
  "tokio",
  "tokio-rustls 0.26.0",
  "tower-service",
@@ -4909,38 +4931,21 @@ dependencies = [
 
 [[package]]
 name = "hyper-rustls"
-version = "0.26.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c"
+version = "0.27.3"
+source = "git+https://github.com/GreptimeTeam/hyper-rustls#a951e03fb914f1830e244400472814d38775118d"
 dependencies = [
  "futures-util",
  "http 1.1.0",
- "hyper 1.3.1",
- "hyper-util",
- "rustls 0.22.4",
- "rustls-pki-types",
- "tokio",
- "tokio-rustls 0.25.0",
- "tower-service",
-]
-
-[[package]]
-name = "hyper-rustls"
-version = "0.27.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155"
-dependencies = [
- "futures-util",
- "http 1.1.0",
- "hyper 1.3.1",
+ "hyper 1.4.1",
  "hyper-util",
  "log",
- "rustls 0.23.10",
- "rustls-native-certs",
+ "rustls 0.23.13",
+ "rustls-native-certs 0.8.0",
  "rustls-pki-types",
  "tokio",
  "tokio-rustls 0.26.0",
  "tower-service",
+ "webpki-roots 0.26.6",
 ]
 
 [[package]]
@@ -4949,7 +4954,7 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
 dependencies = [
- "hyper 0.14.29",
+ "hyper 0.14.30",
  "pin-project-lite",
  "tokio",
  "tokio-io-timeout",
@@ -4961,7 +4966,7 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793"
 dependencies = [
- "hyper 1.3.1",
+ "hyper 1.4.1",
  "hyper-util",
  "pin-project-lite",
  "tokio",
@@ -4970,16 +4975,16 @@ dependencies = [
 
 [[package]]
 name = "hyper-util"
-version = "0.1.5"
+version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56"
+checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba"
 dependencies = [
  "bytes",
  "futures-channel",
  "futures-util",
  "http 1.1.0",
- "http-body 1.0.0",
- "hyper 1.3.1",
+ "http-body 1.0.1",
+ "hyper 1.4.1",
  "pin-project-lite",
  "socket2 0.5.7",
  "tokio",
@@ -4990,9 +4995,9 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.60"
+version = "0.1.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141"
+checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
@@ -5011,124 +5016,6 @@ dependencies = [
  "cc",
 ]
 
-[[package]]
-name = "icu_collections"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
-dependencies = [
- "displaydoc",
- "yoke",
- "zerofrom",
- "zerovec",
-]
-
-[[package]]
-name = "icu_locid"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
-dependencies = [
- "displaydoc",
- "litemap",
- "tinystr",
- "writeable",
- "zerovec",
-]
-
-[[package]]
-name = "icu_locid_transform"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
-dependencies = [
- "displaydoc",
- "icu_locid",
- "icu_locid_transform_data",
- "icu_provider",
- "tinystr",
- "zerovec",
-]
-
-[[package]]
-name = "icu_locid_transform_data"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
-
-[[package]]
-name = "icu_normalizer"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
-dependencies = [
- "displaydoc",
- "icu_collections",
- "icu_normalizer_data",
- "icu_properties",
- "icu_provider",
- "smallvec",
- "utf16_iter",
- "utf8_iter",
- "write16",
- "zerovec",
-]
-
-[[package]]
-name = "icu_normalizer_data"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
-
-[[package]]
-name = "icu_properties"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036"
-dependencies = [
- "displaydoc",
- "icu_collections",
- "icu_locid_transform",
- "icu_properties_data",
- "icu_provider",
- "tinystr",
- "zerovec",
-]
-
-[[package]]
-name = "icu_properties_data"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
-
-[[package]]
-name = "icu_provider"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
-dependencies = [
- "displaydoc",
- "icu_locid",
- "icu_provider_macros",
- "stable_deref_trait",
- "tinystr",
- "writeable",
- "yoke",
- "zerofrom",
- "zerovec",
-]
-
-[[package]]
-name = "icu_provider_macros"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
-
 [[package]]
 name = "ident_case"
 version = "1.0.1"
@@ -5137,14 +5024,12 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
 [[package]]
 name = "idna"
-version = "1.0.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed"
+checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
 dependencies = [
- "icu_normalizer",
- "icu_properties",
- "smallvec",
- "utf8_iter",
+ "unicode-bidi",
+ "unicode-normalization",
 ]
 
 [[package]]
@@ -5155,18 +5040,18 @@ checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed"
 
 [[package]]
 name = "include_dir"
-version = "0.7.3"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18762faeff7122e89e0857b02f7ce6fcc0d101d5e9ad2ad7846cc01d61b7f19e"
+checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd"
 dependencies = [
  "include_dir_macros",
 ]
 
 [[package]]
 name = "include_dir_macros"
-version = "0.7.3"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b139284b5cf57ecfa712bcc66950bb635b31aff41c188e8a4cfc758eca374a3f"
+checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5218,9 +5103,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.2.6"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
+checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5"
 dependencies = [
  "equivalent",
  "hashbrown 0.14.5",
@@ -5235,12 +5120,12 @@ checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5"
 
 [[package]]
 name = "inferno"
-version = "0.11.19"
+version = "0.11.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "321f0f839cd44a4686e9504b0a62b4d69a50b62072144c71c68f5873c167b8d9"
+checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88"
 dependencies = [
  "ahash 0.8.11",
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "is-terminal",
  "itoa",
  "log",
@@ -5312,9 +5197,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
 [[package]]
 name = "integer-encoding"
-version = "4.0.0"
+version = "4.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "924df4f0e24e2e7f9cdd90babb0b96f93b20f3ecfa949ea9e6613756b8c8e1bf"
+checksum = "0d762194228a2f1c11063e46e32e5acb96e66e906382b9eb5441f2e0504bbd5a"
 
 [[package]]
 name = "inventory"
@@ -5324,15 +5209,15 @@ checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767"
 
 [[package]]
 name = "ipnet"
-version = "2.9.0"
+version = "2.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
+checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4"
 
 [[package]]
 name = "iri-string"
-version = "0.7.2"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f5f6c2df22c009ac44f6f1499308e7a3ac7ba42cd2378475cc691510e1eef1b"
+checksum = "9c25163201be6ded9e686703e85532f8f852ea1f92ba625cb3c51f7fe6d07a4a"
 dependencies = [
  "memchr",
  "serde",
@@ -5353,11 +5238,11 @@ dependencies = [
 
 [[package]]
 name = "is-terminal"
-version = "0.4.12"
+version = "0.4.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b"
+checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b"
 dependencies = [
- "hermit-abi 0.3.9",
+ "hermit-abi 0.4.0",
  "libc",
  "windows-sys 0.52.0",
 ]
@@ -5370,9 +5255,9 @@ checksum = "06d198e9919d9822d5f7083ba8530e04de87841eaf21ead9af8f2304efd57c89"
 
 [[package]]
 name = "is_terminal_polyfill"
-version = "1.70.0"
+version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
 
 [[package]]
 name = "iso8601"
@@ -5432,7 +5317,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1e2b0210dc78b49337af9e49d7ae41a39dceac6e5985613f1cf7763e2f76a25"
 dependencies = [
  "cedarwood",
- "derive_builder 0.20.0",
+ "derive_builder 0.20.1",
  "fxhash",
  "lazy_static",
  "phf",
@@ -5464,18 +5349,18 @@ checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130"
 
 [[package]]
 name = "jobserver"
-version = "0.1.31"
+version = "0.1.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e"
+checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
 dependencies = [
  "libc",
 ]
 
 [[package]]
 name = "js-sys"
-version = "0.3.69"
+version = "0.3.70"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
+checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -5512,7 +5397,7 @@ dependencies = [
  "fast-float",
  "itoa",
  "nom",
- "ordered-float 4.2.0",
+ "ordered-float 4.2.2",
  "rand",
  "ryu",
  "serde_json",
@@ -5587,7 +5472,7 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d"
 dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
 ]
 
 [[package]]
@@ -5612,9 +5497,9 @@ dependencies = [
 
 [[package]]
 name = "kube"
-version = "0.92.0"
+version = "0.92.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12dc4487eda98835dcaa7ac92a14165446db29dbd67a743c79fe9f41bf38ee72"
+checksum = "231c5a5392d9e2a9b0d923199760d3f1dd73b95288f2871d16c7c90ba4954506"
 dependencies = [
  "k8s-openapi",
  "kube-client",
@@ -5625,9 +5510,9 @@ dependencies = [
 
 [[package]]
 name = "kube-client"
-version = "0.92.0"
+version = "0.92.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "408f35eab36927d3b883e4ad54c3080ea8c49f899ac84a7856e7182e4ee3b392"
+checksum = "8f4bf54135062ff60e2a0dfb3e7a9c8e931fc4a535b4d6bd561e0a1371321c61"
 dependencies = [
  "base64 0.22.1",
  "bytes",
@@ -5636,19 +5521,19 @@ dependencies = [
  "futures",
  "home",
  "http 1.1.0",
- "http-body 1.0.0",
+ "http-body 1.0.1",
  "http-body-util",
- "hyper 1.3.1",
+ "hyper 1.4.1",
  "hyper-http-proxy",
- "hyper-rustls 0.27.2",
+ "hyper-rustls",
  "hyper-timeout 0.5.1",
  "hyper-util",
  "jsonpath-rust",
  "k8s-openapi",
  "kube-core",
  "pem",
- "rustls 0.23.10",
- "rustls-pemfile 2.1.2",
+ "rustls 0.23.13",
+ "rustls-pemfile 2.1.3",
  "secrecy",
  "serde",
  "serde_json",
@@ -5663,9 +5548,9 @@ dependencies = [
 
 [[package]]
 name = "kube-core"
-version = "0.92.0"
+version = "0.92.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f776624097c1e09e72eb1e9e0c2bb5d17d97c27a6a87734390a9fba246a8f67f"
+checksum = "40fb9bd8141cbc0fe6b0d9112d371679b4cb607b45c31dd68d92e40864a12975"
 dependencies = [
  "chrono",
  "form_urlencoded",
@@ -5680,22 +5565,22 @@ dependencies = [
 
 [[package]]
 name = "kube-derive"
-version = "0.92.0"
+version = "0.92.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ae07adfd7d21b7fa582789206391243f98e155b46c806eb494839569853bcfd"
+checksum = "08fc86f70076921fdf2f433bbd2a796dc08ac537dc1db1f062cfa63ed4fa15fb"
 dependencies = [
- "darling 0.20.9",
+ "darling 0.20.10",
  "proc-macro2",
  "quote",
  "serde_json",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
 name = "kube-runtime"
-version = "0.92.0"
+version = "0.92.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12e5933f2d429f3a05d4cb67f935b25c94a133b0baeb558ab3917c270a11f6ef"
+checksum = "b7eb2fb986f81770eb55ec7f857e197019b31b38768d2410f6c1046ffac34225"
 dependencies = [
  "ahash 0.8.11",
  "async-broadcast",
@@ -5780,16 +5665,16 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
 name = "lazy_static"
-version = "1.4.0"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 dependencies = [
- "spin 0.5.2",
+ "spin 0.9.8",
 ]
 
 [[package]]
@@ -5870,9 +5755,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.155"
+version = "0.2.158"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
 
 [[package]]
 name = "libfuzzer-sys"
@@ -5909,12 +5794,12 @@ dependencies = [
 
 [[package]]
 name = "libloading"
-version = "0.8.3"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19"
+checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
 dependencies = [
  "cfg-if",
- "windows-targets 0.52.5",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -5929,8 +5814,9 @@ version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "libc",
+ "redox_syscall 0.5.4",
 ]
 
 [[package]]
@@ -5946,9 +5832,9 @@ dependencies = [
 
 [[package]]
 name = "libz-sys"
-version = "1.1.18"
+version = "1.1.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e"
+checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472"
 dependencies = [
  "cc",
  "libc",
@@ -5968,12 +5854,6 @@ version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
 
-[[package]]
-name = "litemap"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704"
-
 [[package]]
 name = "lock_api"
 version = "0.4.12"
@@ -5986,9 +5866,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.21"
+version = "0.4.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
+checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
 
 [[package]]
 name = "log-store"
@@ -6058,7 +5938,7 @@ dependencies = [
  "cactus",
  "cfgrammar",
  "filetime",
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "lazy_static",
  "lrtable",
  "num-traits",
@@ -6086,28 +5966,27 @@ dependencies = [
 
 [[package]]
 name = "lru"
-version = "0.12.3"
+version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc"
+checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904"
 dependencies = [
  "hashbrown 0.14.5",
 ]
 
 [[package]]
 name = "lz4"
-version = "1.25.0"
+version = "1.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6eab492fe7f8651add23237ea56dbf11b3c4ff762ab83d40a47f11433421f91"
+checksum = "a231296ca742e418c43660cb68e082486ff2538e8db432bc818580f3965025ed"
 dependencies = [
- "libc",
  "lz4-sys",
 ]
 
 [[package]]
 name = "lz4-sys"
-version = "1.9.5"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9764018d143cc854c9f17f0b907de70f14393b1f502da6375dce70f00514eb3"
+checksum = "fcb44a01837a858d47e5a630d2ccf304c8efcc4b83b8f9f75b7a9ee4fcc6e57d"
 dependencies = [
  "cc",
  "libc",
@@ -6206,9 +6085,9 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
 
 [[package]]
 name = "matrixmultiply"
-version = "0.3.8"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7574c1cf36da4798ab73da5b215bbf444f50718207754cb522201d78d1cd0ff2"
+checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a"
 dependencies = [
  "autocfg",
  "rawpointer",
@@ -6268,9 +6147,9 @@ dependencies = [
 
 [[package]]
 name = "memmap2"
-version = "0.9.4"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fe751422e4a8caa417e13c3ea66452215d7d63e19e604f4980461212f3ae1322"
+checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
 dependencies = [
  "libc",
 ]
@@ -6335,7 +6214,7 @@ dependencies = [
  "api",
  "async-trait",
  "chrono",
- "clap 4.5.7",
+ "clap 4.5.18",
  "client",
  "common-base",
  "common-catalog",
@@ -6379,7 +6258,7 @@ dependencies = [
  "tokio",
  "tokio-postgres",
  "tokio-stream",
- "toml 0.8.14",
+ "toml 0.8.19",
  "tonic 0.11.0",
  "tower",
  "tracing",
@@ -6443,9 +6322,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
 [[package]]
 name = "mime_guess"
-version = "2.0.4"
+version = "2.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef"
+checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
 dependencies = [
  "mime",
  "unicase",
@@ -6468,11 +6347,11 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
 [[package]]
 name = "miniz_oxide"
-version = "0.7.3"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae"
+checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1"
 dependencies = [
- "adler",
+ "adler2",
 ]
 
 [[package]]
@@ -6498,10 +6377,16 @@ dependencies = [
 ]
 
 [[package]]
-name = "mirai-annotations"
-version = "1.12.0"
+name = "mio"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1"
+checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
+dependencies = [
+ "hermit-abi 0.3.9",
+ "libc",
+ "wasi",
+ "windows-sys 0.52.0",
+]
 
 [[package]]
 name = "mito2"
@@ -6569,7 +6454,7 @@ dependencies = [
  "tokio",
  "tokio-stream",
  "tokio-util",
- "toml 0.8.14",
+ "toml 0.8.19",
  "uuid",
 ]
 
@@ -6602,9 +6487,9 @@ dependencies = [
 
 [[package]]
 name = "moka"
-version = "0.12.7"
+version = "0.12.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e0d88686dc561d743b40de8269b26eaf0dc58781bde087b0984646602021d08"
+checksum = "32cf62eb4dd975d2dde76432fb1075c49e3ee2331cf36f1f8fd4b66550d32b6f"
 dependencies = [
  "async-lock",
  "async-trait",
@@ -6675,14 +6560,14 @@ version = "0.30.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "56b0d8a0db9bf6d2213e11f2c701cb91387b0614361625ab7b9743b41aa4938f"
 dependencies = [
- "darling 0.20.9",
+ "darling 0.20.10",
  "heck 0.4.1",
  "num-bigint",
  "proc-macro-crate 1.3.1",
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
  "termcolor",
  "thiserror",
 ]
@@ -6693,14 +6578,14 @@ version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "afe0450cc9344afff34915f8328600ab5ae19260802a334d0f72d2d5bdda3bfe"
 dependencies = [
- "darling 0.20.9",
+ "darling 0.20.10",
  "heck 0.4.1",
  "num-bigint",
- "proc-macro-crate 3.1.0",
+ "proc-macro-crate 3.2.0",
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
  "termcolor",
  "thiserror",
 ]
@@ -6720,7 +6605,7 @@ dependencies = [
  "keyed_priority_queue",
  "lazy_static",
  "lru",
- "mio",
+ "mio 0.8.11",
  "mysql_common 0.31.0",
  "once_cell",
  "pem",
@@ -6750,8 +6635,8 @@ checksum = "06f19e4cfa0ab5a76b627cec2d81331c49b034988eaf302c3bafeada684eadef"
 dependencies = [
  "base64 0.21.7",
  "bigdecimal",
- "bindgen",
- "bitflags 2.5.0",
+ "bindgen 0.70.1",
+ "bitflags 2.6.0",
  "bitvec",
  "btoi",
  "byteorder",
@@ -6789,8 +6674,8 @@ checksum = "478b0ff3f7d67b79da2b96f56f334431aef65e15ba4b29dd74a4236e29582bdc"
 dependencies = [
  "base64 0.21.7",
  "bigdecimal",
- "bindgen",
- "bitflags 2.5.0",
+ "bindgen 0.70.1",
+ "bitflags 2.6.0",
  "bitvec",
  "btoi",
  "byteorder",
@@ -6818,7 +6703,7 @@ dependencies = [
  "thiserror",
  "time",
  "uuid",
- "zstd 0.13.1",
+ "zstd 0.13.2",
 ]
 
 [[package]]
@@ -6904,7 +6789,7 @@ version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cfg-if",
  "cfg_aliases 0.1.1",
  "libc",
@@ -6933,7 +6818,7 @@ version = "6.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "crossbeam-channel",
  "filetime",
  "fsevent-sys",
@@ -6941,7 +6826,7 @@ dependencies = [
  "kqueue",
  "libc",
  "log",
- "mio",
+ "mio 0.8.11",
  "walkdir",
  "windows-sys 0.48.0",
 ]
@@ -6981,9 +6866,9 @@ dependencies = [
 
 [[package]]
 name = "num-bigint"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7"
+checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
 dependencies = [
  "num-integer",
  "num-traits",
@@ -7029,7 +6914,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -7134,9 +7019,9 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.36.0"
+version = "0.36.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434"
+checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a"
 dependencies = [
  "memchr",
 ]
@@ -7203,15 +7088,15 @@ checksum = "e296cf87e61c9cfc1a61c3c63a0f7f286ed4554e0e22be84e8a38e1d264a2a29"
 
 [[package]]
 name = "oorandom"
-version = "11.1.3"
+version = "11.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
+checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9"
 
 [[package]]
 name = "opendal"
-version = "0.49.1"
+version = "0.49.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba615070686c8781ce97376fdafca29d7c42f47b31d2230d7c8c1642ec823950"
+checksum = "9b04d09b9822c2f75a1d2fc513a2c1279c70e91e7407936fffdf6a6976ec530a"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -7228,7 +7113,7 @@ dependencies = [
  "md-5",
  "once_cell",
  "percent-encoding",
- "quick-xml 0.36.1",
+ "quick-xml 0.36.2",
  "reqsign",
  "reqwest",
  "serde",
@@ -7280,7 +7165,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a"
 dependencies = [
  "futures-core",
  "futures-sink",
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "js-sys",
  "once_cell",
  "pin-project-lite",
@@ -7371,7 +7256,7 @@ dependencies = [
  "glob",
  "once_cell",
  "opentelemetry 0.21.0",
- "ordered-float 4.2.0",
+ "ordered-float 4.2.2",
  "percent-encoding",
  "rand",
  "thiserror",
@@ -7393,7 +7278,7 @@ dependencies = [
  "glob",
  "once_cell",
  "opentelemetry 0.22.0",
- "ordered-float 4.2.0",
+ "ordered-float 4.2.2",
  "percent-encoding",
  "rand",
  "thiserror",
@@ -7505,9 +7390,9 @@ dependencies = [
 
 [[package]]
 name = "ordered-float"
-version = "4.2.0"
+version = "4.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e"
+checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6"
 dependencies = [
  "num-traits",
 ]
@@ -7595,9 +7480,9 @@ dependencies = [
 
 [[package]]
 name = "parking"
-version = "2.2.0"
+version = "2.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
+checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
 
 [[package]]
 name = "parking_lot"
@@ -7644,10 +7529,10 @@ dependencies = [
  "cfg-if",
  "libc",
  "petgraph",
- "redox_syscall 0.5.2",
+ "redox_syscall 0.5.4",
  "smallvec",
  "thread-id",
- "windows-targets 0.52.5",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -7682,7 +7567,7 @@ dependencies = [
  "thrift",
  "tokio",
  "twox-hash",
- "zstd 0.13.1",
+ "zstd 0.13.2",
 ]
 
 [[package]]
@@ -7824,9 +7709,9 @@ checksum = "df202b0b0f5b8e389955afd5f27b007b00fb948162953f1db9c70d2c7e3157d7"
 
 [[package]]
 name = "pest"
-version = "2.7.10"
+version = "2.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8"
+checksum = "fdbef9d1d47087a895abd220ed25eb4ad973a5e26f6a4367b038c25e28dfc2d9"
 dependencies = [
  "memchr",
  "thiserror",
@@ -7835,9 +7720,9 @@ dependencies = [
 
 [[package]]
 name = "pest_derive"
-version = "2.7.10"
+version = "2.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459"
+checksum = "4d3a6e3394ec80feb3b6393c725571754c6188490265c61aaf260810d6b95aa0"
 dependencies = [
  "pest",
  "pest_generator",
@@ -7845,22 +7730,22 @@ dependencies = [
 
 [[package]]
 name = "pest_generator"
-version = "2.7.10"
+version = "2.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687"
+checksum = "94429506bde1ca69d1b5601962c73f4172ab4726571a59ea95931218cb0e930e"
 dependencies = [
  "pest",
  "pest_meta",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
 name = "pest_meta"
-version = "2.7.10"
+version = "2.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd"
+checksum = "ac8a071862e93690b6e34e9a5fb8e33ff3734473ac0245b27232222c4906a33f"
 dependencies = [
  "once_cell",
  "pest",
@@ -7874,7 +7759,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
 dependencies = [
  "fixedbitset",
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
 ]
 
 [[package]]
@@ -7965,7 +7850,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -8038,9 +7923,9 @@ dependencies = [
 
 [[package]]
 name = "piper"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae1d5c74c9876f070d3e8fd503d748c7d974c3e48da8f41350fa5222ef9b4391"
+checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066"
 dependencies = [
  "atomic-waker",
  "fastrand",
@@ -8115,9 +8000,9 @@ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
 
 [[package]]
 name = "plotters"
-version = "0.3.6"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3"
+checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
 dependencies = [
  "num-traits",
  "plotters-backend",
@@ -8128,15 +8013,15 @@ dependencies = [
 
 [[package]]
 name = "plotters-backend"
-version = "0.3.6"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7"
+checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
 
 [[package]]
 name = "plotters-svg"
-version = "0.3.6"
+version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705"
+checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
 dependencies = [
  "plotters-backend",
 ]
@@ -8166,9 +8051,9 @@ dependencies = [
 
 [[package]]
 name = "portable-atomic"
-version = "1.6.0"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"
+checksum = "d30538d42559de6b034bc76fd6dd4c38961b1ee5c6c56e3808c50128fdbc22ce"
 
 [[package]]
 name = "postgres-protocol"
@@ -8190,9 +8075,9 @@ dependencies = [
 
 [[package]]
 name = "postgres-types"
-version = "0.2.7"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02048d9e032fb3cc3413bbf7b83a15d84a5d419778e2628751896d856498eee9"
+checksum = "f66ea23a2d0e5734297357705193335e0a957696f34bed2f2faefacb2fec336f"
 dependencies = [
  "array-init",
  "bytes",
@@ -8238,9 +8123,12 @@ dependencies = [
 
 [[package]]
 name = "ppv-lite86"
-version = "0.2.17"
+version = "0.2.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
+checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
+dependencies = [
+ "zerocopy",
+]
 
 [[package]]
 name = "precomputed-hash"
@@ -8264,15 +8152,15 @@ dependencies = [
 
 [[package]]
 name = "predicates-core"
-version = "1.0.6"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174"
+checksum = "ae8177bee8e75d6846599c6b9ff679ed51e882816914eec639944d7c9aa11931"
 
 [[package]]
 name = "predicates-tree"
-version = "1.0.9"
+version = "1.0.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf"
+checksum = "41b740d195ed3166cd147c8047ec98db0e22ec019eb8eeb76d343b795304fb13"
 dependencies = [
  "predicates-core",
  "termtree",
@@ -8280,9 +8168,9 @@ dependencies = [
 
 [[package]]
 name = "pretty_assertions"
-version = "1.4.0"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66"
+checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d"
 dependencies = [
  "diff",
  "yansi",
@@ -8300,12 +8188,12 @@ dependencies = [
 
 [[package]]
 name = "prettyplease"
-version = "0.2.20"
+version = "0.2.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e"
+checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba"
 dependencies = [
  "proc-macro2",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -8320,11 +8208,11 @@ dependencies = [
 
 [[package]]
 name = "proc-macro-crate"
-version = "3.1.0"
+version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284"
+checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b"
 dependencies = [
- "toml_edit 0.21.1",
+ "toml_edit 0.22.21",
 ]
 
 [[package]]
@@ -8353,9 +8241,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.85"
+version = "1.0.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23"
+checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
 dependencies = [
  "unicode-ident",
 ]
@@ -8366,7 +8254,7 @@ version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "hex",
  "lazy_static",
  "procfs-core",
@@ -8379,7 +8267,7 @@ version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "hex",
 ]
 
@@ -8473,12 +8361,12 @@ dependencies = [
 
 [[package]]
 name = "prost"
-version = "0.13.1"
+version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e13db3d3fde688c61e2446b4d843bc27a7e8af269a69440c0308021dc92333cc"
+checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f"
 dependencies = [
  "bytes",
- "prost-derive 0.13.1",
+ "prost-derive 0.13.3",
 ]
 
 [[package]]
@@ -8498,15 +8386,15 @@ dependencies = [
  "prost 0.12.6",
  "prost-types 0.12.6",
  "regex",
- "syn 2.0.66",
+ "syn 2.0.77",
  "tempfile",
 ]
 
 [[package]]
 name = "prost-build"
-version = "0.13.1"
+version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5bb182580f71dd070f88d01ce3de9f4da5021db7115d2e1c3605a754153b77c1"
+checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15"
 dependencies = [
  "bytes",
  "heck 0.5.0",
@@ -8515,8 +8403,8 @@ dependencies = [
  "multimap",
  "once_cell",
  "petgraph",
- "prost 0.13.1",
- "prost-types 0.13.1",
+ "prost 0.13.3",
+ "prost-types 0.13.3",
  "regex",
  "tempfile",
 ]
@@ -8544,20 +8432,20 @@ dependencies = [
  "itertools 0.12.1",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
 name = "prost-derive"
-version = "0.13.1"
+version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18bec9b0adc4eba778b33684b7ba3e7137789434769ee3ce3930463ef904cfca"
+checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5"
 dependencies = [
  "anyhow",
  "itertools 0.13.0",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -8580,11 +8468,11 @@ dependencies = [
 
 [[package]]
 name = "prost-types"
-version = "0.13.1"
+version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cee5168b05f49d4b0ca581206eb14a7b22fafd963efe729ac48eb03266e25cc2"
+checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670"
 dependencies = [
- "prost 0.13.1",
+ "prost 0.13.3",
 ]
 
 [[package]]
@@ -8651,12 +8539,12 @@ dependencies = [
 name = "puffin"
 version = "0.9.3"
 dependencies = [
- "async-compression 0.4.11",
+ "async-compression 0.4.12",
  "async-trait",
  "async-walkdir",
  "auto_impl",
  "base64 0.21.7",
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "common-error",
  "common-macro",
  "common-runtime",
@@ -8738,7 +8626,7 @@ dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -8751,7 +8639,7 @@ dependencies = [
  "proc-macro2",
  "pyo3-build-config",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -8861,19 +8749,67 @@ dependencies = [
 
 [[package]]
 name = "quick-xml"
-version = "0.36.1"
+version = "0.36.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc"
+checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe"
 dependencies = [
  "memchr",
  "serde",
 ]
 
 [[package]]
-name = "quote"
-version = "1.0.36"
+name = "quinn"
+version = "0.11.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684"
+dependencies = [
+ "bytes",
+ "pin-project-lite",
+ "quinn-proto",
+ "quinn-udp",
+ "rustc-hash 2.0.0",
+ "rustls 0.23.13",
+ "socket2 0.5.7",
+ "thiserror",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "quinn-proto"
+version = "0.11.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6"
+dependencies = [
+ "bytes",
+ "rand",
+ "ring 0.17.8",
+ "rustc-hash 2.0.0",
+ "rustls 0.23.13",
+ "slab",
+ "thiserror",
+ "tinyvec",
+ "tracing",
+]
+
+[[package]]
+name = "quinn-udp"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b"
+dependencies = [
+ "libc",
+ "once_cell",
+ "socket2 0.5.7",
+ "tracing",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
 dependencies = [
  "proc-macro2",
 ]
@@ -8971,11 +8907,11 @@ dependencies = [
 
 [[package]]
 name = "raw-cpuid"
-version = "11.0.2"
+version = "11.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e29830cbb1290e404f24c73af91c5d8d631ce7e128691e9477556b540cd01ecd"
+checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
 ]
 
 [[package]]
@@ -9021,27 +8957,18 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.4.1"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
+checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853"
 dependencies = [
- "bitflags 1.3.2",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd"
-dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
 ]
 
 [[package]]
 name = "redox_users"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891"
+checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
 dependencies = [
  "getrandom",
  "libredox",
@@ -9065,14 +8992,14 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
 name = "regex"
-version = "1.10.5"
+version = "1.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f"
+checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -9167,7 +9094,7 @@ dependencies = [
  "rand",
  "reqwest",
  "rsa 0.9.6",
- "rust-ini 0.21.0",
+ "rust-ini 0.21.1",
  "serde",
  "serde_json",
  "sha1",
@@ -9176,19 +9103,19 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.12.4"
+version = "0.12.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10"
+checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63"
 dependencies = [
  "base64 0.22.1",
  "bytes",
  "futures-core",
  "futures-util",
  "http 1.1.0",
- "http-body 1.0.0",
+ "http-body 1.0.1",
  "http-body-util",
- "hyper 1.3.1",
- "hyper-rustls 0.26.0",
+ "hyper 1.4.1",
+ "hyper-rustls",
  "hyper-util",
  "ipnet",
  "js-sys",
@@ -9198,16 +9125,17 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
- "rustls 0.22.4",
- "rustls-native-certs",
- "rustls-pemfile 2.1.2",
+ "quinn",
+ "rustls 0.23.13",
+ "rustls-native-certs 0.7.3",
+ "rustls-pemfile 2.1.3",
  "rustls-pki-types",
  "serde",
  "serde_json",
  "serde_urlencoded",
- "sync_wrapper",
+ "sync_wrapper 1.0.1",
  "tokio",
- "tokio-rustls 0.25.0",
+ "tokio-rustls 0.26.0",
  "tokio-util",
  "tower-service",
  "url",
@@ -9215,8 +9143,8 @@ dependencies = [
  "wasm-bindgen-futures",
  "wasm-streams",
  "web-sys",
- "webpki-roots 0.26.2",
- "winreg 0.52.0",
+ "webpki-roots 0.26.6",
+ "windows-registry",
 ]
 
 [[package]]
@@ -9256,9 +9184,9 @@ dependencies = [
 
 [[package]]
 name = "rgb"
-version = "0.8.37"
+version = "0.8.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05aaa8004b64fd573fc9d002f4e632d51ad4f026c2b5ba95fcb6c2f32c2c47d8"
+checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a"
 dependencies = [
  "bytemuck",
 ]
@@ -9295,9 +9223,9 @@ dependencies = [
 
 [[package]]
 name = "rkyv"
-version = "0.7.44"
+version = "0.7.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cba464629b3394fc4dbc6f940ff8f5b4ff5c7aef40f29166fd4ad12acbc99c0"
+checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b"
 dependencies = [
  "bitvec",
  "bytecheck",
@@ -9313,9 +9241,9 @@ dependencies = [
 
 [[package]]
 name = "rkyv_derive"
-version = "0.7.44"
+version = "0.7.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65"
+checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -9402,18 +9330,18 @@ dependencies = [
  "crc32c",
  "flate2",
  "futures",
- "integer-encoding 4.0.0",
+ "integer-encoding 4.0.2",
  "lz4",
  "parking_lot 0.12.3",
  "rand",
  "rsasl",
- "rustls 0.23.10",
+ "rustls 0.23.13",
  "snap",
  "thiserror",
  "tokio",
  "tokio-rustls 0.26.0",
  "tracing",
- "zstd 0.13.1",
+ "zstd 0.13.2",
 ]
 
 [[package]]
@@ -9436,13 +9364,13 @@ checksum = "4165dfae59a39dd41d8dec720d3cbfbc71f69744efb480a3920f5d4e0cc6798d"
 dependencies = [
  "cfg-if",
  "glob",
- "proc-macro-crate 3.1.0",
+ "proc-macro-crate 3.2.0",
  "proc-macro2",
  "quote",
  "regex",
  "relative-path",
  "rustc_version",
- "syn 2.0.66",
+ "syn 2.0.77",
  "unicode-ident",
 ]
 
@@ -9454,7 +9382,7 @@ checksum = "b3a8fb4672e840a587a66fc577a5491375df51ddb88f2a2c2a792598c326fe14"
 dependencies = [
  "quote",
  "rand",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -9477,7 +9405,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rust-embed-utils",
- "syn 2.0.66",
+ "syn 2.0.77",
  "walkdir",
 ]
 
@@ -9503,9 +9431,9 @@ dependencies = [
 
 [[package]]
 name = "rust-ini"
-version = "0.21.0"
+version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d625ed57d8f49af6cfa514c42e1a71fadcff60eb0b1c517ff82fe41aa025b41"
+checksum = "4e310ef0e1b6eeb79169a1171daf9abcb87a2e17c03bee2c4bb100b55c75409f"
 dependencies = [
  "cfg-if",
  "ordered-multimap 0.7.3",
@@ -9573,9 +9501,9 @@ dependencies = [
 
 [[package]]
 name = "rust_decimal"
-version = "1.35.0"
+version = "1.36.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1790d1c4c0ca81211399e0e0af16333276f375209e71a37b67698a373db5b47a"
+checksum = "b082d80e3e3cc52b2ed634388d436fe1f4de6af5786cc2de9ba9737527bdf555"
 dependencies = [
  "arrayvec",
  "borsh",
@@ -9608,20 +9536,20 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152"
 
 [[package]]
 name = "rustc_version"
-version = "0.4.0"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
 dependencies = [
  "semver",
 ]
 
 [[package]]
 name = "rustix"
-version = "0.38.34"
+version = "0.38.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
+checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -9661,35 +9589,46 @@ dependencies = [
  "log",
  "ring 0.17.8",
  "rustls-pki-types",
- "rustls-webpki 0.102.4",
+ "rustls-webpki 0.102.8",
  "subtle",
  "zeroize",
 ]
 
 [[package]]
 name = "rustls"
-version = "0.23.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402"
+version = "0.23.13"
+source = "git+https://github.com/GreptimeTeam/rustls#e0917109155435821c4416d699babe749b597a03"
 dependencies = [
- "aws-lc-rs",
  "log",
  "once_cell",
  "ring 0.17.8",
  "rustls-pki-types",
- "rustls-webpki 0.102.4",
+ "rustls-webpki 0.102.8",
  "subtle",
  "zeroize",
 ]
 
 [[package]]
 name = "rustls-native-certs"
-version = "0.7.1"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba"
+checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5"
 dependencies = [
  "openssl-probe",
- "rustls-pemfile 2.1.2",
+ "rustls-pemfile 2.1.3",
+ "rustls-pki-types",
+ "schannel",
+ "security-framework",
+]
+
+[[package]]
+name = "rustls-native-certs"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a"
+dependencies = [
+ "openssl-probe",
+ "rustls-pemfile 2.1.3",
  "rustls-pki-types",
  "schannel",
  "security-framework",
@@ -9706,9 +9645,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pemfile"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d"
+checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425"
 dependencies = [
  "base64 0.22.1",
  "rustls-pki-types",
@@ -9716,9 +9655,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.7.0"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d"
+checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0"
 
 [[package]]
 name = "rustls-webpki"
@@ -9732,11 +9671,10 @@ dependencies = [
 
 [[package]]
 name = "rustls-webpki"
-version = "0.102.4"
+version = "0.102.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e"
+checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
 dependencies = [
- "aws-lc-rs",
  "ring 0.17.8",
  "rustls-pki-types",
  "untrusted 0.9.0",
@@ -10022,7 +9960,7 @@ dependencies = [
  "widestring",
  "winapi",
  "windows 0.39.0",
- "winreg 0.10.1",
+ "winreg",
 ]
 
 [[package]]
@@ -10165,11 +10103,11 @@ checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71"
 
 [[package]]
 name = "schannel"
-version = "0.1.23"
+version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534"
+checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -10194,7 +10132,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_derive_internals",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -10295,7 +10233,7 @@ dependencies = [
  "heck 0.4.1",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -10310,11 +10248,11 @@ dependencies = [
 
 [[package]]
 name = "security-framework"
-version = "2.11.0"
+version = "2.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0"
+checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "core-foundation",
  "core-foundation-sys",
  "libc",
@@ -10323,9 +10261,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework-sys"
-version = "2.11.0"
+version = "2.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7"
+checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -10348,9 +10286,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
 
 [[package]]
 name = "serde"
-version = "1.0.203"
+version = "1.0.210"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094"
+checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a"
 dependencies = [
  "serde_derive",
 ]
@@ -10367,13 +10305,13 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.203"
+version = "1.0.210"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba"
+checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -10384,16 +10322,16 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.122"
+version = "1.0.128"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
+checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
 dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "itoa",
  "memchr",
  "ryu",
@@ -10418,28 +10356,28 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
 name = "serde_spanned"
-version = "0.6.6"
+version = "0.6.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0"
+checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d"
 dependencies = [
  "serde",
 ]
 
 [[package]]
 name = "serde_tokenstream"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8790a7c3fe883e443eaa2af6f705952bc5d6e8671a220b9335c8cae92c037e74"
+checksum = "64060d864397305347a78851c51588fd283767e7e7589829e8121d65512340f1"
 dependencies = [
  "proc-macro2",
  "quote",
  "serde",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -10456,15 +10394,15 @@ dependencies = [
 
 [[package]]
 name = "serde_with"
-version = "3.8.1"
+version = "3.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20"
+checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857"
 dependencies = [
  "base64 0.22.1",
  "chrono",
  "hex",
  "indexmap 1.9.3",
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "serde",
  "serde_derive",
  "serde_json",
@@ -10474,14 +10412,14 @@ dependencies = [
 
 [[package]]
 name = "serde_with_macros"
-version = "3.8.1"
+version = "3.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2"
+checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350"
 dependencies = [
- "darling 0.20.9",
+ "darling 0.20.10",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -10490,7 +10428,7 @@ version = "0.9.34+deprecated"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
 dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "itoa",
  "ryu",
  "serde",
@@ -10547,7 +10485,7 @@ dependencies = [
  "http 0.2.12",
  "http-body 0.4.6",
  "humantime-serde",
- "hyper 0.14.29",
+ "hyper 0.14.30",
  "influxdb_line_protocol",
  "itertools 0.10.5",
  "jsonb",
@@ -10575,8 +10513,8 @@ dependencies = [
  "regex",
  "reqwest",
  "rust-embed",
- "rustls 0.23.10",
- "rustls-pemfile 2.1.2",
+ "rustls 0.23.13",
+ "rustls-pemfile 2.1.3",
  "rustls-pki-types",
  "schemars",
  "script",
@@ -10603,7 +10541,7 @@ dependencies = [
  "tower-http 0.4.4",
  "urlencoding",
  "uuid",
- "zstd 0.13.1",
+ "zstd 0.13.2",
 ]
 
 [[package]]
@@ -10669,9 +10607,9 @@ dependencies = [
 
 [[package]]
 name = "shadow-rs"
-version = "0.31.0"
+version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66caf2de9b7e61293c00006cd2807d6c4e4b31018c5ea21d008f44f4852b93c3"
+checksum = "02c282402d25101f9c893e9cd7e4cae535fe7db18b81291de973026c219ddf1e"
 dependencies = [
  "const_format",
  "git2",
@@ -10729,9 +10667,9 @@ dependencies = [
 
 [[package]]
 name = "simdutf8"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
+checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
 
 [[package]]
 name = "similar"
@@ -10854,7 +10792,7 @@ dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -10962,9 +10900,9 @@ dependencies = [
 
 [[package]]
 name = "sqlformat"
-version = "0.2.4"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f895e3734318cc55f1fe66258926c9b910c124d47520339efecbb6c59cec7c1f"
+checksum = "7bba3a93db0cc4f7bdece8bb09e77e2e785c20bfebf79eb8340ed80708048790"
 dependencies = [
  "nom",
  "unicode_categories",
@@ -10993,7 +10931,7 @@ name = "sqlness-runner"
 version = "0.9.3"
 dependencies = [
  "async-trait",
- "clap 4.5.7",
+ "clap 4.5.18",
  "client",
  "common-error",
  "common-query",
@@ -11048,7 +10986,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -11058,7 +10996,7 @@ source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=54a267ac89c09
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -11165,7 +11103,7 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1120e6a8cbd4d85d5532d2e8a245aef2128e1853981f8b6d9943264184843102"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "num_enum",
  "optional",
 ]
@@ -11318,9 +11256,9 @@ dependencies = [
 
 [[package]]
 name = "strum"
-version = "0.26.2"
+version = "0.26.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29"
+checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
 dependencies = [
  "strum_macros 0.26.4",
 ]
@@ -11348,7 +11286,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -11361,7 +11299,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -11414,7 +11352,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_yaml",
- "syn 2.0.66",
+ "syn 2.0.77",
  "typify",
  "walkdir",
 ]
@@ -11427,42 +11365,42 @@ checksum = "9ec889155c56a34200d2c5aee147b8d29545fa7cce7f68b38d927f5d24ced8ef"
 dependencies = [
  "heck 0.5.0",
  "prettyplease",
- "prost 0.13.1",
- "prost-build 0.13.1",
- "prost-types 0.13.1",
+ "prost 0.13.3",
+ "prost-build 0.13.3",
+ "prost-types 0.13.3",
  "schemars",
  "semver",
  "serde",
  "serde_json",
  "serde_yaml",
- "syn 2.0.66",
+ "syn 2.0.77",
  "typify",
  "walkdir",
 ]
 
 [[package]]
 name = "subtle"
-version = "2.5.0"
+version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "symbolic-common"
-version = "12.9.2"
+version = "12.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71297dc3e250f7dbdf8adb99e235da783d690f5819fdeb4cce39d9cfb0aca9f1"
+checksum = "9fdf97c441f18a4f92425b896a4ec7a27e03631a0b1047ec4e34e9916a9a167e"
 dependencies = [
  "debugid",
- "memmap2 0.9.4",
+ "memmap2 0.9.5",
  "stable_deref_trait",
  "uuid",
 ]
 
 [[package]]
 name = "symbolic-demangle"
-version = "12.9.2"
+version = "12.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "424fa2c9bf2c862891b9cfd354a752751a6730fd838a4691e7f6c2c7957b9daf"
+checksum = "bc8ece6b129e97e53d1fbb3f61d33a6a9e5369b11d01228c068094d6d134eaea"
 dependencies = [
  "cpp_demangle",
  "rustc-demangle",
@@ -11482,9 +11420,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.66"
+version = "2.0.77"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5"
+checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -11519,7 +11457,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -11529,21 +11467,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
 
 [[package]]
-name = "synstructure"
-version = "0.13.1"
+name = "sync_wrapper"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
+checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
 dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
+ "futures-core",
 ]
 
 [[package]]
 name = "sysinfo"
-version = "0.30.12"
+version = "0.30.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "732ffa00f53e6b2af46208fba5718d9662a421049204e156328b66791ffa15ae"
+checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3"
 dependencies = [
  "cfg-if",
  "core-foundation-sys",
@@ -11644,7 +11580,7 @@ dependencies = [
  "lru",
  "lz4_flex 0.11.3",
  "measure_time",
- "memmap2 0.9.4",
+ "memmap2 0.9.5",
  "num_cpus",
  "once_cell",
  "oneshot",
@@ -11668,7 +11604,7 @@ dependencies = [
  "time",
  "uuid",
  "winapi",
- "zstd 0.13.1",
+ "zstd 0.13.2",
 ]
 
 [[package]]
@@ -11749,7 +11685,7 @@ dependencies = [
  "tantivy-bitpacker",
  "tantivy-common",
  "tantivy-fst",
- "zstd 0.13.1",
+ "zstd 0.13.2",
 ]
 
 [[package]]
@@ -11780,9 +11716,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.14"
+version = "0.12.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
 
 [[package]]
 name = "temp-env"
@@ -11795,14 +11731,15 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.10.1"
+version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
+checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"
 dependencies = [
  "cfg-if",
  "fastrand",
+ "once_cell",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -11895,7 +11832,7 @@ dependencies = [
  "cache",
  "catalog",
  "chrono",
- "clap 4.5.7",
+ "clap 4.5.18",
  "client",
  "cmd",
  "common-base",
@@ -11957,7 +11894,7 @@ dependencies = [
  "tower",
  "url",
  "uuid",
- "zstd 0.13.1",
+ "zstd 0.13.2",
 ]
 
 [[package]]
@@ -11983,29 +11920,29 @@ checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
 
 [[package]]
 name = "thiserror"
-version = "1.0.63"
+version = "1.0.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724"
+checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.63"
+version = "1.0.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
+checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
 name = "thread-id"
-version = "4.2.1"
+version = "4.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0ec81c46e9eb50deaa257be2f148adf052d1fb7701cfd55ccfab2525280b70b"
+checksum = "cfe8f25bbdd100db7e1d34acf7fd2dc59c4bf8f7483f505eaa7d4f12f76cc0ea"
 dependencies = [
  "libc",
  "winapi",
@@ -12123,16 +12060,6 @@ dependencies = [
  "log",
 ]
 
-[[package]]
-name = "tinystr"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
-dependencies = [
- "displaydoc",
- "zerovec",
-]
-
 [[package]]
 name = "tinytemplate"
 version = "1.2.1"
@@ -12145,9 +12072,9 @@ dependencies = [
 
 [[package]]
 name = "tinyvec"
-version = "1.6.0"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
+checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938"
 dependencies = [
  "tinyvec_macros",
 ]
@@ -12160,22 +12087,21 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.38.0"
+version = "1.40.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a"
+checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998"
 dependencies = [
  "backtrace",
  "bytes",
  "libc",
- "mio",
- "num_cpus",
+ "mio 1.0.2",
  "parking_lot 0.12.3",
  "pin-project-lite",
  "signal-hook-registry",
  "socket2 0.5.7",
  "tokio-macros",
  "tracing",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -12190,13 +12116,13 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "2.3.0"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a"
+checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -12225,9 +12151,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-postgres"
-version = "0.7.11"
+version = "0.7.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03adcf0147e203b6032c0b2d30be1415ba03bc348901f3ff1cc0df6a733e60c3"
+checksum = "3b5d3742945bc7d7f210693b0c58ae542c6fd47b17adbbda0885f3dcb34a6bdb"
 dependencies = [
  "async-trait",
  "byteorder",
@@ -12256,7 +12182,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "04fb792ccd6bbcd4bba408eb8a292f70fc4a3589e5d793626f45190e6454b6ab"
 dependencies = [
  "ring 0.17.8",
- "rustls 0.23.10",
+ "rustls 0.23.13",
  "tokio",
  "tokio-postgres",
  "tokio-rustls 0.26.0",
@@ -12298,19 +12224,18 @@ dependencies = [
 [[package]]
 name = "tokio-rustls"
 version = "0.26.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
+source = "git+https://github.com/GreptimeTeam/tokio-rustls#4604ca6badfd1d10424718e5570cc481ab787fc8"
 dependencies = [
- "rustls 0.23.10",
+ "rustls 0.23.13",
  "rustls-pki-types",
  "tokio",
 ]
 
 [[package]]
 name = "tokio-stream"
-version = "0.1.15"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af"
+checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1"
 dependencies = [
  "futures-core",
  "pin-project-lite",
@@ -12333,9 +12258,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.11"
+version = "0.7.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1"
+checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a"
 dependencies = [
  "bytes",
  "futures-core",
@@ -12357,21 +12282,21 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.8.14"
+version = "0.8.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335"
+checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
 dependencies = [
  "serde",
  "serde_spanned",
  "toml_datetime",
- "toml_edit 0.22.14",
+ "toml_edit 0.22.21",
 ]
 
 [[package]]
 name = "toml_datetime"
-version = "0.6.6"
+version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf"
+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
 dependencies = [
  "serde",
 ]
@@ -12382,33 +12307,22 @@ version = "0.19.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
 dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "toml_datetime",
  "winnow 0.5.40",
 ]
 
 [[package]]
 name = "toml_edit"
-version = "0.21.1"
+version = "0.22.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
+checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf"
 dependencies = [
- "indexmap 2.2.6",
- "toml_datetime",
- "winnow 0.5.40",
-]
-
-[[package]]
-name = "toml_edit"
-version = "0.22.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38"
-dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "serde",
  "serde_spanned",
  "toml_datetime",
- "winnow 0.6.13",
+ "winnow 0.6.18",
 ]
 
 [[package]]
@@ -12426,7 +12340,7 @@ dependencies = [
  "h2",
  "http 0.2.12",
  "http-body 0.4.6",
- "hyper 0.14.29",
+ "hyper 0.14.30",
  "hyper-timeout 0.4.1",
  "percent-encoding",
  "pin-project",
@@ -12454,12 +12368,12 @@ dependencies = [
  "h2",
  "http 0.2.12",
  "http-body 0.4.6",
- "hyper 0.14.29",
+ "hyper 0.14.30",
  "hyper-timeout 0.4.1",
  "percent-encoding",
  "pin-project",
  "prost 0.12.6",
- "rustls-pemfile 2.1.2",
+ "rustls-pemfile 2.1.3",
  "rustls-pki-types",
  "tokio",
  "tokio-rustls 0.25.0",
@@ -12481,7 +12395,7 @@ dependencies = [
  "proc-macro2",
  "prost-build 0.12.6",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -12524,9 +12438,9 @@ version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140"
 dependencies = [
- "async-compression 0.4.11",
+ "async-compression 0.4.12",
  "base64 0.21.7",
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "bytes",
  "futures-core",
  "futures-util",
@@ -12555,10 +12469,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
 dependencies = [
  "base64 0.21.7",
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "bytes",
  "http 1.1.0",
- "http-body 1.0.0",
+ "http-body 1.0.1",
  "http-body-util",
  "mime",
  "pin-project-lite",
@@ -12569,15 +12483,15 @@ dependencies = [
 
 [[package]]
 name = "tower-layer"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
 
 [[package]]
 name = "tower-service"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
 
 [[package]]
 name = "tracing"
@@ -12611,7 +12525,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -12814,9 +12728,9 @@ checksum = "343e926fc669bc8cde4fa3129ab681c63671bae288b1f1081ceee6d9d37904fc"
 
 [[package]]
 name = "triomphe"
-version = "0.1.12"
+version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b2cb4fbb9995eeb36ac86fadf24031ccd58f99d6b4b2d7b911db70bddb80d90"
+checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3"
 
 [[package]]
 name = "try-lock"
@@ -12837,9 +12751,9 @@ dependencies = [
 
 [[package]]
 name = "typeid"
-version = "1.0.0"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "059d83cc991e7a42fc37bd50941885db0888e34209f8cfd9aab07ddec03bc9cf"
+checksum = "0e13db2e0ccd5e14a544e8a246ba2312cd25223f616442d7f2cb0e3db614236e"
 
 [[package]]
 name = "typenum"
@@ -12849,9 +12763,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
 
 [[package]]
 name = "typetag"
-version = "0.2.16"
+version = "0.2.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "661d18414ec032a49ece2d56eee03636e43c4e8d577047ab334c0ba892e29aaf"
+checksum = "52ba3b6e86ffe0054b2c44f2d86407388b933b16cb0a70eea3929420db1d9bbe"
 dependencies = [
  "erased-serde",
  "inventory",
@@ -12862,13 +12776,13 @@ dependencies = [
 
 [[package]]
 name = "typetag-impl"
-version = "0.2.16"
+version = "0.2.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac73887f47b9312552aa90ef477927ff014d63d1920ca8037c6c1951eab64bb1"
+checksum = "70b20a22c42c8f1cd23ce5e34f165d4d37038f5b663ad20fb6adbdf029172483"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -12896,7 +12810,7 @@ dependencies = [
  "semver",
  "serde",
  "serde_json",
- "syn 2.0.66",
+ "syn 2.0.77",
  "thiserror",
  "unicode-ident",
 ]
@@ -12914,7 +12828,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_tokenstream",
- "syn 2.0.66",
+ "syn 2.0.77",
  "typify-impl",
 ]
 
@@ -12940,9 +12854,9 @@ dependencies = [
 
 [[package]]
 name = "tzdb_data"
-version = "0.1.2"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1889fdffac09d65c1d95c42d5202e9b21ad8c758f426e9fe09088817ea998d6"
+checksum = "654c1ec546942ce0594e8d220e6b8e3899e0a0a8fe70ddd54d32a376dfefe3f8"
 dependencies = [
  "tz-rs",
 ]
@@ -13107,42 +13021,42 @@ checksum = "623f59e6af2a98bdafeb93fa277ac8e1e40440973001ca15cf4ae1541cd16d56"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.12"
+version = "1.0.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
 
 [[package]]
 name = "unicode-normalization"
-version = "0.1.23"
+version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
+checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
 dependencies = [
  "tinyvec",
 ]
 
 [[package]]
 name = "unicode-properties"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291"
+checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524"
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.11.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
+checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
 
 [[package]]
 name = "unicode-width"
-version = "0.1.13"
+version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
+checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
 
 [[package]]
 name = "unicode-xid"
-version = "0.2.4"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
+checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
 
 [[package]]
 name = "unicode_categories"
@@ -13184,9 +13098,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
 [[package]]
 name = "url"
-version = "2.5.1"
+version = "2.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56"
+checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
 dependencies = [
  "form_urlencoded",
  "idna",
@@ -13199,12 +13113,6 @@ version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
 
-[[package]]
-name = "utf16_iter"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
-
 [[package]]
 name = "utf8-ranges"
 version = "1.0.5"
@@ -13217,12 +13125,6 @@ version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3"
 
-[[package]]
-name = "utf8_iter"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
-
 [[package]]
 name = "utf8parse"
 version = "0.2.2"
@@ -13231,9 +13133,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "uuid"
-version = "1.8.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0"
+checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314"
 dependencies = [
  "atomic",
  "getrandom",
@@ -13244,13 +13146,13 @@ dependencies = [
 
 [[package]]
 name = "uuid-macro-internal"
-version = "1.8.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9881bea7cbe687e36c9ab3b778c36cd0487402e270304e8b1296d5085303c1a2"
+checksum = "ee1cd046f83ea2c4e920d6ee9f7c3537ef928d75dce5d84a87c2c5d6b3999a3a"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -13281,9 +13183,9 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
 
 [[package]]
 name = "vergen"
-version = "8.3.1"
+version = "8.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e27d6bdd219887a9eadd19e1c34f32e47fa332301184935c6d9bca26f3cca525"
+checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566"
 dependencies = [
  "anyhow",
  "rustversion",
@@ -13292,9 +13194,9 @@ dependencies = [
 
 [[package]]
 name = "version_check"
-version = "0.9.4"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
 [[package]]
 name = "vob"
@@ -13346,34 +13248,35 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
+checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5"
 dependencies = [
  "cfg-if",
+ "once_cell",
  "wasm-bindgen-macro",
 ]
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
+checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b"
 dependencies = [
  "bumpalo",
  "log",
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.42"
+version = "0.4.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0"
+checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -13383,9 +13286,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
+checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -13393,22 +13296,22 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
+checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
+checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
 
 [[package]]
 name = "wasm-streams"
@@ -13425,9 +13328,9 @@ dependencies = [
 
 [[package]]
 name = "web-sys"
-version = "0.3.69"
+version = "0.3.70"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"
+checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -13497,9 +13400,9 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1"
 
 [[package]]
 name = "webpki-roots"
-version = "0.26.2"
+version = "0.26.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c452ad30530b54a4d8e71952716a212b08efd0f3562baa66c29a618b07da7c3"
+checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958"
 dependencies = [
  "rustls-pki-types",
 ]
@@ -13518,20 +13421,20 @@ dependencies = [
 
 [[package]]
 name = "whoami"
-version = "1.5.1"
+version = "1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9"
+checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d"
 dependencies = [
- "redox_syscall 0.4.1",
+ "redox_syscall 0.5.4",
  "wasite",
  "web-sys",
 ]
 
 [[package]]
 name = "wide"
-version = "0.7.24"
+version = "0.7.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a040b111774ab63a19ef46bbc149398ab372b4ccdcfd719e9814dbd7dfd76c8"
+checksum = "b828f995bf1e9622031f8009f8481a85406ce1f4d4588ff746d872043e855690"
 dependencies = [
  "bytemuck",
  "safe_arch",
@@ -13561,11 +13464,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
 [[package]]
 name = "winapi-util"
-version = "0.1.8"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b"
+checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -13594,7 +13497,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"
 dependencies = [
  "windows-core",
- "windows-targets 0.52.5",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -13603,7 +13506,37 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
 dependencies = [
- "windows-targets 0.52.5",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-registry"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
+dependencies = [
+ "windows-result",
+ "windows-strings",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-result"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
+dependencies = [
+ "windows-result",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -13630,7 +13563,16 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
 dependencies = [
- "windows-targets 0.52.5",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -13665,18 +13607,18 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
 dependencies = [
- "windows_aarch64_gnullvm 0.52.5",
- "windows_aarch64_msvc 0.52.5",
- "windows_i686_gnu 0.52.5",
+ "windows_aarch64_gnullvm 0.52.6",
+ "windows_aarch64_msvc 0.52.6",
+ "windows_i686_gnu 0.52.6",
  "windows_i686_gnullvm",
- "windows_i686_msvc 0.52.5",
- "windows_x86_64_gnu 0.52.5",
- "windows_x86_64_gnullvm 0.52.5",
- "windows_x86_64_msvc 0.52.5",
+ "windows_i686_msvc 0.52.6",
+ "windows_x86_64_gnu 0.52.6",
+ "windows_x86_64_gnullvm 0.52.6",
+ "windows_x86_64_msvc 0.52.6",
 ]
 
 [[package]]
@@ -13693,9 +13635,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
 
 [[package]]
 name = "windows_aarch64_msvc"
@@ -13717,9 +13659,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
 
 [[package]]
 name = "windows_i686_gnu"
@@ -13741,15 +13683,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
 
 [[package]]
 name = "windows_i686_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
 
 [[package]]
 name = "windows_i686_msvc"
@@ -13771,9 +13713,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
 
 [[package]]
 name = "windows_x86_64_gnu"
@@ -13795,9 +13737,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
@@ -13813,9 +13755,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
 
 [[package]]
 name = "windows_x86_64_msvc"
@@ -13837,9 +13779,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"
@@ -13852,9 +13794,9 @@ dependencies = [
 
 [[package]]
 name = "winnow"
-version = "0.6.13"
+version = "0.6.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1"
+checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
 dependencies = [
  "memchr",
 ]
@@ -13868,28 +13810,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "winreg"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5"
-dependencies = [
- "cfg-if",
- "windows-sys 0.48.0",
-]
-
-[[package]]
-name = "write16"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
-
-[[package]]
-name = "writeable"
-version = "0.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
-
 [[package]]
 name = "wyz"
 version = "0.5.1"
@@ -13920,9 +13840,9 @@ dependencies = [
 
 [[package]]
 name = "xml-rs"
-version = "0.8.20"
+version = "0.8.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193"
+checksum = "af4e2e2f7cba5a093896c1e150fbfe177d1883e7448200efb81d40b9d339ef26"
 
 [[package]]
 name = "xz2"
@@ -13944,73 +13864,29 @@ dependencies = [
 
 [[package]]
 name = "yansi"
-version = "0.5.1"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec"
-
-[[package]]
-name = "yoke"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5"
-dependencies = [
- "serde",
- "stable_deref_trait",
- "yoke-derive",
- "zerofrom",
-]
-
-[[package]]
-name = "yoke-derive"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
- "synstructure",
-]
+checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
 
 [[package]]
 name = "zerocopy"
-version = "0.7.34"
+version = "0.7.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087"
+checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
 dependencies = [
+ "byteorder",
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.7.34"
+version = "0.7.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b"
+checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
-]
-
-[[package]]
-name = "zerofrom"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55"
-dependencies = [
- "zerofrom-derive",
-]
-
-[[package]]
-name = "zerofrom-derive"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
- "synstructure",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -14030,29 +13906,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
-]
-
-[[package]]
-name = "zerovec"
-version = "0.10.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"
-dependencies = [
- "yoke",
- "zerofrom",
- "zerovec-derive",
-]
-
-[[package]]
-name = "zerovec-derive"
-version = "0.10.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
+ "syn 2.0.77",
 ]
 
 [[package]]
@@ -14075,11 +13929,11 @@ dependencies = [
 
 [[package]]
 name = "zstd"
-version = "0.13.1"
+version = "0.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a"
+checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9"
 dependencies = [
- "zstd-safe 7.1.0",
+ "zstd-safe 7.2.1",
 ]
 
 [[package]]
@@ -14104,18 +13958,18 @@ dependencies = [
 
 [[package]]
 name = "zstd-safe"
-version = "7.1.0"
+version = "7.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a"
+checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059"
 dependencies = [
  "zstd-sys",
 ]
 
 [[package]]
 name = "zstd-sys"
-version = "2.0.10+zstd.1.5.6"
+version = "2.0.13+zstd.1.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa"
+checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"
 dependencies = [
  "cc",
  "pkg-config",
diff --git a/Cargo.toml b/Cargo.toml
index f0b93ac653..44b2cda122 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -245,6 +245,15 @@ store-api = { path = "src/store-api" }
 substrait = { path = "src/common/substrait" }
 table = { path = "src/table" }
 
+[patch.crates-io]
+# change all rustls dependencies to use our fork to default to `ring` to make it "just work"
+hyper-rustls = { git = "https://github.com/GreptimeTeam/hyper-rustls" }
+rustls = { git = "https://github.com/GreptimeTeam/rustls" }
+tokio-rustls = { git = "https://github.com/GreptimeTeam/tokio-rustls" }
+# This is commented, since we are not using aws-lc-sys, if we need to use it, we need to uncomment this line or use a release after this commit, or it wouldn't compile with gcc < 8.1
+# see https://github.com/aws/aws-lc-rs/pull/526
+# aws-lc-sys = { git ="https://github.com/aws/aws-lc-rs", rev = "556558441e3494af4b156ae95ebc07ebc2fd38aa" }
+
 [workspace.dependencies.meter-macros]
 git = "https://github.com/GreptimeTeam/greptime-meter.git"
 rev = "80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd"
diff --git a/Makefile b/Makefile
index da9e8da5c0..c63acb67b4 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,7 @@ CARGO_BUILD_OPTS := --locked
 IMAGE_REGISTRY ?= docker.io
 IMAGE_NAMESPACE ?= greptime
 IMAGE_TAG ?= latest
-DEV_BUILDER_IMAGE_TAG ?= 2024-06-06-1acda74c-20240919113454
+DEV_BUILDER_IMAGE_TAG ?= 2024-06-06-5674c14f-20240920110415
 BUILDX_MULTI_PLATFORM_BUILD ?= false
 BUILDX_BUILDER_NAME ?= gtbuilder
 BASE_IMAGE ?= ubuntu
diff --git a/docker/dev-builder/binstall/pull_binstall.sh b/docker/dev-builder/binstall/pull_binstall.sh
new file mode 100755
index 0000000000..a14ee6c7ea
--- /dev/null
+++ b/docker/dev-builder/binstall/pull_binstall.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+cd "$(mktemp -d)"
+# Fix version to v1.6.6, this is different than the latest version in original install script in
+# https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh
+base_url="https://github.com/cargo-bins/cargo-binstall/releases/download/v1.6.6/cargo-binstall-"
+
+os="$(uname -s)"
+if [ "$os" == "Darwin" ]; then
+    url="${base_url}universal-apple-darwin.zip"
+    curl -LO --proto '=https' --tlsv1.2 -sSf "$url"
+    unzip cargo-binstall-universal-apple-darwin.zip
+elif [ "$os" == "Linux" ]; then
+    machine="$(uname -m)"
+    if [ "$machine" == "armv7l" ]; then
+        machine="armv7"
+    fi
+    target="${machine}-unknown-linux-musl"
+    if [ "$machine" == "armv7" ]; then
+        target="${target}eabihf"
+    fi
+
+    url="${base_url}${target}.tgz"
+    curl -L --proto '=https' --tlsv1.2 -sSf "$url" | tar -xvzf -
+elif [ "${OS-}" = "Windows_NT" ]; then
+    machine="$(uname -m)"
+    target="${machine}-pc-windows-msvc"
+    url="${base_url}${target}.zip"
+    curl -LO --proto '=https' --tlsv1.2 -sSf "$url"
+    unzip "cargo-binstall-${target}.zip"
+else
+    echo "Unsupported OS ${os}"
+    exit 1
+fi
+
+./cargo-binstall -y --force cargo-binstall
+
+CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
+
+if ! [[ ":$PATH:" == *":$CARGO_HOME/bin:"* ]]; then
+    if [ -n "${CI:-}" ] && [ -n "${GITHUB_PATH:-}" ]; then
+        echo "$CARGO_HOME/bin" >> "$GITHUB_PATH"
+    else
+        echo
+        printf "\033[0;31mYour path is missing %s, you might want to add it.\033[0m\n" "$CARGO_HOME/bin"
+        echo
+    fi
+fi
diff --git a/docker/dev-builder/centos/Dockerfile b/docker/dev-builder/centos/Dockerfile
index af7e778e03..04327ab393 100644
--- a/docker/dev-builder/centos/Dockerfile
+++ b/docker/dev-builder/centos/Dockerfile
@@ -32,7 +32,9 @@ RUN rustup toolchain install ${RUST_TOOLCHAIN}
 
 # Install cargo-binstall with a specific version to adapt the current rust toolchain.
 # Note: if we use the latest version, we may encounter the following `use of unstable library feature 'io_error_downcast'` error.
-RUN cargo install cargo-binstall --version 1.6.6 --locked
+# compile from source take too long, so we use the precompiled binary instead
+COPY $DOCKER_BUILD_ROOT/docker/dev-builder/binstall/pull_binstall.sh /usr/local/bin/pull_binstall.sh
+RUN chmod +x /usr/local/bin/pull_binstall.sh && /usr/local/bin/pull_binstall.sh
 
 # Install nextest.
 RUN cargo binstall cargo-nextest --no-confirm
diff --git a/docker/dev-builder/ubuntu/Dockerfile b/docker/dev-builder/ubuntu/Dockerfile
index d70824e911..733bfdab62 100644
--- a/docker/dev-builder/ubuntu/Dockerfile
+++ b/docker/dev-builder/ubuntu/Dockerfile
@@ -26,7 +26,12 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 
 # https://github.com/GreptimeTeam/greptimedb/actions/runs/10935485852/job/30357457188#step:3:7106
 # `aws-lc-sys` require gcc >= 10.3.0 to work, hence alias to use gcc-10
-RUN alias gcc="gcc-10" && alias cc="gcc-10"
+RUN apt-get remove -y gcc-9 g++-9 cpp-9 && \
+    apt-get install -y gcc-10 g++-10 cpp-10 make cmake && \
+    ln -sf /usr/bin/gcc-10 /usr/bin/gcc && ln -sf /usr/bin/g++-10 /usr/bin/g++ && \
+    ln -sf /usr/bin/gcc-10 /usr/bin/cc && \
+    ln -sf /usr/bin/g++-10 /usr/bin/cpp && ln -sf /usr/bin/g++-10 /usr/bin/c++ && \
+    cc --version && gcc --version && g++ --version && cpp --version && c++ --version
 
 # Remove Python 3.8 and install pip.
 RUN apt-get -y purge python3.8 && \
@@ -61,7 +66,9 @@ RUN rustup toolchain install ${RUST_TOOLCHAIN}
 
 # Install cargo-binstall with a specific version to adapt the current rust toolchain.
 # Note: if we use the latest version, we may encounter the following `use of unstable library feature 'io_error_downcast'` error.
-RUN cargo install cargo-binstall --version 1.6.6 --locked
+# compile from source take too long, so we use the precompiled binary instead
+COPY $DOCKER_BUILD_ROOT/docker/dev-builder/binstall/pull_binstall.sh /usr/local/bin/pull_binstall.sh
+RUN chmod +x /usr/local/bin/pull_binstall.sh && /usr/local/bin/pull_binstall.sh
 
 # Install nextest.
 RUN cargo binstall cargo-nextest --no-confirm

From d1b252736d8d82c9e36fce155ab1fbb55f91ed5a Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Tue, 24 Sep 2024 11:48:16 +0800
Subject: [PATCH 033/128] refactor: unify the styling in
 `create_or_alter_tables_on_demand` (#4756)

* refactor: refactor `create_or_alter_tables_on_demand`

* chore: apply suggestions from CR
---
 src/operator/src/insert.rs | 176 +++++++++++++++++--------------------
 1 file changed, 82 insertions(+), 94 deletions(-)

diff --git a/src/operator/src/insert.rs b/src/operator/src/insert.rs
index c56e51c7a3..e0f0d39236 100644
--- a/src/operator/src/insert.rs
+++ b/src/operator/src/insert.rs
@@ -505,28 +505,25 @@ impl Inserter {
                     let table_info = table.table_info();
                     table_name_to_ids.insert(table_info.name.clone(), table_info.table_id());
                     if let Some(alter_expr) =
-                        self.get_alter_table_expr_on_demand(req, table, ctx)?
+                        self.get_alter_table_expr_on_demand(req, &table, ctx)?
                     {
                         alter_tables.push(alter_expr);
                     }
                 }
                 None => {
-                    create_tables.push(req);
+                    let create_expr =
+                        self.get_create_table_expr_on_demand(req, &auto_create_table_type, ctx)?;
+                    create_tables.push(create_expr);
                 }
             }
         }
 
         match auto_create_table_type {
-            AutoCreateTableType::Logical(on_physical_table) => {
+            AutoCreateTableType::Logical(_) => {
                 if !create_tables.is_empty() {
                     // Creates logical tables in batch.
                     let tables = self
-                        .create_logical_tables(
-                            create_tables,
-                            ctx,
-                            &on_physical_table,
-                            statement_executor,
-                        )
+                        .create_logical_tables(create_tables, ctx, statement_executor)
                         .await?;
 
                     for table in tables {
@@ -544,14 +541,9 @@ impl Inserter {
             AutoCreateTableType::Physical
             | AutoCreateTableType::Log
             | AutoCreateTableType::LastNonNull => {
-                for req in create_tables {
+                for create_table in create_tables {
                     let table = self
-                        .create_non_logical_table(
-                            req,
-                            ctx,
-                            statement_executor,
-                            auto_create_table_type.clone(),
-                        )
+                        .create_physical_table(create_table, ctx, statement_executor)
                         .await?;
                     let table_info = table.table_info();
                     table_name_to_ids.insert(table_info.name.clone(), table_info.table_id());
@@ -605,7 +597,8 @@ impl Inserter {
                 options: None,
             },
         ];
-        let create_table_expr = &mut build_create_table_expr(&table_reference, &default_schema)?;
+        let create_table_expr =
+            &mut build_create_table_expr(&table_reference, &default_schema, default_engine())?;
 
         create_table_expr.engine = METRIC_ENGINE_NAME.to_string();
         create_table_expr
@@ -641,10 +634,61 @@ impl Inserter {
             .context(CatalogSnafu)
     }
 
+    fn get_create_table_expr_on_demand(
+        &self,
+        req: &RowInsertRequest,
+        create_type: &AutoCreateTableType,
+        ctx: &QueryContextRef,
+    ) -> Result<CreateTableExpr> {
+        let mut table_options = Vec::with_capacity(4);
+        if let Some(ttl) = ctx.extension(TTL_KEY) {
+            table_options.push((TTL_KEY, ttl));
+        }
+
+        let mut engine_name = default_engine();
+        match create_type {
+            AutoCreateTableType::Logical(physical_table) => {
+                engine_name = METRIC_ENGINE_NAME;
+                table_options.push((LOGICAL_TABLE_METADATA_KEY, physical_table));
+            }
+            AutoCreateTableType::Physical => {
+                if let Some(append_mode) = ctx.extension(APPEND_MODE_KEY) {
+                    table_options.push((APPEND_MODE_KEY, append_mode));
+                }
+                if let Some(merge_mode) = ctx.extension(MERGE_MODE_KEY) {
+                    table_options.push((MERGE_MODE_KEY, merge_mode));
+                }
+            }
+            // Set append_mode to true for log table.
+            // because log tables should keep rows with the same ts and tags.
+            AutoCreateTableType::Log => {
+                table_options.push((APPEND_MODE_KEY, "true"));
+            }
+            AutoCreateTableType::LastNonNull => {
+                table_options.push((MERGE_MODE_KEY, "last_non_null"));
+            }
+        }
+
+        let schema = ctx.current_schema();
+        let table_ref = TableReference::full(ctx.current_catalog(), &schema, &req.table_name);
+        // SAFETY: `req.rows` is guaranteed to be `Some` by `handle_row_inserts_with_create_type()`.
+        let request_schema = req.rows.as_ref().unwrap().schema.as_slice();
+        let mut create_table_expr =
+            build_create_table_expr(&table_ref, request_schema, engine_name)?;
+        info!("Table `{table_ref}` does not exist, try creating table");
+        for (k, v) in table_options {
+            create_table_expr
+                .table_options
+                .insert(k.to_string(), v.to_string());
+        }
+
+        Ok(create_table_expr)
+    }
+
     fn get_alter_table_expr_on_demand(
         &self,
         req: &RowInsertRequest,
-        table: TableRef,
+        table: &TableRef,
         ctx: &QueryContextRef,
     ) -> Result<Option<AlterExpr>> {
         let catalog_name = ctx.current_catalog();
@@ -667,76 +711,37 @@ impl Inserter {
         }))
     }
 
-    /// Creates a non-logical table by create type.
-    /// # Panics
-    /// Panics if `create_type` is `AutoCreateTableType::Logical`.
-    async fn create_non_logical_table(
-        &self,
-        req: &RowInsertRequest,
-        ctx: &QueryContextRef,
-        statement_executor: &StatementExecutor,
-        create_type: AutoCreateTableType,
-    ) -> Result<TableRef> {
-        let mut hint_options = vec![];
-
-        if let Some(ttl) = ctx.extension(TTL_KEY) {
-            hint_options.push((TTL_KEY, ttl));
-        }
-
-        match create_type {
-            AutoCreateTableType::Logical(_) => unreachable!(),
-            AutoCreateTableType::Physical => {
-                if let Some(append_mode) = ctx.extension(APPEND_MODE_KEY) {
-                    hint_options.push((APPEND_MODE_KEY, append_mode));
-                }
-                if let Some(merge_mode) = ctx.extension(MERGE_MODE_KEY) {
-                    hint_options.push((MERGE_MODE_KEY, merge_mode));
-                }
-            }
-            // Set append_mode to true for log table.
-            // because log tables should keep rows with the same ts and tags.
-            AutoCreateTableType::Log => {
-                hint_options.push((APPEND_MODE_KEY, "true"));
-            }
-            AutoCreateTableType::LastNonNull => {
-                hint_options.push((MERGE_MODE_KEY, "last_non_null"));
-            }
-        }
-        let options: &[(&str, &str)] = hint_options.as_slice();
-
-        self.create_table_with_options(req, ctx, statement_executor, options)
-            .await
-    }
-
     /// Creates a table with options.
-    async fn create_table_with_options(
+    async fn create_physical_table(
         &self,
-        req: &RowInsertRequest,
+        mut create_table_expr: CreateTableExpr,
         ctx: &QueryContextRef,
         statement_executor: &StatementExecutor,
-        options: &[(&str, &str)],
     ) -> Result<TableRef> {
-        let schema = ctx.current_schema();
-        let table_ref = TableReference::full(ctx.current_catalog(), &schema, &req.table_name);
-        // SAFETY: `req.rows` is guaranteed to be `Some` by `handle_row_inserts_with_create_type()`.
-        let request_schema = req.rows.as_ref().unwrap().schema.as_slice();
-        let create_table_expr = &mut build_create_table_expr(&table_ref, request_schema)?;
+        {
+            let table_ref = TableReference::full(
+                &create_table_expr.catalog_name,
+                &create_table_expr.schema_name,
+                &create_table_expr.table_name,
+            );
 
-        info!("Table `{table_ref}` does not exist, try creating table");
-        for (k, v) in options {
-            create_table_expr
-                .table_options
-                .insert(k.to_string(), v.to_string());
+            info!("Table `{table_ref}` does not exist, try creating table");
         }
         let res = statement_executor
-            .create_table_inner(create_table_expr, None, ctx.clone())
+            .create_table_inner(&mut create_table_expr, None, ctx.clone())
             .await;
 
+        let table_ref = TableReference::full(
+            &create_table_expr.catalog_name,
+            &create_table_expr.schema_name,
+            &create_table_expr.table_name,
+        );
+
         match res {
             Ok(table) => {
                 info!(
                     "Successfully created table {} with options: {:?}",
-                    table_ref, options
+                    table_ref, create_table_expr.table_options,
                 );
                 Ok(table)
             }
@@ -749,30 +754,12 @@ impl Inserter {
 
     async fn create_logical_tables(
         &self,
-        create_tables: Vec<&RowInsertRequest>,
+        create_table_exprs: Vec<CreateTableExpr>,
         ctx: &QueryContextRef,
-        physical_table: &str,
         statement_executor: &StatementExecutor,
     ) -> Result<Vec<TableRef>> {
         let catalog_name = ctx.current_catalog();
         let schema_name = ctx.current_schema();
-        let create_table_exprs = create_tables
-            .iter()
-            .map(|req| {
-                let table_ref = TableReference::full(catalog_name, &schema_name, &req.table_name);
-                let request_schema = req.rows.as_ref().unwrap().schema.as_slice();
-                let mut create_table_expr = build_create_table_expr(&table_ref, request_schema)?;
-
-                create_table_expr.engine = METRIC_ENGINE_NAME.to_string();
-                create_table_expr.table_options.insert(
-                    LOGICAL_TABLE_METADATA_KEY.to_string(),
-                    physical_table.to_string(),
-                );
-
-                Ok(create_table_expr)
-            })
-            .collect::<Result<Vec<_>>>()?;
-
         let res = statement_executor
             .create_logical_tables(catalog_name, &schema_name, &create_table_exprs, ctx.clone())
             .await;
@@ -827,6 +814,7 @@ fn validate_column_count_match(requests: &RowInsertRequests) -> Result<()> {
 fn build_create_table_expr(
     table: &TableReference,
     request_schema: &[ColumnSchema],
+    engine: &str,
 ) -> Result<CreateTableExpr> {
-    CreateExprFactory.create_table_expr_by_column_schemas(table, request_schema, default_engine())
+    CreateExprFactory.create_table_expr_by_column_schemas(table, request_schema, engine)
 }

From e3c0b5482f379f71433eb2fbd0d8ea63c38018f1 Mon Sep 17 00:00:00 2001
From: Ning Sun <sunng@protonmail.com>
Date: Tue, 24 Sep 2024 16:45:55 +0800
Subject: [PATCH 034/128] feat: returning warning instead of error on
 unsupported `SET` statement (#4761)

* feat: add capability to send warning to pgclient

* fix: refactor query context to carry query scope data

* feat: return a warning for unsupported postgres statement
---
 src/operator/src/statement.rs           | 16 +++++--
 src/servers/src/postgres/handler.rs     | 42 ++++++++++++++++---
 src/servers/src/postgres/types.rs       |  2 +-
 src/servers/src/postgres/types/error.rs | 40 +++++++++---------
 src/session/src/context.rs              | 56 ++++++++++++++++++-------
 src/session/src/lib.rs                  |  2 +-
 6 files changed, 112 insertions(+), 46 deletions(-)

diff --git a/src/operator/src/statement.rs b/src/operator/src/statement.rs
index 4dc43e0d92..7c76d0dcff 100644
--- a/src/operator/src/statement.rs
+++ b/src/operator/src/statement.rs
@@ -46,7 +46,7 @@ use datafusion_expr::LogicalPlan;
 use partition::manager::{PartitionRuleManager, PartitionRuleManagerRef};
 use query::parser::QueryStatement;
 use query::QueryEngineRef;
-use session::context::QueryContextRef;
+use session::context::{Channel, QueryContextRef};
 use session::table_name::table_idents_to_full_name;
 use snafu::{ensure, OptionExt, ResultExt};
 use sql::statements::copy::{CopyDatabase, CopyDatabaseArgument, CopyTable, CopyTableArgument};
@@ -338,10 +338,18 @@ impl StatementExecutor {
 
             "CLIENT_ENCODING" => validate_client_encoding(set_var)?,
             _ => {
-                return NotSupportedSnafu {
-                    feat: format!("Unsupported set variable {}", var_name),
+                // for postgres, we give unknown SET statements a warning with
+                //  success, this is prevent the SET call becoming a blocker
+                //  of connection establishment
+                //
+                if query_ctx.channel() == Channel::Postgres {
+                    query_ctx.set_warning(format!("Unsupported set variable {}", var_name));
+                } else {
+                    return NotSupportedSnafu {
+                        feat: format!("Unsupported set variable {}", var_name),
+                    }
+                    .fail();
                 }
-                .fail()
             }
         }
         Ok(Output::new_with_affected_rows(0))
diff --git a/src/servers/src/postgres/handler.rs b/src/servers/src/postgres/handler.rs
index 158e2cab4d..522c558cdc 100644
--- a/src/servers/src/postgres/handler.rs
+++ b/src/servers/src/postgres/handler.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::fmt::Debug;
 use std::sync::Arc;
 
 use async_trait::async_trait;
@@ -23,7 +24,7 @@ use common_telemetry::{debug, error, tracing};
 use datafusion_common::ParamValues;
 use datatypes::prelude::ConcreteDataType;
 use datatypes::schema::SchemaRef;
-use futures::{future, stream, Stream, StreamExt};
+use futures::{future, stream, Sink, SinkExt, Stream, StreamExt};
 use pgwire::api::portal::{Format, Portal};
 use pgwire::api::query::{ExtendedQueryHandler, SimpleQueryHandler};
 use pgwire::api::results::{
@@ -32,6 +33,7 @@ use pgwire::api::results::{
 use pgwire::api::stmt::{QueryParser, StoredStatement};
 use pgwire::api::{ClientInfo, Type};
 use pgwire::error::{ErrorInfo, PgWireError, PgWireResult};
+use pgwire::messages::PgWireBackendMessage;
 use query::query_engine::DescribeResult;
 use session::context::QueryContextRef;
 use session::Session;
@@ -49,11 +51,13 @@ impl SimpleQueryHandler for PostgresServerHandlerInner {
     #[tracing::instrument(skip_all, fields(protocol = "postgres"))]
     async fn do_query<'a, C>(
         &self,
-        _client: &mut C,
+        client: &mut C,
         query: &'a str,
     ) -> PgWireResult<Vec<Response<'a>>>
     where
-        C: ClientInfo + Unpin + Send + Sync,
+        C: ClientInfo + Sink<PgWireBackendMessage> + Unpin + Send + Sync,
+        C::Error: Debug,
+        PgWireError: From<<C as Sink<PgWireBackendMessage>>::Error>,
     {
         let query_ctx = self.session.new_query_context();
         let db = query_ctx.get_db_string();
@@ -67,6 +71,7 @@ impl SimpleQueryHandler for PostgresServerHandlerInner {
         }
 
         if let Some(resps) = fixtures::process(query, query_ctx.clone()) {
+            send_warning_opt(client, query_ctx).await?;
             Ok(resps)
         } else {
             let outputs = self.query_handler.do_query(query, query_ctx.clone()).await;
@@ -79,11 +84,34 @@ impl SimpleQueryHandler for PostgresServerHandlerInner {
                 results.push(resp);
             }
 
+            send_warning_opt(client, query_ctx).await?;
             Ok(results)
         }
     }
 }
 
+async fn send_warning_opt<C>(client: &mut C, query_context: QueryContextRef) -> PgWireResult<()>
+where
+    C: Sink<PgWireBackendMessage> + Unpin + Send + Sync,
+    C::Error: Debug,
+    PgWireError: From<<C as Sink<PgWireBackendMessage>>::Error>,
+{
+    if let Some(warning) = query_context.warning() {
+        client
+            .feed(PgWireBackendMessage::NoticeResponse(
+                ErrorInfo::new(
+                    PgErrorSeverity::Warning.to_string(),
+                    PgErrorCode::Ec01000.code(),
+                    warning.to_string(),
+                )
+                .into(),
+            ))
+            .await?;
+    }
+
+    Ok(())
+}
+
 pub(crate) fn output_to_query_response<'a>(
     query_ctx: QueryContextRef,
     output: Result<Output>,
@@ -247,12 +275,14 @@ impl ExtendedQueryHandler for PostgresServerHandlerInner {
 
     async fn do_query<'a, C>(
         &self,
-        _client: &mut C,
+        client: &mut C,
         portal: &'a Portal<Self::Statement>,
         _max_rows: usize,
     ) -> PgWireResult<Response<'a>>
     where
-        C: ClientInfo + Unpin + Send + Sync,
+        C: ClientInfo + Sink<PgWireBackendMessage> + Unpin + Send + Sync,
+        C::Error: Debug,
+        PgWireError: From<<C as Sink<PgWireBackendMessage>>::Error>,
     {
         let query_ctx = self.session.new_query_context();
         let db = query_ctx.get_db_string();
@@ -268,6 +298,7 @@ impl ExtendedQueryHandler for PostgresServerHandlerInner {
         }
 
         if let Some(mut resps) = fixtures::process(&sql_plan.query, query_ctx.clone()) {
+            send_warning_opt(client, query_ctx).await?;
             // if the statement matches our predefined rules, return it early
             return Ok(resps.remove(0));
         }
@@ -297,6 +328,7 @@ impl ExtendedQueryHandler for PostgresServerHandlerInner {
                 .remove(0)
         };
 
+        send_warning_opt(client, query_ctx.clone()).await?;
         output_to_query_response(query_ctx, output, &portal.result_column_format)
     }
 
diff --git a/src/servers/src/postgres/types.rs b/src/servers/src/postgres/types.rs
index 2e4a805ef0..a5d1d392ac 100644
--- a/src/servers/src/postgres/types.rs
+++ b/src/servers/src/postgres/types.rs
@@ -37,7 +37,7 @@ use session::session_config::PGByteaOutputValue;
 
 use self::bytea::{EscapeOutputBytea, HexOutputBytea};
 use self::datetime::{StylingDate, StylingDateTime};
-pub use self::error::PgErrorCode;
+pub use self::error::{PgErrorCode, PgErrorSeverity};
 use self::interval::PgInterval;
 use crate::error::{self as server_error, Error, Result};
 use crate::SqlPlan;
diff --git a/src/servers/src/postgres/types/error.rs b/src/servers/src/postgres/types/error.rs
index 928c5454ce..9e6f570f26 100644
--- a/src/servers/src/postgres/types/error.rs
+++ b/src/servers/src/postgres/types/error.rs
@@ -19,7 +19,7 @@ use strum::{AsRefStr, Display, EnumIter, EnumMessage};
 
 #[derive(Display, Debug, PartialEq)]
 #[allow(dead_code)]
-enum ErrorSeverity {
+pub enum PgErrorSeverity {
     #[strum(serialize = "INFO")]
     Info,
     #[strum(serialize = "DEBUG")]
@@ -335,23 +335,23 @@ pub enum PgErrorCode {
 }
 
 impl PgErrorCode {
-    fn severity(&self) -> ErrorSeverity {
+    fn severity(&self) -> PgErrorSeverity {
         match self {
-            PgErrorCode::Ec00000 => ErrorSeverity::Info,
-            PgErrorCode::Ec01000 => ErrorSeverity::Warning,
+            PgErrorCode::Ec00000 => PgErrorSeverity::Info,
+            PgErrorCode::Ec01000 => PgErrorSeverity::Warning,
 
             PgErrorCode::EcXX000 | PgErrorCode::Ec42P14 | PgErrorCode::Ec22023 => {
-                ErrorSeverity::Error
+                PgErrorSeverity::Error
             }
             PgErrorCode::Ec28000 | PgErrorCode::Ec28P01 | PgErrorCode::Ec3D000 => {
-                ErrorSeverity::Fatal
+                PgErrorSeverity::Fatal
             }
 
-            _ => ErrorSeverity::Error,
+            _ => PgErrorSeverity::Error,
         }
     }
 
-    fn code(&self) -> String {
+    pub(crate) fn code(&self) -> String {
         self.as_ref()[2..].to_string()
     }
 
@@ -428,19 +428,19 @@ mod tests {
     use common_error::status_code::StatusCode;
     use strum::{EnumMessage, IntoEnumIterator};
 
-    use super::{ErrorInfo, ErrorSeverity, PgErrorCode};
+    use super::{ErrorInfo, PgErrorCode, PgErrorSeverity};
 
     #[test]
     fn test_error_severity() {
         // test for ErrorSeverity enum
-        assert_eq!("INFO", ErrorSeverity::Info.to_string());
-        assert_eq!("DEBUG", ErrorSeverity::Debug.to_string());
-        assert_eq!("NOTICE", ErrorSeverity::Notice.to_string());
-        assert_eq!("WARNING", ErrorSeverity::Warning.to_string());
+        assert_eq!("INFO", PgErrorSeverity::Info.to_string());
+        assert_eq!("DEBUG", PgErrorSeverity::Debug.to_string());
+        assert_eq!("NOTICE", PgErrorSeverity::Notice.to_string());
+        assert_eq!("WARNING", PgErrorSeverity::Warning.to_string());
 
-        assert_eq!("ERROR", ErrorSeverity::Error.to_string());
-        assert_eq!("FATAL", ErrorSeverity::Fatal.to_string());
-        assert_eq!("PANIC", ErrorSeverity::Panic.to_string());
+        assert_eq!("ERROR", PgErrorSeverity::Error.to_string());
+        assert_eq!("FATAL", PgErrorSeverity::Fatal.to_string());
+        assert_eq!("PANIC", PgErrorSeverity::Panic.to_string());
 
         // test for severity method
         for code in PgErrorCode::iter() {
@@ -448,13 +448,13 @@ mod tests {
             assert_eq!("Ec", &name[0..2]);
 
             if name.starts_with("Ec00") {
-                assert_eq!(ErrorSeverity::Info, code.severity());
+                assert_eq!(PgErrorSeverity::Info, code.severity());
             } else if name.starts_with("Ec01") {
-                assert_eq!(ErrorSeverity::Warning, code.severity());
+                assert_eq!(PgErrorSeverity::Warning, code.severity());
             } else if name.starts_with("Ec28") || name.starts_with("Ec3D") {
-                assert_eq!(ErrorSeverity::Fatal, code.severity());
+                assert_eq!(PgErrorSeverity::Fatal, code.severity());
             } else {
-                assert_eq!(ErrorSeverity::Error, code.severity());
+                assert_eq!(PgErrorSeverity::Error, code.severity());
             }
         }
     }
diff --git a/src/session/src/context.rs b/src/session/src/context.rs
index 70168d9498..f85a8ceea3 100644
--- a/src/session/src/context.rs
+++ b/src/session/src/context.rs
@@ -40,7 +40,9 @@ pub struct QueryContext {
     current_catalog: String,
     // we use Arc<RwLock>> for modifiable fields
     #[builder(default)]
-    mutable_inner: Arc<RwLock<MutableInner>>,
+    mutable_session_data: Arc<RwLock<MutableInner>>,
+    #[builder(default)]
+    mutable_query_context_data: Arc<RwLock<QueryContextMutableFields>>,
     sql_dialect: Arc<dyn Dialect + Send + Sync>,
     #[builder(default)]
     extensions: HashMap<String, String>,
@@ -52,6 +54,12 @@ pub struct QueryContext {
     channel: Channel,
 }
 
+/// This fields hold data that is only valid to current query context
+#[derive(Debug, Builder, Clone, Default)]
+pub struct QueryContextMutableFields {
+    warning: Option<String>,
+}
+
 impl Display for QueryContext {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(
@@ -65,21 +73,26 @@ impl Display for QueryContext {
 
 impl QueryContextBuilder {
     pub fn current_schema(mut self, schema: String) -> Self {
-        if self.mutable_inner.is_none() {
-            self.mutable_inner = Some(Arc::new(RwLock::new(MutableInner::default())));
+        if self.mutable_session_data.is_none() {
+            self.mutable_session_data = Some(Arc::new(RwLock::new(MutableInner::default())));
         }
 
         // safe for unwrap because previous none check
-        self.mutable_inner.as_mut().unwrap().write().unwrap().schema = schema;
+        self.mutable_session_data
+            .as_mut()
+            .unwrap()
+            .write()
+            .unwrap()
+            .schema = schema;
         self
     }
 
     pub fn timezone(mut self, timezone: Timezone) -> Self {
-        if self.mutable_inner.is_none() {
-            self.mutable_inner = Some(Arc::new(RwLock::new(MutableInner::default())));
+        if self.mutable_session_data.is_none() {
+            self.mutable_session_data = Some(Arc::new(RwLock::new(MutableInner::default())));
         }
 
-        self.mutable_inner
+        self.mutable_session_data
             .as_mut()
             .unwrap()
             .write()
@@ -120,7 +133,7 @@ impl From<QueryContext> for api::v1::QueryContext {
     fn from(
         QueryContext {
             current_catalog,
-            mutable_inner,
+            mutable_session_data: mutable_inner,
             extensions,
             channel,
             ..
@@ -182,11 +195,11 @@ impl QueryContext {
     }
 
     pub fn current_schema(&self) -> String {
-        self.mutable_inner.read().unwrap().schema.clone()
+        self.mutable_session_data.read().unwrap().schema.clone()
     }
 
     pub fn set_current_schema(&self, new_schema: &str) {
-        self.mutable_inner.write().unwrap().schema = new_schema.to_string();
+        self.mutable_session_data.write().unwrap().schema = new_schema.to_string();
     }
 
     pub fn current_catalog(&self) -> &str {
@@ -208,19 +221,19 @@ impl QueryContext {
     }
 
     pub fn timezone(&self) -> Timezone {
-        self.mutable_inner.read().unwrap().timezone.clone()
+        self.mutable_session_data.read().unwrap().timezone.clone()
     }
 
     pub fn set_timezone(&self, timezone: Timezone) {
-        self.mutable_inner.write().unwrap().timezone = timezone;
+        self.mutable_session_data.write().unwrap().timezone = timezone;
     }
 
     pub fn current_user(&self) -> UserInfoRef {
-        self.mutable_inner.read().unwrap().user_info.clone()
+        self.mutable_session_data.read().unwrap().user_info.clone()
     }
 
     pub fn set_current_user(&self, user: UserInfoRef) {
-        self.mutable_inner.write().unwrap().user_info = user;
+        self.mutable_session_data.write().unwrap().user_info = user;
     }
 
     pub fn set_extension<S1: Into<String>, S2: Into<String>>(&mut self, key: S1, value: S2) {
@@ -257,6 +270,18 @@ impl QueryContext {
     pub fn set_channel(&mut self, channel: Channel) {
         self.channel = channel;
     }
+
+    pub fn warning(&self) -> Option<String> {
+        self.mutable_query_context_data
+            .read()
+            .unwrap()
+            .warning
+            .clone()
+    }
+
+    pub fn set_warning(&self, msg: String) {
+        self.mutable_query_context_data.write().unwrap().warning = Some(msg);
+    }
 }
 
 impl QueryContextBuilder {
@@ -266,7 +291,8 @@ impl QueryContextBuilder {
             current_catalog: self
                 .current_catalog
                 .unwrap_or_else(|| DEFAULT_CATALOG_NAME.to_string()),
-            mutable_inner: self.mutable_inner.unwrap_or_default(),
+            mutable_session_data: self.mutable_session_data.unwrap_or_default(),
+            mutable_query_context_data: self.mutable_query_context_data.unwrap_or_default(),
             sql_dialect: self
                 .sql_dialect
                 .unwrap_or_else(|| Arc::new(GreptimeDbDialect {})),
diff --git a/src/session/src/lib.rs b/src/session/src/lib.rs
index ecfc02f230..33bd140c70 100644
--- a/src/session/src/lib.rs
+++ b/src/session/src/lib.rs
@@ -76,7 +76,7 @@ impl Session {
             // catalog is not allowed for update in query context so we use
             // string here
             .current_catalog(self.catalog.read().unwrap().clone())
-            .mutable_inner(self.mutable_inner.clone())
+            .mutable_session_data(self.mutable_inner.clone())
             .sql_dialect(self.conn_info.channel.dialect())
             .configuration_parameter(self.configuration_variables.clone())
             .channel(self.conn_info.channel)

From cd4bf239d05f2b9747004c8a7fa3c744c32c2166 Mon Sep 17 00:00:00 2001
From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com>
Date: Wed, 25 Sep 2024 10:45:18 +0800
Subject: [PATCH 035/128] chore: relax table name constraint (#4766)

chore/relax-table-name-constraint: Updated NAME_PATTERN to allow '@' and '#' characters and adjusted tests for new table name validation rules.
---
 src/common/meta/src/key.rs                       |  2 +-
 src/operator/src/statement/ddl.rs                |  8 +++++++-
 .../cases/standalone/common/create/create.result | 16 ++++++++++++++++
 tests/cases/standalone/common/create/create.sql  |  8 ++++++++
 4 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/src/common/meta/src/key.rs b/src/common/meta/src/key.rs
index 3cf6f3a3b7..d864882da6 100644
--- a/src/common/meta/src/key.rs
+++ b/src/common/meta/src/key.rs
@@ -144,7 +144,7 @@ use crate::rpc::router::{region_distribution, RegionRoute, RegionStatus};
 use crate::rpc::store::BatchDeleteRequest;
 use crate::DatanodeId;
 
-pub const NAME_PATTERN: &str = r"[a-zA-Z_:-][a-zA-Z0-9_:\-\.]*";
+pub const NAME_PATTERN: &str = r"[a-zA-Z_:-][a-zA-Z0-9_:\-\.@#]*";
 pub const MAINTENANCE_KEY: &str = "__maintenance";
 
 const DATANODE_TABLE_KEY_PREFIX: &str = "__dn_table";
diff --git a/src/operator/src/statement/ddl.rs b/src/operator/src/statement/ddl.rs
index e9b186bd63..af0f6883d8 100644
--- a/src/operator/src/statement/ddl.rs
+++ b/src/operator/src/statement/ddl.rs
@@ -229,7 +229,7 @@ impl StatementExecutor {
         ensure!(
             NAME_PATTERN_REG.is_match(&create_table.table_name),
             InvalidTableNameSnafu {
-                table_name: create_table.table_name.clone(),
+                table_name: &create_table.table_name,
             }
         );
 
@@ -1516,6 +1516,12 @@ mod test {
         assert!(!NAME_PATTERN_REG.is_match("/adaf"));
         assert!(!NAME_PATTERN_REG.is_match("🈲"));
         assert!(NAME_PATTERN_REG.is_match("hello"));
+        assert!(NAME_PATTERN_REG.is_match("test@"));
+        assert!(!NAME_PATTERN_REG.is_match("@test"));
+        assert!(NAME_PATTERN_REG.is_match("test#"));
+        assert!(!NAME_PATTERN_REG.is_match("#test"));
+        assert!(!NAME_PATTERN_REG.is_match("@"));
+        assert!(!NAME_PATTERN_REG.is_match("#"));
     }
 
     #[tokio::test]
diff --git a/tests/cases/standalone/common/create/create.result b/tests/cases/standalone/common/create/create.result
index 86a89483e8..d01ac4a764 100644
--- a/tests/cases/standalone/common/create/create.result
+++ b/tests/cases/standalone/common/create/create.result
@@ -54,6 +54,22 @@ CREATE TABLE 'N.~' (i TIMESTAMP TIME INDEX);
 
 Error: 1004(InvalidArguments), Invalid table name: N.~
 
+CREATE TABLE `p_perftest001@cc3kvQ_D1D9H9GOZGMIY97TWH20R1LRC8U0SFBA` (i INTEGER, j TIMESTAMP TIME INDEX);
+
+Affected Rows: 0
+
+DROP TABLE `p_perftest001@cc3kvQ_D1D9H9GOZGMIY97TWH20R1LRC8U0SFBA`;
+
+Affected Rows: 0
+
+CREATE TABLE `p_perftest001#cc3kvQ_D1D9H9GOZGMIY97TWH20R1LRC8U0SFBA` (i INTEGER, j TIMESTAMP TIME INDEX);
+
+Affected Rows: 0
+
+DROP TABLE `p_perftest001#cc3kvQ_D1D9H9GOZGMIY97TWH20R1LRC8U0SFBA`;
+
+Affected Rows: 0
+
 CREATE TABLE neg_default_value_min(i TIMESTAMP TIME INDEX, j SMALLINT DEFAULT -32768);
 
 Affected Rows: 0
diff --git a/tests/cases/standalone/common/create/create.sql b/tests/cases/standalone/common/create/create.sql
index f5ae083318..62237019ad 100644
--- a/tests/cases/standalone/common/create/create.sql
+++ b/tests/cases/standalone/common/create/create.sql
@@ -26,6 +26,14 @@ CREATE TABLE test2 (i INTEGER, j TIMESTAMP TIME INDEX);
 
 CREATE TABLE 'N.~' (i TIMESTAMP TIME INDEX);
 
+CREATE TABLE `p_perftest001@cc3kvQ_D1D9H9GOZGMIY97TWH20R1LRC8U0SFBA` (i INTEGER, j TIMESTAMP TIME INDEX);
+
+DROP TABLE `p_perftest001@cc3kvQ_D1D9H9GOZGMIY97TWH20R1LRC8U0SFBA`;
+
+CREATE TABLE `p_perftest001#cc3kvQ_D1D9H9GOZGMIY97TWH20R1LRC8U0SFBA` (i INTEGER, j TIMESTAMP TIME INDEX);
+
+DROP TABLE `p_perftest001#cc3kvQ_D1D9H9GOZGMIY97TWH20R1LRC8U0SFBA`;
+
 CREATE TABLE neg_default_value_min(i TIMESTAMP TIME INDEX, j SMALLINT DEFAULT -32768);
 
 DESC TABLE neg_default_value_min;

From 0274e752aeb1332fbd0a8b51882f7b9da0c73502 Mon Sep 17 00:00:00 2001
From: discord9 <55937128+discord9@users.noreply.github.com>
Date: Thu, 26 Sep 2024 01:11:37 +0800
Subject: [PATCH 036/128] chore: make sure aws-lc-sys wouldn't be built (#4767)

* chore: make sure aws-lc-sys wouldn't be built

* fix: bash

* refactor: multiple line bash
---
 .github/workflows/develop.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
index 5100b89754..073c407486 100644
--- a/.github/workflows/develop.yml
+++ b/.github/workflows/develop.yml
@@ -269,6 +269,13 @@ jobs:
       - name: Install cargo-gc-bin
         shell: bash
         run: cargo install cargo-gc-bin
+      - name: Check aws-lc-sys will not build
+        shell: bash
+        run: |
+             if cargo tree -i aws-lc-sys -e features | grep -q aws-lc-sys; then
+               echo "Found aws-lc-sys, which has compilation problems on older gcc versions. Please replace it with ring until its building experience improves."
+               exit 1
+             fi
       - name: Build greptime bianry
         shell: bash
         # `cargo gc` will invoke `cargo build` with specified args

From 627a3262735eaa490cee8dadf413b61a04948d16 Mon Sep 17 00:00:00 2001
From: localhost <xpaomian@gmail.com>
Date: Thu, 26 Sep 2024 03:32:34 +0800
Subject: [PATCH 037/128] refactor: Change the error type in the pipeline crate
 from String to Error (#4763)

* chore: in process

* chore: change pipeline crate error type

* chore: improve event error

* chore: fix by pr comment

* chore: use snafu context replace ok_or_else

* refactor: update snafu usage

---------

Co-authored-by: Ning Sun <sunng@protonmail.com>
---
 src/pipeline/benches/processor.rs             |   8 +-
 src/pipeline/src/etl.rs                       |  30 +-
 src/pipeline/src/etl/error.rs                 | 552 ++++++++++++++++++
 src/pipeline/src/etl/field.rs                 |  14 +-
 src/pipeline/src/etl/processor.rs             |  71 ++-
 src/pipeline/src/etl/processor/cmcd.rs        |  69 ++-
 src/pipeline/src/etl/processor/csv.rs         |  62 +-
 src/pipeline/src/etl/processor/date.rs        |  63 +-
 src/pipeline/src/etl/processor/dissect.rs     | 222 ++++---
 src/pipeline/src/etl/processor/epoch.rs       |  45 +-
 src/pipeline/src/etl/processor/gsub.rs        |  65 ++-
 src/pipeline/src/etl/processor/join.rs        |  47 +-
 src/pipeline/src/etl/processor/letter.rs      |  44 +-
 src/pipeline/src/etl/processor/regex.rs       |  79 +--
 src/pipeline/src/etl/processor/timestamp.rs   |  93 +--
 src/pipeline/src/etl/processor/urlencoding.rs |  46 +-
 src/pipeline/src/etl/transform.rs             |  45 +-
 src/pipeline/src/etl/transform/index.rs       |  12 +-
 .../src/etl/transform/transformer/greptime.rs |  35 +-
 .../transform/transformer/greptime/coerce.rs  |  80 +--
 src/pipeline/src/etl/value.rs                 |  75 +--
 src/pipeline/src/lib.rs                       |   1 +
 src/pipeline/src/manager/error.rs             |  16 +-
 src/pipeline/src/manager/table.rs             |   3 +-
 src/pipeline/tests/dissect.rs                 |   2 +-
 src/servers/src/http/event.rs                 |  37 +-
 26 files changed, 1275 insertions(+), 541 deletions(-)
 create mode 100644 src/pipeline/src/etl/error.rs

diff --git a/src/pipeline/benches/processor.rs b/src/pipeline/benches/processor.rs
index 281d8ce0ef..09462753d8 100644
--- a/src/pipeline/benches/processor.rs
+++ b/src/pipeline/benches/processor.rs
@@ -13,13 +13,13 @@
 // limitations under the License.
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use pipeline::{parse, Content, GreptimeTransformer, Pipeline};
+use pipeline::{parse, Content, GreptimeTransformer, Pipeline, Result};
 use serde_json::{Deserializer, Value};
 
 fn processor_mut(
     pipeline: &Pipeline<GreptimeTransformer>,
     input_values: Vec<Value>,
-) -> Result<Vec<greptime_proto::v1::Row>, String> {
+) -> Result<Vec<greptime_proto::v1::Row>> {
     let mut payload = pipeline.init_intermediate_state();
     let mut result = Vec::with_capacity(input_values.len());
 
@@ -30,7 +30,7 @@ fn processor_mut(
         pipeline.reset_intermediate_state(&mut payload);
     }
 
-    Ok::<Vec<greptime_proto::v1::Row>, String>(result)
+    Ok(result)
 }
 
 fn prepare_pipeline() -> Pipeline<GreptimeTransformer> {
@@ -230,7 +230,7 @@ fn criterion_benchmark(c: &mut Criterion) {
     let input_value_str = include_str!("./data.log");
     let input_value = Deserializer::from_str(input_value_str)
         .into_iter::<serde_json::Value>()
-        .collect::<Result<Vec<_>, _>>()
+        .collect::<std::result::Result<Vec<_>, _>>()
         .unwrap();
     let pipeline = prepare_pipeline();
     let mut group = c.benchmark_group("pipeline");
diff --git a/src/pipeline/src/etl.rs b/src/pipeline/src/etl.rs
index f6b3efd6e6..f29032e4f8 100644
--- a/src/pipeline/src/etl.rs
+++ b/src/pipeline/src/etl.rs
@@ -14,6 +14,7 @@
 
 #![allow(dead_code)]
 
+pub mod error;
 pub mod field;
 pub mod processor;
 pub mod transform;
@@ -21,12 +22,16 @@ pub mod value;
 
 use ahash::HashSet;
 use common_telemetry::debug;
+use error::{IntermediateKeyIndexSnafu, PrepareValueMustBeObjectSnafu, YamlLoadSnafu};
 use itertools::Itertools;
 use processor::{Processor, ProcessorBuilder, Processors};
+use snafu::{OptionExt, ResultExt};
 use transform::{TransformBuilders, Transformer, Transforms};
 use value::Value;
 use yaml_rust::YamlLoader;
 
+use crate::etl::error::Result;
+
 const DESCRIPTION: &str = "description";
 const PROCESSORS: &str = "processors";
 const TRANSFORM: &str = "transform";
@@ -37,13 +42,13 @@ pub enum Content {
     Yaml(String),
 }
 
-pub fn parse<T>(input: &Content) -> Result<Pipeline<T>, String>
+pub fn parse<T>(input: &Content) -> Result<Pipeline<T>>
 where
     T: Transformer,
 {
     match input {
         Content::Yaml(str) => {
-            let docs = YamlLoader::load_from_str(str).map_err(|e| e.to_string())?;
+            let docs = YamlLoader::load_from_str(str).context(YamlLoadSnafu)?;
 
             let doc = &docs[0];
 
@@ -124,7 +129,7 @@ where
                 .processor_builders
                 .into_iter()
                 .map(|builder| builder.build(&final_intermediate_keys))
-                .collect::<Result<Vec<_>, _>>()?;
+                .collect::<Result<Vec<_>>>()?;
             let processors = Processors {
                 processors: processors_kind_list,
                 required_keys: processors_required_keys.clone(),
@@ -136,7 +141,7 @@ where
                 .builders
                 .into_iter()
                 .map(|builder| builder.build(&final_intermediate_keys, &output_keys))
-                .collect::<Result<Vec<_>, String>>()?;
+                .collect::<Result<Vec<_>>>()?;
 
             let transformers = Transforms {
                 transforms: transfor_list,
@@ -197,7 +202,7 @@ impl<T> Pipeline<T>
 where
     T: Transformer,
 {
-    pub fn exec_mut(&self, val: &mut Vec<Value>) -> Result<T::VecOutput, String> {
+    pub fn exec_mut(&self, val: &mut Vec<Value>) -> Result<T::VecOutput> {
         for processor in self.processors.iter() {
             processor.exec_mut(val)?;
         }
@@ -205,7 +210,7 @@ where
         self.transformer.transform_mut(val)
     }
 
-    pub fn prepare(&self, val: serde_json::Value, result: &mut [Value]) -> Result<(), String> {
+    pub fn prepare(&self, val: serde_json::Value, result: &mut [Value]) -> Result<()> {
         match val {
             serde_json::Value::Object(map) => {
                 let mut search_from = 0;
@@ -230,7 +235,7 @@ where
                 result[0] = val.try_into()?;
             }
             _ => {
-                return Err("expect object".to_string());
+                return PrepareValueMustBeObjectSnafu.fail();
             }
         }
         Ok(())
@@ -274,18 +279,11 @@ where
     }
 }
 
-pub(crate) fn find_key_index(
-    intermediate_keys: &[String],
-    key: &str,
-    kind: &str,
-) -> Result<usize, String> {
+pub(crate) fn find_key_index(intermediate_keys: &[String], key: &str, kind: &str) -> Result<usize> {
     intermediate_keys
         .iter()
         .position(|k| k == key)
-        .ok_or(format!(
-            "{} processor.{} not found in intermediate keys",
-            kind, key
-        ))
+        .context(IntermediateKeyIndexSnafu { kind, key })
 }
 
 #[cfg(test)]
diff --git a/src/pipeline/src/etl/error.rs b/src/pipeline/src/etl/error.rs
new file mode 100644
index 0000000000..3680053ba0
--- /dev/null
+++ b/src/pipeline/src/etl/error.rs
@@ -0,0 +1,552 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+
+use common_error::ext::ErrorExt;
+use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
+use snafu::{Location, Snafu};
+
+#[derive(Snafu)]
+#[snafu(visibility(pub))]
+#[stack_trace_debug]
+pub enum Error {
+    #[snafu(display("Empty input field"))]
+    EmptyInputField {
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Missing input field"))]
+    MissingInputField {
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("Processor must be a map"))]
+    ProcessorMustBeMap {
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("{processor} processor: missing field: {field}"))]
+    ProcessorMissingField {
+        processor: String,
+        field: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("{processor} processor: expect string value, but got {v:?}"))]
+    ProcessorExpectString {
+        processor: String,
+        v: crate::etl::Value,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("{processor} processor: unsupported value {val}"))]
+    ProcessorUnsupportedValue {
+        processor: &'static str,
+        val: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("processor key must be a string"))]
+    ProcessorKeyMustBeString {
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("{kind} processor: failed to parse {value}"))]
+    ProcessorFailedToParseString {
+        kind: String,
+        value: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("processor must have a string key"))]
+    ProcessorMustHaveStringKey {
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("unsupported {processor} processor"))]
+    UnsupportedProcessor {
+        processor: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Field {field} must be a {ty}"))]
+    FieldMustBeType {
+        field: String,
+        ty: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Field parse from string failed: {field}"))]
+    FailedParseFieldFromString {
+        #[snafu(source)]
+        error: Box<dyn std::error::Error + Send + Sync>,
+        field: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("failed to parse {key} as int: {value}"))]
+    FailedToParseIntKey {
+        key: String,
+        value: String,
+        #[snafu(source)]
+        error: std::num::ParseIntError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Failed to parse {value} to int"))]
+    FailedToParseInt {
+        value: String,
+        #[snafu(source)]
+        error: std::num::ParseIntError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("failed to parse {key} as float: {value}"))]
+    FailedToParseFloatKey {
+        key: String,
+        value: String,
+        #[snafu(source)]
+        error: std::num::ParseFloatError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("{kind} processor.{key} not found in intermediate keys"))]
+    IntermediateKeyIndex {
+        kind: String,
+        key: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("{k} missing value in {s}"))]
+    CmcdMissingValue {
+        k: String,
+        s: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("{part} missing key in {s}"))]
+    CmcdMissingKey {
+        part: String,
+        s: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("key must be a string, but got {k:?}"))]
+    KeyMustBeString {
+        k: yaml_rust::Yaml,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("csv read error"))]
+    CsvRead {
+        #[snafu(implicit)]
+        location: Location,
+        #[snafu(source)]
+        error: csv::Error,
+    },
+    #[snafu(display("expected at least one record from csv format, but got none"))]
+    CsvNoRecord {
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("'{separator}' must be a single character, but got '{value}'"))]
+    CsvSeparatorName {
+        separator: &'static str,
+        value: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("'{quote}' must be a single character, but got '{value}'"))]
+    CsvQuoteName {
+        quote: &'static str,
+        value: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Parse date timezone error {value}"))]
+    DateParseTimezone {
+        value: String,
+        #[snafu(source)]
+        error: chrono_tz::ParseError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Parse date error {value}"))]
+    DateParse {
+        value: String,
+        #[snafu(source)]
+        error: chrono::ParseError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("failed to get local timezone"))]
+    DateFailedToGetLocalTimezone {
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("failed to get timestamp"))]
+    DateFailedToGetTimestamp {
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("{processor} processor: invalid format {s}"))]
+    DateInvalidFormat {
+        s: String,
+        processor: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Invalid Pattern: '{s}'. {detail}"))]
+    DissectInvalidPattern {
+        s: String,
+        detail: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Empty pattern is not allowed"))]
+    DissectEmptyPattern {
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("'{split}' exceeds the input"))]
+    DissectSplitExceedsInput {
+        split: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("'{split}' does not match the input '{input}'"))]
+    DissectSplitNotMatchInput {
+        split: String,
+        input: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("consecutive names are not allowed: '{name1}' '{name2}'"))]
+    DissectConsecutiveNames {
+        name1: String,
+        name2: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("No matching pattern found"))]
+    DissectNoMatchingPattern {
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("'{m}' modifier already set, but found {modifier}"))]
+    DissectModifierAlreadySet {
+        m: String,
+        modifier: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Append Order modifier is already set to '{n}', cannot be set to {order}"))]
+    DissectAppendOrderAlreadySet {
+        n: String,
+        order: u32,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("Order can only be set to Append Modifier, current modifier is {m}"))]
+    DissectOrderOnlyAppend {
+        m: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Order can only be set to Append Modifier"))]
+    DissectOrderOnlyAppendModifier {
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("End modifier already set: '{m}'"))]
+    DissectEndModifierAlreadySet {
+        m: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("invalid resolution: {resolution}"))]
+    EpochInvalidResolution {
+        resolution: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("pattern is required"))]
+    GsubPatternRequired {
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("replacement is required"))]
+    GsubReplacementRequired {
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("invalid regex pattern: {pattern}"))]
+    Regex {
+        #[snafu(source)]
+        error: regex::Error,
+        pattern: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("separator is required"))]
+    JoinSeparatorRequired {
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("invalid method: {method}"))]
+    LetterInvalidMethod {
+        method: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("no named group found in regex {origin}"))]
+    RegexNamedGroupNotFound {
+        origin: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("no valid field found in {processor} processor"))]
+    RegexNoValidField {
+        processor: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("no valid pattern found in {processor} processor"))]
+    RegexNoValidPattern {
+        processor: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("invalid method: {s}"))]
+    UrlEncodingInvalidMethod {
+        s: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("url decoding error"))]
+    UrlEncodingDecode {
+        #[snafu(source)]
+        error: std::string::FromUtf8Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("invalid transform on_failure value: {value}"))]
+    TransformOnFailureInvalidValue {
+        value: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("transform element must be a map"))]
+    TransformElementMustBeMap {
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("transform {fields:?} type MUST BE set before default {default}"))]
+    TransformTypeMustBeSet {
+        fields: String,
+        default: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("transform cannot be empty"))]
+    TransformEmpty {
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("column name must be unique, but got duplicated: {duplicates}"))]
+    TransformColumnNameMustBeUnique {
+        duplicates: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display(
+        "Illegal to set multiple timestamp Index columns, please set only one: {columns}"
+    ))]
+    TransformMultipleTimestampIndex {
+        columns: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("transform must have exactly one field specified as timestamp Index, but got {count}: {columns}"))]
+    TransformTimestampIndexCount {
+        count: usize,
+        columns: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("Null type not supported"))]
+    CoerceUnsupportedNullType {
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("Null type not supported when to coerce '{ty}' type"))]
+    CoerceUnsupportedNullTypeTo {
+        ty: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("{ty} value not supported for Epoch"))]
+    CoerceUnsupportedEpochType {
+        ty: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("failed to coerce string value '{s}' to type '{ty}'"))]
+    CoerceStringToType {
+        s: String,
+        ty: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display(
+        "invalid resolution: '{resolution}'. Available resolutions: {valid_resolution}"
+    ))]
+    ValueInvalidResolution {
+        resolution: String,
+        valid_resolution: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("failed to parse type: '{t}'"))]
+    ValueParseType {
+        t: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("failed to parse {ty}: {v}"))]
+    ValueParseInt {
+        ty: String,
+        v: String,
+        #[snafu(source)]
+        error: std::num::ParseIntError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("failed to parse {ty}: {v}"))]
+    ValueParseFloat {
+        ty: String,
+        v: String,
+        #[snafu(source)]
+        error: std::num::ParseFloatError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("failed to parse {ty}: {v}"))]
+    ValueParseBoolean {
+        ty: String,
+        v: String,
+        #[snafu(source)]
+        error: std::str::ParseBoolError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("default value not unsupported for type {value}"))]
+    ValueDefaultValueUnsupported {
+        value: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("unsupported number type: {value}"))]
+    ValueUnsupportedNumberType {
+        value: serde_json::Number,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("unsupported yaml type: {value:?}"))]
+    ValueUnsupportedYamlType {
+        value: yaml_rust::Yaml,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("key in Hash must be a string, but got {value:?}"))]
+    ValueYamlKeyMustBeString {
+        value: yaml_rust::Yaml,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Yaml load error."))]
+    YamlLoad {
+        #[snafu(source)]
+        error: yaml_rust::ScanError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("Prepare value must be an object"))]
+    PrepareValueMustBeObject {
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Column options error"))]
+    ColumnOptions {
+        #[snafu(source)]
+        source: api::error::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("unsupported index type: {value}"))]
+    UnsupportedIndexType {
+        value: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl ErrorExt for Error {
+    fn status_code(&self) -> StatusCode {
+        StatusCode::InvalidArguments
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
diff --git a/src/pipeline/src/etl/field.rs b/src/pipeline/src/etl/field.rs
index ff2f1ee7b5..84d94040b5 100644
--- a/src/pipeline/src/etl/field.rs
+++ b/src/pipeline/src/etl/field.rs
@@ -15,6 +15,10 @@
 use std::ops::Deref;
 use std::str::FromStr;
 
+use snafu::OptionExt;
+
+use super::error::{EmptyInputFieldSnafu, MissingInputFieldSnafu};
+use crate::etl::error::{Error, Result};
 use crate::etl::find_key_index;
 
 /// Information about the input field including the name and index in intermediate keys.
@@ -56,7 +60,7 @@ impl OneInputOneOutputField {
         intermediate_keys: &[String],
         input_field: &str,
         target_field: &str,
-    ) -> Result<Self, String> {
+    ) -> Result<Self> {
         let input_index = find_key_index(intermediate_keys, input_field, processor_kind)?;
 
         let input_field_info = InputFieldInfo::new(input_field, input_index);
@@ -145,19 +149,19 @@ pub struct Field {
 }
 
 impl FromStr for Field {
-    type Err = String;
+    type Err = Error;
 
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
+    fn from_str(s: &str) -> Result<Self> {
         let mut parts = s.split(',');
         let input_field = parts
             .next()
-            .ok_or("input field is missing")?
+            .context(MissingInputFieldSnafu)?
             .trim()
             .to_string();
         let target_field = parts.next().map(|x| x.trim().to_string());
 
         if input_field.is_empty() {
-            return Err("input field is empty".to_string());
+            return EmptyInputFieldSnafu.fail();
         }
 
         Ok(Field {
diff --git a/src/pipeline/src/etl/processor.rs b/src/pipeline/src/etl/processor.rs
index 257cce4dfc..afda8f399e 100644
--- a/src/pipeline/src/etl/processor.rs
+++ b/src/pipeline/src/etl/processor.rs
@@ -36,10 +36,16 @@ use itertools::Itertools;
 use join::{JoinProcessor, JoinProcessorBuilder};
 use letter::{LetterProcessor, LetterProcessorBuilder};
 use regex::{RegexProcessor, RegexProcessorBuilder};
+use snafu::{OptionExt, ResultExt};
 use timestamp::{TimestampProcessor, TimestampProcessorBuilder};
 use urlencoding::{UrlEncodingProcessor, UrlEncodingProcessorBuilder};
 
+use super::error::{
+    FailedParseFieldFromStringSnafu, FieldMustBeTypeSnafu, ProcessorKeyMustBeStringSnafu,
+    ProcessorMustBeMapSnafu, ProcessorMustHaveStringKeySnafu, UnsupportedProcessorSnafu,
+};
 use super::field::{Field, Fields};
+use crate::etl::error::{Error, Result};
 use crate::etl::value::Value;
 
 const FIELD_NAME: &str = "field";
@@ -70,7 +76,7 @@ pub trait Processor: std::fmt::Debug + Send + Sync + 'static {
     fn ignore_missing(&self) -> bool;
 
     /// Execute the processor on a vector which be preprocessed by the pipeline
-    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<(), String>;
+    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<()>;
 }
 
 #[derive(Debug)]
@@ -98,7 +104,7 @@ pub trait ProcessorBuilder: std::fmt::Debug + Send + Sync + 'static {
     /// Get the processor's input keys
     fn input_keys(&self) -> HashSet<&str>;
     /// Build the processor
-    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind, String>;
+    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind>;
 }
 
 #[derive(Debug)]
@@ -171,9 +177,9 @@ impl Processors {
 }
 
 impl TryFrom<&Vec<yaml_rust::Yaml>> for ProcessorBuilderList {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(vec: &Vec<yaml_rust::Yaml>) -> Result<Self, Self::Error> {
+    fn try_from(vec: &Vec<yaml_rust::Yaml>) -> Result<Self> {
         let mut processors_builders = vec![];
         let mut all_output_keys = HashSet::with_capacity(50);
         let mut all_required_keys = HashSet::with_capacity(50);
@@ -226,13 +232,10 @@ impl TryFrom<&Vec<yaml_rust::Yaml>> for ProcessorBuilderList {
     }
 }
 
-fn parse_processor(doc: &yaml_rust::Yaml) -> Result<ProcessorBuilders, String> {
-    let map = doc.as_hash().ok_or("processor must be a map".to_string())?;
+fn parse_processor(doc: &yaml_rust::Yaml) -> Result<ProcessorBuilders> {
+    let map = doc.as_hash().context(ProcessorMustBeMapSnafu)?;
 
-    let key = map
-        .keys()
-        .next()
-        .ok_or("processor must have a string key".to_string())?;
+    let key = map.keys().next().context(ProcessorMustHaveStringKeySnafu)?;
 
     let value = map
         .get(key)
@@ -240,9 +243,7 @@ fn parse_processor(doc: &yaml_rust::Yaml) -> Result<ProcessorBuilders, String> {
         .as_hash()
         .expect("processor value must be a map");
 
-    let str_key = key
-        .as_str()
-        .ok_or("processor key must be a string".to_string())?;
+    let str_key = key.as_str().context(ProcessorKeyMustBeStringSnafu)?;
 
     let processor = match str_key {
         cmcd::PROCESSOR_CMCD => ProcessorBuilders::Cmcd(CmcdProcessorBuilder::try_from(value)?),
@@ -264,58 +265,72 @@ fn parse_processor(doc: &yaml_rust::Yaml) -> Result<ProcessorBuilders, String> {
         urlencoding::PROCESSOR_URL_ENCODING => {
             ProcessorBuilders::UrlEncoding(UrlEncodingProcessorBuilder::try_from(value)?)
         }
-        _ => return Err(format!("unsupported {} processor", str_key)),
+        _ => return UnsupportedProcessorSnafu { processor: str_key }.fail(),
     };
 
     Ok(processor)
 }
 
-pub(crate) fn yaml_string(v: &yaml_rust::Yaml, field: &str) -> Result<String, String> {
+pub(crate) fn yaml_string(v: &yaml_rust::Yaml, field: &str) -> Result<String> {
     v.as_str()
         .map(|s| s.to_string())
-        .ok_or(format!("'{field}' must be a string"))
+        .context(FieldMustBeTypeSnafu {
+            field,
+            ty: "string",
+        })
 }
 
-pub(crate) fn yaml_strings(v: &yaml_rust::Yaml, field: &str) -> Result<Vec<String>, String> {
+pub(crate) fn yaml_strings(v: &yaml_rust::Yaml, field: &str) -> Result<Vec<String>> {
     let vec = v
         .as_vec()
-        .ok_or(format!("'{field}' must be a list of strings",))?
+        .context(FieldMustBeTypeSnafu {
+            field,
+            ty: "list of string",
+        })?
         .iter()
         .map(|v| v.as_str().unwrap_or_default().into())
         .collect();
     Ok(vec)
 }
 
-pub(crate) fn yaml_bool(v: &yaml_rust::Yaml, field: &str) -> Result<bool, String> {
-    v.as_bool().ok_or(format!("'{field}' must be a boolean"))
+pub(crate) fn yaml_bool(v: &yaml_rust::Yaml, field: &str) -> Result<bool> {
+    v.as_bool().context(FieldMustBeTypeSnafu {
+        field,
+        ty: "boolean",
+    })
 }
 
-pub(crate) fn yaml_parse_string<T>(v: &yaml_rust::Yaml, field: &str) -> Result<T, String>
+pub(crate) fn yaml_parse_string<T>(v: &yaml_rust::Yaml, field: &str) -> Result<T>
 where
     T: std::str::FromStr,
-    T::Err: ToString,
+    T::Err: std::error::Error + Send + Sync + 'static,
 {
     yaml_string(v, field)?
         .parse::<T>()
-        .map_err(|e| e.to_string())
+        .map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)
+        .context(FailedParseFieldFromStringSnafu { field })
 }
 
-pub(crate) fn yaml_parse_strings<T>(v: &yaml_rust::Yaml, field: &str) -> Result<Vec<T>, String>
+pub(crate) fn yaml_parse_strings<T>(v: &yaml_rust::Yaml, field: &str) -> Result<Vec<T>>
 where
     T: std::str::FromStr,
-    T::Err: ToString,
+    T::Err: std::error::Error + Send + Sync + 'static,
 {
     yaml_strings(v, field).and_then(|v| {
         v.into_iter()
-            .map(|s| s.parse::<T>().map_err(|e| e.to_string()))
+            .map(|s| {
+                s.parse::<T>()
+                    .map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)
+                    .context(FailedParseFieldFromStringSnafu { field })
+            })
             .collect()
     })
 }
 
-pub(crate) fn yaml_new_fields(v: &yaml_rust::Yaml, field: &str) -> Result<Fields, String> {
+pub(crate) fn yaml_new_fields(v: &yaml_rust::Yaml, field: &str) -> Result<Fields> {
     yaml_parse_strings(v, field).map(Fields::new)
 }
 
-pub(crate) fn yaml_new_field(v: &yaml_rust::Yaml, field: &str) -> Result<Field, String> {
+pub(crate) fn yaml_new_field(v: &yaml_rust::Yaml, field: &str) -> Result<Field> {
     yaml_parse_string(v, field)
 }
diff --git a/src/pipeline/src/etl/processor/cmcd.rs b/src/pipeline/src/etl/processor/cmcd.rs
index 1556829d65..f43186b94a 100644
--- a/src/pipeline/src/etl/processor/cmcd.rs
+++ b/src/pipeline/src/etl/processor/cmcd.rs
@@ -15,8 +15,14 @@
 use std::collections::BTreeMap;
 
 use ahash::HashSet;
+use snafu::{OptionExt, ResultExt};
 use urlencoding::decode;
 
+use crate::etl::error::{
+    CmcdMissingKeySnafu, CmcdMissingValueSnafu, Error, FailedToParseFloatKeySnafu,
+    FailedToParseIntKeySnafu, KeyMustBeStringSnafu, ProcessorExpectStringSnafu,
+    ProcessorMissingFieldSnafu, Result,
+};
 use crate::etl::field::{Field, Fields, InputFieldInfo, OneInputMultiOutputField};
 use crate::etl::find_key_index;
 use crate::etl::processor::{
@@ -82,7 +88,7 @@ impl CmcdProcessorBuilder {
     pub(super) fn build_cmcd_outputs(
         field: &Field,
         intermediate_keys: &[String],
-    ) -> Result<(BTreeMap<String, usize>, Vec<CmcdOutputInfo>), String> {
+    ) -> Result<(BTreeMap<String, usize>, Vec<CmcdOutputInfo>)> {
         let mut output_index = BTreeMap::new();
         let mut cmcd_field_outputs = Vec::with_capacity(CMCD_KEYS.len());
         for cmcd in CMCD_KEYS {
@@ -119,7 +125,7 @@ impl CmcdProcessorBuilder {
     }
 
     /// build CmcdProcessor from CmcdProcessorBuilder
-    pub fn build(self, intermediate_keys: &[String]) -> Result<CmcdProcessor, String> {
+    pub fn build(self, intermediate_keys: &[String]) -> Result<CmcdProcessor> {
         let mut real_fields = vec![];
         let mut cmcd_outputs = Vec::with_capacity(CMCD_KEYS.len());
         for field in self.fields.into_iter() {
@@ -151,7 +157,7 @@ impl ProcessorBuilder for CmcdProcessorBuilder {
         self.fields.iter().map(|f| f.input_field()).collect()
     }
 
-    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind> {
         self.build(intermediate_keys).map(ProcessorKind::Cmcd)
     }
 }
@@ -170,7 +176,7 @@ pub(super) struct CmcdOutputInfo {
     /// index in intermediate_keys
     index: usize,
     /// function to resolve value
-    f: fn(&str, &str, Option<&str>) -> Result<Value, String>,
+    f: fn(&str, &str, Option<&str>) -> Result<Value>,
 }
 
 impl CmcdOutputInfo {
@@ -178,7 +184,7 @@ impl CmcdOutputInfo {
         final_key: String,
         key: &'static str,
         index: usize,
-        f: fn(&str, &str, Option<&str>) -> Result<Value, String>,
+        f: fn(&str, &str, Option<&str>) -> Result<Value>,
     ) -> Self {
         Self {
             final_key,
@@ -201,28 +207,28 @@ impl Default for CmcdOutputInfo {
 }
 
 /// function to resolve CMCD_KEY_BS | CMCD_KEY_SU
-fn bs_su(_: &str, _: &str, _: Option<&str>) -> Result<Value, String> {
+fn bs_su(_: &str, _: &str, _: Option<&str>) -> Result<Value> {
     Ok(Value::Boolean(true))
 }
 
 /// function to resolve CMCD_KEY_BR | CMCD_KEY_BL | CMCD_KEY_D | CMCD_KEY_DL | CMCD_KEY_MTP | CMCD_KEY_RTP | CMCD_KEY_TB
-fn br_tb(s: &str, k: &str, v: Option<&str>) -> Result<Value, String> {
-    let v = v.ok_or(format!("{k} missing value in {s}"))?;
+fn br_tb(s: &str, k: &str, v: Option<&str>) -> Result<Value> {
+    let v = v.context(CmcdMissingValueSnafu { k, s })?;
     let val: i64 = v
         .parse()
-        .map_err(|_| format!("failed to parse {v} as i64"))?;
+        .context(FailedToParseIntKeySnafu { key: k, value: v })?;
     Ok(Value::Int64(val))
 }
 
 /// function to resolve CMCD_KEY_CID | CMCD_KEY_NRR | CMCD_KEY_OT | CMCD_KEY_SF | CMCD_KEY_SID | CMCD_KEY_V
-fn cid_v(s: &str, k: &str, v: Option<&str>) -> Result<Value, String> {
-    let v = v.ok_or(format!("{k} missing value in {s}"))?;
+fn cid_v(s: &str, k: &str, v: Option<&str>) -> Result<Value> {
+    let v = v.context(CmcdMissingValueSnafu { k, s })?;
     Ok(Value::String(v.to_string()))
 }
 
 /// function to resolve CMCD_KEY_NOR
-fn nor(s: &str, k: &str, v: Option<&str>) -> Result<Value, String> {
-    let v = v.ok_or(format!("{k} missing value in {s}"))?;
+fn nor(s: &str, k: &str, v: Option<&str>) -> Result<Value> {
+    let v = v.context(CmcdMissingValueSnafu { k, s })?;
     let val = match decode(v) {
         Ok(val) => val.to_string(),
         Err(_) => v.to_string(),
@@ -231,11 +237,11 @@ fn nor(s: &str, k: &str, v: Option<&str>) -> Result<Value, String> {
 }
 
 /// function to resolve CMCD_KEY_PR
-fn pr(s: &str, k: &str, v: Option<&str>) -> Result<Value, String> {
-    let v = v.ok_or(format!("{k} missing value in {s}"))?;
+fn pr(s: &str, k: &str, v: Option<&str>) -> Result<Value> {
+    let v = v.context(CmcdMissingValueSnafu { k, s })?;
     let val: f64 = v
         .parse()
-        .map_err(|_| format!("failed to parse {v} as f64"))?;
+        .context(FailedToParseFloatKeySnafu { key: k, value: v })?;
     Ok(Value::Float64(val))
 }
 
@@ -287,12 +293,12 @@ impl CmcdProcessor {
         format!("{}_{}", prefix, key)
     }
 
-    fn parse(&self, field_index: usize, s: &str) -> Result<Vec<(usize, Value)>, String> {
+    fn parse(&self, field_index: usize, s: &str) -> Result<Vec<(usize, Value)>> {
         let parts = s.split(',');
         let mut result = Vec::new();
         for part in parts {
             let mut kv = part.split('=');
-            let k = kv.next().ok_or(format!("{part} missing key in {s}"))?;
+            let k = kv.next().context(CmcdMissingKeySnafu { part, s })?;
             let v = kv.next();
 
             for cmcd_key in self.cmcd_outputs[field_index].iter() {
@@ -308,16 +314,16 @@ impl CmcdProcessor {
 }
 
 impl TryFrom<&yaml_rust::yaml::Hash> for CmcdProcessorBuilder {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(value: &yaml_rust::yaml::Hash) -> Result<Self, Self::Error> {
+    fn try_from(value: &yaml_rust::yaml::Hash) -> Result<Self> {
         let mut fields = Fields::default();
         let mut ignore_missing = false;
 
         for (k, v) in value.iter() {
             let key = k
                 .as_str()
-                .ok_or(format!("key must be a string, but got {k:?}"))?;
+                .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
             match key {
                 FIELD_NAME => {
                     fields = Fields::one(yaml_new_field(v, FIELD_NAME)?);
@@ -362,7 +368,7 @@ impl Processor for CmcdProcessor {
         self.ignore_missing
     }
 
-    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<(), String> {
+    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<()> {
         for (field_index, field) in self.fields.iter().enumerate() {
             let field_value_index = field.input_index();
             match val.get(field_value_index) {
@@ -374,18 +380,19 @@ impl Processor for CmcdProcessor {
                 }
                 Some(Value::Null) | None => {
                     if !self.ignore_missing {
-                        return Err(format!(
-                            "{} processor: missing field: {}",
-                            self.kind(),
-                            field.input_name()
-                        ));
+                        return ProcessorMissingFieldSnafu {
+                            processor: self.kind().to_string(),
+                            field: field.input_name().to_string(),
+                        }
+                        .fail();
                     }
                 }
                 Some(v) => {
-                    return Err(format!(
-                        "{} processor: expect string value, but got {v:?}",
-                        self.kind()
-                    ));
+                    return ProcessorExpectStringSnafu {
+                        processor: self.kind().to_string(),
+                        v: v.clone(),
+                    }
+                    .fail();
                 }
             }
         }
diff --git a/src/pipeline/src/etl/processor/csv.rs b/src/pipeline/src/etl/processor/csv.rs
index fb1fca2bfb..c9cb5f847d 100644
--- a/src/pipeline/src/etl/processor/csv.rs
+++ b/src/pipeline/src/etl/processor/csv.rs
@@ -18,7 +18,12 @@ use ahash::HashSet;
 use csv::{ReaderBuilder, Trim};
 use itertools::EitherOrBoth::{Both, Left, Right};
 use itertools::Itertools;
+use snafu::{OptionExt, ResultExt};
 
+use crate::etl::error::{
+    CsvNoRecordSnafu, CsvQuoteNameSnafu, CsvReadSnafu, CsvSeparatorNameSnafu, Error,
+    KeyMustBeStringSnafu, ProcessorExpectStringSnafu, ProcessorMissingFieldSnafu, Result,
+};
 use crate::etl::field::{Fields, InputFieldInfo, OneInputMultiOutputField};
 use crate::etl::find_key_index;
 use crate::etl::processor::{
@@ -53,7 +58,7 @@ pub struct CsvProcessorBuilder {
 }
 
 impl CsvProcessorBuilder {
-    fn build(self, intermediate_keys: &[String]) -> Result<CsvProcessor, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<CsvProcessor> {
         let mut real_fields = vec![];
 
         for field in self.fields {
@@ -68,7 +73,7 @@ impl CsvProcessorBuilder {
             .target_fields
             .iter()
             .map(|f| find_key_index(intermediate_keys, f, "csv"))
-            .collect::<Result<Vec<_>, String>>()?;
+            .collect::<Result<Vec<_>>>()?;
         Ok(CsvProcessor {
             reader: self.reader,
             fields: real_fields,
@@ -88,7 +93,7 @@ impl ProcessorBuilder for CsvProcessorBuilder {
         self.fields.iter().map(|f| f.input_field()).collect()
     }
 
-    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind> {
         self.build(intermediate_keys).map(ProcessorKind::Csv)
     }
 }
@@ -114,11 +119,11 @@ pub struct CsvProcessor {
 
 impl CsvProcessor {
     // process the csv format string to a map with target_fields as keys
-    fn process(&self, val: &str) -> Result<Vec<(usize, Value)>, String> {
+    fn process(&self, val: &str) -> Result<Vec<(usize, Value)>> {
         let mut reader = self.reader.from_reader(val.as_bytes());
 
         if let Some(result) = reader.records().next() {
-            let record: csv::StringRecord = result.map_err(|e| e.to_string())?;
+            let record: csv::StringRecord = result.context(CsvReadSnafu)?;
 
             let values: Vec<(usize, Value)> = self
                 .output_index_info
@@ -142,15 +147,15 @@ impl CsvProcessor {
 
             Ok(values)
         } else {
-            Err("expected at least one record from csv format, but got none".into())
+            CsvNoRecordSnafu.fail()
         }
     }
 }
 
 impl TryFrom<&yaml_rust::yaml::Hash> for CsvProcessorBuilder {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(hash: &yaml_rust::yaml::Hash) -> Result<Self, Self::Error> {
+    fn try_from(hash: &yaml_rust::yaml::Hash) -> Result<Self> {
         let mut reader = ReaderBuilder::new();
         reader.has_headers(false);
 
@@ -162,7 +167,7 @@ impl TryFrom<&yaml_rust::yaml::Hash> for CsvProcessorBuilder {
         for (k, v) in hash {
             let key = k
                 .as_str()
-                .ok_or(format!("key must be a string, but got {k:?}"))?;
+                .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
             match key {
                 FIELD_NAME => {
                     fields = Fields::one(yaml_new_field(v, FIELD_NAME)?);
@@ -180,10 +185,11 @@ impl TryFrom<&yaml_rust::yaml::Hash> for CsvProcessorBuilder {
                 SEPARATOR_NAME => {
                     let separator = yaml_string(v, SEPARATOR_NAME)?;
                     if separator.len() != 1 {
-                        return Err(format!(
-                            "'{}' must be a single character, but got '{}'",
-                            SEPARATOR_NAME, separator
-                        ));
+                        return CsvSeparatorNameSnafu {
+                            separator: SEPARATOR_NAME,
+                            value: separator,
+                        }
+                        .fail();
                     } else {
                         reader.delimiter(separator.as_bytes()[0]);
                     }
@@ -191,10 +197,11 @@ impl TryFrom<&yaml_rust::yaml::Hash> for CsvProcessorBuilder {
                 QUOTE_NAME => {
                     let quote = yaml_string(v, QUOTE_NAME)?;
                     if quote.len() != 1 {
-                        return Err(format!(
-                            "'{}' must be a single character, but got '{}'",
-                            QUOTE_NAME, quote
-                        ));
+                        return CsvQuoteNameSnafu {
+                            quote: QUOTE_NAME,
+                            value: quote,
+                        }
+                        .fail();
                     } else {
                         reader.quote(quote.as_bytes()[0]);
                     }
@@ -240,7 +247,7 @@ impl Processor for CsvProcessor {
         self.ignore_missing
     }
 
-    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<(), String> {
+    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<()> {
         for field in self.fields.iter() {
             let index = field.input_index();
             match val.get(index) {
@@ -252,18 +259,19 @@ impl Processor for CsvProcessor {
                 }
                 Some(Value::Null) | None => {
                     if !self.ignore_missing {
-                        return Err(format!(
-                            "{} processor: missing field: {}",
-                            self.kind(),
-                            field.input_name()
-                        ));
+                        return ProcessorMissingFieldSnafu {
+                            processor: self.kind().to_string(),
+                            field: field.input_name().to_string(),
+                        }
+                        .fail();
                     }
                 }
                 Some(v) => {
-                    return Err(format!(
-                        "{} processor: expect string value, but got {v:?}",
-                        self.kind()
-                    ));
+                    return ProcessorExpectStringSnafu {
+                        processor: self.kind().to_string(),
+                        v: v.clone(),
+                    }
+                    .fail();
                 }
             }
         }
diff --git a/src/pipeline/src/etl/processor/date.rs b/src/pipeline/src/etl/processor/date.rs
index b9bfcf3b6c..fa202a0edf 100644
--- a/src/pipeline/src/etl/processor/date.rs
+++ b/src/pipeline/src/etl/processor/date.rs
@@ -18,7 +18,13 @@ use ahash::HashSet;
 use chrono::{DateTime, NaiveDateTime};
 use chrono_tz::Tz;
 use lazy_static::lazy_static;
+use snafu::{OptionExt, ResultExt};
 
+use crate::etl::error::{
+    DateFailedToGetLocalTimezoneSnafu, DateFailedToGetTimestampSnafu, DateParseSnafu,
+    DateParseTimezoneSnafu, Error, KeyMustBeStringSnafu, ProcessorExpectStringSnafu,
+    ProcessorFailedToParseStringSnafu, ProcessorMissingFieldSnafu, Result,
+};
 use crate::etl::field::{Fields, OneInputOneOutputField};
 use crate::etl::processor::{
     yaml_bool, yaml_new_field, yaml_new_fields, yaml_string, yaml_strings, Processor,
@@ -103,13 +109,13 @@ impl ProcessorBuilder for DateProcessorBuilder {
         self.fields.iter().map(|f| f.input_field()).collect()
     }
 
-    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind> {
         self.build(intermediate_keys).map(ProcessorKind::Date)
     }
 }
 
 impl DateProcessorBuilder {
-    pub fn build(self, intermediate_keys: &[String]) -> Result<DateProcessor, String> {
+    pub fn build(self, intermediate_keys: &[String]) -> Result<DateProcessor> {
         let mut real_fields = vec![];
         for field in self.fields.into_iter() {
             let input = OneInputOneOutputField::build(
@@ -131,9 +137,9 @@ impl DateProcessorBuilder {
 }
 
 impl TryFrom<&yaml_rust::yaml::Hash> for DateProcessorBuilder {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(hash: &yaml_rust::yaml::Hash) -> Result<Self, Self::Error> {
+    fn try_from(hash: &yaml_rust::yaml::Hash) -> Result<Self> {
         let mut fields = Fields::default();
         let mut formats = Formats::default();
         let mut timezone = None;
@@ -143,7 +149,7 @@ impl TryFrom<&yaml_rust::yaml::Hash> for DateProcessorBuilder {
         for (k, v) in hash {
             let key = k
                 .as_str()
-                .ok_or(format!("key must be a string, but got {k:?}"))?;
+                .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
 
             match key {
                 FIELD_NAME => {
@@ -205,10 +211,12 @@ pub struct DateProcessor {
 }
 
 impl DateProcessor {
-    fn parse(&self, val: &str) -> Result<Timestamp, String> {
+    fn parse(&self, val: &str) -> Result<Timestamp> {
         let mut tz = Tz::UTC;
         if let Some(timezone) = &self.timezone {
-            tz = timezone.parse::<Tz>().map_err(|e| e.to_string())?;
+            tz = timezone.parse::<Tz>().context(DateParseTimezoneSnafu {
+                value: timezone.as_ref(),
+            })?;
         }
 
         for fmt in self.formats.iter() {
@@ -217,7 +225,11 @@ impl DateProcessor {
             }
         }
 
-        Err(format!("{} processor: failed to parse {val}", self.kind(),))
+        ProcessorFailedToParseStringSnafu {
+            kind: PROCESSOR_DATE.to_string(),
+            value: val.to_string(),
+        }
+        .fail()
     }
 }
 
@@ -230,7 +242,7 @@ impl Processor for DateProcessor {
         self.ignore_missing
     }
 
-    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<(), String> {
+    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<()> {
         for field in self.fields.iter() {
             let index = field.input_index();
             match val.get(index) {
@@ -241,18 +253,19 @@ impl Processor for DateProcessor {
                 }
                 Some(Value::Null) | None => {
                     if !self.ignore_missing {
-                        return Err(format!(
-                            "{} processor: missing field: {}",
-                            self.kind(),
-                            field.input_name()
-                        ));
+                        return ProcessorMissingFieldSnafu {
+                            processor: self.kind().to_string(),
+                            field: field.input_name().to_string(),
+                        }
+                        .fail();
                     }
                 }
                 Some(v) => {
-                    return Err(format!(
-                        "{} processor: expect string value, but got {v:?}",
-                        self.kind()
-                    ));
+                    return ProcessorExpectStringSnafu {
+                        processor: self.kind().to_string(),
+                        v: v.clone(),
+                    }
+                    .fail();
                 }
             }
         }
@@ -261,16 +274,20 @@ impl Processor for DateProcessor {
 }
 
 /// try to parse val with timezone first, if failed, parse without timezone
-fn try_parse(val: &str, fmt: &str, tz: Tz) -> Result<i64, String> {
+fn try_parse(val: &str, fmt: &str, tz: Tz) -> Result<i64> {
     if let Ok(dt) = DateTime::parse_from_str(val, fmt) {
-        Ok(dt.timestamp_nanos_opt().ok_or("failed to get timestamp")?)
+        Ok(dt
+            .timestamp_nanos_opt()
+            .context(DateFailedToGetTimestampSnafu)?)
     } else {
         let dt = NaiveDateTime::parse_from_str(val, fmt)
-            .map_err(|e| e.to_string())?
+            .context(DateParseSnafu { value: val })?
             .and_local_timezone(tz)
             .single()
-            .ok_or("failed to get local timezone")?;
-        Ok(dt.timestamp_nanos_opt().ok_or("failed to get timestamp")?)
+            .context(DateFailedToGetLocalTimezoneSnafu)?;
+        Ok(dt
+            .timestamp_nanos_opt()
+            .context(DateFailedToGetTimestampSnafu)?)
     }
 }
 
diff --git a/src/pipeline/src/etl/processor/dissect.rs b/src/pipeline/src/etl/processor/dissect.rs
index f9925916fc..09c6fc93d0 100644
--- a/src/pipeline/src/etl/processor/dissect.rs
+++ b/src/pipeline/src/etl/processor/dissect.rs
@@ -16,7 +16,15 @@ use std::ops::Deref;
 
 use ahash::{HashMap, HashMapExt, HashSet, HashSetExt};
 use itertools::Itertools;
+use snafu::OptionExt;
 
+use crate::etl::error::{
+    DissectAppendOrderAlreadySetSnafu, DissectConsecutiveNamesSnafu, DissectEmptyPatternSnafu,
+    DissectEndModifierAlreadySetSnafu, DissectInvalidPatternSnafu, DissectModifierAlreadySetSnafu,
+    DissectNoMatchingPatternSnafu, DissectOrderOnlyAppendModifierSnafu,
+    DissectOrderOnlyAppendSnafu, DissectSplitExceedsInputSnafu, DissectSplitNotMatchInputSnafu,
+    Error, KeyMustBeStringSnafu, ProcessorExpectStringSnafu, ProcessorMissingFieldSnafu, Result,
+};
 use crate::etl::field::{Fields, InputFieldInfo, OneInputMultiOutputField};
 use crate::etl::find_key_index;
 use crate::etl::processor::{
@@ -77,9 +85,13 @@ impl NameInfo {
         self.name.is_empty() && self.start_modifier.is_none() && self.end_modifier.is_none()
     }
 
-    fn try_start_modifier(&mut self, modifier: StartModifier) -> Result<(), String> {
+    fn try_start_modifier(&mut self, modifier: StartModifier) -> Result<()> {
         match &self.start_modifier {
-            Some(m) => Err(format!("'{m}' modifier already set, but found {modifier}",)),
+            Some(m) => DissectModifierAlreadySetSnafu {
+                m: m.to_string(),
+                modifier: modifier.to_string(),
+            }
+            .fail(),
             None => {
                 self.start_modifier = Some(modifier);
                 Ok(())
@@ -87,27 +99,27 @@ impl NameInfo {
         }
     }
 
-    fn try_append_order(&mut self, order: u32) -> Result<(), String> {
+    fn try_append_order(&mut self, order: u32) -> Result<()> {
         match &mut self.start_modifier {
             Some(StartModifier::Append(o)) => match o {
-                Some(n) => Err(format!(
-                    "Append Order modifier is already set to '{n}', cannot be set to {order}"
-                )),
+                Some(n) => DissectAppendOrderAlreadySetSnafu {
+                    n: n.to_string(),
+                    order,
+                }
+                .fail(),
                 None => {
                     *o = Some(order);
                     Ok(())
                 }
             },
-            Some(m) => Err(format!(
-                "Order can only be set to Append Modifier, current modifier is {m}"
-            )),
-            None => Err("Order can only be set to Append Modifier".to_string()),
+            Some(m) => DissectOrderOnlyAppendSnafu { m: m.to_string() }.fail(),
+            None => DissectOrderOnlyAppendModifierSnafu.fail(),
         }
     }
 
-    fn try_end_modifier(&mut self) -> Result<(), String> {
+    fn try_end_modifier(&mut self) -> Result<()> {
         match &self.end_modifier {
-            Some(m) => Err(format!("End modifier already set: '{m}'")),
+            Some(m) => DissectEndModifierAlreadySetSnafu { m: m.to_string() }.fail(),
             None => {
                 self.end_modifier = Some(EndModifier);
                 Ok(())
@@ -290,9 +302,9 @@ impl std::ops::DerefMut for PatternInfo {
 }
 
 impl std::str::FromStr for PatternInfo {
-    type Err = String;
+    type Err = Error;
 
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
+    fn from_str(s: &str) -> Result<Self> {
         let mut parts = vec![];
         let mut cursor = PartInfo::empty_split();
 
@@ -338,9 +350,11 @@ impl std::str::FromStr for PatternInfo {
                     }
 
                     if j == pos + 1 {
-                        return Err(format!(
-                            "Invalid Pattern: '{s}'. Digit order must be set after '/'",
-                        ));
+                        return DissectInvalidPatternSnafu {
+                            s,
+                            detail: "Digit order must be set after '/'",
+                        }
+                        .fail();
                     }
 
                     name.try_append_order(order)?;
@@ -358,14 +372,20 @@ impl std::str::FromStr for PatternInfo {
                 ('-', PartInfo::Name(name)) if !name.is_end_modifier_set() => {
                     if let Some('>') = chars.get(pos + 1) {
                     } else {
-                        return Err(format!(
-                            "Invalid Pattern: '{s}'. expected '->' but only '-'",
-                        ));
+                        return DissectInvalidPatternSnafu {
+                            s,
+                            detail: "Expected '->' but only '-'",
+                        }
+                        .fail();
                     }
 
                     if let Some('}') = chars.get(pos + 2) {
                     } else {
-                        return Err(format!("Invalid Pattern: '{s}'. expected '}}' after '->'",));
+                        return DissectInvalidPatternSnafu {
+                            s,
+                            detail: "Expected '}' after '->'",
+                        }
+                        .fail();
                     }
 
                     name.try_end_modifier()?;
@@ -377,7 +397,7 @@ impl std::str::FromStr for PatternInfo {
                     } else {
                         format!("Invalid '{ch}' in '{name}'")
                     };
-                    return Err(format!("Invalid Pattern: '{s}'. {tail}"));
+                    return DissectInvalidPatternSnafu { s, detail: tail }.fail();
                 }
                 (_, PartInfo::Name(_)) => {
                     cursor.push(ch);
@@ -390,7 +410,11 @@ impl std::str::FromStr for PatternInfo {
         match cursor {
             PartInfo::Split(ref split) if !split.is_empty() => parts.push(cursor),
             PartInfo::Name(name) if !name.is_empty() => {
-                return Err(format!("Invalid Pattern: '{s}'. '{name}' is not closed"))
+                return DissectInvalidPatternSnafu {
+                    s,
+                    detail: format!("'{name}' is not closed"),
+                }
+                .fail();
             }
             _ => {}
         }
@@ -402,9 +426,9 @@ impl std::str::FromStr for PatternInfo {
 }
 
 impl PatternInfo {
-    fn check(&self) -> Result<(), String> {
+    fn check(&self) -> Result<()> {
         if self.len() == 0 {
-            return Err("Empty pattern is not allowed".to_string());
+            return DissectEmptyPatternSnafu.fail();
         }
 
         let mut map_keys = HashSet::new();
@@ -415,42 +439,47 @@ impl PatternInfo {
             let next_part = self.get(i + 1);
             match (this_part, next_part) {
                 (PartInfo::Split(split), _) if split.is_empty() => {
-                    return Err(format!(
-                        "Invalid Pattern: '{}'. Empty split is not allowed",
-                        self.origin
-                    ));
+                    return DissectInvalidPatternSnafu {
+                        s: &self.origin,
+                        detail: "Empty split is not allowed",
+                    }
+                    .fail();
                 }
                 (PartInfo::Name(name1), Some(PartInfo::Name(name2))) => {
-                    return Err(format!(
-                        "Invalid Pattern: '{}'. consecutive names are not allowed: '{}' '{}'",
-                        self.origin, name1, name2
-                    ));
+                    return DissectInvalidPatternSnafu {
+                        s: &self.origin,
+                        detail: format!("consecutive names are not allowed: '{name1}' '{name2}'",),
+                    }
+                    .fail();
                 }
                 (PartInfo::Name(name), _) if name.is_name_empty() => {
                     if let Some(ref m) = name.start_modifier {
-                        return Err(format!(
-                            "Invalid Pattern: '{}'. only '{}' modifier is invalid",
-                            self.origin, m
-                        ));
+                        return DissectInvalidPatternSnafu {
+                            s: &self.origin,
+                            detail: format!("only '{m}' modifier is invalid"),
+                        }
+                        .fail();
                     }
                 }
                 (PartInfo::Name(name), _) => match name.start_modifier {
                     Some(StartModifier::MapKey) => {
                         if map_keys.contains(&name.name) {
-                            return Err(format!(
-                                "Invalid Pattern: '{}'. Duplicate map key: '{}'",
-                                self.origin, name.name
-                            ));
+                            return DissectInvalidPatternSnafu {
+                                s: &self.origin,
+                                detail: format!("Duplicate map key: '{}'", name.name),
+                            }
+                            .fail();
                         } else {
                             map_keys.insert(&name.name);
                         }
                     }
                     Some(StartModifier::MapVal) => {
                         if map_vals.contains(&name.name) {
-                            return Err(format!(
-                                "Invalid Pattern: '{}'. Duplicate map val: '{}'",
-                                self.origin, name.name
-                            ));
+                            return DissectInvalidPatternSnafu {
+                                s: &self.origin,
+                                detail: format!("Duplicate map val: '{}'", name.name),
+                            }
+                            .fail();
                         } else {
                             map_vals.insert(&name.name);
                         }
@@ -462,15 +491,18 @@ impl PatternInfo {
         }
 
         if map_keys != map_vals {
-            return Err(format!(
-                "Invalid Pattern: '{}'. key and value not matched: '{}'",
-                self.origin,
-                map_keys
-                    .symmetric_difference(&map_vals)
-                    .map(|s| s.as_str())
-                    .collect::<Vec<&str>>()
-                    .join(",")
-            ));
+            return DissectInvalidPatternSnafu {
+                s: &self.origin,
+                detail: format!(
+                    "key and value not matched: '{}'",
+                    map_keys
+                        .symmetric_difference(&map_vals)
+                        .map(|s| s.as_str())
+                        .collect::<Vec<&str>>()
+                        .join(",")
+                ),
+            }
+            .fail();
         }
 
         Ok(())
@@ -516,10 +548,7 @@ impl DissectProcessorBuilder {
             .collect()
     }
 
-    fn part_info_to_part(
-        part_info: PartInfo,
-        intermediate_keys: &[String],
-    ) -> Result<Part, String> {
+    fn part_info_to_part(part_info: PartInfo, intermediate_keys: &[String]) -> Result<Part> {
         match part_info {
             PartInfo::Split(s) => Ok(Part::Split(s)),
             PartInfo::Name(n) => match n.start_modifier {
@@ -545,13 +574,13 @@ impl DissectProcessorBuilder {
     fn pattern_info_to_pattern(
         pattern_info: PatternInfo,
         intermediate_keys: &[String],
-    ) -> Result<Pattern, String> {
+    ) -> Result<Pattern> {
         let original = pattern_info.origin;
         let pattern = pattern_info
             .parts
             .into_iter()
             .map(|part_info| Self::part_info_to_part(part_info, intermediate_keys))
-            .collect::<Result<Vec<_>, String>>()?;
+            .collect::<Result<Vec<_>>>()?;
         Ok(Pattern {
             origin: original,
             parts: pattern,
@@ -561,7 +590,7 @@ impl DissectProcessorBuilder {
     fn build_patterns_from_pattern_infos(
         patterns: Vec<PatternInfo>,
         intermediate_keys: &[String],
-    ) -> Result<Vec<Pattern>, String> {
+    ) -> Result<Vec<Pattern>> {
         patterns
             .into_iter()
             .map(|pattern_info| Self::pattern_info_to_pattern(pattern_info, intermediate_keys))
@@ -578,7 +607,7 @@ impl ProcessorBuilder for DissectProcessorBuilder {
         self.fields.iter().map(|f| f.input_field()).collect()
     }
 
-    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind> {
         let mut real_fields = vec![];
         for field in self.fields.into_iter() {
             let input_index = find_key_index(intermediate_keys, field.input_field(), "dissect")?;
@@ -610,11 +639,7 @@ pub struct DissectProcessor {
 }
 
 impl DissectProcessor {
-    fn process_pattern(
-        &self,
-        chs: &[char],
-        pattern: &Pattern,
-    ) -> Result<Vec<(usize, Value)>, String> {
+    fn process_pattern(&self, chs: &[char], pattern: &Pattern) -> Result<Vec<(usize, Value)>> {
         let mut map = Vec::new();
         let mut pos = 0;
 
@@ -668,23 +693,26 @@ impl DissectProcessor {
                     let split_chs = split.chars().collect::<Vec<char>>();
                     let split_len = split_chs.len();
                     if pos + split_len > chs.len() {
-                        return Err(format!("'{split}' exceeds the input",));
+                        return DissectSplitExceedsInputSnafu { split }.fail();
                     }
 
                     if &chs[pos..pos + split_len] != split_chs.as_slice() {
-                        return Err(format!(
-                            "'{split}' does not match the input '{}'",
-                            chs[pos..pos + split_len].iter().collect::<String>()
-                        ));
+                        return DissectSplitNotMatchInputSnafu {
+                            split,
+                            input: chs[pos..pos + split_len].iter().collect::<String>(),
+                        }
+                        .fail();
                     }
 
                     pos += split_len;
                 }
 
                 (Part::Name(name1), Some(Part::Name(name2))) => {
-                    return Err(format!(
-                        "consecutive names are not allowed: '{name1}' '{name2}'"
-                    ));
+                    return DissectConsecutiveNamesSnafu {
+                        name1: name1.to_string(),
+                        name2: name2.to_string(),
+                    }
+                    .fail();
                 }
 
                 // if Name part is the last part, then the rest of the input is the value
@@ -695,10 +723,10 @@ impl DissectProcessor {
 
                 // if Name part, and next part is Split, then find the matched value of the name
                 (Part::Name(name), Some(Part::Split(split))) => {
-                    let stop = split
-                        .chars()
-                        .next()
-                        .ok_or("Empty split is not allowed".to_string())?; // this won't happen
+                    let stop = split.chars().next().context(DissectInvalidPatternSnafu {
+                        s: &pattern.origin,
+                        detail: "Empty split is not allowed",
+                    })?; // this won't happen
                     let mut end = pos;
                     while end < chs.len() && chs[end] != stop {
                         end += 1;
@@ -737,7 +765,7 @@ impl DissectProcessor {
         Ok(map)
     }
 
-    fn process(&self, val: &str) -> Result<Vec<(usize, Value)>, String> {
+    fn process(&self, val: &str) -> Result<Vec<(usize, Value)>> {
         let chs = val.chars().collect::<Vec<char>>();
 
         for pattern in &self.patterns {
@@ -745,15 +773,14 @@ impl DissectProcessor {
                 return Ok(map);
             }
         }
-
-        Err("No matching pattern found".to_string())
+        DissectNoMatchingPatternSnafu.fail()
     }
 }
 
 impl TryFrom<&yaml_rust::yaml::Hash> for DissectProcessorBuilder {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(value: &yaml_rust::yaml::Hash) -> Result<Self, Self::Error> {
+    fn try_from(value: &yaml_rust::yaml::Hash) -> Result<Self> {
         let mut fields = Fields::default();
         let mut patterns = vec![];
         let mut ignore_missing = false;
@@ -762,7 +789,7 @@ impl TryFrom<&yaml_rust::yaml::Hash> for DissectProcessorBuilder {
         for (k, v) in value.iter() {
             let key = k
                 .as_str()
-                .ok_or(format!("key must be a string, but got '{k:?}'"))?;
+                .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
 
             match key {
                 FIELD_NAME => {
@@ -809,7 +836,7 @@ impl Processor for DissectProcessor {
         self.ignore_missing
     }
 
-    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<(), String> {
+    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<()> {
         for field in self.fields.iter() {
             let index = field.input_index();
             match val.get(index) {
@@ -821,18 +848,19 @@ impl Processor for DissectProcessor {
                 }
                 Some(Value::Null) | None => {
                     if !self.ignore_missing {
-                        return Err(format!(
-                            "{} processor: missing field: {}",
-                            self.kind(),
-                            field.input_name()
-                        ));
+                        return ProcessorMissingFieldSnafu {
+                            processor: self.kind(),
+                            field: field.input_name(),
+                        }
+                        .fail();
                     }
                 }
                 Some(v) => {
-                    return Err(format!(
-                        "{} processor: expect string value, but got {v:?}",
-                        self.kind()
-                    ));
+                    return ProcessorExpectStringSnafu {
+                        processor: self.kind(),
+                        v: v.clone(),
+                    }
+                    .fail();
                 }
             }
         }
@@ -1123,7 +1151,7 @@ mod tests {
             ),
             (
                 "%{->clientip} ",
-                "Invalid Pattern: '%{->clientip} '. expected '}' after '->'",
+                "Invalid Pattern: '%{->clientip} '. Expected '}' after '->'",
             ),
             (
                 "%{/clientip} ",
@@ -1185,7 +1213,7 @@ mod tests {
 
         for (pattern, expected) in cases.into_iter() {
             let err = pattern.parse::<PatternInfo>().unwrap_err();
-            assert_eq!(err, expected);
+            assert_eq!(err.to_string(), expected);
         }
     }
 
diff --git a/src/pipeline/src/etl/processor/epoch.rs b/src/pipeline/src/etl/processor/epoch.rs
index 32c7d61786..f2c03fd120 100644
--- a/src/pipeline/src/etl/processor/epoch.rs
+++ b/src/pipeline/src/etl/processor/epoch.rs
@@ -13,7 +13,12 @@
 // limitations under the License.
 
 use ahash::HashSet;
+use snafu::{OptionExt, ResultExt};
 
+use crate::etl::error::{
+    EpochInvalidResolutionSnafu, Error, FailedToParseIntSnafu, KeyMustBeStringSnafu,
+    ProcessorMissingFieldSnafu, ProcessorUnsupportedValueSnafu, Result,
+};
 use crate::etl::field::{Fields, OneInputOneOutputField};
 use crate::etl::processor::{
     yaml_bool, yaml_new_field, yaml_new_fields, yaml_string, Processor, ProcessorBuilder,
@@ -39,15 +44,15 @@ enum Resolution {
 }
 
 impl TryFrom<&str> for Resolution {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(s: &str) -> Result<Self, Self::Error> {
+    fn try_from(s: &str) -> Result<Self> {
         match s {
             SECOND_RESOLUTION | SEC_RESOLUTION | S_RESOLUTION => Ok(Resolution::Second),
             MILLISECOND_RESOLUTION | MILLI_RESOLUTION | MS_RESOLUTION => Ok(Resolution::Milli),
             MICROSECOND_RESOLUTION | MICRO_RESOLUTION | US_RESOLUTION => Ok(Resolution::Micro),
             NANOSECOND_RESOLUTION | NANO_RESOLUTION | NS_RESOLUTION => Ok(Resolution::Nano),
-            _ => Err(format!("invalid resolution: {s}")),
+            _ => EpochInvalidResolutionSnafu { resolution: s }.fail(),
         }
     }
 }
@@ -71,13 +76,13 @@ impl ProcessorBuilder for EpochProcessorBuilder {
         self.fields.iter().map(|f| f.input_field()).collect()
     }
 
-    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind> {
         self.build(intermediate_keys).map(ProcessorKind::Epoch)
     }
 }
 
 impl EpochProcessorBuilder {
-    pub fn build(self, intermediate_keys: &[String]) -> Result<EpochProcessor, String> {
+    pub fn build(self, intermediate_keys: &[String]) -> Result<EpochProcessor> {
         let mut real_fields = vec![];
         for field in self.fields.into_iter() {
             let input = OneInputOneOutputField::build(
@@ -112,11 +117,11 @@ pub struct EpochProcessor {
 }
 
 impl EpochProcessor {
-    fn parse(&self, val: &Value) -> Result<Timestamp, String> {
+    fn parse(&self, val: &Value) -> Result<Timestamp> {
         let t: i64 = match val {
             Value::String(s) => s
                 .parse::<i64>()
-                .map_err(|e| format!("Failed to parse {} to number: {}", s, e))?,
+                .context(FailedToParseIntSnafu { value: s })?,
             Value::Int16(i) => *i as i64,
             Value::Int32(i) => *i as i64,
             Value::Int64(i) => *i,
@@ -135,9 +140,11 @@ impl EpochProcessor {
             },
 
             _ => {
-                return Err(format!(
-                    "{PROCESSOR_EPOCH} processor: unsupported value {val}"
-                ))
+                return ProcessorUnsupportedValueSnafu {
+                    processor: PROCESSOR_EPOCH,
+                    val: val.to_string(),
+                }
+                .fail();
             }
         };
 
@@ -151,9 +158,9 @@ impl EpochProcessor {
 }
 
 impl TryFrom<&yaml_rust::yaml::Hash> for EpochProcessorBuilder {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(hash: &yaml_rust::yaml::Hash) -> Result<Self, Self::Error> {
+    fn try_from(hash: &yaml_rust::yaml::Hash) -> Result<Self> {
         let mut fields = Fields::default();
         let mut resolution = Resolution::default();
         let mut ignore_missing = false;
@@ -161,7 +168,7 @@ impl TryFrom<&yaml_rust::yaml::Hash> for EpochProcessorBuilder {
         for (k, v) in hash {
             let key = k
                 .as_str()
-                .ok_or(format!("key must be a string, but got {k:?}"))?;
+                .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
 
             match key {
                 FIELD_NAME => {
@@ -200,17 +207,17 @@ impl Processor for EpochProcessor {
         self.ignore_missing
     }
 
-    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<(), String> {
+    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<()> {
         for field in self.fields.iter() {
             let index = field.input_index();
             match val.get(index) {
                 Some(Value::Null) | None => {
                     if !self.ignore_missing {
-                        return Err(format!(
-                            "{} processor: missing field: {}",
-                            self.kind(),
-                            field.input_name()
-                        ));
+                        return ProcessorMissingFieldSnafu {
+                            processor: self.kind(),
+                            field: field.input_name(),
+                        }
+                        .fail();
                     }
                 }
                 Some(v) => {
diff --git a/src/pipeline/src/etl/processor/gsub.rs b/src/pipeline/src/etl/processor/gsub.rs
index 1b8e581e6a..b5a328c6fa 100644
--- a/src/pipeline/src/etl/processor/gsub.rs
+++ b/src/pipeline/src/etl/processor/gsub.rs
@@ -14,11 +14,16 @@
 
 use ahash::HashSet;
 use regex::Regex;
+use snafu::{OptionExt, ResultExt};
 
+use crate::etl::error::{
+    Error, GsubPatternRequiredSnafu, GsubReplacementRequiredSnafu, KeyMustBeStringSnafu,
+    ProcessorExpectStringSnafu, ProcessorMissingFieldSnafu, RegexSnafu, Result,
+};
 use crate::etl::field::{Fields, OneInputOneOutputField};
 use crate::etl::processor::{
-    yaml_bool, yaml_new_field, yaml_new_fields, yaml_string, Processor, ProcessorBuilder,
-    ProcessorKind, FIELDS_NAME, FIELD_NAME, IGNORE_MISSING_NAME, PATTERN_NAME,
+    yaml_bool, yaml_new_field, yaml_new_fields, yaml_string, ProcessorBuilder, ProcessorKind,
+    FIELDS_NAME, FIELD_NAME, IGNORE_MISSING_NAME, PATTERN_NAME,
 };
 use crate::etl::value::Value;
 
@@ -46,25 +51,25 @@ impl ProcessorBuilder for GsubProcessorBuilder {
         self.fields.iter().map(|f| f.input_field()).collect()
     }
 
-    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind> {
         self.build(intermediate_keys).map(ProcessorKind::Gsub)
     }
 }
 
 impl GsubProcessorBuilder {
-    fn check(self) -> Result<Self, String> {
+    fn check(self) -> Result<Self> {
         if self.pattern.is_none() {
-            return Err("pattern is required".to_string());
+            return GsubPatternRequiredSnafu.fail();
         }
 
         if self.replacement.is_none() {
-            return Err("replacement is required".to_string());
+            return GsubReplacementRequiredSnafu.fail();
         }
 
         Ok(self)
     }
 
-    fn build(self, intermediate_keys: &[String]) -> Result<GsubProcessor, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<GsubProcessor> {
         let mut real_fields = vec![];
         for field in self.fields.into_iter() {
             let input = OneInputOneOutputField::build(
@@ -94,19 +99,19 @@ pub struct GsubProcessor {
 }
 
 impl GsubProcessor {
-    fn check(self) -> Result<Self, String> {
+    fn check(self) -> Result<Self> {
         if self.pattern.is_none() {
-            return Err("pattern is required".to_string());
+            return GsubPatternRequiredSnafu.fail();
         }
 
         if self.replacement.is_none() {
-            return Err("replacement is required".to_string());
+            return GsubReplacementRequiredSnafu.fail();
         }
 
         Ok(self)
     }
 
-    fn process_string(&self, val: &str) -> Result<Value, String> {
+    fn process_string(&self, val: &str) -> Result<Value> {
         let replacement = self.replacement.as_ref().unwrap();
         let new_val = self
             .pattern
@@ -119,21 +124,26 @@ impl GsubProcessor {
         Ok(val)
     }
 
-    fn process(&self, val: &Value) -> Result<Value, String> {
+    fn process(&self, val: &Value) -> Result<Value> {
         match val {
             Value::String(val) => self.process_string(val),
-            _ => Err(format!(
-                "{} processor: expect string or array string, but got {val:?}",
-                self.kind()
-            )),
+            _ => ProcessorExpectStringSnafu {
+                processor: PROCESSOR_GSUB,
+                v: val.clone(),
+            }
+            .fail(),
+            // Err(format!(
+            //     "{} processor: expect string or array string, but got {val:?}",
+            //     self.kind()
+            // )),
         }
     }
 }
 
 impl TryFrom<&yaml_rust::yaml::Hash> for GsubProcessorBuilder {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(value: &yaml_rust::yaml::Hash) -> Result<Self, Self::Error> {
+    fn try_from(value: &yaml_rust::yaml::Hash) -> Result<Self> {
         let mut fields = Fields::default();
         let mut ignore_missing = false;
         let mut pattern = None;
@@ -142,7 +152,8 @@ impl TryFrom<&yaml_rust::yaml::Hash> for GsubProcessorBuilder {
         for (k, v) in value.iter() {
             let key = k
                 .as_str()
-                .ok_or(format!("key must be a string, but got {k:?}"))?;
+                .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
+
             match key {
                 FIELD_NAME => {
                     fields = Fields::one(yaml_new_field(v, FIELD_NAME)?);
@@ -152,7 +163,9 @@ impl TryFrom<&yaml_rust::yaml::Hash> for GsubProcessorBuilder {
                 }
                 PATTERN_NAME => {
                     let pattern_str = yaml_string(v, PATTERN_NAME)?;
-                    pattern = Some(Regex::new(&pattern_str).map_err(|e| e.to_string())?);
+                    pattern = Some(Regex::new(&pattern_str).context(RegexSnafu {
+                        pattern: pattern_str,
+                    })?);
                 }
                 REPLACEMENT_NAME => {
                     let replacement_str = yaml_string(v, REPLACEMENT_NAME)?;
@@ -187,17 +200,17 @@ impl crate::etl::processor::Processor for GsubProcessor {
         self.ignore_missing
     }
 
-    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<(), String> {
+    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<()> {
         for field in self.fields.iter() {
             let index = field.input_index();
             match val.get(index) {
                 Some(Value::Null) | None => {
                     if !self.ignore_missing {
-                        return Err(format!(
-                            "{} processor: missing field: {}",
-                            self.kind(),
-                            field.input_name()
-                        ));
+                        return ProcessorMissingFieldSnafu {
+                            processor: self.kind(),
+                            field: field.input_name(),
+                        }
+                        .fail();
                     }
                 }
                 Some(v) => {
diff --git a/src/pipeline/src/etl/processor/join.rs b/src/pipeline/src/etl/processor/join.rs
index d4b309d5c2..ddbc086ab8 100644
--- a/src/pipeline/src/etl/processor/join.rs
+++ b/src/pipeline/src/etl/processor/join.rs
@@ -13,7 +13,12 @@
 // limitations under the License.
 
 use ahash::HashSet;
+use snafu::OptionExt;
 
+use crate::etl::error::{
+    Error, JoinSeparatorRequiredSnafu, KeyMustBeStringSnafu, ProcessorExpectStringSnafu,
+    ProcessorMissingFieldSnafu, Result,
+};
 use crate::etl::field::{Fields, OneInputOneOutputField};
 use crate::etl::processor::{
     yaml_bool, yaml_new_field, yaml_new_fields, yaml_string, Processor, ProcessorBuilder,
@@ -42,21 +47,21 @@ impl ProcessorBuilder for JoinProcessorBuilder {
         self.fields.iter().map(|f| f.input_field()).collect()
     }
 
-    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind> {
         self.build(intermediate_keys).map(ProcessorKind::Join)
     }
 }
 
 impl JoinProcessorBuilder {
-    fn check(self) -> Result<Self, String> {
+    fn check(self) -> Result<Self> {
         if self.separator.is_none() {
-            return Err("separator is required".to_string());
+            return JoinSeparatorRequiredSnafu.fail();
         }
 
         Ok(self)
     }
 
-    pub fn build(self, intermediate_keys: &[String]) -> Result<JoinProcessor, String> {
+    pub fn build(self, intermediate_keys: &[String]) -> Result<JoinProcessor> {
         let mut real_fields = vec![];
         for field in self.fields.into_iter() {
             let input = OneInputOneOutputField::build(
@@ -85,7 +90,7 @@ pub struct JoinProcessor {
 }
 
 impl JoinProcessor {
-    fn process(&self, arr: &Array) -> Result<Value, String> {
+    fn process(&self, arr: &Array) -> Result<Value> {
         let sep = self.separator.as_ref().unwrap();
         let val = arr
             .iter()
@@ -96,9 +101,9 @@ impl JoinProcessor {
         Ok(Value::String(val))
     }
 
-    fn check(self) -> Result<Self, String> {
+    fn check(self) -> Result<Self> {
         if self.separator.is_none() {
-            return Err("separator is required".to_string());
+            return JoinSeparatorRequiredSnafu.fail();
         }
 
         Ok(self)
@@ -106,9 +111,9 @@ impl JoinProcessor {
 }
 
 impl TryFrom<&yaml_rust::yaml::Hash> for JoinProcessorBuilder {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(value: &yaml_rust::yaml::Hash) -> Result<Self, Self::Error> {
+    fn try_from(value: &yaml_rust::yaml::Hash) -> Result<Self> {
         let mut fields = Fields::default();
         let mut separator = None;
         let mut ignore_missing = false;
@@ -116,7 +121,8 @@ impl TryFrom<&yaml_rust::yaml::Hash> for JoinProcessorBuilder {
         for (k, v) in value.iter() {
             let key = k
                 .as_str()
-                .ok_or(format!("key must be a string, but got {k:?}"))?;
+                .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
+
             match key {
                 FIELD_NAME => {
                     fields = Fields::one(yaml_new_field(v, FIELD_NAME)?);
@@ -152,7 +158,7 @@ impl Processor for JoinProcessor {
         self.ignore_missing
     }
 
-    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<(), String> {
+    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<()> {
         for field in self.fields.iter() {
             let index = field.input_index();
             match val.get(index) {
@@ -163,18 +169,19 @@ impl Processor for JoinProcessor {
                 }
                 Some(Value::Null) | None => {
                     if !self.ignore_missing {
-                        return Err(format!(
-                            "{} processor: missing field: {}",
-                            self.kind(),
-                            field.input_name()
-                        ));
+                        return ProcessorMissingFieldSnafu {
+                            processor: self.kind(),
+                            field: field.input_name(),
+                        }
+                        .fail();
                     }
                 }
                 Some(v) => {
-                    return Err(format!(
-                        "{} processor: expect string value, but got {v:?}",
-                        self.kind()
-                    ));
+                    return ProcessorExpectStringSnafu {
+                        processor: self.kind(),
+                        v: v.clone(),
+                    }
+                    .fail();
                 }
             }
         }
diff --git a/src/pipeline/src/etl/processor/letter.rs b/src/pipeline/src/etl/processor/letter.rs
index f388b5a2a9..8eb9399181 100644
--- a/src/pipeline/src/etl/processor/letter.rs
+++ b/src/pipeline/src/etl/processor/letter.rs
@@ -13,7 +13,12 @@
 // limitations under the License.
 
 use ahash::HashSet;
+use snafu::OptionExt;
 
+use crate::etl::error::{
+    Error, KeyMustBeStringSnafu, LetterInvalidMethodSnafu, ProcessorExpectStringSnafu,
+    ProcessorMissingFieldSnafu, Result,
+};
 use crate::etl::field::{Fields, OneInputOneOutputField};
 use crate::etl::processor::{
     yaml_bool, yaml_new_field, yaml_new_fields, yaml_string, Processor, ProcessorBuilder,
@@ -42,14 +47,14 @@ impl std::fmt::Display for Method {
 }
 
 impl std::str::FromStr for Method {
-    type Err = String;
+    type Err = Error;
 
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
+    fn from_str(s: &str) -> Result<Self> {
         match s.to_lowercase().as_str() {
             "upper" => Ok(Method::Upper),
             "lower" => Ok(Method::Lower),
             "capital" => Ok(Method::Capital),
-            _ => Err(format!("invalid method: {s}")),
+            _ => LetterInvalidMethodSnafu { method: s }.fail(),
         }
     }
 }
@@ -73,13 +78,13 @@ impl ProcessorBuilder for LetterProcessorBuilder {
         self.fields.iter().map(|f| f.input_field()).collect()
     }
 
-    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind> {
         self.build(intermediate_keys).map(ProcessorKind::Letter)
     }
 }
 
 impl LetterProcessorBuilder {
-    pub fn build(self, intermediate_keys: &[String]) -> Result<LetterProcessor, String> {
+    pub fn build(self, intermediate_keys: &[String]) -> Result<LetterProcessor> {
         let mut real_fields = vec![];
         for field in self.fields.into_iter() {
             let input = OneInputOneOutputField::build(
@@ -108,7 +113,7 @@ pub struct LetterProcessor {
 }
 
 impl LetterProcessor {
-    fn process_field(&self, val: &str) -> Result<Value, String> {
+    fn process_field(&self, val: &str) -> Result<Value> {
         let processed = match self.method {
             Method::Upper => val.to_uppercase(),
             Method::Lower => val.to_lowercase(),
@@ -121,9 +126,9 @@ impl LetterProcessor {
 }
 
 impl TryFrom<&yaml_rust::yaml::Hash> for LetterProcessorBuilder {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(value: &yaml_rust::yaml::Hash) -> Result<Self, Self::Error> {
+    fn try_from(value: &yaml_rust::yaml::Hash) -> Result<Self> {
         let mut fields = Fields::default();
         let mut method = Method::Lower;
         let mut ignore_missing = false;
@@ -131,7 +136,7 @@ impl TryFrom<&yaml_rust::yaml::Hash> for LetterProcessorBuilder {
         for (k, v) in value.iter() {
             let key = k
                 .as_str()
-                .ok_or(format!("key must be a string, but got {k:?}"))?;
+                .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
             match key {
                 FIELD_NAME => {
                     fields = Fields::one(yaml_new_field(v, FIELD_NAME)?);
@@ -166,7 +171,7 @@ impl Processor for LetterProcessor {
         self.ignore_missing
     }
 
-    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<(), String> {
+    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<()> {
         for field in self.fields.iter() {
             let index = field.input_index();
             match val.get(index) {
@@ -177,18 +182,19 @@ impl Processor for LetterProcessor {
                 }
                 Some(Value::Null) | None => {
                     if !self.ignore_missing {
-                        return Err(format!(
-                            "{} processor: missing field: {}",
-                            self.kind(),
-                            &field.input().name
-                        ));
+                        return ProcessorMissingFieldSnafu {
+                            processor: self.kind(),
+                            field: field.input_name(),
+                        }
+                        .fail();
                     }
                 }
                 Some(v) => {
-                    return Err(format!(
-                        "{} processor: expect string value, but got {v:?}",
-                        self.kind()
-                    ));
+                    return ProcessorExpectStringSnafu {
+                        processor: self.kind(),
+                        v: v.clone(),
+                    }
+                    .fail();
                 }
             }
         }
diff --git a/src/pipeline/src/etl/processor/regex.rs b/src/pipeline/src/etl/processor/regex.rs
index a1de2ea76d..a74c19140c 100644
--- a/src/pipeline/src/etl/processor/regex.rs
+++ b/src/pipeline/src/etl/processor/regex.rs
@@ -21,7 +21,13 @@ pub(crate) const PROCESSOR_REGEX: &str = "regex";
 use ahash::{HashSet, HashSetExt};
 use lazy_static::lazy_static;
 use regex::Regex;
+use snafu::{OptionExt, ResultExt};
 
+use crate::etl::error::{
+    Error, KeyMustBeStringSnafu, ProcessorExpectStringSnafu, ProcessorMissingFieldSnafu,
+    RegexNamedGroupNotFoundSnafu, RegexNoValidFieldSnafu, RegexNoValidPatternSnafu, RegexSnafu,
+    Result,
+};
 use crate::etl::field::{Fields, InputFieldInfo, OneInputMultiOutputField};
 use crate::etl::find_key_index;
 use crate::etl::processor::{
@@ -60,15 +66,15 @@ impl std::fmt::Display for GroupRegex {
 }
 
 impl std::str::FromStr for GroupRegex {
-    type Err = String;
+    type Err = Error;
 
-    fn from_str(origin: &str) -> Result<Self, Self::Err> {
+    fn from_str(origin: &str) -> Result<Self> {
         let groups = get_regex_group_names(origin);
         if groups.is_empty() {
-            return Err(format!("no named group found in regex {origin}"));
+            return RegexNamedGroupNotFoundSnafu { origin }.fail();
         }
 
-        let regex = Regex::new(origin).map_err(|e| e.to_string())?;
+        let regex = Regex::new(origin).context(RegexSnafu { pattern: origin })?;
         Ok(GroupRegex {
             origin: origin.into(),
             regex,
@@ -94,25 +100,25 @@ impl ProcessorBuilder for RegexProcessorBuilder {
         self.fields.iter().map(|f| f.input_field()).collect()
     }
 
-    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind> {
         self.build(intermediate_keys).map(ProcessorKind::Regex)
     }
 }
 
 impl RegexProcessorBuilder {
-    fn check(self) -> Result<Self, String> {
+    fn check(self) -> Result<Self> {
         if self.fields.is_empty() {
-            return Err(format!(
-                "no valid field found in {} processor",
-                PROCESSOR_REGEX
-            ));
+            return RegexNoValidFieldSnafu {
+                processor: PROCESSOR_REGEX,
+            }
+            .fail();
         }
 
         if self.patterns.is_empty() {
-            return Err(format!(
-                "no valid pattern found in {} processor",
-                PROCESSOR_REGEX
-            ));
+            return RegexNoValidPatternSnafu {
+                processor: PROCESSOR_REGEX,
+            }
+            .fail();
         }
 
         Ok(self)
@@ -122,7 +128,7 @@ impl RegexProcessorBuilder {
         group_regex: &GroupRegex,
         om_field: &OneInputMultiOutputField,
         intermediate_keys: &[String],
-    ) -> Result<Vec<OutPutInfo>, String> {
+    ) -> Result<Vec<OutPutInfo>> {
         group_regex
             .groups
             .iter()
@@ -135,35 +141,35 @@ impl RegexProcessorBuilder {
                     index,
                 })
             })
-            .collect::<Result<Vec<_>, String>>()
+            .collect::<Result<Vec<_>>>()
     }
 
     fn build_group_output_infos(
         patterns: &[GroupRegex],
         om_field: &OneInputMultiOutputField,
         intermediate_keys: &[String],
-    ) -> Result<Vec<Vec<OutPutInfo>>, String> {
+    ) -> Result<Vec<Vec<OutPutInfo>>> {
         patterns
             .iter()
             .map(|group_regex| {
                 Self::build_group_output_info(group_regex, om_field, intermediate_keys)
             })
-            .collect::<Result<Vec<_>, String>>()
+            .collect::<Result<Vec<_>>>()
     }
 
     fn build_output_info(
         real_fields: &[OneInputMultiOutputField],
         patterns: &[GroupRegex],
         intermediate_keys: &[String],
-    ) -> Result<RegexProcessorOutputInfo, String> {
+    ) -> Result<RegexProcessorOutputInfo> {
         let inner = real_fields
             .iter()
             .map(|om_field| Self::build_group_output_infos(patterns, om_field, intermediate_keys))
-            .collect::<Result<Vec<_>, String>>();
+            .collect::<Result<Vec<_>>>();
         inner.map(|inner| RegexProcessorOutputInfo { inner })
     }
 
-    fn build(self, intermediate_keys: &[String]) -> Result<RegexProcessor, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<RegexProcessor> {
         let mut real_fields = vec![];
         for field in self.fields.into_iter() {
             let input_index = find_key_index(intermediate_keys, field.input_field(), "regex")?;
@@ -184,9 +190,9 @@ impl RegexProcessorBuilder {
 }
 
 impl TryFrom<&yaml_rust::yaml::Hash> for RegexProcessorBuilder {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(value: &yaml_rust::yaml::Hash) -> Result<Self, Self::Error> {
+    fn try_from(value: &yaml_rust::yaml::Hash) -> Result<Self> {
         let mut fields = Fields::default();
         let mut patterns: Vec<GroupRegex> = vec![];
         let mut ignore_missing = false;
@@ -194,7 +200,7 @@ impl TryFrom<&yaml_rust::yaml::Hash> for RegexProcessorBuilder {
         for (k, v) in value.iter() {
             let key = k
                 .as_str()
-                .ok_or(format!("key must be a string, but got {k:?}"))?;
+                .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
             match key {
                 FIELD_NAME => {
                     fields = Fields::one(yaml_new_field(v, FIELD_NAME)?);
@@ -275,7 +281,7 @@ pub struct RegexProcessor {
 }
 
 impl RegexProcessor {
-    fn try_with_patterns(&mut self, patterns: Vec<String>) -> Result<(), String> {
+    fn try_with_patterns(&mut self, patterns: Vec<String>) -> Result<()> {
         let mut rs = vec![];
         for pattern in patterns {
             let gr = pattern.parse()?;
@@ -290,7 +296,7 @@ impl RegexProcessor {
         val: &str,
         gr: &GroupRegex,
         index: (usize, usize),
-    ) -> Result<Vec<(usize, Value)>, String> {
+    ) -> Result<Vec<(usize, Value)>> {
         let mut result = Vec::new();
         if let Some(captures) = gr.regex.captures(val) {
             for (group_index, group) in gr.groups.iter().enumerate() {
@@ -316,7 +322,7 @@ impl Processor for RegexProcessor {
         self.ignore_missing
     }
 
-    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<(), String> {
+    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<()> {
         for (field_index, field) in self.fields.iter().enumerate() {
             let index = field.input_index();
             let mut result_list = None;
@@ -346,18 +352,19 @@ impl Processor for RegexProcessor {
                 }
                 Some(Value::Null) | None => {
                     if !self.ignore_missing {
-                        return Err(format!(
-                            "{} processor: missing field: {}",
-                            self.kind(),
-                            field.input_name()
-                        ));
+                        return ProcessorMissingFieldSnafu {
+                            processor: self.kind(),
+                            field: field.input_name(),
+                        }
+                        .fail();
                     }
                 }
                 Some(v) => {
-                    return Err(format!(
-                        "{} processor: expect string value, but got {v:?}",
-                        self.kind()
-                    ));
+                    return ProcessorExpectStringSnafu {
+                        processor: self.kind(),
+                        v: v.clone(),
+                    }
+                    .fail();
                 }
             }
             // safety here
diff --git a/src/pipeline/src/etl/processor/timestamp.rs b/src/pipeline/src/etl/processor/timestamp.rs
index 7ab9571101..3f9621c19d 100644
--- a/src/pipeline/src/etl/processor/timestamp.rs
+++ b/src/pipeline/src/etl/processor/timestamp.rs
@@ -18,7 +18,14 @@ use ahash::HashSet;
 use chrono::{DateTime, NaiveDateTime};
 use chrono_tz::Tz;
 use lazy_static::lazy_static;
+use snafu::{OptionExt, ResultExt};
 
+use crate::etl::error::{
+    DateFailedToGetLocalTimezoneSnafu, DateFailedToGetTimestampSnafu, DateInvalidFormatSnafu,
+    DateParseSnafu, DateParseTimezoneSnafu, EpochInvalidResolutionSnafu, Error,
+    KeyMustBeStringSnafu, ProcessorFailedToParseStringSnafu, ProcessorMissingFieldSnafu,
+    ProcessorUnsupportedValueSnafu, Result,
+};
 use crate::etl::field::{Fields, OneInputOneOutputField};
 use crate::etl::processor::{
     yaml_bool, yaml_new_field, yaml_new_fields, yaml_string, yaml_strings, Processor,
@@ -69,15 +76,15 @@ enum Resolution {
 }
 
 impl TryFrom<&str> for Resolution {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(s: &str) -> Result<Self, Self::Error> {
+    fn try_from(s: &str) -> Result<Self> {
         match s {
             SECOND_RESOLUTION | SEC_RESOLUTION | S_RESOLUTION => Ok(Resolution::Second),
             MILLISECOND_RESOLUTION | MILLI_RESOLUTION | MS_RESOLUTION => Ok(Resolution::Milli),
             MICROSECOND_RESOLUTION | MICRO_RESOLUTION | US_RESOLUTION => Ok(Resolution::Micro),
             NANOSECOND_RESOLUTION | NANO_RESOLUTION | NS_RESOLUTION => Ok(Resolution::Nano),
-            _ => Err(format!("invalid resolution: {s}")),
+            _ => EpochInvalidResolutionSnafu { resolution: s }.fail(),
         }
     }
 }
@@ -127,13 +134,13 @@ impl ProcessorBuilder for TimestampProcessorBuilder {
         self.fields.iter().map(|f| f.input_field()).collect()
     }
 
-    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind> {
         self.build(intermediate_keys).map(ProcessorKind::Timestamp)
     }
 }
 
 impl TimestampProcessorBuilder {
-    pub fn build(self, intermediate_keys: &[String]) -> Result<TimestampProcessor, String> {
+    pub fn build(self, intermediate_keys: &[String]) -> Result<TimestampProcessor> {
         let mut real_fields = vec![];
         for field in self.fields.into_iter() {
             let input = OneInputOneOutputField::build(
@@ -169,29 +176,37 @@ pub struct TimestampProcessor {
 
 impl TimestampProcessor {
     /// try to parse val with timezone first, if failed, parse without timezone
-    fn try_parse(val: &str, fmt: &str, tz: Tz) -> Result<i64, String> {
+    fn try_parse(val: &str, fmt: &str, tz: Tz) -> Result<i64> {
         if let Ok(dt) = DateTime::parse_from_str(val, fmt) {
-            Ok(dt.timestamp_nanos_opt().ok_or("failed to get timestamp")?)
+            Ok(dt
+                .timestamp_nanos_opt()
+                .context(DateFailedToGetTimestampSnafu)?)
         } else {
             let dt = NaiveDateTime::parse_from_str(val, fmt)
-                .map_err(|e| e.to_string())?
+                .context(DateParseSnafu { value: val })?
                 .and_local_timezone(tz)
                 .single()
-                .ok_or("failed to get local timezone")?;
-            Ok(dt.timestamp_nanos_opt().ok_or("failed to get timestamp")?)
+                .context(DateFailedToGetLocalTimezoneSnafu)?;
+            Ok(dt
+                .timestamp_nanos_opt()
+                .context(DateFailedToGetTimestampSnafu)?)
         }
     }
 
-    fn parse_time_str(&self, val: &str) -> Result<i64, String> {
+    fn parse_time_str(&self, val: &str) -> Result<i64> {
         for (fmt, tz) in self.formats.iter() {
             if let Ok(ns) = Self::try_parse(val, fmt, *tz) {
                 return Ok(ns);
             }
         }
-        Err(format!("{} processor: failed to parse {val}", self.kind(),))
+        ProcessorFailedToParseStringSnafu {
+            kind: PROCESSOR_TIMESTAMP,
+            value: val.to_string(),
+        }
+        .fail()
     }
 
-    fn parse(&self, val: &Value) -> Result<Timestamp, String> {
+    fn parse(&self, val: &Value) -> Result<Timestamp> {
         let t: i64 = match val {
             Value::String(s) => {
                 let t = s.parse::<i64>();
@@ -221,9 +236,11 @@ impl TimestampProcessor {
             },
 
             _ => {
-                return Err(format!(
-                    "{PROCESSOR_TIMESTAMP} processor: unsupported value {val}"
-                ))
+                return ProcessorUnsupportedValueSnafu {
+                    processor: PROCESSOR_TIMESTAMP,
+                    val: val.to_string(),
+                }
+                .fail();
             }
         };
 
@@ -236,7 +253,7 @@ impl TimestampProcessor {
     }
 }
 
-fn parse_formats(yaml: &yaml_rust::yaml::Yaml) -> Result<Vec<(Arc<String>, Tz)>, String> {
+fn parse_formats(yaml: &yaml_rust::yaml::Yaml) -> Result<Vec<(Arc<String>, Tz)>> {
     return match yaml.as_vec() {
         Some(formats_yaml) => {
             let mut formats = Vec::with_capacity(formats_yaml.len());
@@ -244,32 +261,38 @@ fn parse_formats(yaml: &yaml_rust::yaml::Yaml) -> Result<Vec<(Arc<String>, Tz)>,
                 let s = yaml_strings(v, FORMATS_NAME)
                     .or(yaml_string(v, FORMATS_NAME).map(|s| vec![s]))?;
                 if s.len() != 1 && s.len() != 2 {
-                    return Err(format!(
-                        "{PROCESSOR_TIMESTAMP} processor: invalid format {s:?}"
-                    ));
+                    return DateInvalidFormatSnafu {
+                        processor: PROCESSOR_TIMESTAMP,
+                        s: format!("{s:?}"),
+                    }
+                    .fail();
                 }
                 let mut iter = s.into_iter();
                 // safety: unwrap is safe here
                 let formatter = iter.next().unwrap();
                 let tz = iter
                     .next()
-                    .map(|tz| tz.parse::<Tz>())
-                    .unwrap_or(Ok(Tz::UTC))
-                    .map_err(|e| e.to_string())?;
+                    .map(|tz| {
+                        tz.parse::<Tz>()
+                            .context(DateParseTimezoneSnafu { value: tz })
+                    })
+                    .unwrap_or(Ok(Tz::UTC))?;
                 formats.push((Arc::new(formatter), tz));
             }
             Ok(formats)
         }
-        None => Err(format!(
-            "{PROCESSOR_TIMESTAMP} processor: invalid format {yaml:?}"
-        )),
+        None => DateInvalidFormatSnafu {
+            processor: PROCESSOR_TIMESTAMP,
+            s: format!("{yaml:?}"),
+        }
+        .fail(),
     };
 }
 
 impl TryFrom<&yaml_rust::yaml::Hash> for TimestampProcessorBuilder {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(hash: &yaml_rust::yaml::Hash) -> Result<Self, Self::Error> {
+    fn try_from(hash: &yaml_rust::yaml::Hash) -> Result<Self> {
         let mut fields = Fields::default();
         let mut formats = Formats::default();
         let mut resolution = Resolution::default();
@@ -278,7 +301,7 @@ impl TryFrom<&yaml_rust::yaml::Hash> for TimestampProcessorBuilder {
         for (k, v) in hash {
             let key = k
                 .as_str()
-                .ok_or(format!("key must be a string, but got {k:?}"))?;
+                .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
 
             match key {
                 FIELD_NAME => {
@@ -321,17 +344,17 @@ impl Processor for TimestampProcessor {
         self.ignore_missing
     }
 
-    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<(), String> {
+    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<()> {
         for field in self.fields.iter() {
             let index = field.input().index;
             match val.get(index) {
                 Some(Value::Null) | None => {
                     if !self.ignore_missing {
-                        return Err(format!(
-                            "{} processor: missing field: {}",
-                            self.kind(),
-                            &field.input().name
-                        ));
+                        return ProcessorMissingFieldSnafu {
+                            processor: self.kind(),
+                            field: field.input_name(),
+                        }
+                        .fail();
                     }
                 }
                 Some(v) => {
diff --git a/src/pipeline/src/etl/processor/urlencoding.rs b/src/pipeline/src/etl/processor/urlencoding.rs
index 7db9d092f2..ca42aae236 100644
--- a/src/pipeline/src/etl/processor/urlencoding.rs
+++ b/src/pipeline/src/etl/processor/urlencoding.rs
@@ -13,8 +13,13 @@
 // limitations under the License.
 
 use ahash::HashSet;
+use snafu::{OptionExt, ResultExt};
 use urlencoding::{decode, encode};
 
+use crate::etl::error::{
+    Error, KeyMustBeStringSnafu, ProcessorExpectStringSnafu, ProcessorMissingFieldSnafu, Result,
+    UrlEncodingDecodeSnafu, UrlEncodingInvalidMethodSnafu,
+};
 use crate::etl::field::{Fields, OneInputOneOutputField};
 use crate::etl::processor::{
     yaml_bool, yaml_new_field, yaml_new_fields, yaml_string, ProcessorBuilder, ProcessorKind,
@@ -41,13 +46,13 @@ impl std::fmt::Display for Method {
 }
 
 impl std::str::FromStr for Method {
-    type Err = String;
+    type Err = Error;
 
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
+    fn from_str(s: &str) -> Result<Self> {
         match s {
             "decode" => Ok(Method::Decode),
             "encode" => Ok(Method::Encode),
-            _ => Err(format!("invalid method: {s}")),
+            _ => UrlEncodingInvalidMethodSnafu { s }.fail(),
         }
     }
 }
@@ -71,14 +76,14 @@ impl ProcessorBuilder for UrlEncodingProcessorBuilder {
         self.fields.iter().map(|f| f.input_field()).collect()
     }
 
-    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<ProcessorKind> {
         self.build(intermediate_keys)
             .map(ProcessorKind::UrlEncoding)
     }
 }
 
 impl UrlEncodingProcessorBuilder {
-    fn build(self, intermediate_keys: &[String]) -> Result<UrlEncodingProcessor, String> {
+    fn build(self, intermediate_keys: &[String]) -> Result<UrlEncodingProcessor> {
         let mut real_fields = vec![];
         for field in self.fields.into_iter() {
             let input = OneInputOneOutputField::build(
@@ -106,19 +111,19 @@ pub struct UrlEncodingProcessor {
 }
 
 impl UrlEncodingProcessor {
-    fn process_field(&self, val: &str) -> Result<Value, String> {
+    fn process_field(&self, val: &str) -> Result<Value> {
         let processed = match self.method {
             Method::Encode => encode(val).to_string(),
-            Method::Decode => decode(val).map_err(|e| e.to_string())?.into_owned(),
+            Method::Decode => decode(val).context(UrlEncodingDecodeSnafu)?.into_owned(),
         };
         Ok(Value::String(processed))
     }
 }
 
 impl TryFrom<&yaml_rust::yaml::Hash> for UrlEncodingProcessorBuilder {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(value: &yaml_rust::yaml::Hash) -> Result<Self, Self::Error> {
+    fn try_from(value: &yaml_rust::yaml::Hash) -> Result<Self> {
         let mut fields = Fields::default();
         let mut method = Method::Decode;
         let mut ignore_missing = false;
@@ -126,7 +131,7 @@ impl TryFrom<&yaml_rust::yaml::Hash> for UrlEncodingProcessorBuilder {
         for (k, v) in value.iter() {
             let key = k
                 .as_str()
-                .ok_or(format!("key must be a string, but got {k:?}"))?;
+                .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
             match key {
                 FIELD_NAME => {
                     fields = Fields::one(yaml_new_field(v, FIELD_NAME)?);
@@ -166,7 +171,7 @@ impl crate::etl::processor::Processor for UrlEncodingProcessor {
         self.ignore_missing
     }
 
-    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<(), String> {
+    fn exec_mut(&self, val: &mut Vec<Value>) -> Result<()> {
         for field in self.fields.iter() {
             let index = field.input_index();
             match val.get(index) {
@@ -177,18 +182,19 @@ impl crate::etl::processor::Processor for UrlEncodingProcessor {
                 }
                 Some(Value::Null) | None => {
                     if !self.ignore_missing {
-                        return Err(format!(
-                            "{} processor: missing field: {}",
-                            self.kind(),
-                            field.output_name()
-                        ));
+                        return ProcessorMissingFieldSnafu {
+                            processor: self.kind(),
+                            field: field.input_name(),
+                        }
+                        .fail();
                     }
                 }
                 Some(v) => {
-                    return Err(format!(
-                        "{} processor: expect string value, but got {v:?}",
-                        self.kind()
-                    ));
+                    return ProcessorExpectStringSnafu {
+                        processor: self.kind(),
+                        v: v.clone(),
+                    }
+                    .fail();
                 }
             }
         }
diff --git a/src/pipeline/src/etl/transform.rs b/src/pipeline/src/etl/transform.rs
index 15d1bf3378..4408523c27 100644
--- a/src/pipeline/src/etl/transform.rs
+++ b/src/pipeline/src/etl/transform.rs
@@ -16,7 +16,9 @@ pub mod index;
 pub mod transformer;
 
 use itertools::Itertools;
+use snafu::OptionExt;
 
+use crate::etl::error::{Error, Result};
 use crate::etl::find_key_index;
 use crate::etl::processor::yaml_string;
 use crate::etl::transform::index::Index;
@@ -31,6 +33,10 @@ const TRANSFORM_ON_FAILURE: &str = "on_failure";
 
 pub use transformer::greptime::GreptimeTransformer;
 
+use super::error::{
+    KeyMustBeStringSnafu, TransformElementMustBeMapSnafu, TransformOnFailureInvalidValueSnafu,
+    TransformTypeMustBeSetSnafu,
+};
 use super::field::{Fields, InputFieldInfo, OneInputOneOutputField};
 use super::processor::{yaml_new_field, yaml_new_fields};
 
@@ -38,11 +44,11 @@ pub trait Transformer: std::fmt::Display + Sized + Send + Sync + 'static {
     type Output;
     type VecOutput;
 
-    fn new(transforms: Transforms) -> Result<Self, String>;
+    fn new(transforms: Transforms) -> Result<Self>;
     fn schemas(&self) -> &Vec<greptime_proto::v1::ColumnSchema>;
     fn transforms(&self) -> &Transforms;
     fn transforms_mut(&mut self) -> &mut Transforms;
-    fn transform_mut(&self, val: &mut Vec<Value>) -> Result<Self::VecOutput, String>;
+    fn transform_mut(&self, val: &mut Vec<Value>) -> Result<Self::VecOutput>;
 }
 
 /// On Failure behavior when transform fails
@@ -57,13 +63,13 @@ pub enum OnFailure {
 }
 
 impl std::str::FromStr for OnFailure {
-    type Err = String;
+    type Err = Error;
 
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
+    fn from_str(s: &str) -> Result<Self> {
         match s {
             "ignore" => Ok(OnFailure::Ignore),
             "default" => Ok(OnFailure::Default),
-            _ => Err(format!("invalid transform on_failure value: {}", s)),
+            _ => TransformOnFailureInvalidValueSnafu { value: s }.fail(),
         }
     }
 }
@@ -139,16 +145,16 @@ impl std::ops::DerefMut for Transforms {
 }
 
 impl TryFrom<&Vec<yaml_rust::Yaml>> for TransformBuilders {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(docs: &Vec<yaml_rust::Yaml>) -> Result<Self, Self::Error> {
+    fn try_from(docs: &Vec<yaml_rust::Yaml>) -> Result<Self> {
         let mut transforms = Vec::with_capacity(100);
         let mut all_output_keys: Vec<String> = Vec::with_capacity(100);
         let mut all_required_keys = Vec::with_capacity(100);
         for doc in docs {
             let transform_builder: TransformBuilder = doc
                 .as_hash()
-                .ok_or("transform element must be a map".to_string())?
+                .context(TransformElementMustBeMapSnafu)?
                 .try_into()?;
             let mut transform_output_keys = transform_builder
                 .fields
@@ -187,11 +193,7 @@ pub struct TransformBuilder {
 }
 
 impl TransformBuilder {
-    pub fn build(
-        self,
-        intermediate_keys: &[String],
-        output_keys: &[String],
-    ) -> Result<Transform, String> {
+    pub fn build(self, intermediate_keys: &[String], output_keys: &[String]) -> Result<Transform> {
         let mut real_fields = vec![];
         for field in self.fields {
             let input_index = find_key_index(intermediate_keys, field.input_field(), "transform")?;
@@ -277,9 +279,9 @@ impl Transform {
 }
 
 impl TryFrom<&yaml_rust::yaml::Hash> for TransformBuilder {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(hash: &yaml_rust::yaml::Hash) -> Result<Self, Self::Error> {
+    fn try_from(hash: &yaml_rust::yaml::Hash) -> Result<Self> {
         let mut fields = Fields::default();
         let mut type_ = Value::Null;
         let mut default = None;
@@ -287,7 +289,9 @@ impl TryFrom<&yaml_rust::yaml::Hash> for TransformBuilder {
         let mut on_failure = None;
 
         for (k, v) in hash {
-            let key = k.as_str().ok_or("key must be a string")?;
+            let key = k
+                .as_str()
+                .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
             match key {
                 TRANSFORM_FIELD => {
                     fields = Fields::one(yaml_new_field(v, TRANSFORM_FIELD)?);
@@ -324,10 +328,11 @@ impl TryFrom<&yaml_rust::yaml::Hash> for TransformBuilder {
         if let Some(default_value) = default {
             match (&type_, &default_value) {
                 (Value::Null, _) => {
-                    return Err(format!(
-                        "transform {:?} type MUST BE set before default {}",
-                        fields, &default_value,
-                    ));
+                    return TransformTypeMustBeSetSnafu {
+                        fields: format!("{:?}", fields),
+                        default: default_value.to_string(),
+                    }
+                    .fail();
                 }
                 (_, Value::Null) => {} // if default is not set, then it will be regarded as default null
                 (_, _) => {
diff --git a/src/pipeline/src/etl/transform/index.rs b/src/pipeline/src/etl/transform/index.rs
index f26ca4828a..6af41990a0 100644
--- a/src/pipeline/src/etl/transform/index.rs
+++ b/src/pipeline/src/etl/transform/index.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::etl::error::{Error, Result, UnsupportedIndexTypeSnafu};
+
 const INDEX_TIMESTAMP: &str = "timestamp";
 const INDEX_TIMEINDEX: &str = "time";
 const INDEX_TAG: &str = "tag";
@@ -38,22 +40,22 @@ impl std::fmt::Display for Index {
 }
 
 impl TryFrom<String> for Index {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(value: String) -> Result<Self, Self::Error> {
+    fn try_from(value: String) -> Result<Self> {
         Index::try_from(value.as_str())
     }
 }
 
 impl TryFrom<&str> for Index {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(value: &str) -> Result<Self, Self::Error> {
+    fn try_from(value: &str) -> Result<Self> {
         match value {
             INDEX_TIMESTAMP | INDEX_TIMEINDEX => Ok(Index::Time),
             INDEX_TAG => Ok(Index::Tag),
             INDEX_FULLTEXT => Ok(Index::Fulltext),
-            _ => Err(format!("unsupported index type: {}", value)),
+            _ => UnsupportedIndexTypeSnafu { value }.fail(),
         }
     }
 }
diff --git a/src/pipeline/src/etl/transform/transformer/greptime.rs b/src/pipeline/src/etl/transform/transformer/greptime.rs
index 9753b01004..000a2ddc26 100644
--- a/src/pipeline/src/etl/transform/transformer/greptime.rs
+++ b/src/pipeline/src/etl/transform/transformer/greptime.rs
@@ -20,6 +20,10 @@ use coerce::{coerce_columns, coerce_value};
 use greptime_proto::v1::{ColumnSchema, Row, Rows, Value as GreptimeValue};
 use itertools::Itertools;
 
+use crate::etl::error::{
+    Result, TransformColumnNameMustBeUniqueSnafu, TransformEmptySnafu,
+    TransformMultipleTimestampIndexSnafu, TransformTimestampIndexCountSnafu,
+};
 use crate::etl::field::{InputFieldInfo, OneInputOneOutputField};
 use crate::etl::transform::index::Index;
 use crate::etl::transform::{Transform, Transformer, Transforms};
@@ -71,7 +75,7 @@ impl GreptimeTransformer {
     }
 
     /// Generate the schema for the GreptimeTransformer
-    fn schemas(transforms: &Transforms) -> Result<Vec<ColumnSchema>, String> {
+    fn schemas(transforms: &Transforms) -> Result<Vec<ColumnSchema>> {
         let mut schema = vec![];
         for transform in transforms.iter() {
             schema.extend(coerce_columns(transform)?);
@@ -90,9 +94,9 @@ impl Transformer for GreptimeTransformer {
     type Output = Rows;
     type VecOutput = Row;
 
-    fn new(mut transforms: Transforms) -> Result<Self, String> {
+    fn new(mut transforms: Transforms) -> Result<Self> {
         if transforms.is_empty() {
-            return Err("transform cannot be empty".to_string());
+            return TransformEmptySnafu.fail();
         }
 
         let mut column_names_set = HashSet::new();
@@ -108,9 +112,7 @@ impl Transformer for GreptimeTransformer {
             let intersections: Vec<_> = column_names_set.intersection(&target_fields_set).collect();
             if !intersections.is_empty() {
                 let duplicates = intersections.iter().join(",");
-                return Err(format!(
-                    "column name must be unique, but got duplicated: {duplicates}"
-                ));
+                return TransformColumnNameMustBeUniqueSnafu { duplicates }.fail();
             }
 
             column_names_set.extend(target_fields_set);
@@ -121,10 +123,14 @@ impl Transformer for GreptimeTransformer {
                         1 => timestamp_columns
                             .push(transform.real_fields.first().unwrap().input_name()),
                         _ => {
-                            return Err(format!(
-                                "Illegal to set multiple timestamp Index columns, please set only one: {}",
-                                transform.real_fields.iter().map(|x|x.input_name()).join(", ")
-                            ))
+                            return TransformMultipleTimestampIndexSnafu {
+                                columns: transform
+                                    .real_fields
+                                    .iter()
+                                    .map(|x| x.input_name())
+                                    .join(", "),
+                            }
+                            .fail();
                         }
                     }
                 }
@@ -145,14 +151,12 @@ impl Transformer for GreptimeTransformer {
             _ => {
                 let columns: String = timestamp_columns.iter().map(|s| s.to_string()).join(", ");
                 let count = timestamp_columns.len();
-                Err(
-                    format!("transform must have exactly one field specified as timestamp Index, but got {count}: {columns}")
-                )
+                TransformTimestampIndexCountSnafu { count, columns }.fail()
             }
         }
     }
 
-    fn transform_mut(&self, val: &mut Vec<Value>) -> Result<Self::VecOutput, String> {
+    fn transform_mut(&self, val: &mut Vec<Value>) -> Result<Self::VecOutput> {
         let mut values = vec![GreptimeValue { value_data: None }; self.schema.len()];
         for transform in self.transforms.iter() {
             for field in transform.real_fields.iter() {
@@ -160,8 +164,7 @@ impl Transformer for GreptimeTransformer {
                 let output_index = field.output_index();
                 match val.get(index) {
                     Some(v) => {
-                        let value_data = coerce_value(v, transform)
-                            .map_err(|e| format!("{} processor: {}", field.input_name(), e))?;
+                        let value_data = coerce_value(v, transform)?;
                         // every transform fields has only one output field
                         values[output_index] = GreptimeValue { value_data };
                     }
diff --git a/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs b/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs
index 8c7efef22f..827613b02b 100644
--- a/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs
+++ b/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs
@@ -17,17 +17,22 @@ use api::v1::ColumnOptions;
 use datatypes::schema::FulltextOptions;
 use greptime_proto::v1::value::ValueData;
 use greptime_proto::v1::{ColumnDataType, ColumnSchema, SemanticType};
+use snafu::ResultExt;
 
+use crate::etl::error::{
+    CoerceStringToTypeSnafu, CoerceUnsupportedEpochTypeSnafu, CoerceUnsupportedNullTypeSnafu,
+    CoerceUnsupportedNullTypeToSnafu, ColumnOptionsSnafu, Error, Result,
+};
 use crate::etl::transform::index::Index;
 use crate::etl::transform::{OnFailure, Transform};
 use crate::etl::value::{Timestamp, Value};
 
 impl TryFrom<Value> for ValueData {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(value: Value) -> Result<Self, Self::Error> {
+    fn try_from(value: Value) -> Result<Self> {
         match value {
-            Value::Null => Err("Null type not supported".to_string()),
+            Value::Null => CoerceUnsupportedNullTypeSnafu.fail(),
 
             Value::Int8(v) => Ok(ValueData::I32Value(v as i32)),
             Value::Int16(v) => Ok(ValueData::I32Value(v as i32)),
@@ -63,7 +68,7 @@ impl TryFrom<Value> for ValueData {
 }
 
 // TODO(yuanbohan): add fulltext support in datatype_extension
-pub(crate) fn coerce_columns(transform: &Transform) -> Result<Vec<ColumnSchema>, String> {
+pub(crate) fn coerce_columns(transform: &Transform) -> Result<Vec<ColumnSchema>> {
     let mut columns = Vec::new();
 
     for field in transform.real_fields.iter() {
@@ -94,19 +99,19 @@ fn coerce_semantic_type(transform: &Transform) -> SemanticType {
     }
 }
 
-fn coerce_options(transform: &Transform) -> Result<Option<ColumnOptions>, String> {
+fn coerce_options(transform: &Transform) -> Result<Option<ColumnOptions>> {
     if let Some(Index::Fulltext) = transform.index {
         options_from_fulltext(&FulltextOptions {
             enable: true,
             ..Default::default()
         })
-        .map_err(|e| e.to_string())
+        .context(ColumnOptionsSnafu)
     } else {
         Ok(None)
     }
 }
 
-fn coerce_type(transform: &Transform) -> Result<ColumnDataType, String> {
+fn coerce_type(transform: &Transform) -> Result<ColumnDataType> {
     match transform.type_ {
         Value::Int8(_) => Ok(ColumnDataType::Int8),
         Value::Int16(_) => Ok(ColumnDataType::Int16),
@@ -132,17 +137,14 @@ fn coerce_type(transform: &Transform) -> Result<ColumnDataType, String> {
         Value::Array(_) => unimplemented!("Array"),
         Value::Map(_) => unimplemented!("Object"),
 
-        Value::Null => Err(format!(
-            "Null type not supported when to coerce '{}' type",
-            transform.type_.to_str_type()
-        )),
+        Value::Null => CoerceUnsupportedNullTypeToSnafu {
+            ty: transform.type_.to_str_type(),
+        }
+        .fail(),
     }
 }
 
-pub(crate) fn coerce_value(
-    val: &Value,
-    transform: &Transform,
-) -> Result<Option<ValueData>, String> {
+pub(crate) fn coerce_value(val: &Value, transform: &Transform) -> Result<Option<ValueData>> {
     match val {
         Value::Null => match &transform.default {
             Some(default) => coerce_value(default, transform),
@@ -190,7 +192,7 @@ pub(crate) fn coerce_value(
     }
 }
 
-fn coerce_bool_value(b: bool, transform: &Transform) -> Result<Option<ValueData>, String> {
+fn coerce_bool_value(b: bool, transform: &Transform) -> Result<Option<ValueData>> {
     let val = match transform.type_ {
         Value::Int8(_) => ValueData::I8Value(b as i32),
         Value::Int16(_) => ValueData::I16Value(b as i32),
@@ -211,9 +213,11 @@ fn coerce_bool_value(b: bool, transform: &Transform) -> Result<Option<ValueData>
         Value::Timestamp(_) => match transform.on_failure {
             Some(OnFailure::Ignore) => return Ok(None),
             Some(OnFailure::Default) => {
-                return Err("default value not supported for Epoch".to_string())
+                return CoerceUnsupportedEpochTypeSnafu { ty: "Default" }.fail();
+            }
+            None => {
+                return CoerceUnsupportedEpochTypeSnafu { ty: "Boolean" }.fail();
             }
-            None => return Err("Boolean type not supported for Epoch".to_string()),
         },
 
         Value::Array(_) => unimplemented!("Array type not supported"),
@@ -225,7 +229,7 @@ fn coerce_bool_value(b: bool, transform: &Transform) -> Result<Option<ValueData>
     Ok(Some(val))
 }
 
-fn coerce_i64_value(n: i64, transform: &Transform) -> Result<Option<ValueData>, String> {
+fn coerce_i64_value(n: i64, transform: &Transform) -> Result<Option<ValueData>> {
     let val = match transform.type_ {
         Value::Int8(_) => ValueData::I8Value(n as i32),
         Value::Int16(_) => ValueData::I16Value(n as i32),
@@ -246,9 +250,11 @@ fn coerce_i64_value(n: i64, transform: &Transform) -> Result<Option<ValueData>,
         Value::Timestamp(_) => match transform.on_failure {
             Some(OnFailure::Ignore) => return Ok(None),
             Some(OnFailure::Default) => {
-                return Err("default value not supported for Epoch".to_string())
+                return CoerceUnsupportedEpochTypeSnafu { ty: "Default" }.fail();
+            }
+            None => {
+                return CoerceUnsupportedEpochTypeSnafu { ty: "Integer" }.fail();
             }
-            None => return Err("Integer type not supported for Epoch".to_string()),
         },
 
         Value::Array(_) => unimplemented!("Array type not supported"),
@@ -260,7 +266,7 @@ fn coerce_i64_value(n: i64, transform: &Transform) -> Result<Option<ValueData>,
     Ok(Some(val))
 }
 
-fn coerce_u64_value(n: u64, transform: &Transform) -> Result<Option<ValueData>, String> {
+fn coerce_u64_value(n: u64, transform: &Transform) -> Result<Option<ValueData>> {
     let val = match transform.type_ {
         Value::Int8(_) => ValueData::I8Value(n as i32),
         Value::Int16(_) => ValueData::I16Value(n as i32),
@@ -281,9 +287,11 @@ fn coerce_u64_value(n: u64, transform: &Transform) -> Result<Option<ValueData>,
         Value::Timestamp(_) => match transform.on_failure {
             Some(OnFailure::Ignore) => return Ok(None),
             Some(OnFailure::Default) => {
-                return Err("default value not supported for Epoch".to_string())
+                return CoerceUnsupportedEpochTypeSnafu { ty: "Default" }.fail();
+            }
+            None => {
+                return CoerceUnsupportedEpochTypeSnafu { ty: "Integer" }.fail();
             }
-            None => return Err("Integer type not supported for Epoch".to_string()),
         },
 
         Value::Array(_) => unimplemented!("Array type not supported"),
@@ -295,7 +303,7 @@ fn coerce_u64_value(n: u64, transform: &Transform) -> Result<Option<ValueData>,
     Ok(Some(val))
 }
 
-fn coerce_f64_value(n: f64, transform: &Transform) -> Result<Option<ValueData>, String> {
+fn coerce_f64_value(n: f64, transform: &Transform) -> Result<Option<ValueData>> {
     let val = match transform.type_ {
         Value::Int8(_) => ValueData::I8Value(n as i32),
         Value::Int16(_) => ValueData::I16Value(n as i32),
@@ -316,9 +324,11 @@ fn coerce_f64_value(n: f64, transform: &Transform) -> Result<Option<ValueData>,
         Value::Timestamp(_) => match transform.on_failure {
             Some(OnFailure::Ignore) => return Ok(None),
             Some(OnFailure::Default) => {
-                return Err("default value not supported for Epoch".to_string())
+                return CoerceUnsupportedEpochTypeSnafu { ty: "Default" }.fail();
+            }
+            None => {
+                return CoerceUnsupportedEpochTypeSnafu { ty: "Float" }.fail();
             }
-            None => return Err("Float type not supported for Epoch".to_string()),
         },
 
         Value::Array(_) => unimplemented!("Array type not supported"),
@@ -340,17 +350,17 @@ macro_rules! coerce_string_value {
                     Some(default) => coerce_value(default, $transform),
                     None => coerce_value($transform.get_type_matched_default_val(), $transform),
                 },
-                None => Err(format!(
-                    "failed to coerce string value '{}' to type '{}'",
-                    $s,
-                    $transform.type_.to_str_type()
-                )),
+                None => CoerceStringToTypeSnafu {
+                    s: $s,
+                    ty: $transform.type_.to_str_type(),
+                }
+                .fail(),
             },
         }
     };
 }
 
-fn coerce_string_value(s: &String, transform: &Transform) -> Result<Option<ValueData>, String> {
+fn coerce_string_value(s: &String, transform: &Transform) -> Result<Option<ValueData>> {
     match transform.type_ {
         Value::Int8(_) => {
             coerce_string_value!(s, transform, i32, I8Value)
@@ -393,8 +403,8 @@ fn coerce_string_value(s: &String, transform: &Transform) -> Result<Option<Value
 
         Value::Timestamp(_) => match transform.on_failure {
             Some(OnFailure::Ignore) => Ok(None),
-            Some(OnFailure::Default) => Err("default value not supported for Epoch".to_string()),
-            None => Err("String type not supported for Epoch".to_string()),
+            Some(OnFailure::Default) => CoerceUnsupportedEpochTypeSnafu { ty: "Default" }.fail(),
+            None => CoerceUnsupportedEpochTypeSnafu { ty: "String" }.fail(),
         },
 
         Value::Array(_) => unimplemented!("Array type not supported"),
diff --git a/src/pipeline/src/etl/value.rs b/src/pipeline/src/etl/value.rs
index 42390a9b73..3adde2514b 100644
--- a/src/pipeline/src/etl/value.rs
+++ b/src/pipeline/src/etl/value.rs
@@ -19,8 +19,16 @@ pub mod time;
 use ahash::{HashMap, HashMapExt};
 pub use array::Array;
 pub use map::Map;
+use snafu::{OptionExt, ResultExt};
 pub use time::Timestamp;
 
+use super::error::{
+    ValueDefaultValueUnsupportedSnafu, ValueInvalidResolutionSnafu, ValueParseBooleanSnafu,
+    ValueParseFloatSnafu, ValueParseIntSnafu, ValueParseTypeSnafu, ValueUnsupportedNumberTypeSnafu,
+    ValueUnsupportedYamlTypeSnafu, ValueYamlKeyMustBeStringSnafu,
+};
+use crate::etl::error::{Error, Result};
+
 /// Value can be used as type
 /// acts as value: the enclosed value is the actual value
 /// acts as type: the enclosed value is the default value
@@ -58,7 +66,7 @@ impl Value {
         matches!(self, Value::Null)
     }
 
-    pub fn parse_str_type(t: &str) -> Result<Self, String> {
+    pub fn parse_str_type(t: &str) -> Result<Self> {
         let mut parts = t.splitn(2, ',');
         let head = parts.next().unwrap_or_default();
         let tail = parts.next().map(|s| s.trim().to_string());
@@ -93,10 +101,11 @@ impl Value {
                     time::SECOND_RESOLUTION | time::SEC_RESOLUTION | time::S_RESOLUTION => {
                         Ok(Value::Timestamp(Timestamp::Second(0)))
                     }
-                    _ => Err(format!(
-                        "invalid resolution: '{resolution}'. Available resolutions: {}",
-                        time::VALID_RESOLUTIONS.join(",")
-                    )),
+                    _ => ValueInvalidResolutionSnafu {
+                        resolution,
+                        valid_resolution: time::VALID_RESOLUTIONS.join(","),
+                    }
+                    .fail(),
                 },
                 _ => Ok(Value::Timestamp(Timestamp::Nanosecond(0))),
             },
@@ -104,65 +113,68 @@ impl Value {
             "array" => Ok(Value::Array(Array::default())),
             "map" => Ok(Value::Map(Map::default())),
 
-            _ => Err(format!("failed to parse type: '{t}'")),
+            _ => ValueParseTypeSnafu { t }.fail(),
         }
     }
 
     /// only support string, bool, number, null
-    pub fn parse_str_value(&self, v: &str) -> Result<Self, String> {
+    pub fn parse_str_value(&self, v: &str) -> Result<Self> {
         match self {
             Value::Int8(_) => v
                 .parse::<i8>()
                 .map(Value::Int8)
-                .map_err(|e| format!("failed to parse int8: {}", e)),
+                .context(ValueParseIntSnafu { ty: "int8", v }),
             Value::Int16(_) => v
                 .parse::<i16>()
                 .map(Value::Int16)
-                .map_err(|e| format!("failed to parse int16: {}", e)),
+                .context(ValueParseIntSnafu { ty: "int16", v }),
             Value::Int32(_) => v
                 .parse::<i32>()
                 .map(Value::Int32)
-                .map_err(|e| format!("failed to parse int32: {}", e)),
+                .context(ValueParseIntSnafu { ty: "int32", v }),
             Value::Int64(_) => v
                 .parse::<i64>()
                 .map(Value::Int64)
-                .map_err(|e| format!("failed to parse int64: {}", e)),
+                .context(ValueParseIntSnafu { ty: "int64", v }),
 
             Value::Uint8(_) => v
                 .parse::<u8>()
                 .map(Value::Uint8)
-                .map_err(|e| format!("failed to parse uint8: {}", e)),
+                .context(ValueParseIntSnafu { ty: "uint8", v }),
             Value::Uint16(_) => v
                 .parse::<u16>()
                 .map(Value::Uint16)
-                .map_err(|e| format!("failed to parse uint16: {}", e)),
+                .context(ValueParseIntSnafu { ty: "uint16", v }),
             Value::Uint32(_) => v
                 .parse::<u32>()
                 .map(Value::Uint32)
-                .map_err(|e| format!("failed to parse uint32: {}", e)),
+                .context(ValueParseIntSnafu { ty: "uint32", v }),
             Value::Uint64(_) => v
                 .parse::<u64>()
                 .map(Value::Uint64)
-                .map_err(|e| format!("failed to parse uint64: {}", e)),
+                .context(ValueParseIntSnafu { ty: "uint64", v }),
 
             Value::Float32(_) => v
                 .parse::<f32>()
                 .map(Value::Float32)
-                .map_err(|e| format!("failed to parse float32: {}", e)),
+                .context(ValueParseFloatSnafu { ty: "float32", v }),
             Value::Float64(_) => v
                 .parse::<f64>()
                 .map(Value::Float64)
-                .map_err(|e| format!("failed to parse float64: {}", e)),
+                .context(ValueParseFloatSnafu { ty: "float64", v }),
 
             Value::Boolean(_) => v
                 .parse::<bool>()
                 .map(Value::Boolean)
-                .map_err(|e| format!("failed to parse bool: {}", e)),
+                .context(ValueParseBooleanSnafu { ty: "boolean", v }),
             Value::String(_) => Ok(Value::String(v.to_string())),
 
             Value::Null => Ok(Value::Null),
 
-            _ => Err(format!("default value not unsupported for type {}", self)),
+            _ => ValueDefaultValueUnsupportedSnafu {
+                value: format!("{:?}", self),
+            }
+            .fail(),
         }
     }
 
@@ -249,9 +261,9 @@ impl std::fmt::Display for Value {
 }
 
 impl TryFrom<serde_json::Value> for Value {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(v: serde_json::Value) -> Result<Self, Self::Error> {
+    fn try_from(v: serde_json::Value) -> Result<Self> {
         match v {
             serde_json::Value::Null => Ok(Value::Null),
             serde_json::Value::Bool(v) => Ok(Value::Boolean(v)),
@@ -263,7 +275,7 @@ impl TryFrom<serde_json::Value> for Value {
                 } else if let Some(v) = v.as_f64() {
                     Ok(Value::Float64(v))
                 } else {
-                    Err(format!("unsupported number type: {}", v))
+                    ValueUnsupportedNumberTypeSnafu { value: v }.fail()
                 }
             }
             serde_json::Value::String(v) => Ok(Value::String(v)),
@@ -286,20 +298,17 @@ impl TryFrom<serde_json::Value> for Value {
 }
 
 impl TryFrom<&yaml_rust::Yaml> for Value {
-    type Error = String;
+    type Error = Error;
 
-    fn try_from(v: &yaml_rust::Yaml) -> Result<Self, Self::Error> {
+    fn try_from(v: &yaml_rust::Yaml) -> Result<Self> {
         match v {
             yaml_rust::Yaml::Null => Ok(Value::Null),
             yaml_rust::Yaml::Boolean(v) => Ok(Value::Boolean(*v)),
             yaml_rust::Yaml::Integer(v) => Ok(Value::Int64(*v)),
-            yaml_rust::Yaml::Real(v) => {
-                if let Ok(v) = v.parse() {
-                    Ok(Value::Float64(v))
-                } else {
-                    Err(format!("failed to parse float64: {}", v))
-                }
-            }
+            yaml_rust::Yaml::Real(v) => match v.parse::<f64>() {
+                Ok(v) => Ok(Value::Float64(v)),
+                Err(e) => Err(e).context(ValueParseFloatSnafu { ty: "float64", v }),
+            },
             yaml_rust::Yaml::String(v) => Ok(Value::String(v.to_string())),
             yaml_rust::Yaml::Array(arr) => {
                 let mut values = vec![];
@@ -313,12 +322,12 @@ impl TryFrom<&yaml_rust::Yaml> for Value {
                 for (k, v) in v {
                     let key = k
                         .as_str()
-                        .ok_or(format!("key in Hash must be a string, but got {v:?}"))?;
+                        .with_context(|| ValueYamlKeyMustBeStringSnafu { value: v.clone() })?;
                     values.insert(key.to_string(), Value::try_from(v)?);
                 }
                 Ok(Value::Map(Map { values }))
             }
-            _ => Err(format!("unsupported yaml type: {v:?}")),
+            _ => ValueUnsupportedYamlTypeSnafu { value: v.clone() }.fail(),
         }
     }
 }
diff --git a/src/pipeline/src/lib.rs b/src/pipeline/src/lib.rs
index 86367232bd..8fc72c5844 100644
--- a/src/pipeline/src/lib.rs
+++ b/src/pipeline/src/lib.rs
@@ -16,6 +16,7 @@ mod etl;
 mod manager;
 mod metrics;
 
+pub use etl::error::Result;
 pub use etl::processor::Processor;
 pub use etl::transform::{GreptimeTransformer, Transformer};
 pub use etl::value::{Array, Map, Value};
diff --git a/src/pipeline/src/manager/error.rs b/src/pipeline/src/manager/error.rs
index 4467b42b51..a51ad61dac 100644
--- a/src/pipeline/src/manager/error.rs
+++ b/src/pipeline/src/manager/error.rs
@@ -32,14 +32,16 @@ pub enum Error {
 
     #[snafu(display("Failed to insert pipeline to pipelines table"))]
     InsertPipeline {
+        #[snafu(source)]
         source: operator::error::Error,
         #[snafu(implicit)]
         location: Location,
     },
 
-    #[snafu(display("Failed to parse pipeline: {}", reason))]
+    #[snafu(display("Failed to parse pipeline"))]
     CompilePipeline {
-        reason: String,
+        #[snafu(source)]
+        source: crate::etl::error::Error,
         #[snafu(implicit)]
         location: Location,
     },
@@ -56,6 +58,7 @@ pub enum Error {
     CollectRecords {
         #[snafu(implicit)]
         location: Location,
+        #[snafu(source)]
         source: common_recordbatch::error::Error,
     },
 
@@ -76,6 +79,7 @@ pub enum Error {
 
     #[snafu(display("Failed to execute internal statement"))]
     ExecuteInternalStatement {
+        #[snafu(source)]
         source: query::error::Error,
         #[snafu(implicit)]
         location: Location,
@@ -83,6 +87,7 @@ pub enum Error {
 
     #[snafu(display("Failed to create dataframe"))]
     DataFrame {
+        #[snafu(source)]
         source: query::error::Error,
         #[snafu(implicit)]
         location: Location,
@@ -90,6 +95,7 @@ pub enum Error {
 
     #[snafu(display("General catalog error"))]
     Catalog {
+        #[snafu(source)]
         source: catalog::error::Error,
         #[snafu(implicit)]
         location: Location,
@@ -97,14 +103,16 @@ pub enum Error {
 
     #[snafu(display("Failed to create table"))]
     CreateTable {
+        #[snafu(source)]
         source: operator::error::Error,
         #[snafu(implicit)]
         location: Location,
     },
 
-    #[snafu(display("Failed to execute pipeline, reason: {}", reason))]
+    #[snafu(display("Failed to execute pipeline"))]
     PipelineTransform {
-        reason: String,
+        #[snafu(source)]
+        source: crate::etl::error::Error,
         #[snafu(implicit)]
         location: Location,
     },
diff --git a/src/pipeline/src/manager/table.rs b/src/pipeline/src/manager/table.rs
index 8989412c3e..7b3719b667 100644
--- a/src/pipeline/src/manager/table.rs
+++ b/src/pipeline/src/manager/table.rs
@@ -204,8 +204,7 @@ impl PipelineTable {
     /// Compile a pipeline from a string.
     pub fn compile_pipeline(pipeline: &str) -> Result<Pipeline<GreptimeTransformer>> {
         let yaml_content = Content::Yaml(pipeline.into());
-        parse::<GreptimeTransformer>(&yaml_content)
-            .map_err(|e| CompilePipelineSnafu { reason: e }.build())
+        parse::<GreptimeTransformer>(&yaml_content).context(CompilePipelineSnafu)
     }
 
     /// Insert a pipeline into the pipeline table.
diff --git a/src/pipeline/tests/dissect.rs b/src/pipeline/tests/dissect.rs
index 22cf14c46b..7577d58080 100644
--- a/src/pipeline/tests/dissect.rs
+++ b/src/pipeline/tests/dissect.rs
@@ -279,5 +279,5 @@ transform:
     let row = pipeline.exec_mut(&mut result);
 
     assert!(row.is_err());
-    assert_eq!(row.err().unwrap(), "No matching pattern found");
+    assert_eq!(row.err().unwrap().to_string(), "No matching pattern found");
 }
diff --git a/src/servers/src/http/event.rs b/src/servers/src/http/event.rs
index f0a0902837..7877d2b84a 100644
--- a/src/servers/src/http/event.rs
+++ b/src/servers/src/http/event.rs
@@ -115,19 +115,18 @@ pub async fn add_pipeline(
 ) -> Result<GreptimedbManageResponse> {
     let start = Instant::now();
     let handler = state.log_handler;
-    if pipeline_name.is_empty() {
-        return Err(InvalidParameterSnafu {
+    ensure!(
+        !pipeline_name.is_empty(),
+        InvalidParameterSnafu {
             reason: "pipeline_name is required in path",
         }
-        .build());
-    }
-
-    if payload.is_empty() {
-        return Err(InvalidParameterSnafu {
+    );
+    ensure!(
+        !payload.is_empty(),
+        InvalidParameterSnafu {
             reason: "pipeline is required in body",
         }
-        .build());
-    }
+    );
 
     query_ctx.set_channel(Channel::Http);
     let query_ctx = Arc::new(query_ctx);
@@ -252,12 +251,12 @@ pub async fn pipeline_dryrun(
 
     let value = extract_pipeline_value_by_content_type(content_type, payload, ignore_errors)?;
 
-    if value.len() > 10 {
-        return Err(InvalidParameterSnafu {
+    ensure!(
+        value.len() <= 10,
+        InvalidParameterSnafu {
             reason: "too many rows for dryrun",
         }
-        .build());
-    }
+    );
 
     query_ctx.set_channel(Channel::Http);
     let query_ctx = Arc::new(query_ctx);
@@ -272,11 +271,11 @@ pub async fn pipeline_dryrun(
     for v in value {
         pipeline
             .prepare(v, &mut intermediate_state)
-            .map_err(|reason| PipelineTransformSnafu { reason }.build())
+            .context(PipelineTransformSnafu)
             .context(PipelineSnafu)?;
         let r = pipeline
             .exec_mut(&mut intermediate_state)
-            .map_err(|reason| PipelineTransformSnafu { reason }.build())
+            .context(PipelineTransformSnafu)
             .context(PipelineSnafu)?;
         results.push(r);
         pipeline.reset_intermediate_state(&mut intermediate_state);
@@ -438,21 +437,21 @@ async fn ingest_logs_inner(
     for v in pipeline_data {
         pipeline
             .prepare(v, &mut intermediate_state)
-            .map_err(|reason| {
+            .inspect_err(|_| {
                 METRIC_HTTP_LOGS_TRANSFORM_ELAPSED
                     .with_label_values(&[db.as_str(), METRIC_FAILURE_VALUE])
                     .observe(transform_timer.elapsed().as_secs_f64());
-                PipelineTransformSnafu { reason }.build()
             })
+            .context(PipelineTransformSnafu)
             .context(PipelineSnafu)?;
         let r = pipeline
             .exec_mut(&mut intermediate_state)
-            .map_err(|reason| {
+            .inspect_err(|_| {
                 METRIC_HTTP_LOGS_TRANSFORM_ELAPSED
                     .with_label_values(&[db.as_str(), METRIC_FAILURE_VALUE])
                     .observe(transform_timer.elapsed().as_secs_f64());
-                PipelineTransformSnafu { reason }.build()
             })
+            .context(PipelineTransformSnafu)
             .context(PipelineSnafu)?;
         results.push(r);
         pipeline.reset_intermediate_state(&mut intermediate_state);

From cc4106cbd2965f8ffe5bbe9aaa9a7a7baba6ffd0 Mon Sep 17 00:00:00 2001
From: Lanqing Yang <lanqingy@usc.edu>
Date: Thu, 26 Sep 2024 23:12:57 -0700
Subject: [PATCH 038/128] feat: protect datanode with concurrency limit.
 (#4699)

Adding parallelism in region server to protect datanode from query overload.
---
 config/config.md                  |  2 +
 config/datanode.example.toml      |  3 ++
 config/standalone.example.toml    |  3 ++
 src/datanode/src/config.rs        |  2 +
 src/datanode/src/datanode.rs      |  5 ++-
 src/datanode/src/error.rs         | 20 +++++++++
 src/datanode/src/region_server.rs | 75 ++++++++++++++++++++++++++++++-
 7 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/config/config.md b/config/config.md
index 235b1f86fd..641eee4b58 100644
--- a/config/config.md
+++ b/config/config.md
@@ -17,6 +17,7 @@
 | `default_timezone` | String | Unset | The default timezone of the server. |
 | `init_regions_in_background` | Bool | `false` | Initialize all regions in the background during the startup.<br/>By default, it provides services after all regions have been initialized. |
 | `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. |
+| `max_concurrent_queries` | Integer | `0` | The maximum current queries allowed to be executed. Zero means unlimited. |
 | `runtime` | -- | -- | The runtime options. |
 | `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
 | `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute the runtime for global write operations. |
@@ -335,6 +336,7 @@
 | `init_regions_in_background` | Bool | `false` | Initialize all regions in the background during the startup.<br/>By default, it provides services after all regions have been initialized. |
 | `enable_telemetry` | Bool | `true` | Enable telemetry to collect anonymous usage data. |
 | `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. |
+| `max_concurrent_queries` | Integer | `0` | The maximum current queries allowed to be executed. Zero means unlimited. |
 | `rpc_addr` | String | Unset | Deprecated, use `grpc.addr` instead. |
 | `rpc_hostname` | String | Unset | Deprecated, use `grpc.hostname` instead. |
 | `rpc_runtime_size` | Integer | Unset | Deprecated, use `grpc.runtime_size` instead. |
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index 4388c4420f..e4a3dca6d3 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -19,6 +19,9 @@ enable_telemetry = true
 ## Parallelism of initializing regions.
 init_regions_parallelism = 16
 
+## The maximum current queries allowed to be executed. Zero means unlimited.
+max_concurrent_queries = 0
+
 ## Deprecated, use `grpc.addr` instead.
 ## @toml2docs:none-default
 rpc_addr = "127.0.0.1:3001"
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index 040e1e62c2..1cd75e6414 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -15,6 +15,9 @@ init_regions_in_background = false
 ## Parallelism of initializing regions.
 init_regions_parallelism = 16
 
+## The maximum current queries allowed to be executed. Zero means unlimited.
+max_concurrent_queries = 0
+
 ## The runtime options.
 #+ [runtime]
 ## The number of threads to execute the runtime for global read operations.
diff --git a/src/datanode/src/config.rs b/src/datanode/src/config.rs
index 7cb18131ed..70be2513b2 100644
--- a/src/datanode/src/config.rs
+++ b/src/datanode/src/config.rs
@@ -305,6 +305,7 @@ pub struct DatanodeOptions {
     pub meta_client: Option<MetaClientOptions>,
     pub wal: DatanodeWalConfig,
     pub storage: StorageConfig,
+    pub max_concurrent_queries: usize,
     /// Options for different store engines.
     pub region_engine: Vec<RegionEngineConfig>,
     pub logging: LoggingOptions,
@@ -339,6 +340,7 @@ impl Default for DatanodeOptions {
             meta_client: None,
             wal: DatanodeWalConfig::default(),
             storage: StorageConfig::default(),
+            max_concurrent_queries: 0,
             region_engine: vec![
                 RegionEngineConfig::Mito(MitoConfig::default()),
                 RegionEngineConfig::File(FileEngineConfig::default()),
diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs
index 149aa44ebe..f5d7bd9fc6 100644
--- a/src/datanode/src/datanode.rs
+++ b/src/datanode/src/datanode.rs
@@ -314,7 +314,7 @@ impl DatanodeBuilder {
         &self,
         event_listener: RegionServerEventListenerRef,
     ) -> Result<RegionServer> {
-        let opts = &self.opts;
+        let opts: &DatanodeOptions = &self.opts;
 
         let query_engine_factory = QueryEngineFactory::new_with_plugins(
             // query engine in datanode only executes plan with resolved table source.
@@ -334,6 +334,9 @@ impl DatanodeBuilder {
             common_runtime::global_runtime(),
             event_listener,
             table_provider_factory,
+            opts.max_concurrent_queries,
+            //TODO: revaluate the hardcoded timeout on the next version of datanode concurrency limiter.
+            Duration::from_millis(100),
         );
 
         let object_store_manager = Self::build_object_store_manager(&opts.storage).await?;
diff --git a/src/datanode/src/error.rs b/src/datanode/src/error.rs
index 5717d0a403..0b36245924 100644
--- a/src/datanode/src/error.rs
+++ b/src/datanode/src/error.rs
@@ -22,6 +22,7 @@ use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
 use store_api::storage::RegionId;
 use table::error::Error as TableError;
+use tokio::time::error::Elapsed;
 
 /// Business error of datanode.
 #[derive(Snafu)]
@@ -347,6 +348,22 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Failed to acquire permit, source closed"))]
+    ConcurrentQueryLimiterClosed {
+        #[snafu(source)]
+        error: tokio::sync::AcquireError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Failed to acquire permit under timeouts"))]
+    ConcurrentQueryLimiterTimeout {
+        #[snafu(source)]
+        error: Elapsed,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -411,6 +428,9 @@ impl ErrorExt for Error {
 
             FindLogicalRegions { source, .. } => source.status_code(),
             BuildMitoEngine { source, .. } => source.status_code(),
+            ConcurrentQueryLimiterClosed { .. } | ConcurrentQueryLimiterTimeout { .. } => {
+                StatusCode::RegionBusy
+            }
         }
     }
 
diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs
index 54e1cdbafd..4fcdfb86af 100644
--- a/src/datanode/src/region_server.rs
+++ b/src/datanode/src/region_server.rs
@@ -16,6 +16,7 @@ use std::collections::HashMap;
 use std::fmt::Debug;
 use std::ops::Deref;
 use std::sync::{Arc, RwLock};
+use std::time::Duration;
 
 use api::region::RegionResponse;
 use api::v1::region::{region_request, RegionResponse as RegionResponseV1};
@@ -58,10 +59,13 @@ use store_api::region_request::{
     AffectedRows, RegionCloseRequest, RegionOpenRequest, RegionRequest,
 };
 use store_api::storage::RegionId;
+use tokio::sync::{Semaphore, SemaphorePermit};
+use tokio::time::timeout;
 use tonic::{Request, Response, Result as TonicResult};
 
 use crate::error::{
-    self, BuildRegionRequestsSnafu, DataFusionSnafu, DecodeLogicalPlanSnafu,
+    self, BuildRegionRequestsSnafu, ConcurrentQueryLimiterClosedSnafu,
+    ConcurrentQueryLimiterTimeoutSnafu, DataFusionSnafu, DecodeLogicalPlanSnafu,
     ExecuteLogicalPlanSnafu, FindLogicalRegionsSnafu, HandleBatchOpenRequestSnafu,
     HandleRegionRequestSnafu, NewPlanDecoderSnafu, RegionEngineNotFoundSnafu, RegionNotFoundSnafu,
     RegionNotReadySnafu, Result, StopRegionEngineSnafu, UnexpectedSnafu, UnsupportedOutputSnafu,
@@ -90,6 +94,8 @@ impl RegionServer {
             runtime,
             event_listener,
             Arc::new(DummyTableProviderFactory),
+            0,
+            Duration::from_millis(0),
         )
     }
 
@@ -98,6 +104,8 @@ impl RegionServer {
         runtime: Runtime,
         event_listener: RegionServerEventListenerRef,
         table_provider_factory: TableProviderFactoryRef,
+        max_concurrent_queries: usize,
+        concurrent_query_limiter_timeout: Duration,
     ) -> Self {
         Self {
             inner: Arc::new(RegionServerInner::new(
@@ -105,6 +113,10 @@ impl RegionServer {
                 runtime,
                 event_listener,
                 table_provider_factory,
+                RegionServerParallelism::from_opts(
+                    max_concurrent_queries,
+                    concurrent_query_limiter_timeout,
+                ),
             )),
         }
     }
@@ -167,6 +179,11 @@ impl RegionServer {
         &self,
         request: api::v1::region::QueryRequest,
     ) -> Result<SendableRecordBatchStream> {
+        let _permit = if let Some(p) = &self.inner.parallelism {
+            Some(p.acquire().await?)
+        } else {
+            None
+        };
         let region_id = RegionId::from_u64(request.region_id);
         let provider = self.table_provider(region_id).await?;
         let catalog_list = Arc::new(DummyCatalogList::with_table_provider(provider));
@@ -200,6 +217,11 @@ impl RegionServer {
 
     #[tracing::instrument(skip_all)]
     pub async fn handle_read(&self, request: QueryRequest) -> Result<SendableRecordBatchStream> {
+        let _permit = if let Some(p) = &self.inner.parallelism {
+            Some(p.acquire().await?)
+        } else {
+            None
+        };
         let provider = self.table_provider(request.region_id).await?;
 
         struct RegionDataSourceInjector {
@@ -450,6 +472,36 @@ struct RegionServerInner {
     runtime: Runtime,
     event_listener: RegionServerEventListenerRef,
     table_provider_factory: TableProviderFactoryRef,
+    // The number of queries allowed to be executed at the same time.
+    // Act as last line of defense on datanode to prevent query overloading.
+    parallelism: Option<RegionServerParallelism>,
+}
+
+struct RegionServerParallelism {
+    semaphore: Semaphore,
+    timeout: Duration,
+}
+
+impl RegionServerParallelism {
+    pub fn from_opts(
+        max_concurrent_queries: usize,
+        concurrent_query_limiter_timeout: Duration,
+    ) -> Option<Self> {
+        if max_concurrent_queries == 0 {
+            return None;
+        }
+        Some(RegionServerParallelism {
+            semaphore: Semaphore::new(max_concurrent_queries),
+            timeout: concurrent_query_limiter_timeout,
+        })
+    }
+
+    pub async fn acquire(&self) -> Result<SemaphorePermit> {
+        timeout(self.timeout, self.semaphore.acquire())
+            .await
+            .context(ConcurrentQueryLimiterTimeoutSnafu)?
+            .context(ConcurrentQueryLimiterClosedSnafu)
+    }
 }
 
 enum CurrentEngine {
@@ -478,6 +530,7 @@ impl RegionServerInner {
         runtime: Runtime,
         event_listener: RegionServerEventListenerRef,
         table_provider_factory: TableProviderFactoryRef,
+        parallelism: Option<RegionServerParallelism>,
     ) -> Self {
         Self {
             engines: RwLock::new(HashMap::new()),
@@ -486,6 +539,7 @@ impl RegionServerInner {
             runtime,
             event_listener,
             table_provider_factory,
+            parallelism,
         }
     }
 
@@ -1284,4 +1338,23 @@ mod tests {
             assert(result);
         }
     }
+
+    #[tokio::test]
+    async fn test_region_server_parallism() {
+        let p = RegionServerParallelism::from_opts(2, Duration::from_millis(1)).unwrap();
+        let first_query = p.acquire().await;
+        assert!(first_query.is_ok());
+        let second_query = p.acquire().await;
+        assert!(second_query.is_ok());
+        let third_query = p.acquire().await;
+        assert!(third_query.is_err());
+        let err = third_query.unwrap_err();
+        assert_eq!(
+            err.output_msg(),
+            "Failed to acquire permit under timeouts: deadline has elapsed".to_string()
+        );
+        drop(first_query);
+        let forth_query = p.acquire().await;
+        assert!(forth_query.is_ok());
+    }
 }

From 4045298cb2b39ca8dbc98d4c25bafa3fc910bb25 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Fri, 27 Sep 2024 17:54:52 +0800
Subject: [PATCH 039/128] feat: add `region_statistics` table (#4771)

* refactor: introduce `region_statistic`

* refactor: move DatanodeStat related structs to common_meta

* chore: add comments

* feat: implement `list_region_stats` for `ClusterInfo` trait

* feat: add `region_statistics` table

* feat: add table_id and region_number fields

* chore: rename unused snafu

* chore: udpate sqlness results

* chore: avoid to print source in error msg

* chore: move `procedure_info` under `greptime` catalog

* chore: apply suggestions from CR

* Update src/common/meta/src/datanode.rs

Co-authored-by: jeremyhi <jiachun_feng@proton.me>

---------

Co-authored-by: jeremyhi <jiachun_feng@proton.me>
---
 Cargo.lock                                    |   1 +
 src/catalog/src/error.rs                      |  10 +-
 src/catalog/src/kvbackend/manager.rs          |   5 +
 .../src/system_schema/information_schema.rs   |  18 +-
 .../information_schema/region_statistics.rs   | 257 +++++++++++
 .../information_schema/table_names.rs         |   1 +
 src/common/catalog/src/consts.rs              |   3 +
 src/common/meta/src/cluster.rs                |   4 +
 src/common/meta/src/datanode.rs               | 413 ++++++++++++++++++
 src/common/meta/src/error.rs                  |  32 +-
 src/common/meta/src/lib.rs                    |   1 +
 src/datanode/Cargo.toml                       |   1 +
 src/datanode/src/heartbeat.rs                 |  31 +-
 src/datanode/src/region_server.rs             |   6 +-
 src/datanode/src/tests.rs                     |   6 +-
 src/file-engine/src/engine.rs                 |   5 +-
 src/meta-client/src/client.rs                 |  23 +
 src/meta-srv/src/cluster.rs                   |  20 +-
 src/meta-srv/src/error.rs                     |  28 +-
 src/meta-srv/src/handler.rs                   |   3 +-
 .../src/handler/collect_stats_handler.rs      |   8 +-
 .../src/handler/extract_stat_handler.rs       |   2 +-
 src/meta-srv/src/handler/failure_handler.rs   |   6 +-
 src/meta-srv/src/handler/node_stat.rs         | 170 -------
 .../src/handler/region_lease_handler.rs       |   6 +-
 src/meta-srv/src/key/datanode.rs              | 192 +-------
 src/meta-srv/src/region/supervisor.rs         |   2 +-
 src/meta-srv/src/selector/load_based.rs       |   7 +-
 src/meta-srv/src/selector/weight_compute.rs   |  17 +-
 src/meta-srv/src/service/admin/heartbeat.rs   |   6 +-
 src/metric-engine/src/engine.rs               |  12 +-
 src/mito2/src/engine.rs                       |  23 +-
 src/mito2/src/engine/basic_test.rs            |  18 +-
 src/mito2/src/region.rs                       |  17 +-
 src/query/src/optimizer/test_util.rs          |   6 +-
 src/store-api/src/region_engine.rs            |  40 +-
 .../common/show/show_databases_tables.result  |   3 +
 .../common/system/information_schema.result   |   9 +
 .../standalone/common/view/create.result      |   1 +
 39 files changed, 939 insertions(+), 474 deletions(-)
 create mode 100644 src/catalog/src/system_schema/information_schema/region_statistics.rs
 create mode 100644 src/common/meta/src/datanode.rs
 delete mode 100644 src/meta-srv/src/handler/node_stat.rs

diff --git a/Cargo.lock b/Cargo.lock
index 991829a758..3d5783dcd4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3202,6 +3202,7 @@ dependencies = [
  "query",
  "reqwest",
  "serde",
+ "serde_json",
  "servers",
  "session",
  "snafu 0.8.4",
diff --git a/src/catalog/src/error.rs b/src/catalog/src/error.rs
index 14748a4695..dd7071b095 100644
--- a/src/catalog/src/error.rs
+++ b/src/catalog/src/error.rs
@@ -50,13 +50,20 @@ pub enum Error {
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to list nodes in cluster: {source}"))]
+    #[snafu(display("Failed to list nodes in cluster"))]
     ListNodes {
         #[snafu(implicit)]
         location: Location,
         source: BoxedError,
     },
 
+    #[snafu(display("Failed to region stats in cluster"))]
+    ListRegionStats {
+        #[snafu(implicit)]
+        location: Location,
+        source: BoxedError,
+    },
+
     #[snafu(display("Failed to list flows in catalog {catalog}"))]
     ListFlows {
         #[snafu(implicit)]
@@ -314,6 +321,7 @@ impl ErrorExt for Error {
             | Error::ListTables { source, .. }
             | Error::ListFlows { source, .. }
             | Error::ListProcedures { source, .. }
+            | Error::ListRegionStats { source, .. }
             | Error::ConvertProtoData { source, .. } => source.status_code(),
 
             Error::CreateTable { source, .. } => source.status_code(),
diff --git a/src/catalog/src/kvbackend/manager.rs b/src/catalog/src/kvbackend/manager.rs
index 2495a95255..e3377afbe1 100644
--- a/src/catalog/src/kvbackend/manager.rs
+++ b/src/catalog/src/kvbackend/manager.rs
@@ -64,12 +64,17 @@ use crate::CatalogManager;
 #[derive(Clone)]
 pub struct KvBackendCatalogManager {
     mode: Mode,
+    /// Only available in `Distributed` mode.
     meta_client: Option<Arc<MetaClient>>,
+    /// Manages partition rules.
     partition_manager: PartitionRuleManagerRef,
+    /// Manages table metadata.
     table_metadata_manager: TableMetadataManagerRef,
     /// A sub-CatalogManager that handles system tables
     system_catalog: SystemCatalog,
+    /// Cache registry for all caches.
     cache_registry: LayeredCacheRegistryRef,
+    /// Only available in `Standalone` mode.
     procedure_manager: Option<ProcedureManagerRef>,
 }
 
diff --git a/src/catalog/src/system_schema/information_schema.rs b/src/catalog/src/system_schema/information_schema.rs
index 83f2ff4926..9fa31b85fd 100644
--- a/src/catalog/src/system_schema/information_schema.rs
+++ b/src/catalog/src/system_schema/information_schema.rs
@@ -20,6 +20,7 @@ pub mod key_column_usage;
 mod partitions;
 mod procedure_info;
 mod region_peers;
+mod region_statistics;
 mod runtime_metrics;
 pub mod schemata;
 mod table_constraints;
@@ -194,6 +195,11 @@ impl SystemSchemaProviderInner for InformationSchemaProvider {
                     self.catalog_manager.clone(),
                 )) as _,
             ),
+            REGION_STATISTICS => Some(Arc::new(
+                region_statistics::InformationSchemaRegionStatistics::new(
+                    self.catalog_manager.clone(),
+                ),
+            ) as _),
             _ => None,
         }
     }
@@ -241,6 +247,14 @@ impl InformationSchemaProvider {
                 CLUSTER_INFO.to_string(),
                 self.build_table(CLUSTER_INFO).unwrap(),
             );
+            tables.insert(
+                PROCEDURE_INFO.to_string(),
+                self.build_table(PROCEDURE_INFO).unwrap(),
+            );
+            tables.insert(
+                REGION_STATISTICS.to_string(),
+                self.build_table(REGION_STATISTICS).unwrap(),
+            );
         }
 
         tables.insert(TABLES.to_string(), self.build_table(TABLES).unwrap());
@@ -256,10 +270,6 @@ impl InformationSchemaProvider {
             self.build_table(TABLE_CONSTRAINTS).unwrap(),
         );
         tables.insert(FLOWS.to_string(), self.build_table(FLOWS).unwrap());
-        tables.insert(
-            PROCEDURE_INFO.to_string(),
-            self.build_table(PROCEDURE_INFO).unwrap(),
-        );
         // Add memory tables
         for name in MEMORY_TABLES.iter() {
             tables.insert((*name).to_string(), self.build_table(name).expect(name));
diff --git a/src/catalog/src/system_schema/information_schema/region_statistics.rs b/src/catalog/src/system_schema/information_schema/region_statistics.rs
new file mode 100644
index 0000000000..07b94ede54
--- /dev/null
+++ b/src/catalog/src/system_schema/information_schema/region_statistics.rs
@@ -0,0 +1,257 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::{Arc, Weak};
+
+use arrow_schema::SchemaRef as ArrowSchemaRef;
+use common_catalog::consts::INFORMATION_SCHEMA_REGION_STATISTICS_TABLE_ID;
+use common_config::Mode;
+use common_error::ext::BoxedError;
+use common_meta::cluster::ClusterInfo;
+use common_meta::datanode::RegionStat;
+use common_recordbatch::adapter::RecordBatchStreamAdapter;
+use common_recordbatch::{DfSendableRecordBatchStream, RecordBatch, SendableRecordBatchStream};
+use common_telemetry::tracing::warn;
+use datafusion::execution::TaskContext;
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
+use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
+use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder, VectorRef};
+use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
+use datatypes::value::Value;
+use datatypes::vectors::{StringVectorBuilder, UInt32VectorBuilder, UInt64VectorBuilder};
+use snafu::ResultExt;
+use store_api::storage::{ScanRequest, TableId};
+
+use super::{InformationTable, REGION_STATISTICS};
+use crate::error::{CreateRecordBatchSnafu, InternalSnafu, ListRegionStatsSnafu, Result};
+use crate::information_schema::Predicates;
+use crate::system_schema::utils;
+use crate::CatalogManager;
+
+const REGION_ID: &str = "region_id";
+const TABLE_ID: &str = "table_id";
+const REGION_NUMBER: &str = "region_number";
+const MEMTABLE_SIZE: &str = "memtable_size";
+const MANIFEST_SIZE: &str = "manifest_size";
+const SST_SIZE: &str = "sst_size";
+const ENGINE: &str = "engine";
+const REGION_ROLE: &str = "region_role";
+
+const INIT_CAPACITY: usize = 42;
+
+/// The `REGION_STATISTICS` table provides information about the region statistics. Including fields:
+///
+/// - `region_id`: The region id.
+/// - `table_id`: The table id.
+/// - `region_number`: The region number.
+/// - `memtable_size`: The memtable size in bytes.
+/// - `manifest_size`: The manifest size in bytes.
+/// - `sst_size`: The sst size in bytes.
+/// - `engine`: The engine type.
+/// - `region_role`: The region role.
+///
+pub(super) struct InformationSchemaRegionStatistics {
+    schema: SchemaRef,
+    catalog_manager: Weak<dyn CatalogManager>,
+}
+
+impl InformationSchemaRegionStatistics {
+    pub(super) fn new(catalog_manager: Weak<dyn CatalogManager>) -> Self {
+        Self {
+            schema: Self::schema(),
+            catalog_manager,
+        }
+    }
+
+    pub(crate) fn schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![
+            ColumnSchema::new(REGION_ID, ConcreteDataType::uint64_datatype(), false),
+            ColumnSchema::new(TABLE_ID, ConcreteDataType::uint32_datatype(), false),
+            ColumnSchema::new(REGION_NUMBER, ConcreteDataType::uint32_datatype(), false),
+            ColumnSchema::new(MEMTABLE_SIZE, ConcreteDataType::uint64_datatype(), true),
+            ColumnSchema::new(MANIFEST_SIZE, ConcreteDataType::uint64_datatype(), true),
+            ColumnSchema::new(SST_SIZE, ConcreteDataType::uint64_datatype(), true),
+            ColumnSchema::new(ENGINE, ConcreteDataType::string_datatype(), true),
+            ColumnSchema::new(REGION_ROLE, ConcreteDataType::string_datatype(), true),
+        ]))
+    }
+
+    fn builder(&self) -> InformationSchemaRegionStatisticsBuilder {
+        InformationSchemaRegionStatisticsBuilder::new(
+            self.schema.clone(),
+            self.catalog_manager.clone(),
+        )
+    }
+}
+
+impl InformationTable for InformationSchemaRegionStatistics {
+    fn table_id(&self) -> TableId {
+        INFORMATION_SCHEMA_REGION_STATISTICS_TABLE_ID
+    }
+
+    fn table_name(&self) -> &'static str {
+        REGION_STATISTICS
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn to_stream(&self, request: ScanRequest) -> Result<SendableRecordBatchStream> {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+
+        let stream = Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_region_statistics(Some(request))
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ));
+
+        Ok(Box::pin(
+            RecordBatchStreamAdapter::try_new(stream)
+                .map_err(BoxedError::new)
+                .context(InternalSnafu)?,
+        ))
+    }
+}
+
+struct InformationSchemaRegionStatisticsBuilder {
+    schema: SchemaRef,
+    catalog_manager: Weak<dyn CatalogManager>,
+
+    region_ids: UInt64VectorBuilder,
+    table_ids: UInt32VectorBuilder,
+    region_numbers: UInt32VectorBuilder,
+    memtable_sizes: UInt64VectorBuilder,
+    manifest_sizes: UInt64VectorBuilder,
+    sst_sizes: UInt64VectorBuilder,
+    engines: StringVectorBuilder,
+    region_roles: StringVectorBuilder,
+}
+
+impl InformationSchemaRegionStatisticsBuilder {
+    fn new(schema: SchemaRef, catalog_manager: Weak<dyn CatalogManager>) -> Self {
+        Self {
+            schema,
+            catalog_manager,
+            region_ids: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
+            table_ids: UInt32VectorBuilder::with_capacity(INIT_CAPACITY),
+            region_numbers: UInt32VectorBuilder::with_capacity(INIT_CAPACITY),
+            memtable_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
+            manifest_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
+            sst_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
+            engines: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+            region_roles: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+        }
+    }
+
+    /// Construct a new `InformationSchemaRegionStatistics` from the collected data.
+    async fn make_region_statistics(
+        &mut self,
+        request: Option<ScanRequest>,
+    ) -> Result<RecordBatch> {
+        let predicates = Predicates::from_scan_request(&request);
+        let mode = utils::running_mode(&self.catalog_manager)?.unwrap_or(Mode::Standalone);
+
+        match mode {
+            Mode::Standalone => {
+                // TODO(weny): implement it
+            }
+            Mode::Distributed => {
+                if let Some(meta_client) = utils::meta_client(&self.catalog_manager)? {
+                    let region_stats = meta_client
+                        .list_region_stats()
+                        .await
+                        .map_err(BoxedError::new)
+                        .context(ListRegionStatsSnafu)?;
+                    for region_stat in region_stats {
+                        self.add_region_statistic(&predicates, region_stat);
+                    }
+                } else {
+                    warn!("Meta client is not available");
+                }
+            }
+        }
+
+        self.finish()
+    }
+
+    fn add_region_statistic(&mut self, predicate: &Predicates, region_stat: RegionStat) {
+        let row = [
+            (REGION_ID, &Value::from(region_stat.id.as_u64())),
+            (TABLE_ID, &Value::from(region_stat.id.table_id())),
+            (REGION_NUMBER, &Value::from(region_stat.id.region_number())),
+            (MEMTABLE_SIZE, &Value::from(region_stat.memtable_size)),
+            (MANIFEST_SIZE, &Value::from(region_stat.manifest_size)),
+            (SST_SIZE, &Value::from(region_stat.sst_size)),
+            (ENGINE, &Value::from(region_stat.engine.as_str())),
+            (REGION_ROLE, &Value::from(region_stat.role.to_string())),
+        ];
+
+        if !predicate.eval(&row) {
+            return;
+        }
+
+        self.region_ids.push(Some(region_stat.id.as_u64()));
+        self.table_ids.push(Some(region_stat.id.table_id()));
+        self.region_numbers
+            .push(Some(region_stat.id.region_number()));
+        self.memtable_sizes.push(Some(region_stat.memtable_size));
+        self.manifest_sizes.push(Some(region_stat.manifest_size));
+        self.sst_sizes.push(Some(region_stat.sst_size));
+        self.engines.push(Some(&region_stat.engine));
+        self.region_roles.push(Some(&region_stat.role.to_string()));
+    }
+
+    fn finish(&mut self) -> Result<RecordBatch> {
+        let columns: Vec<VectorRef> = vec![
+            Arc::new(self.region_ids.finish()),
+            Arc::new(self.table_ids.finish()),
+            Arc::new(self.region_numbers.finish()),
+            Arc::new(self.memtable_sizes.finish()),
+            Arc::new(self.manifest_sizes.finish()),
+            Arc::new(self.sst_sizes.finish()),
+            Arc::new(self.engines.finish()),
+            Arc::new(self.region_roles.finish()),
+        ];
+
+        RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu)
+    }
+}
+
+impl DfPartitionStream for InformationSchemaRegionStatistics {
+    fn schema(&self) -> &ArrowSchemaRef {
+        self.schema.arrow_schema()
+    }
+
+    fn execute(&self, _: Arc<TaskContext>) -> DfSendableRecordBatchStream {
+        let schema = self.schema.arrow_schema().clone();
+        let mut builder = self.builder();
+        Box::pin(DfRecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::once(async move {
+                builder
+                    .make_region_statistics(None)
+                    .await
+                    .map(|x| x.into_df_record_batch())
+                    .map_err(Into::into)
+            }),
+        ))
+    }
+}
diff --git a/src/catalog/src/system_schema/information_schema/table_names.rs b/src/catalog/src/system_schema/information_schema/table_names.rs
index a62f4ddb40..53541bf955 100644
--- a/src/catalog/src/system_schema/information_schema/table_names.rs
+++ b/src/catalog/src/system_schema/information_schema/table_names.rs
@@ -46,3 +46,4 @@ pub const CLUSTER_INFO: &str = "cluster_info";
 pub const VIEWS: &str = "views";
 pub const FLOWS: &str = "flows";
 pub const PROCEDURE_INFO: &str = "procedure_info";
+pub const REGION_STATISTICS: &str = "region_statistics";
diff --git a/src/common/catalog/src/consts.rs b/src/common/catalog/src/consts.rs
index 2a8e2fc0e4..9cf02b81ee 100644
--- a/src/common/catalog/src/consts.rs
+++ b/src/common/catalog/src/consts.rs
@@ -100,6 +100,9 @@ pub const INFORMATION_SCHEMA_VIEW_TABLE_ID: u32 = 32;
 pub const INFORMATION_SCHEMA_FLOW_TABLE_ID: u32 = 33;
 /// id for information_schema.procedure_info
 pub const INFORMATION_SCHEMA_PROCEDURE_INFO_TABLE_ID: u32 = 34;
+/// id for information_schema.region_statistics
+pub const INFORMATION_SCHEMA_REGION_STATISTICS_TABLE_ID: u32 = 35;
+
 /// ----- End of information_schema tables -----
 
 /// ----- Begin of pg_catalog tables -----
diff --git a/src/common/meta/src/cluster.rs b/src/common/meta/src/cluster.rs
index 0cf593f1ca..785a28f17d 100644
--- a/src/common/meta/src/cluster.rs
+++ b/src/common/meta/src/cluster.rs
@@ -20,6 +20,7 @@ use regex::Regex;
 use serde::{Deserialize, Serialize};
 use snafu::{ensure, OptionExt, ResultExt};
 
+use crate::datanode::RegionStat;
 use crate::error::{
     DecodeJsonSnafu, EncodeJsonSnafu, Error, FromUtf8Snafu, InvalidNodeInfoKeySnafu,
     InvalidRoleSnafu, ParseNumSnafu, Result,
@@ -47,6 +48,9 @@ pub trait ClusterInfo {
         role: Option<Role>,
     ) -> std::result::Result<Vec<NodeInfo>, Self::Error>;
 
+    /// List all region stats in the cluster.
+    async fn list_region_stats(&self) -> std::result::Result<Vec<RegionStat>, Self::Error>;
+
     // TODO(jeremy): Other info, like region status, etc.
 }
 
diff --git a/src/common/meta/src/datanode.rs b/src/common/meta/src/datanode.rs
new file mode 100644
index 0000000000..4551b8de2f
--- /dev/null
+++ b/src/common/meta/src/datanode.rs
@@ -0,0 +1,413 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashSet;
+use std::str::FromStr;
+
+use api::v1::meta::{HeartbeatRequest, RequestHeader};
+use common_time::util as time_util;
+use lazy_static::lazy_static;
+use regex::Regex;
+use serde::{Deserialize, Serialize};
+use snafu::{ensure, OptionExt, ResultExt};
+use store_api::region_engine::{RegionRole, RegionStatistic};
+use store_api::storage::RegionId;
+use table::metadata::TableId;
+
+use crate::error::Result;
+use crate::{error, ClusterId};
+
+pub(crate) const DATANODE_LEASE_PREFIX: &str = "__meta_datanode_lease";
+const INACTIVE_REGION_PREFIX: &str = "__meta_inactive_region";
+
+const DATANODE_STAT_PREFIX: &str = "__meta_datanode_stat";
+
+pub const REGION_STATISTIC_KEY: &str = "__region_statistic";
+
+lazy_static! {
+    pub(crate) static ref DATANODE_LEASE_KEY_PATTERN: Regex =
+        Regex::new(&format!("^{DATANODE_LEASE_PREFIX}-([0-9]+)-([0-9]+)$")).unwrap();
+    static ref DATANODE_STAT_KEY_PATTERN: Regex =
+        Regex::new(&format!("^{DATANODE_STAT_PREFIX}-([0-9]+)-([0-9]+)$")).unwrap();
+    static ref INACTIVE_REGION_KEY_PATTERN: Regex = Regex::new(&format!(
+        "^{INACTIVE_REGION_PREFIX}-([0-9]+)-([0-9]+)-([0-9]+)$"
+    ))
+    .unwrap();
+}
+
+/// The key of the datanode stat in the storage.
+///
+/// The format is `__meta_datanode_stat-{cluster_id}-{node_id}`.
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct Stat {
+    pub timestamp_millis: i64,
+    pub cluster_id: ClusterId,
+    // The datanode Id.
+    pub id: u64,
+    // The datanode address.
+    pub addr: String,
+    /// The read capacity units during this period
+    pub rcus: i64,
+    /// The write capacity units during this period
+    pub wcus: i64,
+    /// How many regions on this node
+    pub region_num: u64,
+    pub region_stats: Vec<RegionStat>,
+    // The node epoch is used to check whether the node has restarted or redeployed.
+    pub node_epoch: u64,
+}
+
+/// The statistics of a region.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RegionStat {
+    /// The region_id.
+    pub id: RegionId,
+    /// The read capacity units during this period
+    pub rcus: i64,
+    /// The write capacity units during this period
+    pub wcus: i64,
+    /// Approximate bytes of this region
+    pub approximate_bytes: i64,
+    /// The engine name.
+    pub engine: String,
+    /// The region role.
+    pub role: RegionRole,
+    /// The size of the memtable in bytes.
+    pub memtable_size: u64,
+    /// The size of the manifest in bytes.
+    pub manifest_size: u64,
+    /// The size of the SST files in bytes.
+    pub sst_size: u64,
+}
+
+impl Stat {
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.region_stats.is_empty()
+    }
+
+    pub fn stat_key(&self) -> DatanodeStatKey {
+        DatanodeStatKey {
+            cluster_id: self.cluster_id,
+            node_id: self.id,
+        }
+    }
+
+    /// Returns a tuple array containing [RegionId] and [RegionRole].
+    pub fn regions(&self) -> Vec<(RegionId, RegionRole)> {
+        self.region_stats.iter().map(|s| (s.id, s.role)).collect()
+    }
+
+    /// Returns all table ids in the region stats.
+    pub fn table_ids(&self) -> HashSet<TableId> {
+        self.region_stats.iter().map(|s| s.id.table_id()).collect()
+    }
+
+    /// Retains the active region stats and updates the rcus, wcus, and region_num.
+    pub fn retain_active_region_stats(&mut self, inactive_region_ids: &HashSet<RegionId>) {
+        if inactive_region_ids.is_empty() {
+            return;
+        }
+
+        self.region_stats
+            .retain(|r| !inactive_region_ids.contains(&r.id));
+        self.rcus = self.region_stats.iter().map(|s| s.rcus).sum();
+        self.wcus = self.region_stats.iter().map(|s| s.wcus).sum();
+        self.region_num = self.region_stats.len() as u64;
+    }
+}
+
+impl TryFrom<&HeartbeatRequest> for Stat {
+    type Error = Option<RequestHeader>;
+
+    fn try_from(value: &HeartbeatRequest) -> std::result::Result<Self, Self::Error> {
+        let HeartbeatRequest {
+            header,
+            peer,
+            region_stats,
+            node_epoch,
+            ..
+        } = value;
+
+        match (header, peer) {
+            (Some(header), Some(peer)) => {
+                let region_stats = region_stats
+                    .iter()
+                    .map(RegionStat::from)
+                    .collect::<Vec<_>>();
+
+                Ok(Self {
+                    timestamp_millis: time_util::current_time_millis(),
+                    cluster_id: header.cluster_id,
+                    // datanode id
+                    id: peer.id,
+                    // datanode address
+                    addr: peer.addr.clone(),
+                    rcus: region_stats.iter().map(|s| s.rcus).sum(),
+                    wcus: region_stats.iter().map(|s| s.wcus).sum(),
+                    region_num: region_stats.len() as u64,
+                    region_stats,
+                    node_epoch: *node_epoch,
+                })
+            }
+            (header, _) => Err(header.clone()),
+        }
+    }
+}
+
+impl From<&api::v1::meta::RegionStat> for RegionStat {
+    fn from(value: &api::v1::meta::RegionStat) -> Self {
+        let region_stat = value
+            .extensions
+            .get(REGION_STATISTIC_KEY)
+            .and_then(|value| RegionStatistic::deserialize_from_slice(value))
+            .unwrap_or_default();
+
+        Self {
+            id: RegionId::from_u64(value.region_id),
+            rcus: value.rcus,
+            wcus: value.wcus,
+            approximate_bytes: value.approximate_bytes,
+            engine: value.engine.to_string(),
+            role: RegionRole::from(value.role()),
+            memtable_size: region_stat.memtable_size,
+            manifest_size: region_stat.manifest_size,
+            sst_size: region_stat.sst_size,
+        }
+    }
+}
+
+/// The key of the datanode stat in the memory store.
+///
+/// The format is `__meta_datanode_stat-{cluster_id}-{node_id}`.
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
+pub struct DatanodeStatKey {
+    pub cluster_id: ClusterId,
+    pub node_id: u64,
+}
+
+impl DatanodeStatKey {
+    /// The key prefix.
+    pub fn prefix_key() -> Vec<u8> {
+        format!("{DATANODE_STAT_PREFIX}-").into_bytes()
+    }
+
+    /// The key prefix with the cluster id.
+    pub fn key_prefix_with_cluster_id(cluster_id: ClusterId) -> String {
+        format!("{DATANODE_STAT_PREFIX}-{cluster_id}-")
+    }
+}
+
+impl From<DatanodeStatKey> for Vec<u8> {
+    fn from(value: DatanodeStatKey) -> Self {
+        format!(
+            "{}-{}-{}",
+            DATANODE_STAT_PREFIX, value.cluster_id, value.node_id
+        )
+        .into_bytes()
+    }
+}
+
+impl FromStr for DatanodeStatKey {
+    type Err = error::Error;
+
+    fn from_str(key: &str) -> Result<Self> {
+        let caps = DATANODE_STAT_KEY_PATTERN
+            .captures(key)
+            .context(error::InvalidStatKeySnafu { key })?;
+
+        ensure!(caps.len() == 3, error::InvalidStatKeySnafu { key });
+
+        let cluster_id = caps[1].to_string();
+        let node_id = caps[2].to_string();
+        let cluster_id: u64 = cluster_id.parse().context(error::ParseNumSnafu {
+            err_msg: format!("invalid cluster_id: {cluster_id}"),
+        })?;
+        let node_id: u64 = node_id.parse().context(error::ParseNumSnafu {
+            err_msg: format!("invalid node_id: {node_id}"),
+        })?;
+
+        Ok(Self {
+            cluster_id,
+            node_id,
+        })
+    }
+}
+
+impl TryFrom<Vec<u8>> for DatanodeStatKey {
+    type Error = error::Error;
+
+    fn try_from(bytes: Vec<u8>) -> Result<Self> {
+        String::from_utf8(bytes)
+            .context(error::FromUtf8Snafu {
+                name: "DatanodeStatKey",
+            })
+            .map(|x| x.parse())?
+    }
+}
+
+/// The value of the datanode stat in the memory store.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct DatanodeStatValue {
+    pub stats: Vec<Stat>,
+}
+
+impl DatanodeStatValue {
+    /// Get the latest number of regions.
+    pub fn region_num(&self) -> Option<u64> {
+        self.stats.last().map(|x| x.region_num)
+    }
+
+    /// Get the latest node addr.
+    pub fn node_addr(&self) -> Option<String> {
+        self.stats.last().map(|x| x.addr.clone())
+    }
+}
+
+impl TryFrom<DatanodeStatValue> for Vec<u8> {
+    type Error = error::Error;
+
+    fn try_from(stats: DatanodeStatValue) -> Result<Self> {
+        Ok(serde_json::to_string(&stats)
+            .context(error::SerializeToJsonSnafu {
+                input: format!("{stats:?}"),
+            })?
+            .into_bytes())
+    }
+}
+
+impl FromStr for DatanodeStatValue {
+    type Err = error::Error;
+
+    fn from_str(value: &str) -> Result<Self> {
+        serde_json::from_str(value).context(error::DeserializeFromJsonSnafu { input: value })
+    }
+}
+
+impl TryFrom<Vec<u8>> for DatanodeStatValue {
+    type Error = error::Error;
+
+    fn try_from(value: Vec<u8>) -> Result<Self> {
+        String::from_utf8(value)
+            .context(error::FromUtf8Snafu {
+                name: "DatanodeStatValue",
+            })
+            .map(|x| x.parse())?
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_stat_key() {
+        let stat = Stat {
+            cluster_id: 3,
+            id: 101,
+            region_num: 10,
+            ..Default::default()
+        };
+
+        let stat_key = stat.stat_key();
+
+        assert_eq!(3, stat_key.cluster_id);
+        assert_eq!(101, stat_key.node_id);
+    }
+
+    #[test]
+    fn test_stat_val_round_trip() {
+        let stat = Stat {
+            cluster_id: 0,
+            id: 101,
+            region_num: 100,
+            ..Default::default()
+        };
+
+        let stat_val = DatanodeStatValue { stats: vec![stat] };
+
+        let bytes: Vec<u8> = stat_val.try_into().unwrap();
+        let stat_val: DatanodeStatValue = bytes.try_into().unwrap();
+        let stats = stat_val.stats;
+
+        assert_eq!(1, stats.len());
+
+        let stat = stats.first().unwrap();
+        assert_eq!(0, stat.cluster_id);
+        assert_eq!(101, stat.id);
+        assert_eq!(100, stat.region_num);
+    }
+
+    #[test]
+    fn test_get_addr_from_stat_val() {
+        let empty = DatanodeStatValue { stats: vec![] };
+        let addr = empty.node_addr();
+        assert!(addr.is_none());
+
+        let stat_val = DatanodeStatValue {
+            stats: vec![
+                Stat {
+                    addr: "1".to_string(),
+                    ..Default::default()
+                },
+                Stat {
+                    addr: "2".to_string(),
+                    ..Default::default()
+                },
+                Stat {
+                    addr: "3".to_string(),
+                    ..Default::default()
+                },
+            ],
+        };
+        let addr = stat_val.node_addr().unwrap();
+        assert_eq!("3", addr);
+    }
+
+    #[test]
+    fn test_get_region_num_from_stat_val() {
+        let empty = DatanodeStatValue { stats: vec![] };
+        let region_num = empty.region_num();
+        assert!(region_num.is_none());
+
+        let wrong = DatanodeStatValue {
+            stats: vec![Stat {
+                region_num: 0,
+                ..Default::default()
+            }],
+        };
+        let right = wrong.region_num();
+        assert_eq!(Some(0), right);
+
+        let stat_val = DatanodeStatValue {
+            stats: vec![
+                Stat {
+                    region_num: 1,
+                    ..Default::default()
+                },
+                Stat {
+                    region_num: 0,
+                    ..Default::default()
+                },
+                Stat {
+                    region_num: 2,
+                    ..Default::default()
+                },
+            ],
+        };
+        let region_num = stat_val.region_num().unwrap();
+        assert_eq!(2, region_num);
+    }
+}
diff --git a/src/common/meta/src/error.rs b/src/common/meta/src/error.rs
index b244b1e310..849ee28948 100644
--- a/src/common/meta/src/error.rs
+++ b/src/common/meta/src/error.rs
@@ -218,6 +218,24 @@ pub enum Error {
         error: JsonError,
     },
 
+    #[snafu(display("Failed to serialize to json: {}", input))]
+    SerializeToJson {
+        input: String,
+        #[snafu(source)]
+        error: serde_json::error::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Failed to deserialize from json: {}", input))]
+    DeserializeFromJson {
+        input: String,
+        #[snafu(source)]
+        error: serde_json::error::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Payload not exist"))]
     PayloadNotExist {
         #[snafu(implicit)]
@@ -531,13 +549,20 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Invalid  node info key: {}", key))]
+    #[snafu(display("Invalid node info key: {}", key))]
     InvalidNodeInfoKey {
         key: String,
         #[snafu(implicit)]
         location: Location,
     },
 
+    #[snafu(display("Invalid node stat key: {}", key))]
+    InvalidStatKey {
+        key: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Failed to parse number: {}", err_msg))]
     ParseNum {
         err_msg: String,
@@ -627,7 +652,9 @@ impl ErrorExt for Error {
             | EtcdTxnFailed { .. }
             | ConnectEtcd { .. }
             | MoveValues { .. }
-            | GetCache { .. } => StatusCode::Internal,
+            | GetCache { .. }
+            | SerializeToJson { .. }
+            | DeserializeFromJson { .. } => StatusCode::Internal,
 
             ValueNotExist { .. } => StatusCode::Unexpected,
 
@@ -700,6 +727,7 @@ impl ErrorExt for Error {
             | InvalidNumTopics { .. }
             | SchemaNotFound { .. }
             | InvalidNodeInfoKey { .. }
+            | InvalidStatKey { .. }
             | ParseNum { .. }
             | InvalidRole { .. }
             | EmptyDdlTasks { .. } => StatusCode::InvalidArguments,
diff --git a/src/common/meta/src/lib.rs b/src/common/meta/src/lib.rs
index 78d111c479..1dd658890c 100644
--- a/src/common/meta/src/lib.rs
+++ b/src/common/meta/src/lib.rs
@@ -22,6 +22,7 @@
 pub mod cache;
 pub mod cache_invalidator;
 pub mod cluster;
+pub mod datanode;
 pub mod ddl;
 pub mod ddl_manager;
 pub mod distributed_time_constants;
diff --git a/src/datanode/Cargo.toml b/src/datanode/Cargo.toml
index 26a7ccb675..c7ef0a7242 100644
--- a/src/datanode/Cargo.toml
+++ b/src/datanode/Cargo.toml
@@ -53,6 +53,7 @@ prost.workspace = true
 query.workspace = true
 reqwest.workspace = true
 serde.workspace = true
+serde_json.workspace = true
 servers.workspace = true
 session.workspace = true
 snafu.workspace = true
diff --git a/src/datanode/src/heartbeat.rs b/src/datanode/src/heartbeat.rs
index 6633d0ab62..d84552a8d2 100644
--- a/src/datanode/src/heartbeat.rs
+++ b/src/datanode/src/heartbeat.rs
@@ -12,11 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
 
 use api::v1::meta::{HeartbeatRequest, NodeInfo, Peer, RegionRole, RegionStat};
+use common_meta::datanode::REGION_STATISTIC_KEY;
 use common_meta::distributed_time_constants::META_KEEP_ALIVE_INTERVAL_SECS;
 use common_meta::heartbeat::handler::parse_mailbox_message::ParseMailboxMessageHandler;
 use common_meta::heartbeat::handler::{
@@ -320,16 +322,25 @@ impl HeartbeatTask {
         region_server
             .reportable_regions()
             .into_iter()
-            .map(|stat| RegionStat {
-                region_id: stat.region_id.as_u64(),
-                engine: stat.engine,
-                role: RegionRole::from(stat.role).into(),
-                // TODO(weny): w/rcus
-                rcus: 0,
-                wcus: 0,
-                approximate_bytes: region_server.region_disk_usage(stat.region_id).unwrap_or(0),
-                // TODO(weny): add extensions
-                extensions: Default::default(),
+            .map(|stat| {
+                let region_stat = region_server
+                    .region_statistic(stat.region_id)
+                    .unwrap_or_default();
+                let mut extensions = HashMap::new();
+                if let Some(serialized) = region_stat.serialize_to_vec() {
+                    extensions.insert(REGION_STATISTIC_KEY.to_string(), serialized);
+                }
+
+                RegionStat {
+                    region_id: stat.region_id.as_u64(),
+                    engine: stat.engine,
+                    role: RegionRole::from(stat.role).into(),
+                    // TODO(weny): w/rcus
+                    rcus: 0,
+                    wcus: 0,
+                    approximate_bytes: region_stat.estimated_disk_size() as i64,
+                    extensions,
+                }
             })
             .collect()
     }
diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs
index 4fcdfb86af..aa80f52a5c 100644
--- a/src/datanode/src/region_server.rs
+++ b/src/datanode/src/region_server.rs
@@ -54,7 +54,7 @@ use snafu::{ensure, OptionExt, ResultExt};
 use store_api::metric_engine_consts::{
     FILE_ENGINE_NAME, LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME,
 };
-use store_api::region_engine::{RegionEngineRef, RegionRole, SetReadonlyResponse};
+use store_api::region_engine::{RegionEngineRef, RegionRole, RegionStatistic, SetReadonlyResponse};
 use store_api::region_request::{
     AffectedRows, RegionCloseRequest, RegionOpenRequest, RegionRequest,
 };
@@ -312,9 +312,9 @@ impl RegionServer {
         self.inner.runtime.clone()
     }
 
-    pub fn region_disk_usage(&self, region_id: RegionId) -> Option<i64> {
+    pub fn region_statistic(&self, region_id: RegionId) -> Option<RegionStatistic> {
         match self.inner.region_map.get(&region_id) {
-            Some(e) => e.region_disk_usage(region_id),
+            Some(e) => e.region_statistic(region_id),
             None => None,
         }
     }
diff --git a/src/datanode/src/tests.rs b/src/datanode/src/tests.rs
index 8966dc4932..35f513bc83 100644
--- a/src/datanode/src/tests.rs
+++ b/src/datanode/src/tests.rs
@@ -31,7 +31,9 @@ use query::query_engine::{DescribeResult, QueryEngineState};
 use query::{QueryEngine, QueryEngineContext};
 use session::context::QueryContextRef;
 use store_api::metadata::RegionMetadataRef;
-use store_api::region_engine::{RegionEngine, RegionRole, RegionScannerRef, SetReadonlyResponse};
+use store_api::region_engine::{
+    RegionEngine, RegionRole, RegionScannerRef, RegionStatistic, SetReadonlyResponse,
+};
 use store_api::region_request::{AffectedRows, RegionRequest};
 use store_api::storage::{RegionId, ScanRequest};
 use table::TableRef;
@@ -210,7 +212,7 @@ impl RegionEngine for MockRegionEngine {
         unimplemented!()
     }
 
-    fn region_disk_usage(&self, _region_id: RegionId) -> Option<i64> {
+    fn region_statistic(&self, _region_id: RegionId) -> Option<RegionStatistic> {
         unimplemented!()
     }
 
diff --git a/src/file-engine/src/engine.rs b/src/file-engine/src/engine.rs
index f8cbbe7ddc..32e1a1d58d 100644
--- a/src/file-engine/src/engine.rs
+++ b/src/file-engine/src/engine.rs
@@ -26,7 +26,8 @@ use object_store::ObjectStore;
 use snafu::{ensure, OptionExt};
 use store_api::metadata::RegionMetadataRef;
 use store_api::region_engine::{
-    RegionEngine, RegionRole, RegionScannerRef, SetReadonlyResponse, SinglePartitionScanner,
+    RegionEngine, RegionRole, RegionScannerRef, RegionStatistic, SetReadonlyResponse,
+    SinglePartitionScanner,
 };
 use store_api::region_request::{
     AffectedRows, RegionCloseRequest, RegionCreateRequest, RegionDropRequest, RegionOpenRequest,
@@ -108,7 +109,7 @@ impl RegionEngine for FileRegionEngine {
         self.inner.stop().await.map_err(BoxedError::new)
     }
 
-    fn region_disk_usage(&self, _: RegionId) -> Option<i64> {
+    fn region_statistic(&self, _: RegionId) -> Option<RegionStatistic> {
         None
     }
 
diff --git a/src/meta-client/src/client.rs b/src/meta-client/src/client.rs
index d5f7c6aaaf..1c5b96a684 100644
--- a/src/meta-client/src/client.rs
+++ b/src/meta-client/src/client.rs
@@ -29,6 +29,7 @@ use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
 use common_meta::cluster::{
     ClusterInfo, MetasrvStatus, NodeInfo, NodeInfoKey, NodeStatus, Role as ClusterRole,
 };
+use common_meta::datanode::{DatanodeStatKey, DatanodeStatValue, RegionStat};
 use common_meta::ddl::{ExecutorContext, ProcedureExecutor};
 use common_meta::error::{self as meta_error, Result as MetaResult};
 use common_meta::rpc::ddl::{SubmitDdlTaskRequest, SubmitDdlTaskResponse};
@@ -327,6 +328,28 @@ impl ClusterInfo for MetaClient {
 
         Ok(nodes)
     }
+
+    async fn list_region_stats(&self) -> Result<Vec<RegionStat>> {
+        let cluster_client = self.cluster_client()?;
+        let range_prefix = DatanodeStatKey::key_prefix_with_cluster_id(self.id.0);
+        let req = RangeRequest::new().with_prefix(range_prefix);
+        let mut datanode_stats = cluster_client
+            .range(req)
+            .await?
+            .kvs
+            .into_iter()
+            .map(|kv| DatanodeStatValue::try_from(kv.value).context(ConvertMetaRequestSnafu))
+            .collect::<Result<Vec<_>>>()?;
+        let region_stats = datanode_stats
+            .iter_mut()
+            .flat_map(|datanode_stat| {
+                let last = datanode_stat.stats.pop();
+                last.map(|stat| stat.region_stats).unwrap_or_default()
+            })
+            .collect::<Vec<_>>();
+
+        Ok(region_stats)
+    }
 }
 
 impl MetaClient {
diff --git a/src/meta-srv/src/cluster.rs b/src/meta-srv/src/cluster.rs
index ebad9e3e6b..9a6cecbd36 100644
--- a/src/meta-srv/src/cluster.rs
+++ b/src/meta-srv/src/cluster.rs
@@ -23,6 +23,7 @@ use api::v1::meta::{
     RangeRequest as PbRangeRequest, RangeResponse as PbRangeResponse, ResponseHeader,
 };
 use common_grpc::channel_manager::ChannelManager;
+use common_meta::datanode::{DatanodeStatKey, DatanodeStatValue};
 use common_meta::kv_backend::{KvBackend, ResettableKvBackendRef, TxnService};
 use common_meta::rpc::store::{
     BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse, BatchPutRequest,
@@ -37,7 +38,6 @@ use snafu::{ensure, OptionExt, ResultExt};
 
 use crate::error;
 use crate::error::{match_for_io_error, Result};
-use crate::key::{DatanodeStatKey, DatanodeStatValue};
 use crate::metasrv::ElectionRef;
 
 pub type MetaPeerClientRef = Arc<MetaPeerClient>;
@@ -217,7 +217,11 @@ impl MetaPeerClient {
     pub async fn get_node_cnt(&self) -> Result<i32> {
         let kvs = self.get_dn_key_value(true).await?;
         kvs.into_iter()
-            .map(|kv| kv.key.try_into())
+            .map(|kv| {
+                kv.key
+                    .try_into()
+                    .context(error::InvalidDatanodeStatFormatSnafu {})
+            })
             .collect::<Result<HashSet<DatanodeStatKey>>>()
             .map(|hash_set| hash_set.len() as i32)
     }
@@ -319,7 +323,14 @@ impl MetaPeerClient {
 fn to_stat_kv_map(kvs: Vec<KeyValue>) -> Result<HashMap<DatanodeStatKey, DatanodeStatValue>> {
     let mut map = HashMap::with_capacity(kvs.len());
     for kv in kvs {
-        let _ = map.insert(kv.key.try_into()?, kv.value.try_into()?);
+        let _ = map.insert(
+            kv.key
+                .try_into()
+                .context(error::InvalidDatanodeStatFormatSnafu {})?,
+            kv.value
+                .try_into()
+                .context(error::InvalidDatanodeStatFormatSnafu {})?,
+        );
     }
     Ok(map)
 }
@@ -356,12 +367,11 @@ fn need_retry(error: &error::Error) -> bool {
 #[cfg(test)]
 mod tests {
     use api::v1::meta::{Error, ErrorCode, ResponseHeader};
+    use common_meta::datanode::{DatanodeStatKey, DatanodeStatValue, Stat};
     use common_meta::rpc::KeyValue;
 
     use super::{check_resp_header, to_stat_kv_map, Context};
     use crate::error;
-    use crate::handler::node_stat::Stat;
-    use crate::key::{DatanodeStatKey, DatanodeStatValue};
 
     #[test]
     fn test_to_stat_kv_map() {
diff --git a/src/meta-srv/src/error.rs b/src/meta-srv/src/error.rs
index 728f326871..7c8d17cbe0 100644
--- a/src/meta-srv/src/error.rs
+++ b/src/meta-srv/src/error.rs
@@ -285,22 +285,6 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to parse stat key from utf8"))]
-    StatKeyFromUtf8 {
-        #[snafu(source)]
-        error: std::string::FromUtf8Error,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Failed to parse stat value from utf8"))]
-    StatValueFromUtf8 {
-        #[snafu(source)]
-        error: std::string::FromUtf8Error,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
     #[snafu(display("Failed to parse invalid region key from utf8"))]
     InvalidRegionKeyFromUtf8 {
         #[snafu(source)]
@@ -719,6 +703,13 @@ pub enum Error {
         source: common_meta::error::Error,
     },
 
+    #[snafu(display("Invalid datanode stat format"))]
+    InvalidDatanodeStatFormat {
+        #[snafu(implicit)]
+        location: Location,
+        source: common_meta::error::Error,
+    },
+
     #[snafu(display("Failed to serialize options to TOML"))]
     TomlFormat {
         #[snafu(implicit)]
@@ -815,8 +806,6 @@ impl ErrorExt for Error {
             | Error::TomlFormat { .. } => StatusCode::InvalidArguments,
             Error::LeaseKeyFromUtf8 { .. }
             | Error::LeaseValueFromUtf8 { .. }
-            | Error::StatKeyFromUtf8 { .. }
-            | Error::StatValueFromUtf8 { .. }
             | Error::InvalidRegionKeyFromUtf8 { .. }
             | Error::TableRouteNotFound { .. }
             | Error::TableInfoNotFound { .. }
@@ -830,7 +819,8 @@ impl ErrorExt for Error {
             | Error::MigrationRunning { .. } => StatusCode::Unexpected,
             Error::TableNotFound { .. } => StatusCode::TableNotFound,
             Error::SaveClusterInfo { source, .. }
-            | Error::InvalidClusterInfoFormat { source, .. } => source.status_code(),
+            | Error::InvalidClusterInfoFormat { source, .. }
+            | Error::InvalidDatanodeStatFormat { source, .. } => source.status_code(),
             Error::InvalidateTableCache { source, .. } => source.status_code(),
             Error::SubmitProcedure { source, .. }
             | Error::WaitProcedure { source, .. }
diff --git a/src/meta-srv/src/handler.rs b/src/meta-srv/src/handler.rs
index d74d49027d..063d3939c1 100644
--- a/src/meta-srv/src/handler.rs
+++ b/src/meta-srv/src/handler.rs
@@ -22,6 +22,7 @@ use api::v1::meta::{
     HeartbeatRequest, HeartbeatResponse, MailboxMessage, RegionLease, RequestHeader,
     ResponseHeader, Role, PROTOCOL_VERSION,
 };
+use common_meta::datanode::Stat;
 use common_meta::instruction::{Instruction, InstructionReply};
 use common_meta::sequence::Sequence;
 use common_telemetry::{debug, info, warn};
@@ -32,7 +33,6 @@ use store_api::storage::RegionId;
 use tokio::sync::mpsc::Sender;
 use tokio::sync::{oneshot, Notify, RwLock};
 
-use self::node_stat::Stat;
 use crate::error::{self, DeserializeFromJsonSnafu, Result, UnexpectedInstructionReplySnafu};
 use crate::metasrv::Context;
 use crate::metrics::{METRIC_META_HANDLER_EXECUTE, METRIC_META_HEARTBEAT_CONNECTION_NUM};
@@ -48,7 +48,6 @@ pub mod failure_handler;
 pub mod filter_inactive_region_stats;
 pub mod keep_lease_handler;
 pub mod mailbox_handler;
-pub mod node_stat;
 pub mod on_leader_start_handler;
 pub mod publish_heartbeat_handler;
 pub mod region_lease_handler;
diff --git a/src/meta-srv/src/handler/collect_stats_handler.rs b/src/meta-srv/src/handler/collect_stats_handler.rs
index ec9fa231e1..b399b961ee 100644
--- a/src/meta-srv/src/handler/collect_stats_handler.rs
+++ b/src/meta-srv/src/handler/collect_stats_handler.rs
@@ -15,6 +15,7 @@
 use std::cmp::Ordering;
 
 use api::v1::meta::{HeartbeatRequest, Role};
+use common_meta::datanode::{DatanodeStatKey, DatanodeStatValue, Stat};
 use common_meta::instruction::CacheIdent;
 use common_meta::key::node_address::{NodeAddressKey, NodeAddressValue};
 use common_meta::key::{MetadataKey, MetadataValue};
@@ -25,9 +26,7 @@ use dashmap::DashMap;
 use snafu::ResultExt;
 
 use crate::error::{self, Result};
-use crate::handler::node_stat::Stat;
 use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
-use crate::key::{DatanodeStatKey, DatanodeStatValue};
 use crate::metasrv::Context;
 
 const MAX_CACHED_STATS_PER_KEY: usize = 10;
@@ -138,7 +137,8 @@ impl HeartbeatHandler for CollectStatsHandler {
         let value: Vec<u8> = DatanodeStatValue {
             stats: epoch_stats.drain_all(),
         }
-        .try_into()?;
+        .try_into()
+        .context(error::InvalidDatanodeStatFormatSnafu {})?;
         let put = PutRequest {
             key,
             value,
@@ -198,6 +198,7 @@ mod tests {
     use std::sync::Arc;
 
     use common_meta::cache_invalidator::DummyCacheInvalidator;
+    use common_meta::datanode::DatanodeStatKey;
     use common_meta::key::TableMetadataManager;
     use common_meta::kv_backend::memory::MemoryKvBackend;
     use common_meta::sequence::SequenceBuilder;
@@ -205,7 +206,6 @@ mod tests {
     use super::*;
     use crate::cluster::MetaPeerClientBuilder;
     use crate::handler::{HeartbeatMailbox, Pushers};
-    use crate::key::DatanodeStatKey;
     use crate::service::store::cached_kv::LeaderCachedKvBackend;
 
     #[tokio::test]
diff --git a/src/meta-srv/src/handler/extract_stat_handler.rs b/src/meta-srv/src/handler/extract_stat_handler.rs
index c23e78314f..b31f41e4ef 100644
--- a/src/meta-srv/src/handler/extract_stat_handler.rs
+++ b/src/meta-srv/src/handler/extract_stat_handler.rs
@@ -13,9 +13,9 @@
 // limitations under the License.
 
 use api::v1::meta::{HeartbeatRequest, Role};
+use common_meta::datanode::Stat;
 use common_telemetry::{info, warn};
 
-use super::node_stat::Stat;
 use crate::error::Result;
 use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
 use crate::metasrv::Context;
diff --git a/src/meta-srv/src/handler/failure_handler.rs b/src/meta-srv/src/handler/failure_handler.rs
index ebeeaf6b7f..02f423c4b4 100644
--- a/src/meta-srv/src/handler/failure_handler.rs
+++ b/src/meta-srv/src/handler/failure_handler.rs
@@ -64,12 +64,12 @@ impl HeartbeatHandler for RegionFailureHandler {
 mod tests {
     use api::v1::meta::HeartbeatRequest;
     use common_catalog::consts::default_engine;
+    use common_meta::datanode::{RegionStat, Stat};
     use store_api::region_engine::RegionRole;
     use store_api::storage::RegionId;
     use tokio::sync::oneshot;
 
     use crate::handler::failure_handler::RegionFailureHandler;
-    use crate::handler::node_stat::{RegionStat, Stat};
     use crate::handler::{HeartbeatAccumulator, HeartbeatHandler};
     use crate::metasrv::builder::MetasrvBuilder;
     use crate::region::supervisor::tests::new_test_supervisor;
@@ -93,7 +93,9 @@ mod tests {
                 approximate_bytes: 0,
                 engine: default_engine().to_string(),
                 role: RegionRole::Follower,
-                extensions: Default::default(),
+                memtable_size: 0,
+                manifest_size: 0,
+                sst_size: 0,
             }
         }
         acc.stat = Some(Stat {
diff --git a/src/meta-srv/src/handler/node_stat.rs b/src/meta-srv/src/handler/node_stat.rs
deleted file mode 100644
index 3a6c6355cd..0000000000
--- a/src/meta-srv/src/handler/node_stat.rs
+++ /dev/null
@@ -1,170 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::{HashMap, HashSet};
-
-use api::v1::meta::{HeartbeatRequest, RequestHeader};
-use common_meta::ClusterId;
-use common_time::util as time_util;
-use serde::{Deserialize, Serialize};
-use store_api::region_engine::RegionRole;
-use store_api::storage::RegionId;
-use table::metadata::TableId;
-
-use crate::key::DatanodeStatKey;
-
-#[derive(Debug, Clone, Default, Serialize, Deserialize)]
-pub struct Stat {
-    pub timestamp_millis: i64,
-    pub cluster_id: ClusterId,
-    // The datanode Id.
-    pub id: u64,
-    // The datanode address.
-    pub addr: String,
-    /// The read capacity units during this period
-    pub rcus: i64,
-    /// The write capacity units during this period
-    pub wcus: i64,
-    /// How many regions on this node
-    pub region_num: u64,
-    pub region_stats: Vec<RegionStat>,
-    // The node epoch is used to check whether the node has restarted or redeployed.
-    pub node_epoch: u64,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct RegionStat {
-    /// The region_id.
-    pub id: RegionId,
-    /// The read capacity units during this period
-    pub rcus: i64,
-    /// The write capacity units during this period
-    pub wcus: i64,
-    /// Approximate bytes of this region
-    pub approximate_bytes: i64,
-    /// The engine name.
-    pub engine: String,
-    /// The region role.
-    pub role: RegionRole,
-    /// The extension info of this region
-    pub extensions: HashMap<String, Vec<u8>>,
-}
-
-impl Stat {
-    #[inline]
-    pub fn is_empty(&self) -> bool {
-        self.region_stats.is_empty()
-    }
-
-    pub fn stat_key(&self) -> DatanodeStatKey {
-        DatanodeStatKey {
-            cluster_id: self.cluster_id,
-            node_id: self.id,
-        }
-    }
-
-    /// Returns a tuple array containing [RegionId] and [RegionRole].
-    pub fn regions(&self) -> Vec<(RegionId, RegionRole)> {
-        self.region_stats.iter().map(|s| (s.id, s.role)).collect()
-    }
-
-    /// Returns all table ids in the region stats.
-    pub fn table_ids(&self) -> HashSet<TableId> {
-        self.region_stats.iter().map(|s| s.id.table_id()).collect()
-    }
-
-    pub fn retain_active_region_stats(&mut self, inactive_region_ids: &HashSet<RegionId>) {
-        if inactive_region_ids.is_empty() {
-            return;
-        }
-
-        self.region_stats
-            .retain(|r| !inactive_region_ids.contains(&r.id));
-        self.rcus = self.region_stats.iter().map(|s| s.rcus).sum();
-        self.wcus = self.region_stats.iter().map(|s| s.wcus).sum();
-        self.region_num = self.region_stats.len() as u64;
-    }
-}
-
-impl TryFrom<&HeartbeatRequest> for Stat {
-    type Error = Option<RequestHeader>;
-
-    fn try_from(value: &HeartbeatRequest) -> Result<Self, Self::Error> {
-        let HeartbeatRequest {
-            header,
-            peer,
-            region_stats,
-            node_epoch,
-            ..
-        } = value;
-
-        match (header, peer) {
-            (Some(header), Some(peer)) => {
-                let region_stats = region_stats
-                    .iter()
-                    .map(RegionStat::from)
-                    .collect::<Vec<_>>();
-
-                Ok(Self {
-                    timestamp_millis: time_util::current_time_millis(),
-                    cluster_id: header.cluster_id,
-                    // datanode id
-                    id: peer.id,
-                    // datanode address
-                    addr: peer.addr.clone(),
-                    rcus: region_stats.iter().map(|s| s.rcus).sum(),
-                    wcus: region_stats.iter().map(|s| s.wcus).sum(),
-                    region_num: region_stats.len() as u64,
-                    region_stats,
-                    node_epoch: *node_epoch,
-                })
-            }
-            (header, _) => Err(header.clone()),
-        }
-    }
-}
-
-impl From<&api::v1::meta::RegionStat> for RegionStat {
-    fn from(value: &api::v1::meta::RegionStat) -> Self {
-        Self {
-            id: RegionId::from_u64(value.region_id),
-            rcus: value.rcus,
-            wcus: value.wcus,
-            approximate_bytes: value.approximate_bytes,
-            engine: value.engine.to_string(),
-            role: RegionRole::from(value.role()),
-            extensions: value.extensions.clone(),
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::handler::node_stat::Stat;
-
-    #[test]
-    fn test_stat_key() {
-        let stat = Stat {
-            cluster_id: 3,
-            id: 101,
-            region_num: 10,
-            ..Default::default()
-        };
-
-        let stat_key = stat.stat_key();
-
-        assert_eq!(3, stat_key.cluster_id);
-        assert_eq!(101, stat_key.node_id);
-    }
-}
diff --git a/src/meta-srv/src/handler/region_lease_handler.rs b/src/meta-srv/src/handler/region_lease_handler.rs
index 28ddb436e0..06cf818d23 100644
--- a/src/meta-srv/src/handler/region_lease_handler.rs
+++ b/src/meta-srv/src/handler/region_lease_handler.rs
@@ -103,6 +103,7 @@ mod test {
     use std::collections::{HashMap, HashSet};
     use std::sync::Arc;
 
+    use common_meta::datanode::{RegionStat, Stat};
     use common_meta::distributed_time_constants;
     use common_meta::key::table_route::TableRouteValue;
     use common_meta::key::test_utils::new_test_table_info;
@@ -115,7 +116,6 @@ mod test {
     use store_api::storage::RegionId;
 
     use super::*;
-    use crate::handler::node_stat::{RegionStat, Stat};
     use crate::metasrv::builder::MetasrvBuilder;
 
     fn new_test_keeper() -> RegionLeaseKeeper {
@@ -135,7 +135,9 @@ mod test {
             wcus: 0,
             approximate_bytes: 0,
             engine: String::new(),
-            extensions: Default::default(),
+            memtable_size: 0,
+            manifest_size: 0,
+            sst_size: 0,
         }
     }
 
diff --git a/src/meta-srv/src/key/datanode.rs b/src/meta-srv/src/key/datanode.rs
index 4fe55685b2..1c4583c233 100644
--- a/src/meta-srv/src/key/datanode.rs
+++ b/src/meta-srv/src/key/datanode.rs
@@ -14,6 +14,7 @@
 
 use std::str::FromStr;
 
+use common_meta::datanode::DatanodeStatKey;
 use common_meta::ClusterId;
 use lazy_static::lazy_static;
 use regex::Regex;
@@ -22,7 +23,6 @@ use snafu::{ensure, OptionExt, ResultExt};
 
 use crate::error;
 use crate::error::Result;
-use crate::handler::node_stat::Stat;
 
 pub(crate) const DATANODE_LEASE_PREFIX: &str = "__meta_datanode_lease";
 const INACTIVE_REGION_PREFIX: &str = "__meta_inactive_region";
@@ -52,18 +52,6 @@ impl DatanodeLeaseKey {
     }
 }
 
-#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
-pub struct DatanodeStatKey {
-    pub cluster_id: ClusterId,
-    pub node_id: u64,
-}
-
-impl DatanodeStatKey {
-    pub fn prefix_key() -> Vec<u8> {
-        format!("{DATANODE_STAT_PREFIX}-").into_bytes()
-    }
-}
-
 impl From<&DatanodeLeaseKey> for DatanodeStatKey {
     fn from(lease_key: &DatanodeLeaseKey) -> Self {
         DatanodeStatKey {
@@ -73,100 +61,6 @@ impl From<&DatanodeLeaseKey> for DatanodeStatKey {
     }
 }
 
-impl From<DatanodeStatKey> for Vec<u8> {
-    fn from(value: DatanodeStatKey) -> Self {
-        format!(
-            "{}-{}-{}",
-            DATANODE_STAT_PREFIX, value.cluster_id, value.node_id
-        )
-        .into_bytes()
-    }
-}
-
-impl FromStr for DatanodeStatKey {
-    type Err = error::Error;
-
-    fn from_str(key: &str) -> Result<Self> {
-        let caps = DATANODE_STAT_KEY_PATTERN
-            .captures(key)
-            .context(error::InvalidStatKeySnafu { key })?;
-
-        ensure!(caps.len() == 3, error::InvalidStatKeySnafu { key });
-
-        let cluster_id = caps[1].to_string();
-        let node_id = caps[2].to_string();
-        let cluster_id: u64 = cluster_id.parse().context(error::ParseNumSnafu {
-            err_msg: format!("invalid cluster_id: {cluster_id}"),
-        })?;
-        let node_id: u64 = node_id.parse().context(error::ParseNumSnafu {
-            err_msg: format!("invalid node_id: {node_id}"),
-        })?;
-
-        Ok(Self {
-            cluster_id,
-            node_id,
-        })
-    }
-}
-
-impl TryFrom<Vec<u8>> for DatanodeStatKey {
-    type Error = error::Error;
-
-    fn try_from(bytes: Vec<u8>) -> Result<Self> {
-        String::from_utf8(bytes)
-            .context(error::StatKeyFromUtf8Snafu {})
-            .map(|x| x.parse())?
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(transparent)]
-pub struct DatanodeStatValue {
-    pub stats: Vec<Stat>,
-}
-
-impl DatanodeStatValue {
-    /// Get the latest number of regions.
-    pub fn region_num(&self) -> Option<u64> {
-        self.stats.last().map(|x| x.region_num)
-    }
-
-    /// Get the latest node addr.
-    pub fn node_addr(&self) -> Option<String> {
-        self.stats.last().map(|x| x.addr.clone())
-    }
-}
-
-impl TryFrom<DatanodeStatValue> for Vec<u8> {
-    type Error = error::Error;
-
-    fn try_from(stats: DatanodeStatValue) -> Result<Self> {
-        Ok(serde_json::to_string(&stats)
-            .context(error::SerializeToJsonSnafu {
-                input: format!("{stats:?}"),
-            })?
-            .into_bytes())
-    }
-}
-
-impl FromStr for DatanodeStatValue {
-    type Err = error::Error;
-
-    fn from_str(value: &str) -> Result<Self> {
-        serde_json::from_str(value).context(error::DeserializeFromJsonSnafu { input: value })
-    }
-}
-
-impl TryFrom<Vec<u8>> for DatanodeStatValue {
-    type Error = error::Error;
-
-    fn try_from(value: Vec<u8>) -> Result<Self> {
-        String::from_utf8(value)
-            .context(error::StatValueFromUtf8Snafu {})
-            .map(|x| x.parse())?
-    }
-}
-
 #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Serialize, Deserialize)]
 pub struct InactiveRegionKey {
     pub cluster_id: ClusterId,
@@ -253,29 +147,6 @@ mod tests {
         assert_eq!(1, new_key.node_id);
     }
 
-    #[test]
-    fn test_stat_val_round_trip() {
-        let stat = Stat {
-            cluster_id: 0,
-            id: 101,
-            region_num: 100,
-            ..Default::default()
-        };
-
-        let stat_val = DatanodeStatValue { stats: vec![stat] };
-
-        let bytes: Vec<u8> = stat_val.try_into().unwrap();
-        let stat_val: DatanodeStatValue = bytes.try_into().unwrap();
-        let stats = stat_val.stats;
-
-        assert_eq!(1, stats.len());
-
-        let stat = stats.first().unwrap();
-        assert_eq!(0, stat.cluster_id);
-        assert_eq!(101, stat.id);
-        assert_eq!(100, stat.region_num);
-    }
-
     #[test]
     fn test_lease_key_round_trip() {
         let key = DatanodeLeaseKey {
@@ -289,67 +160,6 @@ mod tests {
         assert_eq!(new_key, key);
     }
 
-    #[test]
-    fn test_get_addr_from_stat_val() {
-        let empty = DatanodeStatValue { stats: vec![] };
-        let addr = empty.node_addr();
-        assert!(addr.is_none());
-
-        let stat_val = DatanodeStatValue {
-            stats: vec![
-                Stat {
-                    addr: "1".to_string(),
-                    ..Default::default()
-                },
-                Stat {
-                    addr: "2".to_string(),
-                    ..Default::default()
-                },
-                Stat {
-                    addr: "3".to_string(),
-                    ..Default::default()
-                },
-            ],
-        };
-        let addr = stat_val.node_addr().unwrap();
-        assert_eq!("3", addr);
-    }
-
-    #[test]
-    fn test_get_region_num_from_stat_val() {
-        let empty = DatanodeStatValue { stats: vec![] };
-        let region_num = empty.region_num();
-        assert!(region_num.is_none());
-
-        let wrong = DatanodeStatValue {
-            stats: vec![Stat {
-                region_num: 0,
-                ..Default::default()
-            }],
-        };
-        let right = wrong.region_num();
-        assert_eq!(Some(0), right);
-
-        let stat_val = DatanodeStatValue {
-            stats: vec![
-                Stat {
-                    region_num: 1,
-                    ..Default::default()
-                },
-                Stat {
-                    region_num: 0,
-                    ..Default::default()
-                },
-                Stat {
-                    region_num: 2,
-                    ..Default::default()
-                },
-            ],
-        };
-        let region_num = stat_val.region_num().unwrap();
-        assert_eq!(2, region_num);
-    }
-
     #[test]
     fn test_lease_key_to_stat_key() {
         let lease_key = DatanodeLeaseKey {
diff --git a/src/meta-srv/src/region/supervisor.rs b/src/meta-srv/src/region/supervisor.rs
index 79b305bb80..8367acbb26 100644
--- a/src/meta-srv/src/region/supervisor.rs
+++ b/src/meta-srv/src/region/supervisor.rs
@@ -16,6 +16,7 @@ use std::fmt::Debug;
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
+use common_meta::datanode::Stat;
 use common_meta::ddl::{DetectingRegion, RegionFailureDetectorController};
 use common_meta::key::MAINTENANCE_KEY;
 use common_meta::kv_backend::KvBackendRef;
@@ -32,7 +33,6 @@ use tokio::time::{interval, MissedTickBehavior};
 
 use crate::error::{self, Result};
 use crate::failure_detector::PhiAccrualFailureDetectorOptions;
-use crate::handler::node_stat::Stat;
 use crate::metasrv::{SelectorContext, SelectorRef};
 use crate::procedure::region_migration::manager::RegionMigrationManagerRef;
 use crate::procedure::region_migration::RegionMigrationProcedureTask;
diff --git a/src/meta-srv/src/selector/load_based.rs b/src/meta-srv/src/selector/load_based.rs
index 12dea5854b..d7b650deda 100644
--- a/src/meta-srv/src/selector/load_based.rs
+++ b/src/meta-srv/src/selector/load_based.rs
@@ -14,6 +14,7 @@
 
 use std::collections::HashMap;
 
+use common_meta::datanode::{DatanodeStatKey, DatanodeStatValue};
 use common_meta::key::TableMetadataManager;
 use common_meta::peer::Peer;
 use common_meta::rpc::router::find_leaders;
@@ -23,7 +24,7 @@ use snafu::ResultExt;
 use table::metadata::TableId;
 
 use crate::error::{self, Result};
-use crate::key::{DatanodeLeaseKey, DatanodeStatKey, DatanodeStatValue, LeaseValue};
+use crate::key::{DatanodeLeaseKey, LeaseValue};
 use crate::lease;
 use crate::metasrv::SelectorContext;
 use crate::selector::common::choose_peers;
@@ -162,7 +163,9 @@ async fn get_leader_peer_ids(
 mod tests {
     use std::collections::HashMap;
 
-    use crate::key::{DatanodeLeaseKey, DatanodeStatKey, DatanodeStatValue, LeaseValue};
+    use common_meta::datanode::{DatanodeStatKey, DatanodeStatValue};
+
+    use crate::key::{DatanodeLeaseKey, LeaseValue};
     use crate::selector::load_based::filter_out_expired_datanode;
 
     #[test]
diff --git a/src/meta-srv/src/selector/weight_compute.rs b/src/meta-srv/src/selector/weight_compute.rs
index c8c555d204..09d8833e2e 100644
--- a/src/meta-srv/src/selector/weight_compute.rs
+++ b/src/meta-srv/src/selector/weight_compute.rs
@@ -14,10 +14,10 @@
 
 use std::collections::HashMap;
 
+use common_meta::datanode::{DatanodeStatKey, DatanodeStatValue};
 use common_meta::peer::Peer;
 use itertools::{Itertools, MinMaxResult};
 
-use crate::key::{DatanodeStatKey, DatanodeStatValue};
 use crate::selector::weighted_choose::WeightedItem;
 
 /// The [`WeightCompute`] trait is used to compute the weight array by heartbeats.
@@ -95,13 +95,12 @@ impl WeightCompute for RegionNumsBasedWeightCompute {
 mod tests {
     use std::collections::HashMap;
 
+    use common_meta::datanode::{DatanodeStatKey, DatanodeStatValue, RegionStat, Stat};
     use common_meta::peer::Peer;
     use store_api::region_engine::RegionRole;
     use store_api::storage::RegionId;
 
     use super::{RegionNumsBasedWeightCompute, WeightCompute};
-    use crate::handler::node_stat::{RegionStat, Stat};
-    use crate::key::{DatanodeStatKey, DatanodeStatValue};
 
     #[test]
     fn test_weight_compute() {
@@ -199,7 +198,9 @@ mod tests {
                 approximate_bytes: 1,
                 engine: "mito2".to_string(),
                 role: RegionRole::Leader,
-                extensions: Default::default(),
+                memtable_size: 0,
+                manifest_size: 0,
+                sst_size: 0,
             }],
             ..Default::default()
         }
@@ -216,7 +217,9 @@ mod tests {
                 approximate_bytes: 1,
                 engine: "mito2".to_string(),
                 role: RegionRole::Leader,
-                extensions: Default::default(),
+                memtable_size: 0,
+                manifest_size: 0,
+                sst_size: 0,
             }],
             ..Default::default()
         }
@@ -233,7 +236,9 @@ mod tests {
                 approximate_bytes: 1,
                 engine: "mito2".to_string(),
                 role: RegionRole::Leader,
-                extensions: Default::default(),
+                memtable_size: 0,
+                manifest_size: 0,
+                sst_size: 0,
             }],
             ..Default::default()
         }
diff --git a/src/meta-srv/src/service/admin/heartbeat.rs b/src/meta-srv/src/service/admin/heartbeat.rs
index aa4b7cc96d..078d6cdc77 100644
--- a/src/meta-srv/src/service/admin/heartbeat.rs
+++ b/src/meta-srv/src/service/admin/heartbeat.rs
@@ -14,13 +14,13 @@
 
 use std::collections::HashMap;
 
+use common_meta::datanode::DatanodeStatValue;
 use serde::{Deserialize, Serialize};
 use snafu::ResultExt;
 use tonic::codegen::http;
 
 use crate::cluster::MetaPeerClientRef;
 use crate::error::{self, Result};
-use crate::key::DatanodeStatValue;
 use crate::service::admin::{util, HttpHandler};
 
 #[derive(Clone)]
@@ -85,8 +85,8 @@ fn filter_by_addr(stat_vals: Vec<DatanodeStatValue>, addr: &str) -> Vec<Datanode
 
 #[cfg(test)]
 mod tests {
-    use crate::handler::node_stat::Stat;
-    use crate::key::DatanodeStatValue;
+    use common_meta::datanode::{DatanodeStatValue, Stat};
+
     use crate::service::admin::heartbeat::filter_by_addr;
 
     #[tokio::test]
diff --git a/src/metric-engine/src/engine.rs b/src/metric-engine/src/engine.rs
index 08414a97e4..358da1d216 100644
--- a/src/metric-engine/src/engine.rs
+++ b/src/metric-engine/src/engine.rs
@@ -36,7 +36,9 @@ use mito2::engine::MitoEngine;
 use snafu::ResultExt;
 use store_api::metadata::RegionMetadataRef;
 use store_api::metric_engine_consts::METRIC_ENGINE_NAME;
-use store_api::region_engine::{RegionEngine, RegionRole, RegionScannerRef, SetReadonlyResponse};
+use store_api::region_engine::{
+    RegionEngine, RegionRole, RegionScannerRef, RegionStatistic, SetReadonlyResponse,
+};
 use store_api::region_request::RegionRequest;
 use store_api::storage::{RegionId, ScanRequest};
 
@@ -185,9 +187,9 @@ impl RegionEngine for MetricEngine {
     /// Retrieves region's disk usage.
     ///
     /// Note: Returns `None` if it's a logical region.
-    fn region_disk_usage(&self, region_id: RegionId) -> Option<i64> {
+    fn region_statistic(&self, region_id: RegionId) -> Option<RegionStatistic> {
         if self.inner.is_physical_region(region_id) {
-            self.inner.mito.region_disk_usage(region_id)
+            self.inner.mito.region_statistic(region_id)
         } else {
             None
         }
@@ -377,7 +379,7 @@ mod test {
         let logical_region_id = env.default_logical_region_id();
         let physical_region_id = env.default_physical_region_id();
 
-        assert!(env.metric().region_disk_usage(logical_region_id).is_none());
-        assert!(env.metric().region_disk_usage(physical_region_id).is_some());
+        assert!(env.metric().region_statistic(logical_region_id).is_none());
+        assert!(env.metric().region_statistic(physical_region_id).is_some());
     }
 }
diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs
index c0655eb690..e9177d40bf 100644
--- a/src/mito2/src/engine.rs
+++ b/src/mito2/src/engine.rs
@@ -76,7 +76,8 @@ use store_api::logstore::provider::Provider;
 use store_api::logstore::LogStore;
 use store_api::metadata::RegionMetadataRef;
 use store_api::region_engine::{
-    BatchResponses, RegionEngine, RegionRole, RegionScannerRef, SetReadonlyResponse,
+    BatchResponses, RegionEngine, RegionRole, RegionScannerRef, RegionStatistic,
+    SetReadonlyResponse,
 };
 use store_api::region_request::{AffectedRows, RegionOpenRequest, RegionRequest};
 use store_api::storage::{RegionId, ScanRequest};
@@ -89,7 +90,6 @@ use crate::error::{
 use crate::manifest::action::RegionEdit;
 use crate::metrics::HANDLE_REQUEST_ELAPSED;
 use crate::read::scan_region::{ScanParallism, ScanRegion, Scanner};
-use crate::region::RegionUsage;
 use crate::request::{RegionEditRequest, WorkerRequest};
 use crate::wal::entry_distributor::{
     build_wal_entry_distributor_and_receivers, DEFAULT_ENTRY_RECEIVER_BUFFER_SIZE,
@@ -133,15 +133,12 @@ impl MitoEngine {
         self.inner.workers.is_region_opening(region_id)
     }
 
-    /// Returns the region disk/memory usage information.
-    pub fn get_region_usage(&self, region_id: RegionId) -> Result<RegionUsage> {
-        let region = self
-            .inner
+    /// Returns the region disk/memory statistic.
+    pub fn get_region_statistic(&self, region_id: RegionId) -> Option<RegionStatistic> {
+        self.inner
             .workers
             .get_region(region_id)
-            .context(RegionNotFoundSnafu { region_id })?;
-
-        Ok(region.region_usage())
+            .map(|region| region.region_statistic())
     }
 
     /// Handle substrait query and return a stream of record batches
@@ -546,12 +543,8 @@ impl RegionEngine for MitoEngine {
         self.inner.stop().await.map_err(BoxedError::new)
     }
 
-    fn region_disk_usage(&self, region_id: RegionId) -> Option<i64> {
-        let size = self
-            .get_region_usage(region_id)
-            .map(|usage| usage.disk_usage())
-            .ok()?;
-        size.try_into().ok()
+    fn region_statistic(&self, region_id: RegionId) -> Option<RegionStatistic> {
+        self.get_region_statistic(region_id)
     }
 
     fn set_writable(&self, region_id: RegionId, writable: bool) -> Result<(), BoxedError> {
diff --git a/src/mito2/src/engine/basic_test.rs b/src/mito2/src/engine/basic_test.rs
index 9179d8a074..533b6a2ea1 100644
--- a/src/mito2/src/engine/basic_test.rs
+++ b/src/mito2/src/engine/basic_test.rs
@@ -552,8 +552,8 @@ async fn test_region_usage() {
         .unwrap();
     // region is empty now, check manifest size
     let region = engine.get_region(region_id).unwrap();
-    let region_stat = region.region_usage();
-    assert_eq!(region_stat.manifest_usage, 686);
+    let region_stat = region.region_statistic();
+    assert_eq!(region_stat.manifest_size, 686);
 
     // put some rows
     let rows = Rows {
@@ -563,8 +563,8 @@ async fn test_region_usage() {
 
     put_rows(&engine, region_id, rows).await;
 
-    let region_stat = region.region_usage();
-    assert!(region_stat.wal_usage > 0);
+    let region_stat = region.region_statistic();
+    assert!(region_stat.wal_size > 0);
 
     // delete some rows
     let rows = Rows {
@@ -573,18 +573,18 @@ async fn test_region_usage() {
     };
     delete_rows(&engine, region_id, rows).await;
 
-    let region_stat = region.region_usage();
-    assert!(region_stat.wal_usage > 0);
+    let region_stat = region.region_statistic();
+    assert!(region_stat.wal_size > 0);
 
     // flush region
     flush_region(&engine, region_id, None).await;
 
-    let region_stat = region.region_usage();
-    assert_eq!(region_stat.sst_usage, 3010);
+    let region_stat = region.region_statistic();
+    assert_eq!(region_stat.sst_size, 3010);
 
     // region total usage
     // Some memtables may share items.
-    assert!(region_stat.disk_usage() >= 4028);
+    assert!(region_stat.estimated_disk_size() >= 4028);
 }
 
 #[tokio::test]
diff --git a/src/mito2/src/region.rs b/src/mito2/src/region.rs
index e2a4801643..8fc9095ae5 100644
--- a/src/mito2/src/region.rs
+++ b/src/mito2/src/region.rs
@@ -28,6 +28,7 @@ use crossbeam_utils::atomic::AtomicCell;
 use snafu::{ensure, OptionExt};
 use store_api::logstore::provider::Provider;
 use store_api::metadata::RegionMetadataRef;
+use store_api::region_engine::RegionStatistic;
 use store_api::storage::RegionId;
 
 use crate::access_layer::AccessLayerRef;
@@ -252,10 +253,8 @@ impl MitoRegion {
         }
     }
 
-    /// Returns the region usage in bytes.
-    pub(crate) fn region_usage(&self) -> RegionUsage {
-        let region_id = self.region_id;
-
+    /// Returns the region statistic.
+    pub(crate) fn region_statistic(&self) -> RegionStatistic {
         let version = self.version();
         let memtables = &version.memtables;
         let memtable_usage = (memtables.mutable_usage() + memtables.immutables_usage()) as u64;
@@ -265,11 +264,11 @@ impl MitoRegion {
         let wal_usage = self.estimated_wal_usage(memtable_usage);
         let manifest_usage = self.stats.total_manifest_size();
 
-        RegionUsage {
-            region_id,
-            wal_usage,
-            sst_usage,
-            manifest_usage,
+        RegionStatistic {
+            memtable_size: memtable_usage,
+            wal_size: wal_usage,
+            manifest_size: manifest_usage,
+            sst_size: sst_usage,
         }
     }
 
diff --git a/src/query/src/optimizer/test_util.rs b/src/query/src/optimizer/test_util.rs
index 773270351f..6b14a5af52 100644
--- a/src/query/src/optimizer/test_util.rs
+++ b/src/query/src/optimizer/test_util.rs
@@ -27,7 +27,9 @@ use datatypes::schema::ColumnSchema;
 use store_api::metadata::{
     ColumnMetadata, RegionMetadata, RegionMetadataBuilder, RegionMetadataRef,
 };
-use store_api::region_engine::{RegionEngine, RegionRole, RegionScannerRef, SetReadonlyResponse};
+use store_api::region_engine::{
+    RegionEngine, RegionRole, RegionScannerRef, RegionStatistic, SetReadonlyResponse,
+};
 use store_api::region_request::RegionRequest;
 use store_api::storage::{ConcreteDataType, RegionId, ScanRequest};
 
@@ -79,7 +81,7 @@ impl RegionEngine for MetaRegionEngine {
         })
     }
 
-    fn region_disk_usage(&self, _region_id: RegionId) -> Option<i64> {
+    fn region_statistic(&self, _region_id: RegionId) -> Option<RegionStatistic> {
         None
     }
 
diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs
index 84555a595b..483d3cc1ad 100644
--- a/src/store-api/src/region_engine.rs
+++ b/src/store-api/src/region_engine.rs
@@ -242,6 +242,42 @@ pub type RegionScannerRef = Box<dyn RegionScanner>;
 
 pub type BatchResponses = Vec<(RegionId, Result<RegionResponse, BoxedError>)>;
 
+/// Represents the statistics of a region.
+#[derive(Debug, Deserialize, Serialize, Default)]
+pub struct RegionStatistic {
+    /// The size of memtable in bytes.
+    pub memtable_size: u64,
+    /// The size of WAL in bytes.
+    pub wal_size: u64,
+    /// The size of manifest in bytes.
+    pub manifest_size: u64,
+    /// The size of SST files in bytes.
+    pub sst_size: u64,
+}
+
+impl RegionStatistic {
+    /// Deserializes the region statistic to a byte array.
+    ///
+    /// Returns None if the deserialization fails.
+    pub fn deserialize_from_slice(value: &[u8]) -> Option<RegionStatistic> {
+        serde_json::from_slice(value).ok()
+    }
+
+    /// Serializes the region statistic to a byte array.
+    ///
+    /// Returns None if the serialization fails.
+    pub fn serialize_to_vec(&self) -> Option<Vec<u8>> {
+        serde_json::to_vec(self).ok()
+    }
+}
+
+impl RegionStatistic {
+    /// Returns the estimated disk size of the region.
+    pub fn estimated_disk_size(&self) -> u64 {
+        self.wal_size + self.sst_size + self.manifest_size
+    }
+}
+
 #[async_trait]
 pub trait RegionEngine: Send + Sync {
     /// Name of this engine
@@ -289,8 +325,8 @@ pub trait RegionEngine: Send + Sync {
     /// Retrieves region's metadata.
     async fn get_metadata(&self, region_id: RegionId) -> Result<RegionMetadataRef, BoxedError>;
 
-    /// Retrieves region's disk usage.
-    fn region_disk_usage(&self, region_id: RegionId) -> Option<i64>;
+    /// Retrieves region's statistic.
+    fn region_statistic(&self, region_id: RegionId) -> Option<RegionStatistic>;
 
     /// Stops the engine
     async fn stop(&self) -> Result<(), BoxedError>;
diff --git a/tests/cases/standalone/common/show/show_databases_tables.result b/tests/cases/standalone/common/show/show_databases_tables.result
index c92a5b33c8..6dc267b61b 100644
--- a/tests/cases/standalone/common/show/show_databases_tables.result
+++ b/tests/cases/standalone/common/show/show_databases_tables.result
@@ -49,6 +49,7 @@ SHOW TABLES;
 | profiling                             |
 | referential_constraints               |
 | region_peers                          |
+| region_statistics                     |
 | routines                              |
 | runtime_metrics                       |
 | schema_privileges                     |
@@ -96,6 +97,7 @@ SHOW FULL TABLES;
 | profiling                             | LOCAL TEMPORARY |
 | referential_constraints               | LOCAL TEMPORARY |
 | region_peers                          | LOCAL TEMPORARY |
+| region_statistics                     | LOCAL TEMPORARY |
 | routines                              | LOCAL TEMPORARY |
 | runtime_metrics                       | LOCAL TEMPORARY |
 | schema_privileges                     | LOCAL TEMPORARY |
@@ -137,6 +139,7 @@ SHOW TABLE STATUS;
 |profiling||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0|||
 |referential_constraints||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0|||
 |region_peers||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0|||
+|region_statistics||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0|||
 |routines||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0|||
 |runtime_metrics||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0|||
 |schema_privileges||11|Fixed|0|0|0|0|0|0|0|DATETIME|||utf8_bin|0|||
diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result
index 03b65cf390..882607f842 100644
--- a/tests/cases/standalone/common/system/information_schema.result
+++ b/tests/cases/standalone/common/system/information_schema.result
@@ -36,6 +36,7 @@ order by table_schema, table_name;
 |greptime|information_schema|profiling|LOCALTEMPORARY|19|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y|
 |greptime|information_schema|referential_constraints|LOCALTEMPORARY|20|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y|
 |greptime|information_schema|region_peers|LOCALTEMPORARY|29|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y|
+|greptime|information_schema|region_statistics|LOCALTEMPORARY|35|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y|
 |greptime|information_schema|routines|LOCALTEMPORARY|21|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y|
 |greptime|information_schema|runtime_metrics|LOCALTEMPORARY|27|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y|
 |greptime|information_schema|schema_privileges|LOCALTEMPORARY|22|0|0|0|0|0||11|Fixed|0|0|0|DATETIME|||utf8_bin|0|||Y|
@@ -298,6 +299,14 @@ select * from information_schema.columns order by table_schema, table_name, colu
 | greptime      | information_schema | region_peers                          | peer_id                           | 2                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | Yes         | bigint unsigned |                |        |
 | greptime      | information_schema | region_peers                          | region_id                         | 1                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | No          | bigint unsigned |                |        |
 | greptime      | information_schema | region_peers                          | status                            | 5                | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | Yes         | string          |                |        |
+| greptime      | information_schema | region_statistics                     | engine                            | 7                | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | Yes         | string          |                |        |
+| greptime      | information_schema | region_statistics                     | manifest_size                     | 5                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | Yes         | bigint unsigned |                |        |
+| greptime      | information_schema | region_statistics                     | memtable_size                     | 4                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | Yes         | bigint unsigned |                |        |
+| greptime      | information_schema | region_statistics                     | region_id                         | 1                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | No          | bigint unsigned |                |        |
+| greptime      | information_schema | region_statistics                     | region_number                     | 3                |                          |                        | 10                | 0             |                    |                    |                |            |       | select,insert |                       | UInt32               | int unsigned    | FIELD         |                | No          | int unsigned    |                |        |
+| greptime      | information_schema | region_statistics                     | region_role                       | 8                | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | Yes         | string          |                |        |
+| greptime      | information_schema | region_statistics                     | sst_size                          | 6                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | Yes         | bigint unsigned |                |        |
+| greptime      | information_schema | region_statistics                     | table_id                          | 2                |                          |                        | 10                | 0             |                    |                    |                |            |       | select,insert |                       | UInt32               | int unsigned    | FIELD         |                | No          | int unsigned    |                |        |
 | greptime      | information_schema | routines                              | character_maximum_length          | 7                |                          |                        | 19                | 0             |                    |                    |                |            |       | select,insert |                       | Int64                | bigint          | FIELD         |                | No          | bigint          |                |        |
 | greptime      | information_schema | routines                              | character_octet_length            | 8                |                          |                        | 19                | 0             |                    |                    |                |            |       | select,insert |                       | Int64                | bigint          | FIELD         |                | No          | bigint          |                |        |
 | greptime      | information_schema | routines                              | character_set_client              | 29               | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | No          | string          |                |        |
diff --git a/tests/cases/standalone/common/view/create.result b/tests/cases/standalone/common/view/create.result
index 4fd2f59356..06d892d189 100644
--- a/tests/cases/standalone/common/view/create.result
+++ b/tests/cases/standalone/common/view/create.result
@@ -110,6 +110,7 @@ SELECT * FROM INFORMATION_SCHEMA.TABLES ORDER BY TABLE_NAME, TABLE_TYPE;
 |greptime|information_schema|profiling|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y|
 |greptime|information_schema|referential_constraints|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y|
 |greptime|information_schema|region_peers|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y|
+|greptime|information_schema|region_statistics|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y|
 |greptime|information_schema|routines|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y|
 |greptime|information_schema|runtime_metrics|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y|
 |greptime|information_schema|schema_privileges|LOCALTEMPORARY|ID|ID|ID|ID|ID|ID||ID|Fixed|ID|ID|ID|DATETIME|||utf8_bin|ID|||Y|

From 934bc139677be572f816ee28e80708cf7ac85eab Mon Sep 17 00:00:00 2001
From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com>
Date: Fri, 27 Sep 2024 19:17:36 +0800
Subject: [PATCH 040/128] feat(mito): limit compaction output file size (#4754)

* Commit Message

 Clarify documentation for CompactionOutput struct

 Updated the documentation for the `CompactionOutput` struct to specify that the output time range is only relevant for windowed compaction.

* Add max_output_file_size to TwcsPicker and TwcsOptions

 - Introduced `max_output_file_size` to `TwcsPicker` struct and its logic to enforce output file size limits during compaction.
 - Updated `TwcsOptions` to include `max_output_file_size` and adjusted related tests.
 - Modified `new_picker` function to initialize `TwcsPicker` with the new `max_output_file_size` field.

* feat/limit-compaction-output-size:
 Refactor compaction picker and TWCS to support append mode and improve options handling

 - Update compaction picker to accept a reference to options and append mode flag
 - Modify TWCS picker logic to consider append mode when filtering deleted rows
 - Remove VersionControl usage in compactor and simplify return type
 - Adjust enforce_max_output_size logic in TWCS picker to handle max output file size
 - Add append mode flag to TwcsPicker struct
 - Fix incorrect condition in TWCS picker for enforcing max output size
 - Update region options tests to reflect new max output file size format (1GB and 7MB)
 - Simplify InvalidTableOptionSnafu error handling in create_parser
 - Add `compaction.twcs.max_output_file_size` to mito engine option keys

* resolve some comments
---
 src/mito2/src/compaction.rs              |   8 +-
 src/mito2/src/compaction/compactor.rs    |   8 +-
 src/mito2/src/compaction/picker.rs       |  21 +--
 src/mito2/src/compaction/twcs.rs         | 198 +++++++++++++++++------
 src/mito2/src/region/options.rs          |   8 +
 src/sql/src/parsers/create_parser.rs     |   7 +-
 src/store-api/src/mito_engine_options.rs |   1 +
 7 files changed, 185 insertions(+), 66 deletions(-)

diff --git a/src/mito2/src/compaction.rs b/src/mito2/src/compaction.rs
index 0f33471b21..3cea492071 100644
--- a/src/mito2/src/compaction.rs
+++ b/src/mito2/src/compaction.rs
@@ -240,7 +240,11 @@ impl CompactionScheduler {
         request: CompactionRequest,
         options: compact_request::Options,
     ) -> Result<()> {
-        let picker = new_picker(options.clone(), &request.current_version.options.compaction);
+        let picker = new_picker(
+            &options,
+            &request.current_version.options.compaction,
+            request.current_version.options.append_mode,
+        );
         let region_id = request.region_id();
         let CompactionRequest {
             engine_config,
@@ -500,7 +504,7 @@ pub struct CompactionOutput {
     pub inputs: Vec<FileHandle>,
     /// Whether to remove deletion markers.
     pub filter_deleted: bool,
-    /// Compaction output time range.
+    /// Compaction output time range. Only windowed compaction specifies output time range.
     pub output_time_range: Option<TimestampRange>,
 }
 
diff --git a/src/mito2/src/compaction/compactor.rs b/src/mito2/src/compaction/compactor.rs
index 9c8c0e02bd..12b9dd5fef 100644
--- a/src/mito2/src/compaction/compactor.rs
+++ b/src/mito2/src/compaction/compactor.rs
@@ -38,7 +38,7 @@ use crate::memtable::MemtableBuilderProvider;
 use crate::read::Source;
 use crate::region::opener::new_manifest_dir;
 use crate::region::options::RegionOptions;
-use crate::region::version::{VersionBuilder, VersionControl, VersionRef};
+use crate::region::version::{VersionBuilder, VersionRef};
 use crate::region::ManifestContext;
 use crate::region::RegionState::Writable;
 use crate::schedule::scheduler::LocalScheduler;
@@ -164,8 +164,7 @@ pub async fn open_compaction_region(
             .compaction_time_window(manifest.compaction_time_window)
             .options(req.region_options.clone())
             .build();
-        let version_control = Arc::new(VersionControl::new(version));
-        version_control.current().version
+        Arc::new(version)
     };
 
     Ok(CompactionRegion {
@@ -395,8 +394,9 @@ impl Compactor for DefaultCompactor {
     ) -> Result<()> {
         let picker_output = {
             let picker_output = new_picker(
-                compact_request_options,
+                &compact_request_options,
                 &compaction_region.region_options.compaction,
+                compaction_region.region_options.append_mode,
             )
             .pick(compaction_region);
 
diff --git a/src/mito2/src/compaction/picker.rs b/src/mito2/src/compaction/picker.rs
index 30c8d28446..9397c2bf64 100644
--- a/src/mito2/src/compaction/picker.rs
+++ b/src/mito2/src/compaction/picker.rs
@@ -119,10 +119,11 @@ impl PickerOutput {
 
 /// Create a new picker based on the compaction request options and compaction options.
 pub fn new_picker(
-    compact_request_options: compact_request::Options,
+    compact_request_options: &compact_request::Options,
     compaction_options: &CompactionOptions,
+    append_mode: bool,
 ) -> Arc<dyn Picker> {
-    if let compact_request::Options::StrictWindow(window) = &compact_request_options {
+    if let compact_request::Options::StrictWindow(window) = compact_request_options {
         let window = if window.window_seconds == 0 {
             None
         } else {
@@ -131,13 +132,15 @@ pub fn new_picker(
         Arc::new(WindowedCompactionPicker::new(window)) as Arc<_>
     } else {
         match compaction_options {
-            CompactionOptions::Twcs(twcs_opts) => Arc::new(TwcsPicker::new(
-                twcs_opts.max_active_window_runs,
-                twcs_opts.max_active_window_files,
-                twcs_opts.max_inactive_window_runs,
-                twcs_opts.max_inactive_window_files,
-                twcs_opts.time_window_seconds(),
-            )) as Arc<_>,
+            CompactionOptions::Twcs(twcs_opts) => Arc::new(TwcsPicker {
+                max_active_window_runs: twcs_opts.max_active_window_runs,
+                max_active_window_files: twcs_opts.max_active_window_files,
+                max_inactive_window_runs: twcs_opts.max_inactive_window_runs,
+                max_inactive_window_files: twcs_opts.max_inactive_window_files,
+                time_window_seconds: twcs_opts.time_window_seconds(),
+                max_output_file_size: twcs_opts.max_output_file_size.map(|r| r.as_bytes()),
+                append_mode,
+            }) as Arc<_>,
         }
     }
 }
diff --git a/src/mito2/src/compaction/twcs.rs b/src/mito2/src/compaction/twcs.rs
index 4bbad692f0..c6d2a112aa 100644
--- a/src/mito2/src/compaction/twcs.rs
+++ b/src/mito2/src/compaction/twcs.rs
@@ -35,30 +35,23 @@ const LEVEL_COMPACTED: Level = 1;
 /// candidates.
 #[derive(Debug)]
 pub struct TwcsPicker {
-    max_active_window_runs: usize,
-    max_active_window_files: usize,
-    max_inactive_window_runs: usize,
-    max_inactive_window_files: usize,
-    time_window_seconds: Option<i64>,
+    /// Max allowed sorted runs in active window.
+    pub max_active_window_runs: usize,
+    /// Max allowed files in active window.
+    pub max_active_window_files: usize,
+    /// Max allowed sorted runs in inactive windows.
+    pub max_inactive_window_runs: usize,
+    /// Max allowed files in inactive windows.
+    pub max_inactive_window_files: usize,
+    /// Compaction time window in seconds.
+    pub time_window_seconds: Option<i64>,
+    /// Max allowed compaction output file size.
+    pub max_output_file_size: Option<u64>,
+    /// Whether the target region is in append mode.
+    pub append_mode: bool,
 }
 
 impl TwcsPicker {
-    pub fn new(
-        max_active_window_runs: usize,
-        max_active_window_files: usize,
-        max_inactive_window_runs: usize,
-        max_inactive_window_files: usize,
-        time_window_seconds: Option<i64>,
-    ) -> Self {
-        Self {
-            max_inactive_window_runs,
-            max_active_window_runs,
-            time_window_seconds,
-            max_active_window_files,
-            max_inactive_window_files,
-        }
-    }
-
     /// Builds compaction output from files.
     /// For active writing window, we allow for at most `max_active_window_runs` files to alleviate
     /// fragmentation. For other windows, we allow at most 1 file at each window.
@@ -82,47 +75,114 @@ impl TwcsPicker {
                 )
             };
 
-            // we only remove deletion markers once no file in current window overlaps with any other window.
             let found_runs = sorted_runs.len();
-            let filter_deleted = !files.overlapping && (found_runs == 1 || max_runs == 1);
+            // We only remove deletion markers once no file in current window overlaps with any other window
+            // and region is not in append mode.
+            let filter_deleted =
+                !files.overlapping && (found_runs == 1 || max_runs == 1) && !self.append_mode;
 
-            if found_runs > max_runs {
+            let inputs = if found_runs > max_runs {
                 let files_to_compact = reduce_runs(sorted_runs, max_runs);
-                info!("Building compaction output, active window: {:?}, current window: {}, max runs: {}, found runs: {}, output size: {}, remove deletion markers: {}", active_window, *window,max_runs, found_runs, files_to_compact.len(), filter_deleted);
-                for inputs in files_to_compact {
-                    output.push(CompactionOutput {
-                        output_file_id: FileId::random(),
-                        output_level: LEVEL_COMPACTED, // always compact to l1
-                        inputs,
-                        filter_deleted,
-                        output_time_range: None, // we do not enforce output time range in twcs compactions.
-                    });
-                }
+                let files_to_compact_len = files_to_compact.len();
+                info!(
+                    "Building compaction output, active window: {:?}, \
+                        current window: {}, \
+                        max runs: {}, \
+                        found runs: {}, \
+                        output size: {}, \
+                        max output size: {:?}, \
+                        remove deletion markers: {}",
+                    active_window,
+                    *window,
+                    max_runs,
+                    found_runs,
+                    files_to_compact_len,
+                    self.max_output_file_size,
+                    filter_deleted
+                );
+                files_to_compact
             } else if files.files.len() > max_files {
-                debug!(
-                    "Enforcing max file num in window: {}, active: {:?}, max: {}, current: {}",
+                info!(
+                    "Enforcing max file num in window: {}, active: {:?}, max: {}, current: {}, max output size: {:?}, filter delete: {}",
                     *window,
                     active_window,
                     max_files,
-                    files.files.len()
+                    files.files.len(),
+                    self.max_output_file_size,
+                    filter_deleted,
                 );
                 // Files in window exceeds file num limit
-                let to_merge = enforce_file_num(&files.files, max_files);
+                vec![enforce_file_num(&files.files, max_files)]
+            } else {
+                debug!("Skip building compaction output, active window: {:?}, current window: {}, max runs: {}, found runs: {}, ", active_window, *window, max_runs, found_runs);
+                continue;
+            };
+
+            let split_inputs = if !filter_deleted
+                && let Some(max_output_file_size) = self.max_output_file_size
+            {
+                let len_before_split = inputs.len();
+                let maybe_split = enforce_max_output_size(inputs, max_output_file_size);
+                if maybe_split.len() != len_before_split {
+                    info!("Compaction output file size exceeds threshold {}, split compaction inputs to: {:?}", max_output_file_size, maybe_split);
+                }
+                maybe_split
+            } else {
+                inputs
+            };
+
+            for input in split_inputs {
+                debug_assert!(input.len() > 1);
                 output.push(CompactionOutput {
                     output_file_id: FileId::random(),
                     output_level: LEVEL_COMPACTED, // always compact to l1
-                    inputs: to_merge,
+                    inputs: input,
                     filter_deleted,
-                    output_time_range: None,
+                    output_time_range: None, // we do not enforce output time range in twcs compactions.
                 });
-            } else {
-                debug!("Skip building compaction output, active window: {:?}, current window: {}, max runs: {}, found runs: {}, ", active_window, *window, max_runs, found_runs);
             }
         }
         output
     }
 }
 
+/// Limits the size of compaction output in a naive manner.
+/// todo(hl): we can find the output file size more precisely by checking the time range
+/// of each row group and adding the sizes of those non-overlapping row groups. But now
+/// we'd better not to expose the SST details in this level.
+fn enforce_max_output_size(
+    inputs: Vec<Vec<FileHandle>>,
+    max_output_file_size: u64,
+) -> Vec<Vec<FileHandle>> {
+    inputs
+        .into_iter()
+        .flat_map(|input| {
+            debug_assert!(input.len() > 1);
+            let estimated_output_size = input.iter().map(|f| f.size()).sum::<u64>();
+            if estimated_output_size < max_output_file_size {
+                // total file size does not exceed the threshold, just return the original input.
+                return vec![input];
+            }
+            let mut splits = vec![];
+            let mut new_input = vec![];
+            let mut new_input_size = 0;
+            for f in input {
+                if new_input_size + f.size() > max_output_file_size {
+                    splits.push(std::mem::take(&mut new_input));
+                    new_input_size = 0;
+                }
+                new_input_size += f.size();
+                new_input.push(f);
+            }
+            if !new_input.is_empty() {
+                splits.push(new_input);
+            }
+            splits
+        })
+        .filter(|p| p.len() > 1)
+        .collect()
+}
+
 /// Merges consecutive files so that file num does not exceed `max_file_num`, and chooses
 /// the solution with minimum overhead according to files sizes to be merged.
 /// `enforce_file_num` only merges consecutive files so that it won't create overlapping outputs.
@@ -305,10 +365,12 @@ fn find_latest_window_in_seconds<'a>(
 #[cfg(test)]
 mod tests {
     use std::collections::HashSet;
+    use std::sync::Arc;
 
     use super::*;
     use crate::compaction::test_util::{new_file_handle, new_file_handles};
-    use crate::sst::file::Level;
+    use crate::sst::file::{FileMeta, Level};
+    use crate::test_util::NoopFilePurger;
 
     #[test]
     fn test_get_latest_window_in_seconds() {
@@ -525,8 +587,16 @@ mod tests {
             let mut windows = assign_to_windows(self.input_files.iter(), self.window_size);
             let active_window =
                 find_latest_window_in_seconds(self.input_files.iter(), self.window_size);
-            let output = TwcsPicker::new(4, usize::MAX, 1, usize::MAX, None)
-                .build_output(&mut windows, active_window);
+            let output = TwcsPicker {
+                max_active_window_runs: 4,
+                max_active_window_files: usize::MAX,
+                max_inactive_window_runs: 1,
+                max_inactive_window_files: usize::MAX,
+                time_window_seconds: None,
+                max_output_file_size: None,
+                append_mode: false,
+            }
+            .build_output(&mut windows, active_window);
 
             let output = output
                 .iter()
@@ -641,5 +711,43 @@ mod tests {
         .check();
     }
 
+    fn make_file_handles(inputs: &[(i64, i64, u64)]) -> Vec<FileHandle> {
+        inputs
+            .iter()
+            .map(|(start, end, size)| {
+                FileHandle::new(
+                    FileMeta {
+                        region_id: Default::default(),
+                        file_id: Default::default(),
+                        time_range: (
+                            Timestamp::new_millisecond(*start),
+                            Timestamp::new_millisecond(*end),
+                        ),
+                        level: 0,
+                        file_size: *size,
+                        available_indexes: Default::default(),
+                        index_file_size: 0,
+                        num_rows: 0,
+                        num_row_groups: 0,
+                    },
+                    Arc::new(NoopFilePurger),
+                )
+            })
+            .collect()
+    }
+
+    #[test]
+    fn test_limit_output_size() {
+        let mut files = make_file_handles(&[(1, 1, 1)].repeat(6));
+        let runs = find_sorted_runs(&mut files);
+        assert_eq!(6, runs.len());
+        let files_to_merge = reduce_runs(runs, 2);
+
+        let enforced = enforce_max_output_size(files_to_merge, 2);
+        assert_eq!(2, enforced.len());
+        assert_eq!(2, enforced[0].len());
+        assert_eq!(2, enforced[1].len());
+    }
+
     // TODO(hl): TTL tester that checks if get_expired_ssts function works as expected.
 }
diff --git a/src/mito2/src/region/options.rs b/src/mito2/src/region/options.rs
index cc866e5502..4abc5925b7 100644
--- a/src/mito2/src/region/options.rs
+++ b/src/mito2/src/region/options.rs
@@ -204,6 +204,8 @@ pub struct TwcsOptions {
     /// Compaction time window defined when creating tables.
     #[serde(with = "humantime_serde")]
     pub time_window: Option<Duration>,
+    /// Compaction time window defined when creating tables.
+    pub max_output_file_size: Option<ReadableSize>,
     /// Whether to use remote compaction.
     #[serde_as(as = "DisplayFromStr")]
     pub remote_compaction: bool,
@@ -236,6 +238,7 @@ impl Default for TwcsOptions {
             max_inactive_window_runs: 1,
             max_inactive_window_files: 1,
             time_window: None,
+            max_output_file_size: None,
             remote_compaction: false,
             fallback_to_local: true,
         }
@@ -597,6 +600,7 @@ mod tests {
             ("compaction.twcs.max_active_window_files", "11"),
             ("compaction.twcs.max_inactive_window_runs", "2"),
             ("compaction.twcs.max_inactive_window_files", "3"),
+            ("compaction.twcs.max_output_file_size", "1GB"),
             ("compaction.twcs.time_window", "2h"),
             ("compaction.type", "twcs"),
             ("compaction.twcs.remote_compaction", "false"),
@@ -624,6 +628,7 @@ mod tests {
                 max_inactive_window_runs: 2,
                 max_inactive_window_files: 3,
                 time_window: Some(Duration::from_secs(3600 * 2)),
+                max_output_file_size: Some(ReadableSize::gb(1)),
                 remote_compaction: false,
                 fallback_to_local: true,
             }),
@@ -656,6 +661,7 @@ mod tests {
                 max_inactive_window_runs: 2,
                 max_inactive_window_files: usize::MAX,
                 time_window: Some(Duration::from_secs(3600 * 2)),
+                max_output_file_size: None,
                 remote_compaction: false,
                 fallback_to_local: true,
             }),
@@ -693,6 +699,7 @@ mod tests {
     "compaction.twcs.max_active_window_files": "11",
     "compaction.twcs.max_inactive_window_runs": "2",
     "compaction.twcs.max_inactive_window_files": "7",
+    "compaction.twcs.max_output_file_size": "7MB",
     "compaction.twcs.time_window": "2h"
   },
   "storage": "S3",
@@ -722,6 +729,7 @@ mod tests {
                 max_inactive_window_runs: 2,
                 max_inactive_window_files: 7,
                 time_window: Some(Duration::from_secs(3600 * 2)),
+                max_output_file_size: Some(ReadableSize::mb(7)),
                 remote_compaction: false,
                 fallback_to_local: true,
             }),
diff --git a/src/sql/src/parsers/create_parser.rs b/src/sql/src/parsers/create_parser.rs
index de81bd1a21..8b7eb420d2 100644
--- a/src/sql/src/parsers/create_parser.rs
+++ b/src/sql/src/parsers/create_parser.rs
@@ -370,12 +370,7 @@ impl<'a> ParserContext<'a> {
             .map(parse_option_string)
             .collect::<Result<HashMap<String, String>>>()?;
         for key in options.keys() {
-            ensure!(
-                validate_table_option(key),
-                InvalidTableOptionSnafu {
-                    key: key.to_string()
-                }
-            );
+            ensure!(validate_table_option(key), InvalidTableOptionSnafu { key });
         }
         Ok(options.into())
     }
diff --git a/src/store-api/src/mito_engine_options.rs b/src/store-api/src/mito_engine_options.rs
index e641a1d2fc..0e0f3fdac7 100644
--- a/src/store-api/src/mito_engine_options.rs
+++ b/src/store-api/src/mito_engine_options.rs
@@ -33,6 +33,7 @@ pub fn is_mito_engine_option_key(key: &str) -> bool {
         "compaction.twcs.max_active_window_files",
         "compaction.twcs.max_inactive_window_runs",
         "compaction.twcs.max_inactive_window_files",
+        "compaction.twcs.max_output_file_size",
         "compaction.twcs.time_window",
         "compaction.twcs.remote_compaction",
         "compaction.twcs.fallback_to_local",

From 3a46c1b2352ba52955653dd0e33f597f1c02171d Mon Sep 17 00:00:00 2001
From: JohnsonLee <0xjohnsonlee@gmail.com>
Date: Sat, 28 Sep 2024 07:15:48 +0800
Subject: [PATCH 041/128] fix: use information_schema returns Unknown database 
 (#4774)

* fix: use information_schema returns Unknown database 'information_schema'

* test: make sure 'use information_schma' successful
---
 src/frontend/src/instance.rs                                  | 1 -
 .../cases/standalone/common/system/information_schema.result  | 4 ++++
 tests/cases/standalone/common/system/information_schema.sql   | 2 ++
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs
index dcc56f7581..fc46ae9e59 100644
--- a/src/frontend/src/instance.rs
+++ b/src/frontend/src/instance.rs
@@ -359,7 +359,6 @@ impl SqlQueryHandler for Instance {
             .schema_exists(catalog, schema, None)
             .await
             .context(error::CatalogSnafu)
-            .map(|b| b && !self.catalog_manager.is_reserved_schema_name(schema))
     }
 }
 
diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result
index 882607f842..4264bd8df6 100644
--- a/tests/cases/standalone/common/system/information_schema.result
+++ b/tests/cases/standalone/common/system/information_schema.result
@@ -434,6 +434,10 @@ database my_db;
 
 Affected Rows: 1
 
+use information_schema;
+
+Affected Rows: 0
+
 use my_db;
 
 Affected Rows: 0
diff --git a/tests/cases/standalone/common/system/information_schema.sql b/tests/cases/standalone/common/system/information_schema.sql
index 49007cb44b..b913050e70 100644
--- a/tests/cases/standalone/common/system/information_schema.sql
+++ b/tests/cases/standalone/common/system/information_schema.sql
@@ -14,6 +14,8 @@ select * from information_schema.columns order by table_schema, table_name, colu
 create
 database my_db;
 
+use information_schema;
+
 use my_db;
 
 create table foo

From d6be44bc7f57c0ba6ea72fa00846d4c765fd166c Mon Sep 17 00:00:00 2001
From: Ning Sun <sunng@protonmail.com>
Date: Sat, 28 Sep 2024 09:39:29 +0800
Subject: [PATCH 042/128] fix: dead loop on detecting postgres ssl handshake
 (#4778)

---
 Cargo.lock             | 4 ++--
 src/servers/Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 3d5783dcd4..25ba1eb378 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7765,9 +7765,9 @@ dependencies = [
 
 [[package]]
 name = "pgwire"
-version = "0.24.2"
+version = "0.24.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed4ca46dd335b3a030d977be54dfe121b1b9fe22aa8bbd69161ac2434524fc68"
+checksum = "4895c8e98cbe81496692ae3d2262a9fb0d26302af7c9ea483cf708000106aa0a"
 dependencies = [
  "async-trait",
  "bytes",
diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml
index b94fa17d44..0de8d0904a 100644
--- a/src/servers/Cargo.toml
+++ b/src/servers/Cargo.toml
@@ -71,7 +71,7 @@ openmetrics-parser = "0.4"
 opensrv-mysql = { git = "https://github.com/datafuselabs/opensrv", rev = "6bbc3b65e6b19212c4f7fc4f40c20daf6f452deb" }
 opentelemetry-proto.workspace = true
 parking_lot = "0.12"
-pgwire = { version = "0.24.2", default-features = false, features = ["server-api-ring"] }
+pgwire = { version = "0.24.3", default-features = false, features = ["server-api-ring"] }
 pin-project = "1.0"
 pipeline.workspace = true
 postgres-types = { version = "0.2", features = ["with-chrono-0_4", "with-serde_json-1"] }

From cedbbcf2b8262037225218a130002abd3fb1804c Mon Sep 17 00:00:00 2001
From: Ning Sun <sunng@protonmail.com>
Date: Sun, 29 Sep 2024 03:27:31 +0800
Subject: [PATCH 043/128] fix: update pgwire for potential issue with
 connection establish (#4783)

---
 Cargo.lock             | 4 ++--
 src/servers/Cargo.toml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 25ba1eb378..2ae89afc74 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7765,9 +7765,9 @@ dependencies = [
 
 [[package]]
 name = "pgwire"
-version = "0.24.3"
+version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4895c8e98cbe81496692ae3d2262a9fb0d26302af7c9ea483cf708000106aa0a"
+checksum = "5e63bc3945a17010ff93677589c656c5e8fb4183b00bc86360de8e187d2a86cb"
 dependencies = [
  "async-trait",
  "bytes",
diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml
index 0de8d0904a..725ff497a4 100644
--- a/src/servers/Cargo.toml
+++ b/src/servers/Cargo.toml
@@ -71,7 +71,7 @@ openmetrics-parser = "0.4"
 opensrv-mysql = { git = "https://github.com/datafuselabs/opensrv", rev = "6bbc3b65e6b19212c4f7fc4f40c20daf6f452deb" }
 opentelemetry-proto.workspace = true
 parking_lot = "0.12"
-pgwire = { version = "0.24.3", default-features = false, features = ["server-api-ring"] }
+pgwire = { version = "0.25.0", default-features = false, features = ["server-api-ring"] }
 pin-project = "1.0"
 pipeline.workspace = true
 postgres-types = { version = "0.2", features = ["with-chrono-0_4", "with-serde_json-1"] }

From d9f2f0ccf0dde73341a6959b36248205bd764c49 Mon Sep 17 00:00:00 2001
From: LFC <990479+MichaelScofield@users.noreply.github.com>
Date: Sun, 29 Sep 2024 11:38:50 +0800
Subject: [PATCH 044/128] feat: add a new status code for "external" errors
 (#4775)

* feat: add a new status code for "external" errors

* Update src/auth/src/error.rs

Co-authored-by: shuiyisong <113876041+shuiyisong@users.noreply.github.com>

* support mysql cli cleartext auth

* resolve PR comments

---------

Co-authored-by: shuiyisong <113876041+shuiyisong@users.noreply.github.com>
---
 src/auth/src/common.rs                  | 10 +++++++
 src/auth/src/error.rs                   |  2 +-
 src/auth/src/user_provider.rs           |  5 ++++
 src/common/error/src/status_code.rs     | 11 +++++---
 src/servers/src/http/error_result.rs    |  3 ++-
 src/servers/src/mysql/handler.rs        | 36 ++++++++++++++++++++++++-
 src/servers/src/mysql/writer.rs         |  2 +-
 src/servers/src/postgres/types/error.rs |  1 +
 8 files changed, 63 insertions(+), 7 deletions(-)

diff --git a/src/auth/src/common.rs b/src/auth/src/common.rs
index 3aad89920d..8a13e3fc3d 100644
--- a/src/auth/src/common.rs
+++ b/src/auth/src/common.rs
@@ -75,6 +75,16 @@ pub enum Password<'a> {
     PgMD5(HashedPassword<'a>, Salt<'a>),
 }
 
+impl Password<'_> {
+    pub fn r#type(&self) -> &str {
+        match self {
+            Password::PlainText(_) => "plain_text",
+            Password::MysqlNativePassword(_, _) => "mysql_native_password",
+            Password::PgMD5(_, _) => "pg_md5",
+        }
+    }
+}
+
 pub fn auth_mysql(
     auth_data: HashedPassword,
     salt: Salt,
diff --git a/src/auth/src/error.rs b/src/auth/src/error.rs
index 281c45234d..7ed748559b 100644
--- a/src/auth/src/error.rs
+++ b/src/auth/src/error.rs
@@ -89,7 +89,7 @@ impl ErrorExt for Error {
             Error::FileWatch { .. } => StatusCode::InvalidArguments,
             Error::InternalState { .. } => StatusCode::Unexpected,
             Error::Io { .. } => StatusCode::StorageUnavailable,
-            Error::AuthBackend { .. } => StatusCode::Internal,
+            Error::AuthBackend { source, .. } => source.status_code(),
 
             Error::UserNotFound { .. } => StatusCode::UserNotFound,
             Error::UnsupportedPasswordType { .. } => StatusCode::UnsupportedPasswordType,
diff --git a/src/auth/src/user_provider.rs b/src/auth/src/user_provider.rs
index b00f3cf29d..526e72b775 100644
--- a/src/auth/src/user_provider.rs
+++ b/src/auth/src/user_provider.rs
@@ -57,6 +57,11 @@ pub trait UserProvider: Send + Sync {
         self.authorize(catalog, schema, &user_info).await?;
         Ok(user_info)
     }
+
+    /// Returns whether this user provider implementation is backed by an external system.
+    fn external(&self) -> bool {
+        false
+    }
 }
 
 fn load_credential_from_file(filepath: &str) -> Result<Option<HashMap<String, Vec<u8>>>> {
diff --git a/src/common/error/src/status_code.rs b/src/common/error/src/status_code.rs
index 11b4c7587e..239871e1d1 100644
--- a/src/common/error/src/status_code.rs
+++ b/src/common/error/src/status_code.rs
@@ -38,6 +38,8 @@ pub enum StatusCode {
     Cancelled = 1005,
     /// Illegal state, can be exposed to users.
     IllegalState = 1006,
+    /// Caused by some error originated from external system.
+    External = 1007,
     // ====== End of common status code ================
 
     // ====== Begin of SQL related status code =========
@@ -162,7 +164,8 @@ impl StatusCode {
             | StatusCode::InvalidAuthHeader
             | StatusCode::AccessDenied
             | StatusCode::PermissionDenied
-            | StatusCode::RequestOutdated => false,
+            | StatusCode::RequestOutdated
+            | StatusCode::External => false,
         }
     }
 
@@ -177,7 +180,9 @@ impl StatusCode {
             | StatusCode::IllegalState
             | StatusCode::EngineExecuteQuery
             | StatusCode::StorageUnavailable
-            | StatusCode::RuntimeResourcesExhausted => true,
+            | StatusCode::RuntimeResourcesExhausted
+            | StatusCode::External => true,
+
             StatusCode::Success
             | StatusCode::Unsupported
             | StatusCode::InvalidArguments
@@ -256,7 +261,7 @@ macro_rules! define_into_tonic_status {
 pub fn status_to_tonic_code(status_code: StatusCode) -> Code {
     match status_code {
         StatusCode::Success => Code::Ok,
-        StatusCode::Unknown => Code::Unknown,
+        StatusCode::Unknown | StatusCode::External => Code::Unknown,
         StatusCode::Unsupported => Code::Unimplemented,
         StatusCode::Unexpected
         | StatusCode::IllegalState
diff --git a/src/servers/src/http/error_result.rs b/src/servers/src/http/error_result.rs
index 518f16cde0..40766d2cbf 100644
--- a/src/servers/src/http/error_result.rs
+++ b/src/servers/src/http/error_result.rs
@@ -122,7 +122,8 @@ pub fn status_code_to_http_status(status_code: &StatusCode) -> HttpStatusCode {
         StatusCode::RegionNotReady
         | StatusCode::TableUnavailable
         | StatusCode::RegionBusy
-        | StatusCode::StorageUnavailable => HttpStatusCode::SERVICE_UNAVAILABLE,
+        | StatusCode::StorageUnavailable
+        | StatusCode::External => HttpStatusCode::SERVICE_UNAVAILABLE,
 
         StatusCode::Internal
         | StatusCode::Unexpected
diff --git a/src/servers/src/mysql/handler.rs b/src/servers/src/mysql/handler.rs
index 3c0ac36f4c..abf417996c 100644
--- a/src/servers/src/mysql/handler.rs
+++ b/src/servers/src/mysql/handler.rs
@@ -54,6 +54,9 @@ use crate::mysql::writer::{create_mysql_column, handle_err};
 use crate::query_handler::sql::ServerSqlQueryHandlerRef;
 use crate::SqlPlan;
 
+const MYSQL_NATIVE_PASSWORD: &str = "mysql_native_password";
+const MYSQL_CLEAR_PASSWORD: &str = "mysql_clear_password";
+
 // An intermediate shim for executing MySQL queries.
 pub struct MysqlInstanceShim {
     query_handler: ServerSqlQueryHandlerRef,
@@ -219,6 +222,19 @@ impl MysqlInstanceShim {
         let mut guard = self.prepared_stmts.write();
         let _ = guard.remove(&stmt_key);
     }
+
+    fn auth_plugin(&self) -> &str {
+        if self
+            .user_provider
+            .as_ref()
+            .map(|x| x.external())
+            .unwrap_or(false)
+        {
+            MYSQL_CLEAR_PASSWORD
+        } else {
+            MYSQL_NATIVE_PASSWORD
+        }
+    }
 }
 
 #[async_trait]
@@ -229,6 +245,14 @@ impl<W: AsyncWrite + Send + Sync + Unpin> AsyncMysqlShim<W> for MysqlInstanceShi
         std::env::var("GREPTIMEDB_MYSQL_SERVER_VERSION").unwrap_or_else(|_| "8.4.2".to_string())
     }
 
+    fn default_auth_plugin(&self) -> &str {
+        self.auth_plugin()
+    }
+
+    async fn auth_plugin_for_username(&self, _user: &[u8]) -> &str {
+        self.auth_plugin()
+    }
+
     fn salt(&self) -> [u8; 20] {
         self.salt
     }
@@ -253,7 +277,17 @@ impl<W: AsyncWrite + Send + Sync + Unpin> AsyncMysqlShim<W> for MysqlInstanceShi
             let user_id = Identity::UserId(&username, addr.as_deref());
 
             let password = match auth_plugin {
-                "mysql_native_password" => Password::MysqlNativePassword(auth_data, salt),
+                MYSQL_NATIVE_PASSWORD => Password::MysqlNativePassword(auth_data, salt),
+                MYSQL_CLEAR_PASSWORD => {
+                    // The raw bytes received could be represented in C-like string, ended in '\0'.
+                    // We must "trim" it to get the real password string.
+                    let password = if let &[password @ .., 0] = &auth_data {
+                        password
+                    } else {
+                        auth_data
+                    };
+                    Password::PlainText(String::from_utf8_lossy(password).to_string().into())
+                }
                 other => {
                     error!("Unsupported mysql auth plugin: {}", other);
                     return false;
diff --git a/src/servers/src/mysql/writer.rs b/src/servers/src/mysql/writer.rs
index d957edaa55..8a5814d8fe 100644
--- a/src/servers/src/mysql/writer.rs
+++ b/src/servers/src/mysql/writer.rs
@@ -328,7 +328,7 @@ pub fn create_mysql_column_def(schema: &SchemaRef) -> Result<Vec<Column>> {
 fn mysql_error_kind(status_code: &StatusCode) -> ErrorKind {
     match status_code {
         StatusCode::Success => ErrorKind::ER_YES,
-        StatusCode::Unknown => ErrorKind::ER_UNKNOWN_ERROR,
+        StatusCode::Unknown | StatusCode::External => ErrorKind::ER_UNKNOWN_ERROR,
         StatusCode::Unsupported => ErrorKind::ER_NOT_SUPPORTED_YET,
         StatusCode::Cancelled => ErrorKind::ER_QUERY_INTERRUPTED,
         StatusCode::RuntimeResourcesExhausted => ErrorKind::ER_OUT_OF_RESOURCES,
diff --git a/src/servers/src/postgres/types/error.rs b/src/servers/src/postgres/types/error.rs
index 9e6f570f26..033f4f6b87 100644
--- a/src/servers/src/postgres/types/error.rs
+++ b/src/servers/src/postgres/types/error.rs
@@ -374,6 +374,7 @@ impl From<StatusCode> for PgErrorCode {
             StatusCode::Unsupported => PgErrorCode::Ec0A000,
             StatusCode::InvalidArguments => PgErrorCode::Ec22023,
             StatusCode::Cancelled => PgErrorCode::Ec57000,
+            StatusCode::External => PgErrorCode::Ec58000,
 
             StatusCode::Unknown
             | StatusCode::Unexpected

From 0a82b12d087d29e4a75a70612b72cb4a79e71340 Mon Sep 17 00:00:00 2001
From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com>
Date: Sun, 29 Sep 2024 13:54:01 +0800
Subject: [PATCH 045/128] fix(sqlness): sqlness isolation (#4780)

* fix: isolate logs

* fix: copy cases

* fix: clippy
---
 .../copy/copy_database_from_fs_parquet.result | 10 ++---
 .../copy/copy_database_from_fs_parquet.sql    | 10 ++---
 .../common/copy/copy_from_fs_csv.result       |  8 ++--
 .../common/copy/copy_from_fs_csv.sql          |  8 ++--
 .../common/copy/copy_from_fs_json.result      |  8 ++--
 .../common/copy/copy_from_fs_json.sql         |  8 ++--
 .../common/copy/copy_from_fs_parquet.result   | 18 ++++----
 .../common/copy/copy_from_fs_parquet.sql      | 16 +++----
 .../standalone/common/copy/copy_to_fs.result  |  6 +--
 .../standalone/common/copy/copy_to_fs.sql     |  6 +--
 tests/runner/src/env.rs                       | 44 ++++++++++++++-----
 tests/runner/src/main.rs                      |  8 ++--
 12 files changed, 86 insertions(+), 64 deletions(-)

diff --git a/tests/cases/standalone/common/copy/copy_database_from_fs_parquet.result b/tests/cases/standalone/common/copy/copy_database_from_fs_parquet.result
index 18badf93f1..999271da8d 100644
--- a/tests/cases/standalone/common/copy/copy_database_from_fs_parquet.result
+++ b/tests/cases/standalone/common/copy/copy_database_from_fs_parquet.result
@@ -6,11 +6,11 @@ INSERT INTO demo(host, cpu, memory, ts) values ('host1', 66.6, 1024, 16552765570
 
 Affected Rows: 2
 
-COPY DATABASE public TO '/tmp/demo/export/parquet/' WITH (FORMAT="parquet");
+COPY DATABASE public TO '${SQLNESS_HOME}/demo/export/parquet/' WITH (FORMAT="parquet");
 
 Affected Rows: 2
 
-COPY DATABASE public TO '/tmp/demo/export/parquet_range/' WITH (FORMAT="parquet", start_time='2022-06-15 07:02:37.000Z', end_time='2022-06-15 07:02:37.1Z');
+COPY DATABASE public TO '${SQLNESS_HOME}/demo/export/parquet_range/' WITH (FORMAT="parquet", start_time='2022-06-15 07:02:37.000Z', end_time='2022-06-15 07:02:37.1Z');
 
 Affected Rows: 1
 
@@ -23,7 +23,7 @@ SELECT * FROM demo ORDER BY ts;
 ++
 ++
 
-COPY DATABASE public FROM '/tmp/demo/export/parquet/';
+COPY DATABASE public FROM '${SQLNESS_HOME}/demo/export/parquet/';
 
 Affected Rows: 2
 
@@ -40,7 +40,7 @@ DELETE FROM demo;
 
 Affected Rows: 2
 
-COPY DATABASE public FROM '/tmp/demo/export/parquet_range/';
+COPY DATABASE public FROM '${SQLNESS_HOME}/demo/export/parquet_range/';
 
 Affected Rows: 1
 
@@ -56,7 +56,7 @@ DELETE FROM demo;
 
 Affected Rows: 1
 
-COPY DATABASE public FROM '/tmp/demo/export/parquet_range/' LIMIT 2;
+COPY DATABASE public FROM '${SQLNESS_HOME}/demo/export/parquet_range/' LIMIT 2;
 
 Error: 2000(InvalidSyntax), Invalid SQL, error: limit is not supported
 
diff --git a/tests/cases/standalone/common/copy/copy_database_from_fs_parquet.sql b/tests/cases/standalone/common/copy/copy_database_from_fs_parquet.sql
index 5d3716dc22..671070e07b 100644
--- a/tests/cases/standalone/common/copy/copy_database_from_fs_parquet.sql
+++ b/tests/cases/standalone/common/copy/copy_database_from_fs_parquet.sql
@@ -2,26 +2,26 @@ CREATE TABLE demo(host string, cpu double, memory double, ts TIMESTAMP time inde
 
 INSERT INTO demo(host, cpu, memory, ts) values ('host1', 66.6, 1024, 1655276557000), ('host2', 88.8,  333.3, 1655276558000);
 
-COPY DATABASE public TO '/tmp/demo/export/parquet/' WITH (FORMAT="parquet");
+COPY DATABASE public TO '${SQLNESS_HOME}/demo/export/parquet/' WITH (FORMAT="parquet");
 
-COPY DATABASE public TO '/tmp/demo/export/parquet_range/' WITH (FORMAT="parquet", start_time='2022-06-15 07:02:37.000Z', end_time='2022-06-15 07:02:37.1Z');
+COPY DATABASE public TO '${SQLNESS_HOME}/demo/export/parquet_range/' WITH (FORMAT="parquet", start_time='2022-06-15 07:02:37.000Z', end_time='2022-06-15 07:02:37.1Z');
 
 DELETE FROM demo;
 
 SELECT * FROM demo ORDER BY ts;
 
-COPY DATABASE public FROM '/tmp/demo/export/parquet/';
+COPY DATABASE public FROM '${SQLNESS_HOME}/demo/export/parquet/';
 
 SELECT * FROM demo ORDER BY ts;
 
 DELETE FROM demo;
 
-COPY DATABASE public FROM '/tmp/demo/export/parquet_range/';
+COPY DATABASE public FROM '${SQLNESS_HOME}/demo/export/parquet_range/';
 
 SELECT * FROM demo ORDER BY ts;
 
 DELETE FROM demo;
 
-COPY DATABASE public FROM '/tmp/demo/export/parquet_range/' LIMIT 2;
+COPY DATABASE public FROM '${SQLNESS_HOME}/demo/export/parquet_range/' LIMIT 2;
 
 DROP TABLE demo;
diff --git a/tests/cases/standalone/common/copy/copy_from_fs_csv.result b/tests/cases/standalone/common/copy/copy_from_fs_csv.result
index 19b3fda3f9..f412677a32 100644
--- a/tests/cases/standalone/common/copy/copy_from_fs_csv.result
+++ b/tests/cases/standalone/common/copy/copy_from_fs_csv.result
@@ -11,7 +11,7 @@ values
 
 Affected Rows: 3
 
-Copy demo TO '/tmp/demo/export/csv/demo.csv' with (format='csv');
+Copy demo TO '${SQLNESS_HOME}/demo/export/csv/demo.csv' with (format='csv');
 
 Affected Rows: 3
 
@@ -19,7 +19,7 @@ CREATE TABLE with_filename(host string, cpu double, memory double, ts timestamp
 
 Affected Rows: 0
 
-Copy with_filename FROM '/tmp/demo/export/csv/demo.csv' with (format='csv', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
+Copy with_filename FROM '${SQLNESS_HOME}/demo/export/csv/demo.csv' with (format='csv', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
 
 Affected Rows: 2
 
@@ -36,7 +36,7 @@ CREATE TABLE with_path(host string, cpu double, memory double, ts timestamp time
 
 Affected Rows: 0
 
-Copy with_path FROM '/tmp/demo/export/csv/' with (format='csv', start_time='2023-06-15 07:02:37');
+Copy with_path FROM '${SQLNESS_HOME}/demo/export/csv/' with (format='csv', start_time='2023-06-15 07:02:37');
 
 Affected Rows: 1
 
@@ -52,7 +52,7 @@ CREATE TABLE with_pattern(host string, cpu double, memory double, ts timestamp t
 
 Affected Rows: 0
 
-Copy with_pattern FROM '/tmp/demo/export/csv/' WITH (pattern = 'demo.*', format='csv', end_time='2025-06-15 07:02:39');
+Copy with_pattern FROM '${SQLNESS_HOME}/demo/export/csv/' WITH (pattern = 'demo.*', format='csv', end_time='2025-06-15 07:02:39');
 
 Affected Rows: 3
 
diff --git a/tests/cases/standalone/common/copy/copy_from_fs_csv.sql b/tests/cases/standalone/common/copy/copy_from_fs_csv.sql
index cd6b91f4f8..f2c6ccf5bd 100644
--- a/tests/cases/standalone/common/copy/copy_from_fs_csv.sql
+++ b/tests/cases/standalone/common/copy/copy_from_fs_csv.sql
@@ -7,23 +7,23 @@ values
     ('host2', 88.8, 333.3, 1655276558000),
     ('host3', 99.9, 444.4, 1722077263000);
 
-Copy demo TO '/tmp/demo/export/csv/demo.csv' with (format='csv');
+Copy demo TO '${SQLNESS_HOME}/demo/export/csv/demo.csv' with (format='csv');
 
 CREATE TABLE with_filename(host string, cpu double, memory double, ts timestamp time index);
 
-Copy with_filename FROM '/tmp/demo/export/csv/demo.csv' with (format='csv', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
+Copy with_filename FROM '${SQLNESS_HOME}/demo/export/csv/demo.csv' with (format='csv', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
 
 select * from with_filename order by ts;
 
 CREATE TABLE with_path(host string, cpu double, memory double, ts timestamp time index);
 
-Copy with_path FROM '/tmp/demo/export/csv/' with (format='csv', start_time='2023-06-15 07:02:37');
+Copy with_path FROM '${SQLNESS_HOME}/demo/export/csv/' with (format='csv', start_time='2023-06-15 07:02:37');
 
 select * from with_path order by ts;
 
 CREATE TABLE with_pattern(host string, cpu double, memory double, ts timestamp time index);
 
-Copy with_pattern FROM '/tmp/demo/export/csv/' WITH (pattern = 'demo.*', format='csv', end_time='2025-06-15 07:02:39');
+Copy with_pattern FROM '${SQLNESS_HOME}/demo/export/csv/' WITH (pattern = 'demo.*', format='csv', end_time='2025-06-15 07:02:39');
 
 select * from with_pattern order by ts;
 
diff --git a/tests/cases/standalone/common/copy/copy_from_fs_json.result b/tests/cases/standalone/common/copy/copy_from_fs_json.result
index bd71b5d624..e1e3a810d5 100644
--- a/tests/cases/standalone/common/copy/copy_from_fs_json.result
+++ b/tests/cases/standalone/common/copy/copy_from_fs_json.result
@@ -11,7 +11,7 @@ values
 
 Affected Rows: 3
 
-Copy demo TO '/tmp/demo/export/json/demo.json' with (format='json');
+Copy demo TO '${SQLNESS_HOME}/demo/export/json/demo.json' with (format='json');
 
 Affected Rows: 3
 
@@ -19,7 +19,7 @@ CREATE TABLE with_filename(host string, cpu double, memory double, ts timestamp
 
 Affected Rows: 0
 
-Copy with_filename FROM '/tmp/demo/export/json/demo.json' with (format='json', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
+Copy with_filename FROM '${SQLNESS_HOME}/demo/export/json/demo.json' with (format='json', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
 
 Affected Rows: 2
 
@@ -36,7 +36,7 @@ CREATE TABLE with_path(host string, cpu double, memory double, ts timestamp time
 
 Affected Rows: 0
 
-Copy with_path FROM '/tmp/demo/export/json/' with (format='json', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
+Copy with_path FROM '${SQLNESS_HOME}/demo/export/json/' with (format='json', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
 
 Affected Rows: 2
 
@@ -53,7 +53,7 @@ CREATE TABLE with_pattern(host string, cpu double, memory double, ts timestamp t
 
 Affected Rows: 0
 
-Copy with_pattern FROM '/tmp/demo/export/json/' WITH (pattern = 'demo.*',format='json', end_time='2022-06-15 07:02:39');
+Copy with_pattern FROM '${SQLNESS_HOME}/demo/export/json/' WITH (pattern = 'demo.*',format='json', end_time='2022-06-15 07:02:39');
 
 Affected Rows: 2
 
diff --git a/tests/cases/standalone/common/copy/copy_from_fs_json.sql b/tests/cases/standalone/common/copy/copy_from_fs_json.sql
index c182bb82dc..55e8a55ebf 100644
--- a/tests/cases/standalone/common/copy/copy_from_fs_json.sql
+++ b/tests/cases/standalone/common/copy/copy_from_fs_json.sql
@@ -7,23 +7,23 @@ values
     ('host2', 88.8,  333.3, 1655276558000),
     ('host3', 99.9, 444.4, 1722077263000);
 
-Copy demo TO '/tmp/demo/export/json/demo.json' with (format='json');
+Copy demo TO '${SQLNESS_HOME}/demo/export/json/demo.json' with (format='json');
 
 CREATE TABLE with_filename(host string, cpu double, memory double, ts timestamp time index);
 
-Copy with_filename FROM '/tmp/demo/export/json/demo.json' with (format='json', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
+Copy with_filename FROM '${SQLNESS_HOME}/demo/export/json/demo.json' with (format='json', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
 
 select * from with_filename order by ts;
 
 CREATE TABLE with_path(host string, cpu double, memory double, ts timestamp time index);
 
-Copy with_path FROM '/tmp/demo/export/json/' with (format='json', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
+Copy with_path FROM '${SQLNESS_HOME}/demo/export/json/' with (format='json', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
 
 select * from with_path order by ts;
 
 CREATE TABLE with_pattern(host string, cpu double, memory double, ts timestamp time index);
 
-Copy with_pattern FROM '/tmp/demo/export/json/' WITH (pattern = 'demo.*',format='json', end_time='2022-06-15 07:02:39');
+Copy with_pattern FROM '${SQLNESS_HOME}/demo/export/json/' WITH (pattern = 'demo.*',format='json', end_time='2022-06-15 07:02:39');
 
 select * from with_pattern order by ts;
 
diff --git a/tests/cases/standalone/common/copy/copy_from_fs_parquet.result b/tests/cases/standalone/common/copy/copy_from_fs_parquet.result
index 82299d48b3..44e1422d5b 100644
--- a/tests/cases/standalone/common/copy/copy_from_fs_parquet.result
+++ b/tests/cases/standalone/common/copy/copy_from_fs_parquet.result
@@ -24,11 +24,11 @@ values
 
 Affected Rows: 3
 
-Copy demo TO '/tmp/demo/export/parquet_files/demo.parquet';
+Copy demo TO '${SQLNESS_HOME}/demo/export/parquet_files/demo.parquet';
 
 Affected Rows: 3
 
-Copy demo_2 TO '/tmp/demo/export/parquet_files/demo_2.parquet';
+Copy demo_2 TO '${SQLNESS_HOME}/demo/export/parquet_files/demo_2.parquet';
 
 Affected Rows: 3
 
@@ -36,7 +36,7 @@ CREATE TABLE with_filename(host string, cpu double, memory double, ts timestamp
 
 Affected Rows: 0
 
-Copy with_filename FROM '/tmp/demo/export/parquet_files/demo.parquet' with (start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
+Copy with_filename FROM '${SQLNESS_HOME}/demo/export/parquet_files/demo.parquet' with (start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
 
 Affected Rows: 2
 
@@ -53,7 +53,7 @@ CREATE TABLE with_path(host string, cpu double, memory double, ts timestamp time
 
 Affected Rows: 0
 
-Copy with_path FROM '/tmp/demo/export/parquet_files/';
+Copy with_path FROM '${SQLNESS_HOME}/demo/export/parquet_files/';
 
 Affected Rows: 6
 
@@ -74,7 +74,7 @@ CREATE TABLE with_pattern(host string, cpu double, memory double, ts timestamp t
 
 Affected Rows: 0
 
-Copy with_pattern FROM '/tmp/demo/export/parquet_files/' WITH (PATTERN = 'demo.*', start_time='2022-06-15 07:02:39');
+Copy with_pattern FROM '${SQLNESS_HOME}/demo/export/parquet_files/' WITH (PATTERN = 'demo.*', start_time='2022-06-15 07:02:39');
 
 Affected Rows: 2
 
@@ -91,7 +91,7 @@ CREATE TABLE without_limit_rows(host string, cpu double, memory double, ts times
 
 Affected Rows: 0
 
-Copy without_limit_rows FROM '/tmp/demo/export/parquet_files/';
+Copy without_limit_rows FROM '${SQLNESS_HOME}/demo/export/parquet_files/';
 
 Affected Rows: 6
 
@@ -107,7 +107,7 @@ CREATE TABLE with_limit_rows_segment(host string, cpu double, memory double, ts
 
 Affected Rows: 0
 
-Copy with_limit_rows_segment FROM '/tmp/demo/export/parquet_files/' LIMIT 3;
+Copy with_limit_rows_segment FROM '${SQLNESS_HOME}/demo/export/parquet_files/' LIMIT 3;
 
 Affected Rows: 3
 
@@ -119,9 +119,9 @@ select count(*) from with_limit_rows_segment;
 | 3        |
 +----------+
 
-Copy with_limit_rows_segment FROM '/tmp/demo/export/parquet_files/' LIMIT hello;
+Copy with_limit_rows_segment FROM '${SQLNESS_HOME}/demo/export/parquet_files/' LIMIT hello;
 
-Error: 2000(InvalidSyntax), Unexpected token while parsing SQL statement, expected: 'the number of maximum rows', found: ;: sql parser error: Expected literal int, found: hello at Line: 1, Column 75
+Error: 2000(InvalidSyntax), Unexpected token while parsing SQL statement, expected: 'the number of maximum rows', found: ;: sql parser error: Expected literal int, found: hello at Line: 1, Column 86
 
 drop table demo;
 
diff --git a/tests/cases/standalone/common/copy/copy_from_fs_parquet.sql b/tests/cases/standalone/common/copy/copy_from_fs_parquet.sql
index 10319e1281..0db5e119d8 100644
--- a/tests/cases/standalone/common/copy/copy_from_fs_parquet.sql
+++ b/tests/cases/standalone/common/copy/copy_from_fs_parquet.sql
@@ -16,41 +16,41 @@ values
     ('host5', 99.9, 444.4, 1655276556000),
     ('host6', 222.2, 555.5, 1722077264000);
 
-Copy demo TO '/tmp/demo/export/parquet_files/demo.parquet';
+Copy demo TO '${SQLNESS_HOME}/demo/export/parquet_files/demo.parquet';
 
-Copy demo_2 TO '/tmp/demo/export/parquet_files/demo_2.parquet';
+Copy demo_2 TO '${SQLNESS_HOME}/demo/export/parquet_files/demo_2.parquet';
 
 CREATE TABLE with_filename(host string, cpu double, memory double, ts timestamp time index);
 
-Copy with_filename FROM '/tmp/demo/export/parquet_files/demo.parquet' with (start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
+Copy with_filename FROM '${SQLNESS_HOME}/demo/export/parquet_files/demo.parquet' with (start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:39');
 
 select * from with_filename order by ts;
 
 CREATE TABLE with_path(host string, cpu double, memory double, ts timestamp time index);
 
-Copy with_path FROM '/tmp/demo/export/parquet_files/';
+Copy with_path FROM '${SQLNESS_HOME}/demo/export/parquet_files/';
 
 select * from with_path order by ts;
 
 CREATE TABLE with_pattern(host string, cpu double, memory double, ts timestamp time index);
 
-Copy with_pattern FROM '/tmp/demo/export/parquet_files/' WITH (PATTERN = 'demo.*', start_time='2022-06-15 07:02:39');
+Copy with_pattern FROM '${SQLNESS_HOME}/demo/export/parquet_files/' WITH (PATTERN = 'demo.*', start_time='2022-06-15 07:02:39');
 
 select * from with_pattern order by ts;
 
 CREATE TABLE without_limit_rows(host string, cpu double, memory double, ts timestamp time index);
 
-Copy without_limit_rows FROM '/tmp/demo/export/parquet_files/';
+Copy without_limit_rows FROM '${SQLNESS_HOME}/demo/export/parquet_files/';
 
 select count(*) from without_limit_rows;
 
 CREATE TABLE with_limit_rows_segment(host string, cpu double, memory double, ts timestamp time index);
 
-Copy with_limit_rows_segment FROM '/tmp/demo/export/parquet_files/' LIMIT 3;
+Copy with_limit_rows_segment FROM '${SQLNESS_HOME}/demo/export/parquet_files/' LIMIT 3;
 
 select count(*) from with_limit_rows_segment;
 
-Copy with_limit_rows_segment FROM '/tmp/demo/export/parquet_files/' LIMIT hello;
+Copy with_limit_rows_segment FROM '${SQLNESS_HOME}/demo/export/parquet_files/' LIMIT hello;
 
 drop table demo;
 
diff --git a/tests/cases/standalone/common/copy/copy_to_fs.result b/tests/cases/standalone/common/copy/copy_to_fs.result
index f6bdc22a20..a3b73ab4ad 100644
--- a/tests/cases/standalone/common/copy/copy_to_fs.result
+++ b/tests/cases/standalone/common/copy/copy_to_fs.result
@@ -6,15 +6,15 @@ insert into demo(host, cpu, memory, ts) values ('host1', 66.6, 1024, 16552765570
 
 Affected Rows: 2
 
-COPY demo TO '/tmp/export/demo.parquet' WITH (start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:38');
+COPY demo TO '${SQLNESS_HOME}/export/demo.parquet' WITH (start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:38');
 
 Affected Rows: 1
 
-COPY demo TO '/tmp/export/demo.csv' WITH (format='csv', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:38');
+COPY demo TO '${SQLNESS_HOME}/export/demo.csv' WITH (format='csv', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:38');
 
 Affected Rows: 1
 
-COPY demo TO '/tmp/export/demo.json' WITH (format='json', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:38');
+COPY demo TO '${SQLNESS_HOME}/export/demo.json' WITH (format='json', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:38');
 
 Affected Rows: 1
 
diff --git a/tests/cases/standalone/common/copy/copy_to_fs.sql b/tests/cases/standalone/common/copy/copy_to_fs.sql
index d095c2d795..82e3ae1fb3 100644
--- a/tests/cases/standalone/common/copy/copy_to_fs.sql
+++ b/tests/cases/standalone/common/copy/copy_to_fs.sql
@@ -2,10 +2,10 @@ CREATE TABLE demo(host string, cpu DOUBLE, memory DOUBLE, ts TIMESTAMP TIME INDE
 
 insert into demo(host, cpu, memory, ts) values ('host1', 66.6, 1024, 1655276557000), ('host2', 88.8,  333.3, 1655276558000);
 
-COPY demo TO '/tmp/export/demo.parquet' WITH (start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:38');
+COPY demo TO '${SQLNESS_HOME}/export/demo.parquet' WITH (start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:38');
 
-COPY demo TO '/tmp/export/demo.csv' WITH (format='csv', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:38');
+COPY demo TO '${SQLNESS_HOME}/export/demo.csv' WITH (format='csv', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:38');
 
-COPY demo TO '/tmp/export/demo.json' WITH (format='json', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:38');
+COPY demo TO '${SQLNESS_HOME}/export/demo.json' WITH (format='json', start_time='2022-06-15 07:02:37', end_time='2022-06-15 07:02:38');
 
 drop table demo;
diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs
index d8ed5713f5..9d71489f8c 100644
--- a/tests/runner/src/env.rs
+++ b/tests/runner/src/env.rs
@@ -54,7 +54,7 @@ pub enum WalConfig {
 
 #[derive(Clone)]
 pub struct Env {
-    data_home: PathBuf,
+    sqlness_home: PathBuf,
     server_addr: Option<String>,
     wal: WalConfig,
 
@@ -69,6 +69,7 @@ impl EnvController for Env {
     type DB = GreptimeDB;
 
     async fn start(&self, mode: &str, _config: Option<&Path>) -> Self::DB {
+        std::env::set_var("SQLNESS_HOME", self.sqlness_home.display().to_string());
         match mode {
             "standalone" => self.start_standalone().await,
             "distributed" => self.start_distributed().await,
@@ -90,7 +91,7 @@ impl Env {
         bins_dir: Option<PathBuf>,
     ) -> Self {
         Self {
-            data_home,
+            sqlness_home: data_home,
             server_addr,
             wal,
             bins_dir: Arc::new(Mutex::new(bins_dir)),
@@ -204,26 +205,30 @@ impl Env {
             "standalone" => "greptime-sqlness-standalone.log".to_string(),
             _ => panic!("Unexpected subcommand: {subcommand}"),
         };
-        let log_file_name = self.data_home.join(log_file_name).display().to_string();
+        let stdout_file_name = self.sqlness_home.join(log_file_name).display().to_string();
 
-        println!("{subcommand} log file at {log_file_name}");
+        println!("{subcommand} log file at {stdout_file_name}");
 
-        let log_file = OpenOptions::new()
+        let stdout_file = OpenOptions::new()
             .create(true)
             .write(true)
             .truncate(truncate_log)
             .append(!truncate_log)
-            .open(log_file_name)
+            .open(stdout_file_name)
             .unwrap();
 
         let (args, check_ip_addr) = match subcommand {
             "datanode" => self.datanode_start_args(db_ctx),
-            "flownode" => self.flownode_start_args(db_ctx),
+            "flownode" => self.flownode_start_args(db_ctx, &self.sqlness_home),
             "standalone" => {
                 let args = vec![
                     DEFAULT_LOG_LEVEL.to_string(),
                     subcommand.to_string(),
                     "start".to_string(),
+                    format!(
+                        "--log-dir={}/greptimedb-flownode/logs",
+                        self.sqlness_home.display()
+                    ),
                     "-c".to_string(),
                     self.generate_config_file(subcommand, db_ctx),
                     "--http-addr=127.0.0.1:5002".to_string(),
@@ -237,6 +242,10 @@ impl Env {
                     "start".to_string(),
                     "--metasrv-addrs=127.0.0.1:3002".to_string(),
                     "--http-addr=127.0.0.1:5003".to_string(),
+                    format!(
+                        "--log-dir={}/greptimedb-frontend/logs",
+                        self.sqlness_home.display()
+                    ),
                 ];
                 (args, SERVER_ADDR.to_string())
             }
@@ -250,6 +259,10 @@ impl Env {
                     "--enable-region-failover".to_string(),
                     "false".to_string(),
                     "--http-addr=127.0.0.1:5002".to_string(),
+                    format!(
+                        "--log-dir={}/greptimedb-metasrv/logs",
+                        self.sqlness_home.display()
+                    ),
                     "-c".to_string(),
                     self.generate_config_file(subcommand, db_ctx),
                 ];
@@ -278,7 +291,7 @@ impl Env {
             .current_dir(bins_dir)
             .env("TZ", "UTC")
             .args(args)
-            .stdout(log_file)
+            .stdout(stdout_file)
             .spawn()
             .unwrap_or_else(|error| {
                 panic!("Failed to start the DB with subcommand {subcommand},Error: {error}")
@@ -296,7 +309,7 @@ impl Env {
         let id = db_ctx.datanode_id();
 
         let data_home = self
-            .data_home
+            .sqlness_home
             .join(format!("greptimedb_datanode_{}_{id}", db_ctx.time));
 
         let subcommand = "datanode";
@@ -308,6 +321,7 @@ impl Env {
         args.push(format!("--rpc-addr=127.0.0.1:410{id}"));
         args.push(format!("--http-addr=127.0.0.1:430{id}"));
         args.push(format!("--data-home={}", data_home.display()));
+        args.push(format!("--log-dir={}/logs", data_home.display()));
         args.push(format!("--node-id={id}"));
         args.push("-c".to_string());
         args.push(self.generate_config_file(subcommand, db_ctx));
@@ -315,7 +329,11 @@ impl Env {
         (args, format!("127.0.0.1:410{id}"))
     }
 
-    fn flownode_start_args(&self, _db_ctx: &GreptimeDBContext) -> (Vec<String>, String) {
+    fn flownode_start_args(
+        &self,
+        _db_ctx: &GreptimeDBContext,
+        sqlness_home: &Path,
+    ) -> (Vec<String>, String) {
         let id = 0;
 
         let subcommand = "flownode";
@@ -326,6 +344,10 @@ impl Env {
         ];
         args.push(format!("--rpc-addr=127.0.0.1:680{id}"));
         args.push(format!("--node-id={id}"));
+        args.push(format!(
+            "--log-dir={}/greptimedb-flownode/logs",
+            sqlness_home.display()
+        ));
         args.push("--metasrv-addrs=127.0.0.1:3002".to_string());
         (args, format!("127.0.0.1:680{id}"))
     }
@@ -387,7 +409,7 @@ impl Env {
             kafka_wal_broker_endpoints: String,
         }
 
-        let data_home = self.data_home.join(format!("greptimedb-{subcommand}"));
+        let data_home = self.sqlness_home.join(format!("greptimedb-{subcommand}"));
         std::fs::create_dir_all(data_home.as_path()).unwrap();
 
         let wal_dir = data_home.join("wal").display().to_string();
diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs
index 1684962645..4474676802 100644
--- a/tests/runner/src/main.rs
+++ b/tests/runner/src/main.rs
@@ -82,7 +82,7 @@ async fn main() {
         .prefix("sqlness")
         .tempdir()
         .unwrap();
-    let data_home = temp_dir.into_path();
+    let sqlness_home = temp_dir.into_path();
 
     let config = ConfigBuilder::default()
         .case_dir(util::get_case_dir(args.case_dir))
@@ -107,13 +107,13 @@ async fn main() {
 
     let runner = Runner::new(
         config,
-        Env::new(data_home.clone(), args.server_addr, wal, args.bins_dir),
+        Env::new(sqlness_home.clone(), args.server_addr, wal, args.bins_dir),
     );
     runner.run().await.unwrap();
 
     // clean up and exit
     if !args.preserve_state {
-        println!("Removing state in {:?}", data_home);
-        tokio::fs::remove_dir_all(data_home).await.unwrap();
+        println!("Removing state in {:?}", sqlness_home);
+        tokio::fs::remove_dir_all(sqlness_home).await.unwrap();
     }
 }

From 50cb59587db67bb0248ce51d2f93eac860060b14 Mon Sep 17 00:00:00 2001
From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com>
Date: Sun, 29 Sep 2024 14:27:35 +0800
Subject: [PATCH 046/128] chore: replace anymap with anymap2 (#4781)

---
 Cargo.lock                     |  2 +-
 src/common/base/Cargo.toml     |  2 +-
 src/common/base/src/plugins.rs | 13 +++++++------
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2ae89afc74..4533257660 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1868,7 +1868,7 @@ checksum = "55b672471b4e9f9e95499ea597ff64941a309b2cdbffcc46f2cc5e2d971fd335"
 name = "common-base"
 version = "0.9.3"
 dependencies = [
- "anymap",
+ "anymap2",
  "async-trait",
  "bitvec",
  "bytes",
diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml
index 5afbc3b88c..5dfde47bf2 100644
--- a/src/common/base/Cargo.toml
+++ b/src/common/base/Cargo.toml
@@ -8,7 +8,7 @@ license.workspace = true
 workspace = true
 
 [dependencies]
-anymap = "1.0.0-beta.2"
+anymap2 = "0.13"
 async-trait.workspace = true
 bitvec = "1.0"
 bytes.workspace = true
diff --git a/src/common/base/src/plugins.rs b/src/common/base/src/plugins.rs
index 84d78b0c91..98b40cd059 100644
--- a/src/common/base/src/plugins.rs
+++ b/src/common/base/src/plugins.rs
@@ -12,20 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::any::Any;
 use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
 
-/// [`Plugins`] is a wrapper of [AnyMap](https://github.com/chris-morgan/anymap) and provides a thread-safe way to store and retrieve plugins.
+use anymap2::SendSyncAnyMap;
+
+/// [`Plugins`] is a wrapper of [anymap2](https://github.com/azriel91/anymap2) and provides a thread-safe way to store and retrieve plugins.
 /// Make it Cloneable and we can treat it like an Arc struct.
 #[derive(Default, Clone)]
 pub struct Plugins {
-    inner: Arc<RwLock<anymap::Map<dyn Any + Send + Sync>>>,
+    inner: Arc<RwLock<SendSyncAnyMap>>,
 }
 
 impl Plugins {
     pub fn new() -> Self {
         Self {
-            inner: Arc::new(RwLock::new(anymap::Map::new())),
+            inner: Arc::new(RwLock::new(SendSyncAnyMap::new())),
         }
     }
 
@@ -61,11 +62,11 @@ impl Plugins {
         self.read().is_empty()
     }
 
-    fn read(&self) -> RwLockReadGuard<anymap::Map<dyn Any + Send + Sync>> {
+    fn read(&self) -> RwLockReadGuard<SendSyncAnyMap> {
         self.inner.read().unwrap()
     }
 
-    fn write(&self) -> RwLockWriteGuard<anymap::Map<dyn Any + Send + Sync>> {
+    fn write(&self) -> RwLockWriteGuard<SendSyncAnyMap> {
         self.inner.write().unwrap()
     }
 }

From cd552021362c45f2ef0292ac6e85fb775d2d8449 Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Sun, 29 Sep 2024 15:14:48 +0800
Subject: [PATCH 047/128] feat: unordered scanner scans data by time ranges
 (#4757)

* feat: define range meta

* feat: group ranges

* feat: split range

* feat: build ranges from the scan input

* feat: get partition range from range meta

* feat: build file range

* feat: unordered scan read by ranges

* feat: wip for mem ranges

* feat: build ranges

* feat: remove unused codes

* chore: update comments

* feat: update metrics

* chore: address review comments

* chore: debug assertion
---
 src/mito2/src/memtable.rs                |  12 +-
 src/mito2/src/memtable/bulk.rs           |   3 +-
 src/mito2/src/memtable/partition_tree.rs |   7 +-
 src/mito2/src/memtable/time_series.rs    |   6 +-
 src/mito2/src/read.rs                    |   5 +
 src/mito2/src/read/dedup.rs              |   1 -
 src/mito2/src/read/range.rs              | 266 ++++++++++++++++
 src/mito2/src/read/scan_region.rs        | 298 +++++++++++++++---
 src/mito2/src/read/seq_scan.rs           |   7 +-
 src/mito2/src/read/unordered_scan.rs     | 374 ++++++++++-------------
 src/mito2/src/sst/parquet.rs             |   2 +-
 src/mito2/src/sst/parquet/reader.rs      |   1 +
 src/mito2/src/test_util/memtable_util.rs |   5 +-
 13 files changed, 711 insertions(+), 276 deletions(-)
 create mode 100644 src/mito2/src/read/range.rs

diff --git a/src/mito2/src/memtable.rs b/src/mito2/src/memtable.rs
index e79a3290be..668eb51023 100644
--- a/src/mito2/src/memtable.rs
+++ b/src/mito2/src/memtable.rs
@@ -14,6 +14,7 @@
 
 //! Memtables are write buffers for regions.
 
+use std::collections::BTreeMap;
 use std::fmt;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use std::sync::Arc;
@@ -71,6 +72,8 @@ pub struct MemtableStats {
     time_range: Option<(Timestamp, Timestamp)>,
     /// Total rows in memtable
     num_rows: usize,
+    /// Total number of ranges in the memtable.
+    num_ranges: usize,
 }
 
 impl MemtableStats {
@@ -95,6 +98,11 @@ impl MemtableStats {
     pub fn num_rows(&self) -> usize {
         self.num_rows
     }
+
+    /// Returns the number of ranges in the memtable.
+    pub fn num_ranges(&self) -> usize {
+        self.num_ranges
+    }
 }
 
 pub type BoxedBatchIterator = Box<dyn Iterator<Item = Result<Batch>> + Send>;
@@ -123,11 +131,12 @@ pub trait Memtable: Send + Sync + fmt::Debug {
     ) -> Result<BoxedBatchIterator>;
 
     /// Returns the ranges in the memtable.
+    /// The returned map contains the range id and the range after applying the predicate.
     fn ranges(
         &self,
         projection: Option<&[ColumnId]>,
         predicate: Option<Predicate>,
-    ) -> Vec<MemtableRange>;
+    ) -> BTreeMap<usize, MemtableRange>;
 
     /// Returns true if the memtable is empty.
     fn is_empty(&self) -> bool;
@@ -332,7 +341,6 @@ impl MemtableRangeContext {
 pub struct MemtableRange {
     /// Shared context.
     context: MemtableRangeContextRef,
-    // TODO(yingwen): Id to identify the range in the memtable.
 }
 
 impl MemtableRange {
diff --git a/src/mito2/src/memtable/bulk.rs b/src/mito2/src/memtable/bulk.rs
index 3ed2ed1347..46e757f3df 100644
--- a/src/mito2/src/memtable/bulk.rs
+++ b/src/mito2/src/memtable/bulk.rs
@@ -14,6 +14,7 @@
 
 //! Memtable implementation for bulk load
 
+use std::collections::BTreeMap;
 use std::sync::{Arc, RwLock};
 
 use store_api::metadata::RegionMetadataRef;
@@ -67,7 +68,7 @@ impl Memtable for BulkMemtable {
         &self,
         _projection: Option<&[ColumnId]>,
         _predicate: Option<Predicate>,
-    ) -> Vec<MemtableRange> {
+    ) -> BTreeMap<usize, MemtableRange> {
         todo!()
     }
 
diff --git a/src/mito2/src/memtable/partition_tree.rs b/src/mito2/src/memtable/partition_tree.rs
index e320503886..3309e66b7e 100644
--- a/src/mito2/src/memtable/partition_tree.rs
+++ b/src/mito2/src/memtable/partition_tree.rs
@@ -24,6 +24,7 @@ mod shard;
 mod shard_builder;
 mod tree;
 
+use std::collections::BTreeMap;
 use std::fmt;
 use std::sync::atomic::{AtomicI64, AtomicUsize, Ordering};
 use std::sync::Arc;
@@ -176,7 +177,7 @@ impl Memtable for PartitionTreeMemtable {
         &self,
         projection: Option<&[ColumnId]>,
         predicate: Option<Predicate>,
-    ) -> Vec<MemtableRange> {
+    ) -> BTreeMap<usize, MemtableRange> {
         let projection = projection.map(|ids| ids.to_vec());
         let builder = Box::new(PartitionTreeIterBuilder {
             tree: self.tree.clone(),
@@ -185,7 +186,7 @@ impl Memtable for PartitionTreeMemtable {
         });
         let context = Arc::new(MemtableRangeContext::new(self.id, builder));
 
-        vec![MemtableRange::new(context)]
+        [(0, MemtableRange::new(context))].into()
     }
 
     fn is_empty(&self) -> bool {
@@ -207,6 +208,7 @@ impl Memtable for PartitionTreeMemtable {
                 estimated_bytes,
                 time_range: None,
                 num_rows: 0,
+                num_ranges: 0,
             };
         }
 
@@ -225,6 +227,7 @@ impl Memtable for PartitionTreeMemtable {
             estimated_bytes,
             time_range: Some((min_timestamp, max_timestamp)),
             num_rows: self.num_rows.load(Ordering::Relaxed),
+            num_ranges: 1,
         }
     }
 
diff --git a/src/mito2/src/memtable/time_series.rs b/src/mito2/src/memtable/time_series.rs
index 6d5fbb33a0..d82db55730 100644
--- a/src/mito2/src/memtable/time_series.rs
+++ b/src/mito2/src/memtable/time_series.rs
@@ -287,7 +287,7 @@ impl Memtable for TimeSeriesMemtable {
         &self,
         projection: Option<&[ColumnId]>,
         predicate: Option<Predicate>,
-    ) -> Vec<MemtableRange> {
+    ) -> BTreeMap<usize, MemtableRange> {
         let projection = if let Some(projection) = projection {
             projection.iter().copied().collect()
         } else {
@@ -305,7 +305,7 @@ impl Memtable for TimeSeriesMemtable {
         });
         let context = Arc::new(MemtableRangeContext::new(self.id, builder));
 
-        vec![MemtableRange::new(context)]
+        [(0, MemtableRange::new(context))].into()
     }
 
     fn is_empty(&self) -> bool {
@@ -327,6 +327,7 @@ impl Memtable for TimeSeriesMemtable {
                 estimated_bytes,
                 time_range: None,
                 num_rows: 0,
+                num_ranges: 0,
             };
         }
         let ts_type = self
@@ -343,6 +344,7 @@ impl Memtable for TimeSeriesMemtable {
             estimated_bytes,
             time_range: Some((min_timestamp, max_timestamp)),
             num_rows: self.num_rows.load(Ordering::Relaxed),
+            num_ranges: 1,
         }
     }
 
diff --git a/src/mito2/src/read.rs b/src/mito2/src/read.rs
index edde28dab8..d9a6fae746 100644
--- a/src/mito2/src/read.rs
+++ b/src/mito2/src/read.rs
@@ -20,6 +20,7 @@ pub mod last_row;
 pub mod merge;
 pub mod projection;
 pub(crate) mod prune;
+pub(crate) mod range;
 pub(crate) mod scan_region;
 pub(crate) mod seq_scan;
 pub(crate) mod unordered_scan;
@@ -753,6 +754,10 @@ pub(crate) struct ScannerMetrics {
     num_batches: usize,
     /// Number of rows returned.
     num_rows: usize,
+    /// Number of mem ranges scanned.
+    num_mem_ranges: usize,
+    /// Number of file ranges scanned.
+    num_file_ranges: usize,
     /// Filter related metrics for readers.
     filter_metrics: ReaderFilterMetrics,
 }
diff --git a/src/mito2/src/read/dedup.rs b/src/mito2/src/read/dedup.rs
index ddc96049e7..c77d0c3fab 100644
--- a/src/mito2/src/read/dedup.rs
+++ b/src/mito2/src/read/dedup.rs
@@ -400,7 +400,6 @@ pub(crate) struct LastNonNull {
 
 impl LastNonNull {
     /// Creates a new strategy with the given `filter_deleted` flag.
-    #[allow(dead_code)]
     pub(crate) fn new(filter_deleted: bool) -> Self {
         Self {
             buffer: None,
diff --git a/src/mito2/src/read/range.rs b/src/mito2/src/read/range.rs
new file mode 100644
index 0000000000..8408aad7b2
--- /dev/null
+++ b/src/mito2/src/read/range.rs
@@ -0,0 +1,266 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Structs for partition ranges.
+
+use smallvec::{smallvec, SmallVec};
+
+use crate::memtable::MemtableRef;
+use crate::read::scan_region::ScanInput;
+use crate::sst::file::{overlaps, FileHandle, FileTimeRange};
+use crate::sst::parquet::DEFAULT_ROW_GROUP_SIZE;
+
+const ALL_ROW_GROUPS: i64 = -1;
+
+/// Index to access a row group.
+#[derive(Clone, Copy, PartialEq)]
+pub(crate) struct RowGroupIndex {
+    /// Index to the memtable/file.
+    pub(crate) index: usize,
+    /// Row group index in the file.
+    /// Negative index indicates all row groups.
+    pub(crate) row_group_index: i64,
+}
+
+/// Meta data of a partition range.
+/// If the scanner is [UnorderedScan], each meta only has one row group or memtable.
+/// If the scanner is [SeqScan], each meta may have multiple row groups and memtables.
+pub(crate) struct RangeMeta {
+    /// The time range of the range.
+    pub(crate) time_range: FileTimeRange,
+    /// Indices to memtables or files.
+    indices: SmallVec<[usize; 2]>,
+    /// Indices to memtable/file row groups that this range scans.
+    pub(crate) row_group_indices: SmallVec<[RowGroupIndex; 2]>,
+    /// Estimated number of rows in the range. This can be 0 if the statistics are not available.
+    pub(crate) num_rows: usize,
+}
+
+impl RangeMeta {
+    /// Creates a list of ranges from the `input` for seq scan.
+    pub(crate) fn seq_scan_ranges(input: &ScanInput) -> Vec<RangeMeta> {
+        let mut ranges = Vec::with_capacity(input.memtables.len() + input.files.len());
+        Self::push_seq_mem_ranges(&input.memtables, &mut ranges);
+        Self::push_seq_file_ranges(input.memtables.len(), &input.files, &mut ranges);
+
+        let ranges = group_ranges_for_seq_scan(ranges);
+        maybe_split_ranges_for_seq_scan(ranges)
+    }
+
+    /// Creates a list of ranges from the `input` for unordered scan.
+    pub(crate) fn unordered_scan_ranges(input: &ScanInput) -> Vec<RangeMeta> {
+        let mut ranges = Vec::with_capacity(input.memtables.len() + input.files.len());
+        Self::push_unordered_mem_ranges(&input.memtables, &mut ranges);
+        Self::push_unordered_file_ranges(input.memtables.len(), &input.files, &mut ranges);
+
+        ranges
+    }
+
+    /// Returns true if the time range of given `meta` overlaps with the time range of this meta.
+    pub(crate) fn overlaps(&self, meta: &RangeMeta) -> bool {
+        overlaps(&self.time_range, &meta.time_range)
+    }
+
+    /// Merges given `meta` to this meta.
+    /// It assumes that the time ranges overlap and they don't have the same file or memtable index.
+    pub(crate) fn merge(&mut self, mut other: RangeMeta) {
+        debug_assert!(self.overlaps(&other));
+        debug_assert!(self.indices.iter().all(|idx| !other.indices.contains(idx)));
+        debug_assert!(self
+            .row_group_indices
+            .iter()
+            .all(|idx| !other.row_group_indices.contains(idx)));
+
+        self.time_range = (
+            self.time_range.0.min(other.time_range.0),
+            self.time_range.1.max(other.time_range.1),
+        );
+        self.indices.append(&mut other.indices);
+        self.row_group_indices.append(&mut other.row_group_indices);
+        self.num_rows += other.num_rows;
+    }
+
+    /// Returns true if we can split the range into multiple smaller ranges and
+    /// still preserve the order for [SeqScan].
+    pub(crate) fn can_split_preserve_order(&self) -> bool {
+        // Only one source and multiple row groups.
+        self.indices.len() == 1 && self.row_group_indices.len() > 1
+    }
+
+    /// Splits the range if it can preserve the order.
+    pub(crate) fn maybe_split(self, output: &mut Vec<RangeMeta>) {
+        if self.can_split_preserve_order() {
+            output.reserve(self.row_group_indices.len());
+            let num_rows = self.num_rows / self.row_group_indices.len();
+            // Splits by row group.
+            for index in self.row_group_indices {
+                output.push(RangeMeta {
+                    time_range: self.time_range,
+                    indices: self.indices.clone(),
+                    row_group_indices: smallvec![index],
+                    num_rows,
+                });
+            }
+        } else {
+            output.push(self);
+        }
+    }
+
+    fn push_unordered_mem_ranges(memtables: &[MemtableRef], ranges: &mut Vec<RangeMeta>) {
+        // For append mode, we can parallelize reading memtables.
+        for (memtable_index, memtable) in memtables.iter().enumerate() {
+            let stats = memtable.stats();
+            let Some(time_range) = stats.time_range() else {
+                continue;
+            };
+            for row_group_index in 0..stats.num_ranges() {
+                let num_rows = stats.num_rows() / stats.num_ranges();
+                ranges.push(RangeMeta {
+                    time_range,
+                    indices: smallvec![memtable_index],
+                    row_group_indices: smallvec![RowGroupIndex {
+                        index: memtable_index,
+                        row_group_index: row_group_index as i64,
+                    }],
+                    num_rows,
+                });
+            }
+        }
+    }
+
+    fn push_unordered_file_ranges(
+        num_memtables: usize,
+        files: &[FileHandle],
+        ranges: &mut Vec<RangeMeta>,
+    ) {
+        // For append mode, we can parallelize reading row groups.
+        for (i, file) in files.iter().enumerate() {
+            let file_index = num_memtables + i;
+            if file.meta_ref().num_row_groups > 0 {
+                // Scans each row group.
+                for row_group_index in 0..file.meta_ref().num_row_groups {
+                    ranges.push(RangeMeta {
+                        time_range: file.time_range(),
+                        indices: smallvec![file_index],
+                        row_group_indices: smallvec![RowGroupIndex {
+                            index: file_index,
+                            row_group_index: row_group_index as i64,
+                        }],
+                        num_rows: DEFAULT_ROW_GROUP_SIZE,
+                    });
+                }
+            } else {
+                // If we don't known the number of row groups in advance, scan all row groups.
+                ranges.push(RangeMeta {
+                    time_range: file.time_range(),
+                    indices: smallvec![file_index],
+                    row_group_indices: smallvec![RowGroupIndex {
+                        index: file_index,
+                        row_group_index: ALL_ROW_GROUPS,
+                    }],
+                    // This may be 0.
+                    num_rows: file.meta_ref().num_rows as usize,
+                });
+            }
+        }
+    }
+
+    fn push_seq_mem_ranges(memtables: &[MemtableRef], ranges: &mut Vec<RangeMeta>) {
+        // For non append-only mode, each range only contains one memtable.
+        for (i, memtable) in memtables.iter().enumerate() {
+            let stats = memtable.stats();
+            let Some(time_range) = stats.time_range() else {
+                continue;
+            };
+            ranges.push(RangeMeta {
+                time_range,
+                indices: smallvec![i],
+                row_group_indices: smallvec![RowGroupIndex {
+                    index: i,
+                    row_group_index: ALL_ROW_GROUPS,
+                }],
+                num_rows: stats.num_rows(),
+            });
+        }
+    }
+
+    fn push_seq_file_ranges(
+        num_memtables: usize,
+        files: &[FileHandle],
+        ranges: &mut Vec<RangeMeta>,
+    ) {
+        // For non append-only mode, each range only contains one file.
+        for (i, file) in files.iter().enumerate() {
+            let file_index = num_memtables + i;
+            ranges.push(RangeMeta {
+                time_range: file.time_range(),
+                indices: smallvec![file_index],
+                row_group_indices: smallvec![RowGroupIndex {
+                    index: file_index,
+                    row_group_index: ALL_ROW_GROUPS,
+                }],
+                num_rows: file.meta_ref().num_rows as usize,
+            });
+        }
+    }
+}
+
+/// Groups ranges by time range.
+/// It assumes each input range only contains a file or a memtable.
+fn group_ranges_for_seq_scan(mut ranges: Vec<RangeMeta>) -> Vec<RangeMeta> {
+    if ranges.is_empty() {
+        return ranges;
+    }
+
+    // Sorts ranges by time range (start asc, end desc).
+    ranges.sort_unstable_by(|a, b| {
+        let l = a.time_range;
+        let r = b.time_range;
+        l.0.cmp(&r.0).then_with(|| r.1.cmp(&l.1))
+    });
+    let mut range_in_progress = None;
+    // Parts with exclusive time ranges.
+    let mut exclusive_ranges = Vec::with_capacity(ranges.len());
+    for range in ranges {
+        let Some(mut prev_range) = range_in_progress.take() else {
+            // This is the new range to process.
+            range_in_progress = Some(range);
+            continue;
+        };
+
+        if prev_range.overlaps(&range) {
+            prev_range.merge(range);
+            range_in_progress = Some(prev_range);
+        } else {
+            exclusive_ranges.push(prev_range);
+            range_in_progress = Some(range);
+        }
+    }
+    if let Some(range) = range_in_progress {
+        exclusive_ranges.push(range);
+    }
+
+    exclusive_ranges
+}
+
+/// Splits the range into multiple smaller ranges.
+/// It assumes the input `ranges` list is created by [group_ranges_for_seq_scan()].
+fn maybe_split_ranges_for_seq_scan(ranges: Vec<RangeMeta>) -> Vec<RangeMeta> {
+    let mut new_ranges = Vec::with_capacity(ranges.len());
+    for range in ranges {
+        range.maybe_split(&mut new_ranges);
+    }
+
+    new_ranges
+}
diff --git a/src/mito2/src/read/scan_region.rs b/src/mito2/src/read/scan_region.rs
index d28562c554..28deb1186e 100644
--- a/src/mito2/src/read/scan_region.rs
+++ b/src/mito2/src/read/scan_region.rs
@@ -14,9 +14,9 @@
 
 //! Scans a region according to the scan request.
 
-use std::collections::HashSet;
+use std::collections::{BTreeMap, HashSet};
 use std::fmt;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex as StdMutex};
 use std::time::{Duration, Instant};
 
 use common_error::ext::BoxedError;
@@ -26,6 +26,7 @@ use common_time::range::TimestampRange;
 use common_time::Timestamp;
 use datafusion::physical_plan::DisplayFormatType;
 use datafusion_expr::utils::expr_to_columns;
+use parquet::arrow::arrow_reader::RowSelection;
 use smallvec::SmallVec;
 use store_api::region_engine::{PartitionRange, RegionScannerRef};
 use store_api::storage::{ScanRequest, TimeSeriesRowSelector};
@@ -41,6 +42,7 @@ use crate::memtable::{MemtableRange, MemtableRef};
 use crate::metrics::READ_SST_COUNT;
 use crate::read::compat::{self, CompatBatch};
 use crate::read::projection::ProjectionMapper;
+use crate::read::range::{RangeMeta, RowGroupIndex};
 use crate::read::seq_scan::SeqScan;
 use crate::read::unordered_scan::UnorderedScan;
 use crate::read::{Batch, Source};
@@ -51,7 +53,7 @@ use crate::sst::index::fulltext_index::applier::builder::FulltextIndexApplierBui
 use crate::sst::index::fulltext_index::applier::FulltextIndexApplierRef;
 use crate::sst::index::inverted_index::applier::builder::InvertedIndexApplierBuilder;
 use crate::sst::index::inverted_index::applier::InvertedIndexApplierRef;
-use crate::sst::parquet::file_range::FileRange;
+use crate::sst::parquet::file_range::{FileRange, FileRangeContextRef};
 use crate::sst::parquet::reader::ReaderMetrics;
 
 /// A scanner scans a region and returns a [SendableRecordBatchStream].
@@ -643,6 +645,61 @@ impl ScanInput {
         Ok(sources)
     }
 
+    /// Prunes a memtable to scan and returns the builder to build readers.
+    fn prune_memtable(&self, mem_index: usize) -> MemRangeBuilder {
+        let memtable = &self.memtables[mem_index];
+        let row_groups = memtable.ranges(Some(self.mapper.column_ids()), self.predicate.clone());
+        MemRangeBuilder { row_groups }
+    }
+
+    /// Prunes a file to scan and returns the builder to build readers.
+    async fn prune_file(
+        &self,
+        file_index: usize,
+        reader_metrics: &mut ReaderMetrics,
+    ) -> Result<FileRangeBuilder> {
+        let file = &self.files[file_index];
+        let res = self
+            .access_layer
+            .read_sst(file.clone())
+            .predicate(self.predicate.clone())
+            .time_range(self.time_range)
+            .projection(Some(self.mapper.column_ids().to_vec()))
+            .cache(self.cache_manager.clone())
+            .inverted_index_applier(self.inverted_index_applier.clone())
+            .fulltext_index_applier(self.fulltext_index_applier.clone())
+            .expected_metadata(Some(self.mapper.metadata().clone()))
+            .build_reader_input(reader_metrics)
+            .await;
+        let (mut file_range_ctx, row_groups) = match res {
+            Ok(x) => x,
+            Err(e) => {
+                if e.is_object_not_found() && self.ignore_file_not_found {
+                    error!(e; "File to scan does not exist, region_id: {}, file: {}", file.region_id(), file.file_id());
+                    return Ok(FileRangeBuilder::default());
+                } else {
+                    return Err(e);
+                }
+            }
+        };
+        if !compat::has_same_columns(
+            self.mapper.metadata(),
+            file_range_ctx.read_format().metadata(),
+        ) {
+            // They have different schema. We need to adapt the batch first so the
+            // mapper can convert it.
+            let compat = CompatBatch::new(
+                &self.mapper,
+                file_range_ctx.read_format().metadata().clone(),
+            )?;
+            file_range_ctx.set_compat_batch(Some(compat));
+        }
+        Ok(FileRangeBuilder {
+            context: Some(Arc::new(file_range_ctx)),
+            row_groups,
+        })
+    }
+
     /// Prunes file ranges to scan and adds them to the `collector`.
     pub(crate) async fn prune_file_ranges(
         &self,
@@ -749,51 +806,6 @@ impl ScanInput {
     pub(crate) fn predicate(&self) -> Option<Predicate> {
         self.predicate.clone()
     }
-
-    /// Retrieves [`PartitionRange`] from memtable and files
-    pub(crate) fn partition_ranges(&self) -> Vec<PartitionRange> {
-        let mut id = 0;
-        let mut container = Vec::with_capacity(self.memtables.len() + self.files.len());
-
-        for memtable in &self.memtables {
-            let range = PartitionRange {
-                // TODO(ruihang): filter out empty memtables in the future.
-                start: memtable.stats().time_range().unwrap().0,
-                end: memtable.stats().time_range().unwrap().1,
-                num_rows: memtable.stats().num_rows(),
-                identifier: id,
-            };
-            id += 1;
-            container.push(range);
-        }
-
-        for file in &self.files {
-            if self.append_mode {
-                // For append mode, we can parallelize reading row groups.
-                for _ in 0..file.meta_ref().num_row_groups {
-                    let range = PartitionRange {
-                        start: file.time_range().0,
-                        end: file.time_range().1,
-                        num_rows: file.num_rows(),
-                        identifier: id,
-                    };
-                    id += 1;
-                    container.push(range);
-                }
-            } else {
-                let range = PartitionRange {
-                    start: file.meta_ref().time_range.0,
-                    end: file.meta_ref().time_range.1,
-                    num_rows: file.meta_ref().num_rows as usize,
-                    identifier: id,
-                };
-                id += 1;
-                container.push(range);
-            }
-        }
-
-        container
-    }
 }
 
 #[cfg(test)]
@@ -967,6 +979,10 @@ pub(crate) struct StreamContext {
     /// The scanner builds parts to scan from the input lazily.
     /// The mutex is used to ensure the parts are only built once.
     pub(crate) parts: Mutex<(ScanPartList, Duration)>,
+    /// Metadata for partition ranges.
+    pub(crate) ranges: Vec<RangeMeta>,
+    /// Lists of range builders.
+    range_builders: RangeBuilderList,
 
     // Metrics:
     /// The start time of the query.
@@ -974,17 +990,77 @@ pub(crate) struct StreamContext {
 }
 
 impl StreamContext {
-    /// Creates a new [StreamContext].
-    pub(crate) fn new(input: ScanInput) -> Self {
+    /// Creates a new [StreamContext] for [SeqScan].
+    pub(crate) fn seq_scan_ctx(input: ScanInput) -> Self {
         let query_start = input.query_start.unwrap_or_else(Instant::now);
+        let ranges = RangeMeta::seq_scan_ranges(&input);
+        READ_SST_COUNT.observe(input.files.len() as f64);
+        let range_builders = RangeBuilderList::new(input.memtables.len(), input.files.len());
 
         Self {
             input,
             parts: Mutex::new((ScanPartList::default(), Duration::default())),
+            ranges,
+            range_builders,
             query_start,
         }
     }
 
+    /// Creates a new [StreamContext] for [UnorderedScan].
+    pub(crate) fn unordered_scan_ctx(input: ScanInput) -> Self {
+        let query_start = input.query_start.unwrap_or_else(Instant::now);
+        let ranges = RangeMeta::unordered_scan_ranges(&input);
+        READ_SST_COUNT.observe(input.files.len() as f64);
+        let range_builders = RangeBuilderList::new(input.memtables.len(), input.files.len());
+
+        Self {
+            input,
+            parts: Mutex::new((ScanPartList::default(), Duration::default())),
+            ranges,
+            range_builders,
+            query_start,
+        }
+    }
+
+    /// Returns true if the index refers to a memtable.
+    pub(crate) fn is_mem_range_index(&self, index: RowGroupIndex) -> bool {
+        self.input.memtables.len() > index.index
+    }
+
+    /// Creates file ranges to scan.
+    pub(crate) async fn build_file_ranges(
+        &self,
+        index: RowGroupIndex,
+        ranges: &mut Vec<FileRange>,
+        reader_metrics: &mut ReaderMetrics,
+    ) -> Result<()> {
+        ranges.clear();
+        self.range_builders
+            .build_file_ranges(&self.input, index, ranges, reader_metrics)
+            .await
+    }
+
+    /// Creates memtable ranges to scan.
+    pub(crate) fn build_mem_ranges(&self, index: RowGroupIndex, ranges: &mut Vec<MemtableRange>) {
+        ranges.clear();
+        self.range_builders
+            .build_mem_ranges(&self.input, index, ranges)
+    }
+
+    /// Retrieves the partition ranges.
+    pub(crate) fn partition_ranges(&self) -> Vec<PartitionRange> {
+        self.ranges
+            .iter()
+            .enumerate()
+            .map(|(idx, range_meta)| PartitionRange {
+                start: range_meta.time_range.0,
+                end: range_meta.time_range.1,
+                num_rows: range_meta.num_rows,
+                identifier: idx,
+            })
+            .collect()
+    }
+
     /// Format the context for explain.
     pub(crate) fn format_for_explain(
         &self,
@@ -1011,3 +1087,125 @@ impl StreamContext {
         Ok(())
     }
 }
+
+/// List to manages the builders to create file ranges.
+struct RangeBuilderList {
+    mem_builders: Vec<StdMutex<Option<MemRangeBuilder>>>,
+    file_builders: Vec<Mutex<Option<FileRangeBuilder>>>,
+}
+
+impl RangeBuilderList {
+    /// Creates a new [ReaderBuilderList] with the given number of memtables and files.
+    fn new(num_memtables: usize, num_files: usize) -> Self {
+        let mem_builders = (0..num_memtables).map(|_| StdMutex::new(None)).collect();
+        let file_builders = (0..num_files).map(|_| Mutex::new(None)).collect();
+        Self {
+            mem_builders,
+            file_builders,
+        }
+    }
+
+    /// Builds file ranges to read the row group at `index`.
+    async fn build_file_ranges(
+        &self,
+        input: &ScanInput,
+        index: RowGroupIndex,
+        ranges: &mut Vec<FileRange>,
+        reader_metrics: &mut ReaderMetrics,
+    ) -> Result<()> {
+        let file_index = index.index - self.mem_builders.len();
+        let mut builder_opt = self.file_builders[file_index].lock().await;
+        match &mut *builder_opt {
+            Some(builder) => builder.build_ranges(index.row_group_index, ranges),
+            None => {
+                let builder = input.prune_file(file_index, reader_metrics).await?;
+                builder.build_ranges(index.row_group_index, ranges);
+                *builder_opt = Some(builder);
+            }
+        }
+        Ok(())
+    }
+
+    /// Builds mem ranges to read the row group at `index`.
+    fn build_mem_ranges(
+        &self,
+        input: &ScanInput,
+        index: RowGroupIndex,
+        ranges: &mut Vec<MemtableRange>,
+    ) {
+        let mut builder_opt = self.mem_builders[index.index].lock().unwrap();
+        match &mut *builder_opt {
+            Some(builder) => builder.build_ranges(index.row_group_index, ranges),
+            None => {
+                let builder = input.prune_memtable(index.index);
+                builder.build_ranges(index.row_group_index, ranges);
+                *builder_opt = Some(builder);
+            }
+        }
+    }
+}
+
+/// Builder to create file ranges.
+#[derive(Default)]
+struct FileRangeBuilder {
+    /// Context for the file.
+    /// None indicates nothing to read.
+    context: Option<FileRangeContextRef>,
+    /// Row selections for each row group to read.
+    /// It skips the row group if it is not in the map.
+    row_groups: BTreeMap<usize, Option<RowSelection>>,
+}
+
+impl FileRangeBuilder {
+    /// Builds file ranges to read.
+    /// Negative `row_group_index` indicates all row groups.
+    fn build_ranges(&self, row_group_index: i64, ranges: &mut Vec<FileRange>) {
+        let Some(context) = self.context.clone() else {
+            return;
+        };
+        if row_group_index >= 0 {
+            let row_group_index = row_group_index as usize;
+            // Scans one row group.
+            let Some(row_selection) = self.row_groups.get(&row_group_index) else {
+                return;
+            };
+            ranges.push(FileRange::new(
+                context,
+                row_group_index,
+                row_selection.clone(),
+            ));
+        } else {
+            // Scans all row groups.
+            ranges.extend(
+                self.row_groups
+                    .iter()
+                    .map(|(row_group_index, row_selection)| {
+                        FileRange::new(context.clone(), *row_group_index, row_selection.clone())
+                    }),
+            );
+        }
+    }
+}
+
+/// Builder to create mem ranges.
+struct MemRangeBuilder {
+    /// Ranges of a memtable.
+    row_groups: BTreeMap<usize, MemtableRange>,
+}
+
+impl MemRangeBuilder {
+    /// Builds mem ranges to read in the memtable.
+    /// Negative `row_group_index` indicates all row groups.
+    fn build_ranges(&self, row_group_index: i64, ranges: &mut Vec<MemtableRange>) {
+        if row_group_index >= 0 {
+            let row_group_index = row_group_index as usize;
+            // Scans one row group.
+            let Some(range) = self.row_groups.get(&row_group_index) else {
+                return;
+            };
+            ranges.push(range.clone());
+        } else {
+            ranges.extend(self.row_groups.values().cloned());
+        }
+    }
+}
diff --git a/src/mito2/src/read/seq_scan.rs b/src/mito2/src/read/seq_scan.rs
index 296d55250b..6e45c8ce4a 100644
--- a/src/mito2/src/read/seq_scan.rs
+++ b/src/mito2/src/read/seq_scan.rs
@@ -68,11 +68,10 @@ impl SeqScan {
     pub(crate) fn new(input: ScanInput) -> Self {
         let parallelism = input.parallelism.parallelism.max(1);
         let mut properties = ScannerProperties::default()
-            .with_parallelism(parallelism)
             .with_append_mode(input.append_mode)
             .with_total_rows(input.total_rows());
-        properties.partitions = vec![input.partition_ranges()];
-        let stream_ctx = Arc::new(StreamContext::new(input));
+        let stream_ctx = Arc::new(StreamContext::seq_scan_ctx(input));
+        properties.partitions = vec![stream_ctx.partition_ranges()];
 
         Self {
             properties,
@@ -594,7 +593,7 @@ impl SeqDistributor {
                 continue;
             }
             let part = ScanPart {
-                memtable_ranges: mem_ranges,
+                memtable_ranges: mem_ranges.into_values().collect(),
                 file_ranges: smallvec![],
                 time_range: stats.time_range(),
             };
diff --git a/src/mito2/src/read/unordered_scan.rs b/src/mito2/src/read/unordered_scan.rs
index 67e87197a6..8a019132d7 100644
--- a/src/mito2/src/read/unordered_scan.rs
+++ b/src/mito2/src/read/unordered_scan.rs
@@ -16,7 +16,7 @@
 
 use std::fmt;
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::Instant;
 
 use async_stream::{stream, try_stream};
 use common_error::ext::BoxedError;
@@ -25,23 +25,18 @@ use common_recordbatch::{RecordBatch, RecordBatchStreamWrapper, SendableRecordBa
 use common_telemetry::debug;
 use datafusion::physical_plan::{DisplayAs, DisplayFormatType};
 use datatypes::schema::SchemaRef;
-use futures::StreamExt;
-use smallvec::smallvec;
+use futures::{Stream, StreamExt};
 use snafu::ResultExt;
 use store_api::region_engine::{PartitionRange, RegionScanner, ScannerProperties};
-use store_api::storage::ColumnId;
-use table::predicate::Predicate;
 
 use crate::cache::CacheManager;
 use crate::error::Result;
-use crate::memtable::{MemtableRange, MemtableRef};
+use crate::memtable::MemtableRange;
 use crate::read::compat::CompatBatch;
 use crate::read::projection::ProjectionMapper;
-use crate::read::scan_region::{
-    FileRangeCollector, ScanInput, ScanPart, ScanPartList, StreamContext,
-};
+use crate::read::range::RowGroupIndex;
+use crate::read::scan_region::{ScanInput, StreamContext};
 use crate::read::{ScannerMetrics, Source};
-use crate::sst::file::FileMeta;
 use crate::sst::parquet::file_range::FileRange;
 use crate::sst::parquet::reader::ReaderMetrics;
 
@@ -58,13 +53,11 @@ pub struct UnorderedScan {
 impl UnorderedScan {
     /// Creates a new [UnorderedScan].
     pub(crate) fn new(input: ScanInput) -> Self {
-        let parallelism = input.parallelism.parallelism.max(1);
         let mut properties = ScannerProperties::default()
-            .with_parallelism(parallelism)
             .with_append_mode(input.append_mode)
             .with_total_rows(input.total_rows());
-        properties.partitions = vec![input.partition_ranges()];
-        let stream_ctx = Arc::new(StreamContext::new(input));
+        let stream_ctx = Arc::new(StreamContext::unordered_scan_ctx(input));
+        properties.partitions = vec![stream_ctx.partition_ranges()];
 
         Self {
             properties,
@@ -127,6 +120,161 @@ impl UnorderedScan {
 
         Ok(Some(record_batch))
     }
+
+    /// Scans a [PartitionRange] and returns a stream.
+    fn scan_partition_range<'a>(
+        stream_ctx: &'a StreamContext,
+        part_range: &'a PartitionRange,
+        mem_ranges: &'a mut Vec<MemtableRange>,
+        file_ranges: &'a mut Vec<FileRange>,
+        reader_metrics: &'a mut ReaderMetrics,
+        metrics: &'a mut ScannerMetrics,
+    ) -> impl Stream<Item = common_recordbatch::error::Result<RecordBatch>> + 'a {
+        stream! {
+            // Gets range meta.
+            let range_meta = &stream_ctx.ranges[part_range.identifier];
+            for index in &range_meta.row_group_indices {
+                if stream_ctx.is_mem_range_index(*index) {
+                    let stream = Self::scan_mem_ranges(stream_ctx, *index, mem_ranges, metrics);
+                    for await batch in stream {
+                        yield batch;
+                    }
+                } else {
+                    let stream = Self::scan_file_ranges(stream_ctx, *index, file_ranges, reader_metrics, metrics);
+                    for await batch in stream {
+                        yield batch;
+                    }
+                }
+            }
+        }
+    }
+
+    /// Scans memtable ranges at `index`.
+    fn scan_mem_ranges<'a>(
+        stream_ctx: &'a StreamContext,
+        index: RowGroupIndex,
+        ranges: &'a mut Vec<MemtableRange>,
+        metrics: &'a mut ScannerMetrics,
+    ) -> impl Stream<Item = common_recordbatch::error::Result<RecordBatch>> + 'a {
+        try_stream! {
+            let mapper = &stream_ctx.input.mapper;
+            let cache = stream_ctx.input.cache_manager.as_deref();
+            stream_ctx.build_mem_ranges(index, ranges);
+            metrics.num_mem_ranges += ranges.len();
+            for range in ranges {
+                let build_reader_start = Instant::now();
+                let iter = range.build_iter().map_err(BoxedError::new).context(ExternalSnafu)?;
+                metrics.build_reader_cost = build_reader_start.elapsed();
+
+                let mut source = Source::Iter(iter);
+                while let Some(batch) =
+                    Self::fetch_from_source(&mut source, mapper, cache, None, metrics).await?
+                {
+                    metrics.num_batches += 1;
+                    metrics.num_rows += batch.num_rows();
+                    let yield_start = Instant::now();
+                    yield batch;
+                    metrics.yield_cost += yield_start.elapsed();
+                }
+            }
+        }
+    }
+
+    /// Scans file ranges at `index`.
+    fn scan_file_ranges<'a>(
+        stream_ctx: &'a StreamContext,
+        index: RowGroupIndex,
+        ranges: &'a mut Vec<FileRange>,
+        reader_metrics: &'a mut ReaderMetrics,
+        metrics: &'a mut ScannerMetrics,
+    ) -> impl Stream<Item = common_recordbatch::error::Result<RecordBatch>> + 'a {
+        try_stream! {
+            let mapper = &stream_ctx.input.mapper;
+            let cache = stream_ctx.input.cache_manager.as_deref();
+            stream_ctx
+                .build_file_ranges(index, ranges, reader_metrics)
+                .await
+                .map_err(BoxedError::new)
+                .context(ExternalSnafu)?;
+            metrics.num_file_ranges += ranges.len();
+            for range in ranges {
+                let build_reader_start = Instant::now();
+                let reader = range
+                    .reader(None)
+                    .await
+                    .map_err(BoxedError::new)
+                    .context(ExternalSnafu)?;
+                metrics.build_reader_cost += build_reader_start.elapsed();
+                let compat_batch = range.compat_batch();
+                let mut source = Source::PruneReader(reader);
+                while let Some(batch) =
+                    Self::fetch_from_source(&mut source, mapper, cache, compat_batch, metrics)
+                        .await?
+                {
+                    metrics.num_batches += 1;
+                    metrics.num_rows += batch.num_rows();
+                    let yield_start = Instant::now();
+                    yield batch;
+                    metrics.yield_cost += yield_start.elapsed();
+                }
+                if let Source::PruneReader(mut reader) = source {
+                    reader_metrics.merge_from(reader.metrics());
+                }
+            }
+        }
+    }
+
+    fn scan_partition_impl(
+        &self,
+        partition: usize,
+    ) -> Result<SendableRecordBatchStream, BoxedError> {
+        let mut metrics = ScannerMetrics {
+            prepare_scan_cost: self.stream_ctx.query_start.elapsed(),
+            ..Default::default()
+        };
+        let stream_ctx = self.stream_ctx.clone();
+        let ranges_opt = self.properties.partitions.get(partition).cloned();
+
+        let stream = stream! {
+            let first_poll = stream_ctx.query_start.elapsed();
+            let Some(part_ranges) = ranges_opt else {
+                return;
+            };
+
+            let mut mem_ranges = Vec::new();
+            let mut file_ranges = Vec::new();
+            let mut reader_metrics = ReaderMetrics::default();
+            // Scans each part.
+            for part_range in part_ranges {
+                let stream = Self::scan_partition_range(
+                    &stream_ctx,
+                    &part_range,
+                    &mut mem_ranges,
+                    &mut file_ranges,
+                    &mut reader_metrics,
+                    &mut metrics,
+                );
+                for await batch in stream {
+                    yield batch;
+                }
+            }
+
+            reader_metrics.observe_rows("unordered_scan_files");
+            metrics.total_cost = stream_ctx.query_start.elapsed();
+            metrics.observe_metrics_on_finish();
+            let mapper = &stream_ctx.input.mapper;
+            debug!(
+                "Unordered scan partition {} finished, region_id: {}, metrics: {:?}, reader_metrics: {:?}, first_poll: {:?}",
+                partition, mapper.metadata().region_id, metrics, reader_metrics, first_poll,
+            );
+        };
+        let stream = Box::pin(RecordBatchStreamWrapper::new(
+            self.stream_ctx.input.mapper.output_schema(),
+            Box::pin(stream),
+        ));
+
+        Ok(stream)
+    }
 }
 
 impl RegionScanner for UnorderedScan {
@@ -144,89 +292,7 @@ impl RegionScanner for UnorderedScan {
     }
 
     fn scan_partition(&self, partition: usize) -> Result<SendableRecordBatchStream, BoxedError> {
-        let mut metrics = ScannerMetrics {
-            prepare_scan_cost: self.stream_ctx.query_start.elapsed(),
-            ..Default::default()
-        };
-        let stream_ctx = self.stream_ctx.clone();
-        let parallelism = self.properties.num_partitions();
-        let stream = try_stream! {
-            let first_poll = stream_ctx.query_start.elapsed();
-            let part = {
-                let mut parts = stream_ctx.parts.lock().await;
-                maybe_init_parts(&stream_ctx.input, &mut parts, &mut metrics, parallelism)
-                    .await
-                    .map_err(BoxedError::new)
-                    .context(ExternalSnafu)?;
-                // Clone the part and releases the lock.
-                // TODO(yingwen): We might wrap the part in an Arc in the future if cloning is too expensive.
-                let Some(part) = parts.0.get_part(partition).cloned() else {
-                    return;
-                };
-                part
-            };
-
-            let build_reader_start = Instant::now();
-            let mapper = &stream_ctx.input.mapper;
-            let memtable_sources = part
-                .memtable_ranges
-                .iter()
-                .map(|mem| {
-                    let iter = mem.build_iter()?;
-                    Ok(Source::Iter(iter))
-                })
-                .collect::<Result<Vec<_>>>()
-                .map_err(BoxedError::new)
-                .context(ExternalSnafu)?;
-            metrics.build_reader_cost = build_reader_start.elapsed();
-
-            let query_start = stream_ctx.query_start;
-            let cache = stream_ctx.input.cache_manager.as_deref();
-            // Scans memtables first.
-            for mut source in memtable_sources {
-                while let Some(batch) = Self::fetch_from_source(&mut source, mapper, cache, None, &mut metrics).await? {
-                    metrics.num_batches += 1;
-                    metrics.num_rows += batch.num_rows();
-                    let yield_start = Instant::now();
-                    yield batch;
-                    metrics.yield_cost += yield_start.elapsed();
-                }
-            }
-            // Then scans file ranges.
-            let mut reader_metrics = ReaderMetrics::default();
-            // Safety: UnorderedDistributor::build_parts() ensures this.
-            for file_range in &part.file_ranges[0] {
-                let build_reader_start = Instant::now();
-                let reader = file_range.reader(None).await.map_err(BoxedError::new).context(ExternalSnafu)?;
-                metrics.build_reader_cost += build_reader_start.elapsed();
-                let compat_batch = file_range.compat_batch();
-                let mut source = Source::PruneReader(reader);
-                while let Some(batch) = Self::fetch_from_source(&mut source, mapper, cache, compat_batch, &mut metrics).await? {
-                    metrics.num_batches += 1;
-                    metrics.num_rows += batch.num_rows();
-                    let yield_start = Instant::now();
-                    yield batch;
-                    metrics.yield_cost += yield_start.elapsed();
-                }
-                if let Source::PruneReader(mut reader) = source {
-                    reader_metrics.merge_from(reader.metrics());
-                }
-            }
-
-            reader_metrics.observe_rows("unordered_scan_files");
-            metrics.total_cost = query_start.elapsed();
-            metrics.observe_metrics_on_finish();
-            debug!(
-                "Unordered scan partition {} finished, region_id: {}, metrics: {:?}, reader_metrics: {:?}, first_poll: {:?}, ranges: {}",
-                partition, mapper.metadata().region_id, metrics, reader_metrics, first_poll, part.file_ranges[0].len(),
-            );
-        };
-        let stream = Box::pin(RecordBatchStreamWrapper::new(
-            self.stream_ctx.input.mapper.output_schema(),
-            Box::pin(stream),
-        ));
-
-        Ok(stream)
+        self.scan_partition_impl(partition)
     }
 
     fn has_predicate(&self) -> bool {
@@ -261,117 +327,3 @@ impl UnorderedScan {
         &self.stream_ctx.input
     }
 }
-
-/// Initializes parts if they are not built yet.
-async fn maybe_init_parts(
-    input: &ScanInput,
-    part_list: &mut (ScanPartList, Duration),
-    metrics: &mut ScannerMetrics,
-    parallelism: usize,
-) -> Result<()> {
-    if part_list.0.is_none() {
-        let now = Instant::now();
-        let mut distributor = UnorderedDistributor::default();
-        let reader_metrics = input.prune_file_ranges(&mut distributor).await?;
-        distributor.append_mem_ranges(
-            &input.memtables,
-            Some(input.mapper.column_ids()),
-            input.predicate.clone(),
-        );
-        part_list.0.set_parts(distributor.build_parts(parallelism));
-        let build_part_cost = now.elapsed();
-        part_list.1 = build_part_cost;
-
-        metrics.observe_init_part(build_part_cost, &reader_metrics);
-    } else {
-        // Updates the cost of building parts.
-        metrics.build_parts_cost = part_list.1;
-    }
-    Ok(())
-}
-
-/// Builds [ScanPart]s without preserving order. It distributes file ranges and memtables
-/// across partitions. Each partition scans a subset of memtables and file ranges. There
-/// is no output ordering guarantee of each partition.
-#[derive(Default)]
-struct UnorderedDistributor {
-    mem_ranges: Vec<MemtableRange>,
-    file_ranges: Vec<FileRange>,
-}
-
-impl FileRangeCollector for UnorderedDistributor {
-    fn append_file_ranges(
-        &mut self,
-        _file_meta: &FileMeta,
-        file_ranges: impl Iterator<Item = FileRange>,
-    ) {
-        self.file_ranges.extend(file_ranges);
-    }
-}
-
-impl UnorderedDistributor {
-    /// Appends memtable ranges to the distributor.
-    fn append_mem_ranges(
-        &mut self,
-        memtables: &[MemtableRef],
-        projection: Option<&[ColumnId]>,
-        predicate: Option<Predicate>,
-    ) {
-        for mem in memtables {
-            let mut mem_ranges = mem.ranges(projection, predicate.clone());
-            if mem_ranges.is_empty() {
-                continue;
-            }
-            self.mem_ranges.append(&mut mem_ranges);
-        }
-    }
-
-    /// Distributes file ranges and memtables across partitions according to the `parallelism`.
-    /// The output number of parts may be `<= parallelism`.
-    ///
-    /// [ScanPart] created by this distributor only contains one group of file ranges.
-    fn build_parts(self, parallelism: usize) -> Vec<ScanPart> {
-        if parallelism <= 1 {
-            // Returns a single part.
-            let part = ScanPart {
-                memtable_ranges: self.mem_ranges.clone(),
-                file_ranges: smallvec![self.file_ranges],
-                time_range: None,
-            };
-            return vec![part];
-        }
-
-        let mems_per_part = ((self.mem_ranges.len() + parallelism - 1) / parallelism).max(1);
-        let ranges_per_part = ((self.file_ranges.len() + parallelism - 1) / parallelism).max(1);
-        debug!(
-            "Parallel scan is enabled, parallelism: {}, {} mem_ranges, {} file_ranges, mems_per_part: {}, ranges_per_part: {}",
-            parallelism,
-            self.mem_ranges.len(),
-            self.file_ranges.len(),
-            mems_per_part,
-            ranges_per_part
-        );
-        let mut scan_parts = self
-            .mem_ranges
-            .chunks(mems_per_part)
-            .map(|mems| ScanPart {
-                memtable_ranges: mems.to_vec(),
-                file_ranges: smallvec![Vec::new()], // Ensures there is always one group.
-                time_range: None,
-            })
-            .collect::<Vec<_>>();
-        for (i, ranges) in self.file_ranges.chunks(ranges_per_part).enumerate() {
-            if i == scan_parts.len() {
-                scan_parts.push(ScanPart {
-                    memtable_ranges: Vec::new(),
-                    file_ranges: smallvec![ranges.to_vec()],
-                    time_range: None,
-                });
-            } else {
-                scan_parts[i].file_ranges = smallvec![ranges.to_vec()];
-            }
-        }
-
-        scan_parts
-    }
-}
diff --git a/src/mito2/src/sst/parquet.rs b/src/mito2/src/sst/parquet.rs
index cce9965539..cc0cff605b 100644
--- a/src/mito2/src/sst/parquet.rs
+++ b/src/mito2/src/sst/parquet.rs
@@ -40,7 +40,7 @@ pub const PARQUET_METADATA_KEY: &str = "greptime:metadata";
 /// Default batch size to read parquet files.
 pub(crate) const DEFAULT_READ_BATCH_SIZE: usize = 1024;
 /// Default row group size for parquet files.
-const DEFAULT_ROW_GROUP_SIZE: usize = 100 * DEFAULT_READ_BATCH_SIZE;
+pub(crate) const DEFAULT_ROW_GROUP_SIZE: usize = 100 * DEFAULT_READ_BATCH_SIZE;
 
 /// Parquet write options.
 #[derive(Debug)]
diff --git a/src/mito2/src/sst/parquet/reader.rs b/src/mito2/src/sst/parquet/reader.rs
index 766dab0429..ff7fe77e7a 100644
--- a/src/mito2/src/sst/parquet/reader.rs
+++ b/src/mito2/src/sst/parquet/reader.rs
@@ -238,6 +238,7 @@ impl ParquetReaderBuilder {
             cache_manager: self.cache_manager.clone(),
         };
 
+        // TODO(yingwen): count the cost of the method.
         metrics.build_cost = start.elapsed();
 
         let mut filters = if let Some(predicate) = &self.predicate {
diff --git a/src/mito2/src/test_util/memtable_util.rs b/src/mito2/src/test_util/memtable_util.rs
index 235e9694c7..70dec3cad4 100644
--- a/src/mito2/src/test_util/memtable_util.rs
+++ b/src/mito2/src/test_util/memtable_util.rs
@@ -14,6 +14,7 @@
 
 //! Memtable test utilities.
 
+use std::collections::BTreeMap;
 use std::sync::Arc;
 
 use api::helper::ColumnDataTypeWrapper;
@@ -92,8 +93,8 @@ impl Memtable for EmptyMemtable {
         &self,
         _projection: Option<&[ColumnId]>,
         _predicate: Option<Predicate>,
-    ) -> Vec<MemtableRange> {
-        vec![]
+    ) -> BTreeMap<usize, MemtableRange> {
+        BTreeMap::new()
     }
 
     fn is_empty(&self) -> bool {

From 77af4fd981f2816ca05e2a94f2d8632e6f7aeedc Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Mon, 30 Sep 2024 10:56:53 +0800
Subject: [PATCH 048/128] refactor: introduce `HeartbeatHandlerGroupBuilder`
 (#4785)

---
 src/meta-srv/src/handler.rs           | 154 +++++++++++++++++++++-----
 src/meta-srv/src/metasrv.rs           |   6 +-
 src/meta-srv/src/metasrv/builder.rs   |  53 ++-------
 src/meta-srv/src/service/heartbeat.rs |   4 +-
 4 files changed, 142 insertions(+), 75 deletions(-)

diff --git a/src/meta-srv/src/handler.rs b/src/meta-srv/src/handler.rs
index 063d3939c1..5363b6c548 100644
--- a/src/meta-srv/src/handler.rs
+++ b/src/meta-srv/src/handler.rs
@@ -22,12 +22,28 @@ use api::v1::meta::{
     HeartbeatRequest, HeartbeatResponse, MailboxMessage, RegionLease, RequestHeader,
     ResponseHeader, Role, PROTOCOL_VERSION,
 };
+use check_leader_handler::CheckLeaderHandler;
+use collect_cluster_info_handler::{
+    CollectDatanodeClusterInfoHandler, CollectFlownodeClusterInfoHandler,
+    CollectFrontendClusterInfoHandler,
+};
+use collect_stats_handler::CollectStatsHandler;
+use common_base::Plugins;
 use common_meta::datanode::Stat;
 use common_meta::instruction::{Instruction, InstructionReply};
 use common_meta::sequence::Sequence;
 use common_telemetry::{debug, info, warn};
 use dashmap::DashMap;
+use extract_stat_handler::ExtractStatHandler;
+use failure_handler::RegionFailureHandler;
+use filter_inactive_region_stats::FilterInactiveRegionStatsHandler;
 use futures::future::join_all;
+use keep_lease_handler::{DatanodeKeepLeaseHandler, FlownodeKeepLeaseHandler};
+use mailbox_handler::MailboxHandler;
+use on_leader_start_handler::OnLeaderStartHandler;
+use publish_heartbeat_handler::PublishHeartbeatHandler;
+use region_lease_handler::RegionLeaseHandler;
+use response_header_handler::ResponseHeaderHandler;
 use snafu::{OptionExt, ResultExt};
 use store_api::storage::RegionId;
 use tokio::sync::mpsc::Sender;
@@ -36,6 +52,7 @@ use tokio::sync::{oneshot, Notify, RwLock};
 use crate::error::{self, DeserializeFromJsonSnafu, Result, UnexpectedInstructionReplySnafu};
 use crate::metasrv::Context;
 use crate::metrics::{METRIC_META_HANDLER_EXECUTE, METRIC_META_HEARTBEAT_CONNECTION_NUM};
+use crate::pubsub::PublisherRef;
 use crate::service::mailbox::{
     BroadcastChannel, Channel, Mailbox, MailboxReceiver, MailboxRef, MessageId,
 };
@@ -96,6 +113,7 @@ impl HeartbeatAccumulator {
     }
 }
 
+/// The pusher of the heartbeat response.
 pub struct Pusher {
     sender: Sender<std::result::Result<HeartbeatResponse, tonic::Status>>,
     res_header: ResponseHeader,
@@ -131,6 +149,7 @@ impl Pusher {
     }
 }
 
+/// The group of heartbeat pushers.
 #[derive(Clone, Default)]
 pub struct Pushers(Arc<RwLock<BTreeMap<String, Pusher>>>);
 
@@ -203,50 +222,57 @@ impl NameCachedHandler {
     }
 }
 
-#[derive(Clone, Default)]
+pub type HeartbeatHandlerGroupRef = Arc<HeartbeatHandlerGroup>;
+
+/// The group of heartbeat handlers.
+#[derive(Default)]
 pub struct HeartbeatHandlerGroup {
-    handlers: Arc<RwLock<Vec<NameCachedHandler>>>,
+    handlers: Vec<NameCachedHandler>,
     pushers: Pushers,
 }
 
 impl HeartbeatHandlerGroup {
     pub(crate) fn new(pushers: Pushers) -> Self {
         Self {
-            handlers: Arc::new(RwLock::new(vec![])),
+            handlers: vec![],
             pushers,
         }
     }
 
-    pub async fn add_handler(&self, handler: impl HeartbeatHandler + 'static) {
-        let mut handlers = self.handlers.write().await;
-        handlers.push(NameCachedHandler::new(handler));
+    fn add_handler(&mut self, handler: impl HeartbeatHandler + 'static) {
+        self.handlers.push(NameCachedHandler::new(handler));
     }
 
-    pub async fn register(&self, key: impl AsRef<str>, pusher: Pusher) {
+    /// Registers the heartbeat response [`Pusher`] with the given key to the group.
+    pub async fn register_pusher(&self, key: impl AsRef<str>, pusher: Pusher) {
         let key = key.as_ref();
         METRIC_META_HEARTBEAT_CONNECTION_NUM.inc();
         info!("Pusher register: {}", key);
         let _ = self.pushers.insert(key.to_string(), pusher).await;
     }
 
-    pub async fn deregister(&self, key: impl AsRef<str>) -> Option<Pusher> {
+    /// Deregisters the heartbeat response [`Pusher`] with the given key from the group.
+    ///
+    /// Returns the [`Pusher`] if it exists.
+    pub async fn deregister_push(&self, key: impl AsRef<str>) -> Option<Pusher> {
         let key = key.as_ref();
         METRIC_META_HEARTBEAT_CONNECTION_NUM.dec();
         info!("Pusher unregister: {}", key);
         self.pushers.remove(key).await
     }
 
+    /// Returns the [`Pushers`] of the group.
     pub fn pushers(&self) -> Pushers {
         self.pushers.clone()
     }
 
+    /// Handles the heartbeat request.
     pub async fn handle(
         &self,
         req: HeartbeatRequest,
         mut ctx: Context,
     ) -> Result<HeartbeatResponse> {
         let mut acc = HeartbeatAccumulator::default();
-        let handlers = self.handlers.read().await;
         let role = req
             .header
             .as_ref()
@@ -255,7 +281,7 @@ impl HeartbeatHandlerGroup {
                 err_msg: format!("invalid role: {:?}", req.header),
             })?;
 
-        for NameCachedHandler { name, handler } in handlers.iter() {
+        for NameCachedHandler { name, handler } in self.handlers.iter() {
             if !handler.is_acceptable(role) {
                 continue;
             }
@@ -426,6 +452,84 @@ impl Mailbox for HeartbeatMailbox {
     }
 }
 
+/// The builder to build the group of heartbeat handlers.
+pub struct HeartbeatHandlerGroupBuilder {
+    /// The handler to handle region failure.
+    region_failure_handler: Option<RegionFailureHandler>,
+
+    /// The handler to handle region lease.
+    region_lease_handler: RegionLeaseHandler,
+
+    /// The plugins.
+    plugins: Option<Plugins>,
+
+    /// The heartbeat response pushers.
+    pushers: Pushers,
+}
+
+impl HeartbeatHandlerGroupBuilder {
+    pub fn new(pushers: Pushers, region_lease_handler: RegionLeaseHandler) -> Self {
+        Self {
+            region_failure_handler: None,
+            region_lease_handler,
+            plugins: None,
+            pushers,
+        }
+    }
+
+    /// Sets the [`RegionFailureHandler`].
+    pub fn with_region_failure_handler(mut self, handler: Option<RegionFailureHandler>) -> Self {
+        self.region_failure_handler = handler;
+        self
+    }
+
+    /// Sets the [`Plugins`].
+    pub fn with_plugins(mut self, plugins: Option<Plugins>) -> Self {
+        self.plugins = plugins;
+        self
+    }
+
+    /// Builds the group of heartbeat handlers.
+    pub fn build(self) -> HeartbeatHandlerGroup {
+        // Extract the `PublishHeartbeatHandler` from the plugins.
+        let publish_heartbeat_handler = if let Some(plugins) = self.plugins {
+            plugins
+                .get::<PublisherRef>()
+                .map(|publish| PublishHeartbeatHandler::new(publish.clone()))
+        } else {
+            None
+        };
+
+        // TODO(weny): Considers classifying handlers
+        // to make it easier for upper layers to customize handler groups.
+        let mut group = HeartbeatHandlerGroup::new(self.pushers);
+        group.add_handler(ResponseHeaderHandler);
+        // `KeepLeaseHandler` should preferably be in front of `CheckLeaderHandler`,
+        // because even if the current meta-server node is no longer the leader it can
+        // still help the datanode to keep lease.
+        group.add_handler(DatanodeKeepLeaseHandler);
+        group.add_handler(FlownodeKeepLeaseHandler);
+        group.add_handler(CheckLeaderHandler);
+        group.add_handler(OnLeaderStartHandler);
+        group.add_handler(ExtractStatHandler);
+        group.add_handler(CollectDatanodeClusterInfoHandler);
+        group.add_handler(CollectFrontendClusterInfoHandler);
+        group.add_handler(CollectFlownodeClusterInfoHandler);
+        group.add_handler(MailboxHandler);
+        group.add_handler(self.region_lease_handler);
+        group.add_handler(FilterInactiveRegionStatsHandler);
+        if let Some(region_failure_handler) = self.region_failure_handler {
+            group.add_handler(region_failure_handler);
+        }
+        if let Some(publish_heartbeat_handler) = publish_heartbeat_handler {
+            group.add_handler(publish_heartbeat_handler);
+        }
+        group.add_handler(CollectStatsHandler::default());
+
+        group
+    }
+}
+
 #[cfg(test)]
 mod tests {
 
@@ -489,7 +593,7 @@ mod tests {
         let pusher: Pusher = Pusher::new(pusher_tx, &res_header);
         let handler_group = HeartbeatHandlerGroup::default();
         handler_group
-            .register(format!("{}-{}", Role::Datanode as i32, datanode_id), pusher)
+            .register_pusher(format!("{}-{}", Role::Datanode as i32, datanode_id), pusher)
             .await;
 
         let kv_backend = Arc::new(MemoryKvBackend::new());
@@ -519,21 +623,21 @@ mod tests {
 
     #[tokio::test]
     async fn test_handler_name() {
-        let group = HeartbeatHandlerGroup::default();
-        group.add_handler(ResponseHeaderHandler).await;
-        group.add_handler(DatanodeKeepLeaseHandler).await;
-        group.add_handler(FlownodeKeepLeaseHandler).await;
-        group.add_handler(CheckLeaderHandler).await;
-        group.add_handler(OnLeaderStartHandler).await;
-        group.add_handler(ExtractStatHandler).await;
-        group.add_handler(CollectDatanodeClusterInfoHandler).await;
-        group.add_handler(CollectFrontendClusterInfoHandler).await;
-        group.add_handler(CollectFlownodeClusterInfoHandler).await;
-        group.add_handler(MailboxHandler).await;
-        group.add_handler(FilterInactiveRegionStatsHandler).await;
-        group.add_handler(CollectStatsHandler::default()).await;
+        let mut group = HeartbeatHandlerGroup::default();
+        group.add_handler(ResponseHeaderHandler);
+        group.add_handler(DatanodeKeepLeaseHandler);
+        group.add_handler(FlownodeKeepLeaseHandler);
+        group.add_handler(CheckLeaderHandler);
+        group.add_handler(OnLeaderStartHandler);
+        group.add_handler(ExtractStatHandler);
+        group.add_handler(CollectDatanodeClusterInfoHandler);
+        group.add_handler(CollectFrontendClusterInfoHandler);
+        group.add_handler(CollectFlownodeClusterInfoHandler);
+        group.add_handler(MailboxHandler);
+        group.add_handler(FilterInactiveRegionStatsHandler);
+        group.add_handler(CollectStatsHandler::default());
 
-        let handlers = group.handlers.read().await;
+        let handlers = group.handlers;
 
         assert_eq!(12, handlers.len());
 
diff --git a/src/meta-srv/src/metasrv.rs b/src/meta-srv/src/metasrv.rs
index 2beb09859b..de7d54aa65 100644
--- a/src/meta-srv/src/metasrv.rs
+++ b/src/meta-srv/src/metasrv.rs
@@ -52,7 +52,7 @@ use crate::error::{
     StopProcedureManagerSnafu,
 };
 use crate::failure_detector::PhiAccrualFailureDetectorOptions;
-use crate::handler::HeartbeatHandlerGroup;
+use crate::handler::HeartbeatHandlerGroupRef;
 use crate::lease::lookup_datanode_peer;
 use crate::lock::DistLockRef;
 use crate::procedure::region_migration::manager::RegionMigrationManagerRef;
@@ -366,7 +366,7 @@ pub struct Metasrv {
     selector: SelectorRef,
     // The flow selector is used to select a target flownode.
     flow_selector: SelectorRef,
-    handler_group: HeartbeatHandlerGroup,
+    handler_group: HeartbeatHandlerGroupRef,
     election: Option<ElectionRef>,
     lock: DistLockRef,
     procedure_manager: ProcedureManagerRef,
@@ -562,7 +562,7 @@ impl Metasrv {
         &self.flow_selector
     }
 
-    pub fn handler_group(&self) -> &HeartbeatHandlerGroup {
+    pub fn handler_group(&self) -> &HeartbeatHandlerGroupRef {
         &self.handler_group
     }
 
diff --git a/src/meta-srv/src/metasrv/builder.rs b/src/meta-srv/src/metasrv/builder.rs
index 6b06bab867..662de433ab 100644
--- a/src/meta-srv/src/metasrv/builder.rs
+++ b/src/meta-srv/src/metasrv/builder.rs
@@ -46,22 +46,11 @@ use crate::cluster::{MetaPeerClientBuilder, MetaPeerClientRef};
 use crate::error::{self, Result};
 use crate::flow_meta_alloc::FlowPeerAllocator;
 use crate::greptimedb_telemetry::get_greptimedb_telemetry_task;
-use crate::handler::check_leader_handler::CheckLeaderHandler;
-use crate::handler::collect_cluster_info_handler::{
-    CollectDatanodeClusterInfoHandler, CollectFlownodeClusterInfoHandler,
-    CollectFrontendClusterInfoHandler,
-};
-use crate::handler::collect_stats_handler::CollectStatsHandler;
-use crate::handler::extract_stat_handler::ExtractStatHandler;
 use crate::handler::failure_handler::RegionFailureHandler;
-use crate::handler::filter_inactive_region_stats::FilterInactiveRegionStatsHandler;
-use crate::handler::keep_lease_handler::{DatanodeKeepLeaseHandler, FlownodeKeepLeaseHandler};
-use crate::handler::mailbox_handler::MailboxHandler;
-use crate::handler::on_leader_start_handler::OnLeaderStartHandler;
-use crate::handler::publish_heartbeat_handler::PublishHeartbeatHandler;
 use crate::handler::region_lease_handler::RegionLeaseHandler;
-use crate::handler::response_header_handler::ResponseHeaderHandler;
-use crate::handler::{HeartbeatHandlerGroup, HeartbeatMailbox, Pushers};
+use crate::handler::{
+    HeartbeatHandlerGroup, HeartbeatHandlerGroupBuilder, HeartbeatMailbox, Pushers,
+};
 use crate::lease::MetaPeerLookupService;
 use crate::lock::memory::MemLock;
 use crate::lock::DistLockRef;
@@ -70,7 +59,6 @@ use crate::metasrv::{
 };
 use crate::procedure::region_migration::manager::RegionMigrationManager;
 use crate::procedure::region_migration::DefaultContextFactory;
-use crate::pubsub::PublisherRef;
 use crate::region::supervisor::{
     HeartbeatAcceptor, RegionFailureDetectorControl, RegionSupervisor, RegionSupervisorTicker,
     DEFAULT_TICK_INTERVAL,
@@ -364,41 +352,16 @@ impl MetasrvBuilder {
         let handler_group = match handler_group {
             Some(handler_group) => handler_group,
             None => {
-                let publish_heartbeat_handler = plugins
-                    .clone()
-                    .and_then(|plugins| plugins.get::<PublisherRef>())
-                    .map(|publish| PublishHeartbeatHandler::new(publish.clone()));
-
                 let region_lease_handler = RegionLeaseHandler::new(
                     distributed_time_constants::REGION_LEASE_SECS,
                     table_metadata_manager.clone(),
                     memory_region_keeper.clone(),
                 );
 
-                let group = HeartbeatHandlerGroup::new(pushers);
-                group.add_handler(ResponseHeaderHandler).await;
-                // `KeepLeaseHandler` should preferably be in front of `CheckLeaderHandler`,
-                // because even if the current meta-server node is no longer the leader it can
-                // still help the datanode to keep lease.
-                group.add_handler(DatanodeKeepLeaseHandler).await;
-                group.add_handler(FlownodeKeepLeaseHandler).await;
-                group.add_handler(CheckLeaderHandler).await;
-                group.add_handler(OnLeaderStartHandler).await;
-                group.add_handler(ExtractStatHandler).await;
-                group.add_handler(CollectDatanodeClusterInfoHandler).await;
-                group.add_handler(CollectFrontendClusterInfoHandler).await;
-                group.add_handler(CollectFlownodeClusterInfoHandler).await;
-                group.add_handler(MailboxHandler).await;
-                group.add_handler(region_lease_handler).await;
-                group.add_handler(FilterInactiveRegionStatsHandler).await;
-                if let Some(region_failover_handler) = region_failover_handler {
-                    group.add_handler(region_failover_handler).await;
-                }
-                if let Some(publish_heartbeat_handler) = publish_heartbeat_handler {
-                    group.add_handler(publish_heartbeat_handler).await;
-                }
-                group.add_handler(CollectStatsHandler::default()).await;
-                group
+                HeartbeatHandlerGroupBuilder::new(pushers, region_lease_handler)
+                    .with_plugins(plugins.clone())
+                    .with_region_failure_handler(region_failover_handler)
+                    .build()
             }
         };
 
@@ -417,7 +380,7 @@ impl MetasrvBuilder {
             selector,
             // TODO(jeremy): We do not allow configuring the flow selector.
             flow_selector: Arc::new(RoundRobinSelector::new(SelectTarget::Flownode)),
-            handler_group,
+            handler_group: Arc::new(handler_group),
             election,
             lock,
             procedure_manager,
diff --git a/src/meta-srv/src/service/heartbeat.rs b/src/meta-srv/src/service/heartbeat.rs
index f5ac74a4b5..569d6a8089 100644
--- a/src/meta-srv/src/service/heartbeat.rs
+++ b/src/meta-srv/src/service/heartbeat.rs
@@ -113,7 +113,7 @@ impl heartbeat_server::Heartbeat for Metasrv {
             );
 
             if let Some(key) = pusher_key {
-                let _ = handler_group.deregister(&key).await;
+                let _ = handler_group.deregister_push(&key).await;
             }
         });
 
@@ -177,7 +177,7 @@ async fn register_pusher(
     let node_id = get_node_id(header);
     let key = format!("{}-{}", role, node_id);
     let pusher = Pusher::new(sender, header);
-    handler_group.register(&key, pusher).await;
+    handler_group.register_pusher(&key, pusher).await;
     Some(key)
 }
 

From e39a9e6feb30dcd7f58742e1bc296856a9b27b04 Mon Sep 17 00:00:00 2001
From: zyy17 <zyylsxm@gmail.com>
Date: Mon, 30 Sep 2024 11:26:50 +0800
Subject: [PATCH 049/128] feat: add `StatementStatistics` for slow query
 logging implementation (#4719)

* feat: log slow query

* feat: log slow query for sql

* refactor: add slow query logging options

* ci: fix errors

* feat: add StatementStatistics

* chore: revert modification of servers crate

* docs: update config docs

* fix: clippy errors
---
 Cargo.lock                           |  1 +
 config/config.md                     | 20 ++++++
 config/datanode.example.toml         | 13 ++++
 config/flownode.example.toml         | 13 ++++
 config/frontend.example.toml         | 13 ++++
 config/metasrv.example.toml          | 13 ++++
 config/standalone.example.toml       | 13 ++++
 src/cmd/src/frontend.rs              |  2 +
 src/cmd/src/standalone.rs            |  2 +
 src/common/telemetry/Cargo.toml      |  1 +
 src/common/telemetry/src/logging.rs  | 62 +++++++++++++++++-
 src/common/telemetry/src/macros.rs   | 11 ++++
 src/flow/src/server.rs               |  2 +
 src/frontend/src/instance/builder.rs |  5 ++
 src/operator/src/statement.rs        |  6 ++
 src/query/Cargo.toml                 |  1 +
 src/query/src/lib.rs                 |  1 +
 src/query/src/stats.rs               | 98 ++++++++++++++++++++++++++++
 tests-integration/src/cluster.rs     |  2 +
 tests-integration/src/standalone.rs  |  2 +
 tests-integration/tests/http.rs      |  3 +
 21 files changed, 282 insertions(+), 2 deletions(-)
 create mode 100644 src/query/src/stats.rs

diff --git a/Cargo.lock b/Cargo.lock
index 4533257660..7aebe2e719 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2289,6 +2289,7 @@ dependencies = [
  "common-error",
  "console-subscriber",
  "greptime-proto",
+ "humantime-serde",
  "lazy_static",
  "once_cell",
  "opentelemetry 0.21.0",
diff --git a/config/config.md b/config/config.md
index 641eee4b58..d413936ff3 100644
--- a/config/config.md
+++ b/config/config.md
@@ -163,6 +163,10 @@
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
+| `logging.slow_query` | -- | -- | The slow query log options. |
+| `logging.slow_query.enable` | Bool | `false` | Whether to enable slow query log. |
+| `logging.slow_query.threshold` | String | Unset | The threshold of slow query. |
+| `logging.slow_query.sample_ratio` | Float | Unset | The sampling ratio of slow query log. The value should be in the range of (0, 1]. |
 | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
@@ -249,6 +253,10 @@
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
+| `logging.slow_query` | -- | -- | The slow query log options. |
+| `logging.slow_query.enable` | Bool | `false` | Whether to enable slow query log. |
+| `logging.slow_query.threshold` | String | Unset | The threshold of slow query. |
+| `logging.slow_query.sample_ratio` | Float | Unset | The sampling ratio of slow query log. The value should be in the range of (0, 1]. |
 | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
@@ -314,6 +322,10 @@
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
+| `logging.slow_query` | -- | -- | The slow query log options. |
+| `logging.slow_query.enable` | Bool | `false` | Whether to enable slow query log. |
+| `logging.slow_query.threshold` | String | Unset | The threshold of slow query. |
+| `logging.slow_query.sample_ratio` | Float | Unset | The sampling ratio of slow query log. The value should be in the range of (0, 1]. |
 | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
@@ -466,6 +478,10 @@
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
+| `logging.slow_query` | -- | -- | The slow query log options. |
+| `logging.slow_query.enable` | Bool | `false` | Whether to enable slow query log. |
+| `logging.slow_query.threshold` | String | Unset | The threshold of slow query. |
+| `logging.slow_query.sample_ratio` | Float | Unset | The sampling ratio of slow query log. The value should be in the range of (0, 1]. |
 | `export_metrics` | -- | -- | The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.<br/>This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape. |
 | `export_metrics.enable` | Bool | `false` | whether enable export metrics. |
 | `export_metrics.write_interval` | String | `30s` | The interval of export metrics. |
@@ -512,5 +528,9 @@
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
+| `logging.slow_query` | -- | -- | The slow query log options. |
+| `logging.slow_query.enable` | Bool | `false` | Whether to enable slow query log. |
+| `logging.slow_query.threshold` | String | Unset | The threshold of slow query. |
+| `logging.slow_query.sample_ratio` | Float | Unset | The sampling ratio of slow query log. The value should be in the range of (0, 1]. |
 | `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
 | `tracing.tokio_console_addr` | String | Unset | The tokio console address. |
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index e4a3dca6d3..06a59ebd6f 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -586,6 +586,19 @@ log_format = "text"
 [logging.tracing_sample_ratio]
 default_ratio = 1.0
 
+## The slow query log options.
+[logging.slow_query]
+## Whether to enable slow query log.
+enable = false
+
+## The threshold of slow query.
+## @toml2docs:none-default
+threshold = "10s"
+
+## The sampling ratio of slow query log. The value should be in the range of (0, 1].
+## @toml2docs:none-default
+sample_ratio = 1.0
+
 ## The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
 ## This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
 [export_metrics]
diff --git a/config/flownode.example.toml b/config/flownode.example.toml
index a3a414fb98..9d6d2fe5aa 100644
--- a/config/flownode.example.toml
+++ b/config/flownode.example.toml
@@ -84,6 +84,19 @@ log_format = "text"
 [logging.tracing_sample_ratio]
 default_ratio = 1.0
 
+## The slow query log options.
+[logging.slow_query]
+## Whether to enable slow query log.
+enable = false
+
+## The threshold of slow query.
+## @toml2docs:none-default
+threshold = "10s"
+
+## The sampling ratio of slow query log. The value should be in the range of (0, 1].
+## @toml2docs:none-default
+sample_ratio = 1.0
+
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
 [tracing]
 ## The tokio console address.
diff --git a/config/frontend.example.toml b/config/frontend.example.toml
index eae001a2db..cc9698f61e 100644
--- a/config/frontend.example.toml
+++ b/config/frontend.example.toml
@@ -191,6 +191,19 @@ log_format = "text"
 [logging.tracing_sample_ratio]
 default_ratio = 1.0
 
+## The slow query log options.
+[logging.slow_query]
+## Whether to enable slow query log.
+enable = false
+
+## The threshold of slow query.
+## @toml2docs:none-default
+threshold = "10s"
+
+## The sampling ratio of slow query log. The value should be in the range of (0, 1].
+## @toml2docs:none-default
+sample_ratio = 1.0
+
 ## The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
 ## This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
 [export_metrics]
diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml
index e95a9fa7f2..8431940b45 100644
--- a/config/metasrv.example.toml
+++ b/config/metasrv.example.toml
@@ -178,6 +178,19 @@ log_format = "text"
 [logging.tracing_sample_ratio]
 default_ratio = 1.0
 
+## The slow query log options.
+[logging.slow_query]
+## Whether to enable slow query log.
+enable = false
+
+## The threshold of slow query.
+## @toml2docs:none-default
+threshold = "10s"
+
+## The sampling ratio of slow query log. The value should be in the range of (0, 1].
+## @toml2docs:none-default
+sample_ratio = 1.0
+
 ## The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
 ## This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
 [export_metrics]
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index 1cd75e6414..2d14b6550f 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -630,6 +630,19 @@ log_format = "text"
 [logging.tracing_sample_ratio]
 default_ratio = 1.0
 
+## The slow query log options.
+[logging.slow_query]
+## Whether to enable slow query log.
+enable = false
+
+## The threshold of slow query.
+## @toml2docs:none-default
+threshold = "10s"
+
+## The sampling ratio of slow query log. The value should be in the range of (0, 1].
+## @toml2docs:none-default
+sample_ratio = 1.0
+
 ## The datanode can export its metrics and send to Prometheus compatible service (e.g. send to `greptimedb` itself) from remote-write API.
 ## This is only used for `greptimedb` to export its own metrics internally. It's different from prometheus scrape.
 [export_metrics]
diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs
index 320dc49c19..7678e90c88 100644
--- a/src/cmd/src/frontend.rs
+++ b/src/cmd/src/frontend.rs
@@ -36,6 +36,7 @@ use frontend::instance::builder::FrontendBuilder;
 use frontend::instance::{FrontendInstance, Instance as FeInstance};
 use frontend::server::Services;
 use meta_client::{MetaClientOptions, MetaClientType};
+use query::stats::StatementStatistics;
 use servers::tls::{TlsMode, TlsOption};
 use servers::Mode;
 use snafu::{OptionExt, ResultExt};
@@ -352,6 +353,7 @@ impl StartCommand {
             catalog_manager,
             Arc::new(client),
             meta_client,
+            StatementStatistics::new(opts.logging.slow_query.clone()),
         )
         .with_plugin(plugins.clone())
         .with_local_cache_invalidator(layered_cache_registry)
diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
index 80b38ebaa7..4335bd5447 100644
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -55,6 +55,7 @@ use frontend::service_config::{
 };
 use meta_srv::metasrv::{FLOW_ID_SEQ, TABLE_ID_SEQ};
 use mito2::config::MitoConfig;
+use query::stats::StatementStatistics;
 use serde::{Deserialize, Serialize};
 use servers::export_metrics::ExportMetricsOption;
 use servers::grpc::GrpcOptions;
@@ -557,6 +558,7 @@ impl StartCommand {
             catalog_manager.clone(),
             node_manager.clone(),
             ddl_task_executor.clone(),
+            StatementStatistics::new(opts.logging.slow_query.clone()),
         )
         .with_plugin(plugins.clone())
         .try_build()
diff --git a/src/common/telemetry/Cargo.toml b/src/common/telemetry/Cargo.toml
index 20fc52a763..2b4023cf7c 100644
--- a/src/common/telemetry/Cargo.toml
+++ b/src/common/telemetry/Cargo.toml
@@ -17,6 +17,7 @@ backtrace = "0.3"
 common-error.workspace = true
 console-subscriber = { version = "0.1", optional = true }
 greptime-proto.workspace = true
+humantime-serde.workspace = true
 lazy_static.workspace = true
 once_cell.workspace = true
 opentelemetry = { version = "0.21.0", default-features = false, features = [
diff --git a/src/common/telemetry/src/logging.rs b/src/common/telemetry/src/logging.rs
index de018aa4b6..c21766f04d 100644
--- a/src/common/telemetry/src/logging.rs
+++ b/src/common/telemetry/src/logging.rs
@@ -15,6 +15,7 @@
 //! logging stuffs, inspired by databend
 use std::env;
 use std::sync::{Arc, Mutex, Once};
+use std::time::Duration;
 
 use once_cell::sync::{Lazy, OnceCell};
 use opentelemetry::{global, KeyValue};
@@ -26,7 +27,7 @@ use serde::{Deserialize, Serialize};
 use tracing_appender::non_blocking::WorkerGuard;
 use tracing_appender::rolling::{RollingFileAppender, Rotation};
 use tracing_log::LogTracer;
-use tracing_subscriber::filter::Targets;
+use tracing_subscriber::filter::{FilterFn, Targets};
 use tracing_subscriber::fmt::Layer;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::prelude::*;
@@ -64,6 +65,24 @@ pub struct LoggingOptions {
 
     /// The tracing sample ratio.
     pub tracing_sample_ratio: Option<TracingSampleOptions>,
+
+    /// The logging options of slow query.
+    pub slow_query: SlowQueryOptions,
+}
+
+/// The options of slow query.
+#[derive(Clone, Debug, Serialize, Deserialize, Default)]
+#[serde(default)]
+pub struct SlowQueryOptions {
+    /// Whether to enable slow query log.
+    pub enable: bool,
+
+    /// The threshold of slow queries.
+    #[serde(with = "humantime_serde")]
+    pub threshold: Option<Duration>,
+
+    /// The sample ratio of slow queries.
+    pub sample_ratio: Option<f64>,
 }
 
 #[derive(Clone, Debug, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -96,6 +115,7 @@ impl Default for LoggingOptions {
             otlp_endpoint: None,
             tracing_sample_ratio: None,
             append_stdout: true,
+            slow_query: SlowQueryOptions::default(),
         }
     }
 }
@@ -235,6 +255,42 @@ pub fn init_global_logging(
             None
         };
 
+        let slow_query_logging_layer = if !opts.dir.is_empty() && opts.slow_query.enable {
+            let rolling_appender =
+                RollingFileAppender::new(Rotation::HOURLY, &opts.dir, "greptimedb-slow-queries");
+            let (writer, guard) = tracing_appender::non_blocking(rolling_appender);
+            guards.push(guard);
+
+            // Only logs if the field contains "slow".
+            let slow_query_filter = FilterFn::new(|metadata| {
+                metadata
+                    .fields()
+                    .iter()
+                    .any(|field| field.name().contains("slow"))
+            });
+
+            if opts.log_format == LogFormat::Json {
+                Some(
+                    Layer::new()
+                        .json()
+                        .with_writer(writer)
+                        .with_ansi(false)
+                        .with_filter(slow_query_filter)
+                        .boxed(),
+                )
+            } else {
+                Some(
+                    Layer::new()
+                        .with_writer(writer)
+                        .with_ansi(false)
+                        .with_filter(slow_query_filter)
+                        .boxed(),
+                )
+            }
+        } else {
+            None
+        };
+
         // resolve log level settings from:
         // - options from command line or config files
         // - environment variable: RUST_LOG
@@ -279,6 +335,7 @@ pub fn init_global_logging(
                 .with(stdout_logging_layer)
                 .with(file_logging_layer)
                 .with(err_file_logging_layer)
+                .with(slow_query_logging_layer)
         };
 
         // consume the `tracing_opts` to avoid "unused" warnings.
@@ -289,7 +346,8 @@ pub fn init_global_logging(
             .with(dyn_filter)
             .with(stdout_logging_layer)
             .with(file_logging_layer)
-            .with(err_file_logging_layer);
+            .with(err_file_logging_layer)
+            .with(slow_query_logging_layer);
 
         if opts.enable_otlp_tracing {
             global::set_text_map_propagator(TraceContextPropagator::new());
diff --git a/src/common/telemetry/src/macros.rs b/src/common/telemetry/src/macros.rs
index cb838db6fe..2846bd5d20 100644
--- a/src/common/telemetry/src/macros.rs
+++ b/src/common/telemetry/src/macros.rs
@@ -152,6 +152,17 @@ macro_rules! trace {
     };
 }
 
+#[macro_export]
+macro_rules! slow {
+    (target: $target:expr, $($arg:tt)+) => {
+        $crate::log!(target: $target, slow = true, $crate::tracing::Level::INFO, $($arg)+)
+    };
+
+    ($($arg:tt)+) => {
+        $crate::log!($crate::tracing::Level::INFO, slow = true, $($arg)+)
+    };
+}
+
 #[cfg(test)]
 mod tests {
     use common_error::mock::MockError;
diff --git a/src/flow/src/server.rs b/src/flow/src/server.rs
index d78f9219cb..4381dd06a0 100644
--- a/src/flow/src/server.rs
+++ b/src/flow/src/server.rs
@@ -37,6 +37,7 @@ use operator::delete::Deleter;
 use operator::insert::Inserter;
 use operator::statement::StatementExecutor;
 use partition::manager::PartitionRuleManager;
+use query::stats::StatementStatistics;
 use query::{QueryEngine, QueryEngineFactory};
 use servers::error::{AlreadyStartedSnafu, StartGrpcSnafu, TcpBindSnafu, TcpIncomingSnafu};
 use servers::server::Server;
@@ -475,6 +476,7 @@ impl FrontendInvoker {
             layered_cache_registry.clone(),
             inserter.clone(),
             table_route_cache,
+            StatementStatistics::default(),
         ));
 
         let invoker = FrontendInvoker::new(inserter, deleter, statement_executor);
diff --git a/src/frontend/src/instance/builder.rs b/src/frontend/src/instance/builder.rs
index 5450e55ce2..a9513121d8 100644
--- a/src/frontend/src/instance/builder.rs
+++ b/src/frontend/src/instance/builder.rs
@@ -33,6 +33,7 @@ use operator::statement::{StatementExecutor, StatementExecutorRef};
 use operator::table::TableMutationOperator;
 use partition::manager::PartitionRuleManager;
 use pipeline::pipeline_operator::PipelineOperator;
+use query::stats::StatementStatistics;
 use query::QueryEngineFactory;
 use servers::server::ServerHandlers;
 use snafu::OptionExt;
@@ -55,6 +56,7 @@ pub struct FrontendBuilder {
     plugins: Option<Plugins>,
     procedure_executor: ProcedureExecutorRef,
     heartbeat_task: Option<HeartbeatTask>,
+    stats: StatementStatistics,
 }
 
 impl FrontendBuilder {
@@ -65,6 +67,7 @@ impl FrontendBuilder {
         catalog_manager: CatalogManagerRef,
         node_manager: NodeManagerRef,
         procedure_executor: ProcedureExecutorRef,
+        stats: StatementStatistics,
     ) -> Self {
         Self {
             options,
@@ -76,6 +79,7 @@ impl FrontendBuilder {
             plugins: None,
             procedure_executor,
             heartbeat_task: None,
+            stats,
         }
     }
 
@@ -181,6 +185,7 @@ impl FrontendBuilder {
             local_cache_invalidator,
             inserter.clone(),
             table_route_cache,
+            self.stats,
         ));
 
         let pipeline_operator = Arc::new(PipelineOperator::new(
diff --git a/src/operator/src/statement.rs b/src/operator/src/statement.rs
index 7c76d0dcff..033bd14e9c 100644
--- a/src/operator/src/statement.rs
+++ b/src/operator/src/statement.rs
@@ -45,6 +45,7 @@ use common_time::Timestamp;
 use datafusion_expr::LogicalPlan;
 use partition::manager::{PartitionRuleManager, PartitionRuleManagerRef};
 use query::parser::QueryStatement;
+use query::stats::StatementStatistics;
 use query::QueryEngineRef;
 use session::context::{Channel, QueryContextRef};
 use session::table_name::table_idents_to_full_name;
@@ -80,11 +81,13 @@ pub struct StatementExecutor {
     partition_manager: PartitionRuleManagerRef,
     cache_invalidator: CacheInvalidatorRef,
     inserter: InserterRef,
+    stats: StatementStatistics,
 }
 
 pub type StatementExecutorRef = Arc<StatementExecutor>;
 
 impl StatementExecutor {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         catalog_manager: CatalogManagerRef,
         query_engine: QueryEngineRef,
@@ -93,6 +96,7 @@ impl StatementExecutor {
         cache_invalidator: CacheInvalidatorRef,
         inserter: InserterRef,
         table_route_cache: TableRouteCacheRef,
+        stats: StatementStatistics,
     ) -> Self {
         Self {
             catalog_manager,
@@ -104,6 +108,7 @@ impl StatementExecutor {
             partition_manager: Arc::new(PartitionRuleManager::new(kv_backend, table_route_cache)),
             cache_invalidator,
             inserter,
+            stats,
         }
     }
 
@@ -113,6 +118,7 @@ impl StatementExecutor {
         stmt: QueryStatement,
         query_ctx: QueryContextRef,
     ) -> Result<Output> {
+        let _slow_query_timer = self.stats.start_slow_query_timer(stmt.clone());
         match stmt {
             QueryStatement::Sql(stmt) => self.execute_sql(stmt, query_ctx).await,
             QueryStatement::Promql(_) => self.plan_exec(stmt, query_ctx).await,
diff --git a/src/query/Cargo.toml b/src/query/Cargo.toml
index eed0d5a339..4bcda7b0a7 100644
--- a/src/query/Cargo.toml
+++ b/src/query/Cargo.toml
@@ -54,6 +54,7 @@ prometheus.workspace = true
 promql.workspace = true
 promql-parser.workspace = true
 prost.workspace = true
+rand.workspace = true
 regex.workspace = true
 session.workspace = true
 snafu.workspace = true
diff --git a/src/query/src/lib.rs b/src/query/src/lib.rs
index d6dfc5e097..e2843c708e 100644
--- a/src/query/src/lib.rs
+++ b/src/query/src/lib.rs
@@ -35,6 +35,7 @@ pub mod query_engine;
 mod range_select;
 pub mod region_query;
 pub mod sql;
+pub mod stats;
 
 #[cfg(test)]
 mod tests;
diff --git a/src/query/src/stats.rs b/src/query/src/stats.rs
new file mode 100644
index 0000000000..896271897d
--- /dev/null
+++ b/src/query/src/stats.rs
@@ -0,0 +1,98 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Duration;
+
+use common_telemetry::logging::SlowQueryOptions;
+use common_telemetry::slow;
+use rand::random;
+
+use crate::parser::QueryStatement;
+
+/// StatementStatistics is used to collect statistics for a statement.
+#[derive(Default, Clone, Debug)]
+pub struct StatementStatistics {
+    /// slow_query is used to configure slow query log.
+    pub slow_query: SlowQueryOptions,
+}
+
+impl StatementStatistics {
+    pub fn new(slow_query_options: SlowQueryOptions) -> Self {
+        Self {
+            slow_query: slow_query_options,
+        }
+    }
+
+    pub fn start_slow_query_timer(&self, stmt: QueryStatement) -> Option<SlowQueryTimer> {
+        if self.slow_query.enable {
+            Some(SlowQueryTimer {
+                start: std::time::Instant::now(),
+                stmt,
+                threshold: self.slow_query.threshold,
+                sample_ratio: self.slow_query.sample_ratio,
+            })
+        } else {
+            None
+        }
+    }
+}
+
+/// SlowQueryTimer is used to log slow query when it's dropped.
+pub struct SlowQueryTimer {
+    start: std::time::Instant,
+    stmt: QueryStatement,
+    threshold: Option<Duration>,
+    sample_ratio: Option<f64>,
+}
+
+impl SlowQueryTimer {
+    fn log_slow_query(&self, elapsed: Duration, threshold: Duration) {
+        match &self.stmt {
+            QueryStatement::Sql(stmt) => {
+                slow!(
+                    cost = elapsed.as_millis() as u64,
+                    threshold = threshold.as_millis() as u64,
+                    sql = stmt.to_string()
+                );
+            }
+            QueryStatement::Promql(stmt) => {
+                slow!(
+                    cost = elapsed.as_millis() as u64,
+                    threshold = threshold.as_millis() as u64,
+                    // TODO(zyy17): It's better to implement Display for EvalStmt for pretty print.
+                    promql = format!("{:?}", stmt)
+                );
+            }
+        }
+    }
+}
+
+impl Drop for SlowQueryTimer {
+    fn drop(&mut self) {
+        if let Some(threshold) = self.threshold {
+            let elapsed = self.start.elapsed();
+            if elapsed > threshold {
+                if let Some(ratio) = self.sample_ratio {
+                    // Generate a random number in [0, 1) and compare it with sample_ratio.
+                    if ratio >= 1.0 || random::<f64>() <= ratio {
+                        self.log_slow_query(elapsed, threshold);
+                    }
+                } else {
+                    // If sample_ratio is not set, log all slow queries.
+                    self.log_slow_query(elapsed, threshold);
+                }
+            }
+        }
+    }
+}
diff --git a/tests-integration/src/cluster.rs b/tests-integration/src/cluster.rs
index ce2803996a..ad2c3e369f 100644
--- a/tests-integration/src/cluster.rs
+++ b/tests-integration/src/cluster.rs
@@ -48,6 +48,7 @@ use meta_client::client::MetaClientBuilder;
 use meta_srv::cluster::MetaPeerClientRef;
 use meta_srv::metasrv::{Metasrv, MetasrvOptions, SelectorRef};
 use meta_srv::mocks::MockInfo;
+use query::stats::StatementStatistics;
 use servers::grpc::flight::FlightCraftWrapper;
 use servers::grpc::region_server::RegionServerRequestHandler;
 use servers::heartbeat_options::HeartbeatOptions;
@@ -393,6 +394,7 @@ impl GreptimeDbClusterBuilder {
             catalog_manager,
             datanode_clients,
             meta_client,
+            StatementStatistics::default(),
         )
         .with_local_cache_invalidator(cache_registry)
         .with_heartbeat_task(heartbeat_task)
diff --git a/tests-integration/src/standalone.rs b/tests-integration/src/standalone.rs
index fa6e8f2a9e..123614e436 100644
--- a/tests-integration/src/standalone.rs
+++ b/tests-integration/src/standalone.rs
@@ -40,6 +40,7 @@ use flow::FlownodeBuilder;
 use frontend::instance::builder::FrontendBuilder;
 use frontend::instance::{FrontendInstance, Instance, StandaloneDatanodeManager};
 use meta_srv::metasrv::{FLOW_ID_SEQ, TABLE_ID_SEQ};
+use query::stats::StatementStatistics;
 use servers::Mode;
 use snafu::ResultExt;
 
@@ -215,6 +216,7 @@ impl GreptimeDbStandaloneBuilder {
             catalog_manager.clone(),
             node_manager.clone(),
             ddl_task_executor.clone(),
+            StatementStatistics::default(),
         )
         .with_plugin(plugins)
         .try_build()
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index e11060fbbd..5f3eb854bb 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -836,6 +836,9 @@ retry_delay = "500ms"
 append_stdout = true
 enable_otlp_tracing = false
 
+[logging.slow_query]
+enable = false
+
 [[region_engine]]
 
 [region_engine.mito]

From 6e776d5f988c54ce8481064c5179c387985ec6cd Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Mon, 30 Sep 2024 16:28:51 +0800
Subject: [PATCH 050/128] feat: support to reject write after flushing (#4759)

* refactor: use `RegionRoleState` instead of `RegionState`

* feat: introducing `RegionLeaderState::Downgrading`

* refactor: introduce `set_region_role_state_gracefully`

* refactor: use `set_region_role` instead of `set_writable`

* feat: support to reject write after flushing

* fix: fix unit tests

* test: add unit test for `should_reject_write`

* chore: add comments

* chore: refine comments

* fix: fix unit test

* test: enable fuzz tests for Local WAL

* chore: add logs

* chore: rename `RegionStatus` to `RegionState`

* feat: introduce `DowngradingLeader`

* chore: rename

* refactor: refactor `set_role_state` tests

* test: ensure downgrading region will reject write

* chore: enhance logs

* chore: refine name

* chore: refine comment

* test: add tests for `set_role_role_state`

* fix: fix unit tests

* chore: apply suggestions from CR

* chore: apply suggestions from CR
---
 .github/workflows/develop.yml                 |   9 +-
 Cargo.lock                                    |   2 +-
 Cargo.toml                                    |   2 +-
 .../information_schema/region_peers.rs        |   6 +-
 src/cmd/src/cli/bench.rs                      |   2 +-
 .../src/ddl/alter_table/region_request.rs     |   2 +-
 src/common/meta/src/ddl/tests/alter_table.rs  |  12 +-
 src/common/meta/src/ddl/tests/drop_table.rs   |   6 +-
 src/common/meta/src/instruction.rs            |   6 +-
 src/common/meta/src/key.rs                    |  40 +--
 src/common/meta/src/key/table_route.rs        |  43 ++-
 src/common/meta/src/region_keeper.rs          |   2 +-
 src/common/meta/src/rpc/router.rs             |  90 ++---
 src/datanode/src/alive_keeper.rs              |  16 +-
 src/datanode/src/datanode.rs                  |   6 +-
 src/datanode/src/heartbeat.rs                 |   4 +-
 src/datanode/src/heartbeat/handler.rs         |   8 +-
 .../src/heartbeat/handler/downgrade_region.rs | 163 +++++----
 .../src/heartbeat/handler/upgrade_region.rs   |   2 +-
 src/datanode/src/region_server.rs             |  33 +-
 src/datanode/src/tests.rs                     |  12 +-
 src/file-engine/src/engine.rs                 |  19 +-
 .../src/handler/region_lease_handler.rs       |   6 +-
 .../downgrade_leader_region.rs                |  96 +++++-
 .../src/procedure/region_migration/manager.rs |   2 +-
 .../procedure/region_migration/test_util.rs   |   2 +-
 .../downgrade_leader_region.rs                |  12 +-
 .../rollback_downgraded_region.rs             |  16 +-
 .../upgrade_candidate_region.rs               |  24 +-
 src/meta-srv/src/region/lease_keeper.rs       |  12 +-
 src/meta-srv/src/test_util.rs                 |   2 +-
 src/metric-engine/src/engine.rs               |  17 +-
 src/mito2/src/compaction/compactor.rs         |  10 +-
 src/mito2/src/engine.rs                       |  34 +-
 src/mito2/src/engine/alter_test.rs            |   8 +-
 src/mito2/src/engine/catchup_test.rs          |  16 +-
 src/mito2/src/engine/compaction_test.rs       |   8 +-
 src/mito2/src/engine/open_test.rs             |  10 +-
 src/mito2/src/engine/set_readonly_test.rs     |  98 ------
 src/mito2/src/engine/set_role_state_test.rs   | 159 +++++++++
 src/mito2/src/error.rs                        |  23 +-
 src/mito2/src/flush.rs                        |  20 +-
 src/mito2/src/region.rs                       | 321 ++++++++++++++----
 src/mito2/src/region/opener.rs                |  23 +-
 src/mito2/src/request.rs                      |  17 +-
 src/mito2/src/test_util.rs                    |   6 +-
 src/mito2/src/test_util/scheduler_util.rs     |   4 +-
 src/mito2/src/worker.rs                       |  24 +-
 src/mito2/src/worker/handle_catchup.rs        |   6 +-
 src/mito2/src/worker/handle_drop.rs           |   4 +-
 src/mito2/src/worker/handle_flush.rs          |  16 +-
 src/mito2/src/worker/handle_manifest.rs       |  21 +-
 src/mito2/src/worker/handle_truncate.rs       |   4 +-
 src/operator/src/tests/partition_manager.rs   |   6 +-
 src/query/src/optimizer/test_util.rs          |  10 +-
 src/store-api/src/region_engine.rs            |  52 ++-
 56 files changed, 1078 insertions(+), 496 deletions(-)
 delete mode 100644 src/mito2/src/engine/set_readonly_test.rs
 create mode 100644 src/mito2/src/engine/set_role_state_test.rs

diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
index 073c407486..13c155d617 100644
--- a/.github/workflows/develop.yml
+++ b/.github/workflows/develop.yml
@@ -442,6 +442,13 @@ jobs:
             minio: true
             kafka: true
             values: "with-remote-wal.yaml"
+        include:
+          - target: "fuzz_migrate_mito_regions"
+            mode:
+              name: "Local WAL"
+              minio: true
+              kafka: false
+              values: "with-minio.yaml"
     steps:
       - name: Remove unused software
         run: |
@@ -530,7 +537,7 @@ jobs:
         with:
           image-registry: localhost:5001
           values-filename: ${{ matrix.mode.values }}
-          enable-region-failover: true
+          enable-region-failover: ${{ matrix.mode.kafka }}
       - name: Port forward (mysql)
         run: |
           kubectl port-forward service/my-greptimedb-frontend 4002:4002 -n my-greptimedb&
diff --git a/Cargo.lock b/Cargo.lock
index 7aebe2e719..ab5043939a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4414,7 +4414,7 @@ dependencies = [
 [[package]]
 name = "greptime-proto"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=36334744c7020734dcb4a6b8d24d52ae7ed53fe1#36334744c7020734dcb4a6b8d24d52ae7ed53fe1"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=0b4f7c8ab06399f6b90e1626e8d5b9697cb33bb9#0b4f7c8ab06399f6b90e1626e8d5b9697cb33bb9"
 dependencies = [
  "prost 0.12.6",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index 44b2cda122..63d7ad3ba7 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -120,7 +120,7 @@ etcd-client = { version = "0.13" }
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "36334744c7020734dcb4a6b8d24d52ae7ed53fe1" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "0b4f7c8ab06399f6b90e1626e8d5b9697cb33bb9" }
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"
diff --git a/src/catalog/src/system_schema/information_schema/region_peers.rs b/src/catalog/src/system_schema/information_schema/region_peers.rs
index 5496879af0..50c2593f86 100644
--- a/src/catalog/src/system_schema/information_schema/region_peers.rs
+++ b/src/catalog/src/system_schema/information_schema/region_peers.rs
@@ -224,8 +224,8 @@ impl InformationSchemaRegionPeersBuilder {
             let region_id = RegionId::new(table_id, route.region.id.region_number()).as_u64();
             let peer_id = route.leader_peer.clone().map(|p| p.id);
             let peer_addr = route.leader_peer.clone().map(|p| p.addr);
-            let status = if let Some(status) = route.leader_status {
-                Some(status.as_ref().to_string())
+            let state = if let Some(state) = route.leader_state {
+                Some(state.as_ref().to_string())
             } else {
                 // Alive by default
                 Some("ALIVE".to_string())
@@ -242,7 +242,7 @@ impl InformationSchemaRegionPeersBuilder {
             self.peer_ids.push(peer_id);
             self.peer_addrs.push(peer_addr.as_deref());
             self.is_leaders.push(Some("Yes"));
-            self.statuses.push(status.as_deref());
+            self.statuses.push(state.as_deref());
             self.down_seconds
                 .push(route.leader_down_millis().map(|m| m / 1000));
         }
diff --git a/src/cmd/src/cli/bench.rs b/src/cmd/src/cli/bench.rs
index bf5a6825f0..f3d1d0f809 100644
--- a/src/cmd/src/cli/bench.rs
+++ b/src/cmd/src/cli/bench.rs
@@ -158,7 +158,7 @@ fn create_region_routes(regions: Vec<RegionNumber>) -> Vec<RegionRoute> {
                 addr: String::new(),
             }),
             follower_peers: vec![],
-            leader_status: None,
+            leader_state: None,
             leader_down_since: None,
         });
     }
diff --git a/src/common/meta/src/ddl/alter_table/region_request.rs b/src/common/meta/src/ddl/alter_table/region_request.rs
index b4223b8ea0..0756360395 100644
--- a/src/common/meta/src/ddl/alter_table/region_request.rs
+++ b/src/common/meta/src/ddl/alter_table/region_request.rs
@@ -187,7 +187,7 @@ mod tests {
                     region: Region::new_test(region_id),
                     leader_peer: Some(Peer::empty(1)),
                     follower_peers: vec![],
-                    leader_status: None,
+                    leader_state: None,
                     leader_down_since: None,
                 }]),
                 HashMap::new(),
diff --git a/src/common/meta/src/ddl/tests/alter_table.rs b/src/common/meta/src/ddl/tests/alter_table.rs
index 06654cfe0f..36a1ff0ece 100644
--- a/src/common/meta/src/ddl/tests/alter_table.rs
+++ b/src/common/meta/src/ddl/tests/alter_table.rs
@@ -107,21 +107,21 @@ async fn test_on_submit_alter_request() {
                     region: Region::new_test(RegionId::new(table_id, 1)),
                     leader_peer: Some(Peer::empty(1)),
                     follower_peers: vec![Peer::empty(5)],
-                    leader_status: None,
+                    leader_state: None,
                     leader_down_since: None,
                 },
                 RegionRoute {
                     region: Region::new_test(RegionId::new(table_id, 2)),
                     leader_peer: Some(Peer::empty(2)),
                     follower_peers: vec![Peer::empty(4)],
-                    leader_status: None,
+                    leader_state: None,
                     leader_down_since: None,
                 },
                 RegionRoute {
                     region: Region::new_test(RegionId::new(table_id, 3)),
                     leader_peer: Some(Peer::empty(3)),
                     follower_peers: vec![],
-                    leader_status: None,
+                    leader_state: None,
                     leader_down_since: None,
                 },
             ]),
@@ -193,21 +193,21 @@ async fn test_on_submit_alter_request_with_outdated_request() {
                     region: Region::new_test(RegionId::new(table_id, 1)),
                     leader_peer: Some(Peer::empty(1)),
                     follower_peers: vec![Peer::empty(5)],
-                    leader_status: None,
+                    leader_state: None,
                     leader_down_since: None,
                 },
                 RegionRoute {
                     region: Region::new_test(RegionId::new(table_id, 2)),
                     leader_peer: Some(Peer::empty(2)),
                     follower_peers: vec![Peer::empty(4)],
-                    leader_status: None,
+                    leader_state: None,
                     leader_down_since: None,
                 },
                 RegionRoute {
                     region: Region::new_test(RegionId::new(table_id, 3)),
                     leader_peer: Some(Peer::empty(3)),
                     follower_peers: vec![],
-                    leader_status: None,
+                    leader_state: None,
                     leader_down_since: None,
                 },
             ]),
diff --git a/src/common/meta/src/ddl/tests/drop_table.rs b/src/common/meta/src/ddl/tests/drop_table.rs
index aff1237472..c3a5f5875c 100644
--- a/src/common/meta/src/ddl/tests/drop_table.rs
+++ b/src/common/meta/src/ddl/tests/drop_table.rs
@@ -119,21 +119,21 @@ async fn test_on_datanode_drop_regions() {
                     region: Region::new_test(RegionId::new(table_id, 1)),
                     leader_peer: Some(Peer::empty(1)),
                     follower_peers: vec![Peer::empty(5)],
-                    leader_status: None,
+                    leader_state: None,
                     leader_down_since: None,
                 },
                 RegionRoute {
                     region: Region::new_test(RegionId::new(table_id, 2)),
                     leader_peer: Some(Peer::empty(2)),
                     follower_peers: vec![Peer::empty(4)],
-                    leader_status: None,
+                    leader_state: None,
                     leader_down_since: None,
                 },
                 RegionRoute {
                     region: Region::new_test(RegionId::new(table_id, 3)),
                     leader_peer: Some(Peer::empty(3)),
                     follower_peers: vec![],
-                    leader_status: None,
+                    leader_state: None,
                     leader_down_since: None,
                 },
             ]),
diff --git a/src/common/meta/src/instruction.rs b/src/common/meta/src/instruction.rs
index 61e2811e72..4864f7562d 100644
--- a/src/common/meta/src/instruction.rs
+++ b/src/common/meta/src/instruction.rs
@@ -137,14 +137,16 @@ pub struct DowngradeRegion {
     /// `None` stands for don't flush before downgrading the region.
     #[serde(default)]
     pub flush_timeout: Option<Duration>,
+    /// Rejects all write requests after flushing.
+    pub reject_write: bool,
 }
 
 impl Display for DowngradeRegion {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
-            "DowngradeRegion(region_id={}, flush_timeout={:?})",
-            self.region_id, self.flush_timeout,
+            "DowngradeRegion(region_id={}, flush_timeout={:?}, rejct_write={})",
+            self.region_id, self.flush_timeout, self.reject_write
         )
     }
 }
diff --git a/src/common/meta/src/key.rs b/src/common/meta/src/key.rs
index d864882da6..0f703b9430 100644
--- a/src/common/meta/src/key.rs
+++ b/src/common/meta/src/key.rs
@@ -140,7 +140,7 @@ use crate::key::table_route::TableRouteKey;
 use crate::key::txn_helper::TxnOpGetResponseSet;
 use crate::kv_backend::txn::{Txn, TxnOp};
 use crate::kv_backend::KvBackendRef;
-use crate::rpc::router::{region_distribution, RegionRoute, RegionStatus};
+use crate::rpc::router::{region_distribution, LeaderState, RegionRoute};
 use crate::rpc::store::BatchDeleteRequest;
 use crate::DatanodeId;
 
@@ -1126,14 +1126,14 @@ impl TableMetadataManager {
         next_region_route_status: F,
     ) -> Result<()>
     where
-        F: Fn(&RegionRoute) -> Option<Option<RegionStatus>>,
+        F: Fn(&RegionRoute) -> Option<Option<LeaderState>>,
     {
         let mut new_region_routes = current_table_route_value.region_routes()?.clone();
 
         let mut updated = 0;
         for route in &mut new_region_routes {
-            if let Some(status) = next_region_route_status(route) {
-                if route.set_leader_status(status) {
+            if let Some(state) = next_region_route_status(route) {
+                if route.set_leader_state(state) {
                     updated += 1;
                 }
             }
@@ -1280,7 +1280,7 @@ mod tests {
     use crate::key::{DeserializedValueWithBytes, TableMetadataManager, ViewInfoValue};
     use crate::kv_backend::memory::MemoryKvBackend;
     use crate::peer::Peer;
-    use crate::rpc::router::{region_distribution, Region, RegionRoute, RegionStatus};
+    use crate::rpc::router::{region_distribution, LeaderState, Region, RegionRoute};
 
     #[test]
     fn test_deserialized_value_with_bytes() {
@@ -1324,7 +1324,7 @@ mod tests {
             },
             leader_peer: Some(Peer::new(datanode, "a2")),
             follower_peers: vec![],
-            leader_status: None,
+            leader_state: None,
             leader_down_since: None,
         }
     }
@@ -1715,7 +1715,7 @@ mod tests {
                     attrs: BTreeMap::new(),
                 },
                 leader_peer: Some(Peer::new(datanode, "a2")),
-                leader_status: Some(RegionStatus::Downgraded),
+                leader_state: Some(LeaderState::Downgrading),
                 follower_peers: vec![],
                 leader_down_since: Some(current_time_millis()),
             },
@@ -1727,7 +1727,7 @@ mod tests {
                     attrs: BTreeMap::new(),
                 },
                 leader_peer: Some(Peer::new(datanode, "a1")),
-                leader_status: None,
+                leader_state: None,
                 follower_peers: vec![],
                 leader_down_since: None,
             },
@@ -1750,10 +1750,10 @@ mod tests {
 
         table_metadata_manager
             .update_leader_region_status(table_id, &current_table_route_value, |region_route| {
-                if region_route.leader_status.is_some() {
+                if region_route.leader_state.is_some() {
                     None
                 } else {
-                    Some(Some(RegionStatus::Downgraded))
+                    Some(Some(LeaderState::Downgrading))
                 }
             })
             .await
@@ -1768,8 +1768,8 @@ mod tests {
             .unwrap();
 
         assert_eq!(
-            updated_route_value.region_routes().unwrap()[0].leader_status,
-            Some(RegionStatus::Downgraded)
+            updated_route_value.region_routes().unwrap()[0].leader_state,
+            Some(LeaderState::Downgrading)
         );
 
         assert!(updated_route_value.region_routes().unwrap()[0]
@@ -1777,8 +1777,8 @@ mod tests {
             .is_some());
 
         assert_eq!(
-            updated_route_value.region_routes().unwrap()[1].leader_status,
-            Some(RegionStatus::Downgraded)
+            updated_route_value.region_routes().unwrap()[1].leader_state,
+            Some(LeaderState::Downgrading)
         );
         assert!(updated_route_value.region_routes().unwrap()[1]
             .leader_down_since
@@ -1943,21 +1943,21 @@ mod tests {
                         region: Region::new_test(RegionId::new(table_id, 1)),
                         leader_peer: Some(Peer::empty(1)),
                         follower_peers: vec![Peer::empty(5)],
-                        leader_status: None,
+                        leader_state: None,
                         leader_down_since: None,
                     },
                     RegionRoute {
                         region: Region::new_test(RegionId::new(table_id, 2)),
                         leader_peer: Some(Peer::empty(2)),
                         follower_peers: vec![Peer::empty(4)],
-                        leader_status: None,
+                        leader_state: None,
                         leader_down_since: None,
                     },
                     RegionRoute {
                         region: Region::new_test(RegionId::new(table_id, 3)),
                         leader_peer: Some(Peer::empty(3)),
                         follower_peers: vec![],
-                        leader_status: None,
+                        leader_state: None,
                         leader_down_since: None,
                     },
                 ]),
@@ -1996,21 +1996,21 @@ mod tests {
                         region: Region::new_test(RegionId::new(table_id, 1)),
                         leader_peer: Some(Peer::empty(1)),
                         follower_peers: vec![Peer::empty(5)],
-                        leader_status: None,
+                        leader_state: None,
                         leader_down_since: None,
                     },
                     RegionRoute {
                         region: Region::new_test(RegionId::new(table_id, 2)),
                         leader_peer: Some(Peer::empty(2)),
                         follower_peers: vec![Peer::empty(4)],
-                        leader_status: None,
+                        leader_state: None,
                         leader_down_since: None,
                     },
                     RegionRoute {
                         region: Region::new_test(RegionId::new(table_id, 3)),
                         leader_peer: Some(Peer::empty(3)),
                         follower_peers: vec![],
-                        leader_status: None,
+                        leader_state: None,
                         leader_down_since: None,
                     },
                 ]),
diff --git a/src/common/meta/src/key/table_route.rs b/src/common/meta/src/key/table_route.rs
index 0be0aab3aa..c9990ab121 100644
--- a/src/common/meta/src/key/table_route.rs
+++ b/src/common/meta/src/key/table_route.rs
@@ -744,6 +744,7 @@ mod tests {
     use crate::kv_backend::memory::MemoryKvBackend;
     use crate::kv_backend::{KvBackend, TxnService};
     use crate::peer::Peer;
+    use crate::rpc::router::Region;
     use crate::rpc::store::PutRequest;
 
     #[test]
@@ -751,11 +752,43 @@ mod tests {
         let old_raw_v = r#"{"region_routes":[{"region":{"id":1,"name":"r1","partition":null,"attrs":{}},"leader_peer":{"id":2,"addr":"a2"},"follower_peers":[]},{"region":{"id":1,"name":"r1","partition":null,"attrs":{}},"leader_peer":{"id":2,"addr":"a2"},"follower_peers":[]}],"version":0}"#;
         let v = TableRouteValue::try_from_raw_value(old_raw_v.as_bytes()).unwrap();
 
-        let new_raw_v = format!("{:?}", v);
-        assert_eq!(
-            new_raw_v,
-            r#"Physical(PhysicalTableRouteValue { region_routes: [RegionRoute { region: Region { id: 1(0, 1), name: "r1", partition: None, attrs: {} }, leader_peer: Some(Peer { id: 2, addr: "a2" }), follower_peers: [], leader_status: None, leader_down_since: None }, RegionRoute { region: Region { id: 1(0, 1), name: "r1", partition: None, attrs: {} }, leader_peer: Some(Peer { id: 2, addr: "a2" }), follower_peers: [], leader_status: None, leader_down_since: None }], version: 0 })"#
-        );
+        let expected_table_route = TableRouteValue::Physical(PhysicalTableRouteValue {
+            region_routes: vec![
+                RegionRoute {
+                    region: Region {
+                        id: RegionId::new(0, 1),
+                        name: "r1".to_string(),
+                        partition: None,
+                        attrs: Default::default(),
+                    },
+                    leader_peer: Some(Peer {
+                        id: 2,
+                        addr: "a2".to_string(),
+                    }),
+                    follower_peers: vec![],
+                    leader_state: None,
+                    leader_down_since: None,
+                },
+                RegionRoute {
+                    region: Region {
+                        id: RegionId::new(0, 1),
+                        name: "r1".to_string(),
+                        partition: None,
+                        attrs: Default::default(),
+                    },
+                    leader_peer: Some(Peer {
+                        id: 2,
+                        addr: "a2".to_string(),
+                    }),
+                    follower_peers: vec![],
+                    leader_state: None,
+                    leader_down_since: None,
+                },
+            ],
+            version: 0,
+        });
+
+        assert_eq!(v, expected_table_route);
     }
 
     #[test]
diff --git a/src/common/meta/src/region_keeper.rs b/src/common/meta/src/region_keeper.rs
index a0d53b8477..54d5d6cc11 100644
--- a/src/common/meta/src/region_keeper.rs
+++ b/src/common/meta/src/region_keeper.rs
@@ -58,7 +58,7 @@ impl MemoryRegionKeeper {
         Default::default()
     }
 
-    /// Returns [OpeningRegionGuard] if Region(`region_id`) on Peer(`datanode_id`) does not exist.
+    /// Returns [OperatingRegionGuard] if Region(`region_id`) on Peer(`datanode_id`) does not exist.
     pub fn register(
         &self,
         datanode_id: DatanodeId,
diff --git a/src/common/meta/src/rpc/router.rs b/src/common/meta/src/rpc/router.rs
index 3e609e4af4..8dc409c8be 100644
--- a/src/common/meta/src/rpc/router.rs
+++ b/src/common/meta/src/rpc/router.rs
@@ -108,16 +108,16 @@ pub fn convert_to_region_peer_map(
         .collect::<HashMap<_, _>>()
 }
 
-/// Returns the HashMap<[RegionNumber], [RegionStatus]>;
-pub fn convert_to_region_leader_status_map(
+/// Returns the HashMap<[RegionNumber], [LeaderState]>;
+pub fn convert_to_region_leader_state_map(
     region_routes: &[RegionRoute],
-) -> HashMap<RegionNumber, RegionStatus> {
+) -> HashMap<RegionNumber, LeaderState> {
     region_routes
         .iter()
         .filter_map(|x| {
-            x.leader_status
+            x.leader_state
                 .as_ref()
-                .map(|status| (x.region.id.region_number(), *status))
+                .map(|state| (x.region.id.region_number(), *state))
         })
         .collect::<HashMap<_, _>>()
 }
@@ -205,7 +205,7 @@ impl TableRoute {
                 region,
                 leader_peer,
                 follower_peers,
-                leader_status: None,
+                leader_state: None,
                 leader_down_since: None,
             });
         }
@@ -259,9 +259,13 @@ pub struct RegionRoute {
     pub follower_peers: Vec<Peer>,
     /// `None` by default.
     #[builder(setter(into, strip_option), default)]
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub leader_status: Option<RegionStatus>,
-    /// The start time when the leader is in `Downgraded` status.
+    #[serde(
+        default,
+        alias = "leader_status",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub leader_state: Option<LeaderState>,
+    /// The start time when the leader is in `Downgraded` state.
     #[serde(default)]
     #[builder(default = "self.default_leader_down_since()")]
     pub leader_down_since: Option<i64>,
@@ -269,76 +273,78 @@ pub struct RegionRoute {
 
 impl RegionRouteBuilder {
     fn default_leader_down_since(&self) -> Option<i64> {
-        match self.leader_status {
-            Some(Some(RegionStatus::Downgraded)) => Some(current_time_millis()),
+        match self.leader_state {
+            Some(Some(LeaderState::Downgrading)) => Some(current_time_millis()),
             _ => None,
         }
     }
 }
 
-/// The Status of the [Region].
+/// The State of the [`Region`] Leader.
 /// TODO(dennis): It's better to add more fine-grained statuses such as `PENDING` etc.
 #[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, AsRefStr)]
 #[strum(serialize_all = "UPPERCASE")]
-pub enum RegionStatus {
-    /// The following cases in which the [Region] will be downgraded.
+pub enum LeaderState {
+    /// The following cases in which the [`Region`] will be downgraded.
     ///
-    /// - The [Region] is unavailable(e.g., Crashed, Network disconnected).
-    /// - The [Region] was planned to migrate to another [Peer].
-    Downgraded,
+    /// - The [`Region`] may be unavailable (e.g., Crashed, Network disconnected).
+    /// - The [`Region`] was planned to migrate to another [`Peer`].
+    Downgrading,
 }
 
 impl RegionRoute {
-    /// Returns true if the Leader [Region] is downgraded.
+    /// Returns true if the Leader [`Region`] is downgraded.
     ///
-    /// The following cases in which the [Region] will be downgraded.
+    /// The following cases in which the [`Region`] will be downgraded.
     ///
-    /// - The [Region] is unavailable(e.g., Crashed, Network disconnected).
-    /// - The [Region] was planned to migrate to another [Peer].
+    /// - The [`Region`] is unavailable(e.g., Crashed, Network disconnected).
+    /// - The [`Region`] was planned to migrate to another [`Peer`].
     ///
-    pub fn is_leader_downgraded(&self) -> bool {
-        matches!(self.leader_status, Some(RegionStatus::Downgraded))
+    pub fn is_leader_downgrading(&self) -> bool {
+        matches!(self.leader_state, Some(LeaderState::Downgrading))
     }
 
-    /// Marks the Leader [Region] as downgraded.
+    /// Marks the Leader [`Region`] as [`RegionState::Downgrading`].
     ///
-    /// We should downgrade a [Region] before deactivating it:
+    /// We should downgrade a [`Region`] before deactivating it:
     ///
-    /// - During the [Region] Failover Procedure.
-    /// - Migrating a [Region].
+    /// - During the [`Region`] Failover Procedure.
+    /// - Migrating a [`Region`].
     ///
-    /// **Notes:** Meta Server will stop renewing the lease for the downgraded [Region].
+    /// **Notes:** Meta Server will renewing a special lease(`Downgrading`) for the downgrading [`Region`].
+    ///
+    /// A downgrading region will reject any write requests, and only allow memetable to be flushed to object storage
     ///
     pub fn downgrade_leader(&mut self) {
         self.leader_down_since = Some(current_time_millis());
-        self.leader_status = Some(RegionStatus::Downgraded)
+        self.leader_state = Some(LeaderState::Downgrading)
     }
 
-    /// Returns how long since the leader is in `Downgraded` status.
+    /// Returns how long since the leader is in `Downgraded` state.
     pub fn leader_down_millis(&self) -> Option<i64> {
         self.leader_down_since
             .map(|start| current_time_millis() - start)
     }
 
-    /// Sets the leader status.
+    /// Sets the leader state.
     ///
     /// Returns true if updated.
-    pub fn set_leader_status(&mut self, status: Option<RegionStatus>) -> bool {
-        let updated = self.leader_status != status;
+    pub fn set_leader_state(&mut self, state: Option<LeaderState>) -> bool {
+        let updated = self.leader_state != state;
 
-        match (status, updated) {
-            (Some(RegionStatus::Downgraded), true) => {
+        match (state, updated) {
+            (Some(LeaderState::Downgrading), true) => {
                 self.leader_down_since = Some(current_time_millis());
             }
-            (Some(RegionStatus::Downgraded), false) => {
-                // Do nothing if leader is still in `Downgraded` status.
+            (Some(LeaderState::Downgrading), false) => {
+                // Do nothing if leader is still in `Downgraded` state.
             }
             _ => {
                 self.leader_down_since = None;
             }
         }
 
-        self.leader_status = status;
+        self.leader_state = state;
         updated
     }
 }
@@ -477,15 +483,15 @@ mod tests {
             },
             leader_peer: Some(Peer::new(1, "a1")),
             follower_peers: vec![Peer::new(2, "a2"), Peer::new(3, "a3")],
-            leader_status: None,
+            leader_state: None,
             leader_down_since: None,
         };
 
-        assert!(!region_route.is_leader_downgraded());
+        assert!(!region_route.is_leader_downgrading());
 
         region_route.downgrade_leader();
 
-        assert!(region_route.is_leader_downgraded());
+        assert!(region_route.is_leader_downgrading());
     }
 
     #[test]
@@ -499,7 +505,7 @@ mod tests {
             },
             leader_peer: Some(Peer::new(1, "a1")),
             follower_peers: vec![Peer::new(2, "a2"), Peer::new(3, "a3")],
-            leader_status: None,
+            leader_state: None,
             leader_down_since: None,
         };
 
diff --git a/src/datanode/src/alive_keeper.rs b/src/datanode/src/alive_keeper.rs
index c6ef6cb3f6..a0ea2c0188 100644
--- a/src/datanode/src/alive_keeper.rs
+++ b/src/datanode/src/alive_keeper.rs
@@ -129,8 +129,10 @@ impl RegionAliveKeeper {
         let request = RegionRequest::Close(RegionCloseRequest {});
         if let Err(e) = self.region_server.handle_request(region_id, request).await {
             if e.status_code() != StatusCode::RegionNotFound {
-                let _ = self.region_server.set_writable(region_id, false);
-                error!(e; "Failed to close staled region {}, set region to readonly.",region_id);
+                let _ = self
+                    .region_server
+                    .set_region_role(region_id, RegionRole::Follower);
+                error!(e; "Failed to close staled region {}, convert region to follower.", region_id);
             }
         }
     }
@@ -378,7 +380,7 @@ impl CountdownTask {
                             }
                         },
                         Some(CountdownCommand::Reset((role, deadline))) => {
-                            let _ = self.region_server.set_writable(self.region_id, role.writable());
+                            let _ = self.region_server.set_region_role(self.region_id, role);
                             trace!(
                                 "Reset deadline of region {region_id} to approximately {} seconds later.",
                                 (deadline - Instant::now()).as_secs_f32(),
@@ -399,8 +401,8 @@ impl CountdownTask {
                     }
                 }
                 () = &mut countdown => {
-                    warn!("The region {region_id} lease is expired, set region to readonly.");
-                    let _ = self.region_server.set_writable(self.region_id, false);
+                    warn!("The region {region_id} lease is expired, convert region to follower.");
+                    let _ = self.region_server.set_region_role(self.region_id, RegionRole::Follower);
                     // resets the countdown.
                     let far_future = Instant::now() + Duration::from_secs(86400 * 365 * 30);
                     countdown.as_mut().reset(far_future);
@@ -436,7 +438,9 @@ mod test {
             .handle_request(region_id, RegionRequest::Create(builder.build()))
             .await
             .unwrap();
-        region_server.set_writable(region_id, true).unwrap();
+        region_server
+            .set_region_role(region_id, RegionRole::Leader)
+            .unwrap();
 
         // Register a region before starting.
         alive_keeper.register_region(region_id).await;
diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs
index f5d7bd9fc6..128a60ab9b 100644
--- a/src/datanode/src/datanode.rs
+++ b/src/datanode/src/datanode.rs
@@ -47,7 +47,7 @@ use servers::server::ServerHandlers;
 use servers::Mode;
 use snafu::{ensure, OptionExt, ResultExt};
 use store_api::path_utils::{region_dir, WAL_DIR};
-use store_api::region_engine::RegionEngineRef;
+use store_api::region_engine::{RegionEngineRef, RegionRole};
 use store_api::region_request::RegionOpenRequest;
 use store_api::storage::RegionId;
 use tokio::fs;
@@ -546,9 +546,9 @@ async fn open_all_regions(
 
     for region_id in open_regions {
         if open_with_writable {
-            if let Err(e) = region_server.set_writable(region_id, true) {
+            if let Err(e) = region_server.set_region_role(region_id, RegionRole::Leader) {
                 error!(
-                    e; "failed to set writable for region {region_id}"
+                    e; "failed to convert region {region_id} to leader"
                 );
             }
         }
diff --git a/src/datanode/src/heartbeat.rs b/src/datanode/src/heartbeat.rs
index d84552a8d2..ef9af0acdd 100644
--- a/src/datanode/src/heartbeat.rs
+++ b/src/datanode/src/heartbeat.rs
@@ -126,7 +126,9 @@ impl HeartbeatTask {
                     let mut follower_region_lease_count = 0;
                     for lease in &lease.regions {
                         match lease.role() {
-                            RegionRole::Leader => leader_region_lease_count += 1,
+                            RegionRole::Leader | RegionRole::DowngradingLeader => {
+                                leader_region_lease_count += 1
+                            }
                             RegionRole::Follower => follower_region_lease_count += 1,
                         }
                     }
diff --git a/src/datanode/src/heartbeat/handler.rs b/src/datanode/src/heartbeat/handler.rs
index d23615eb13..89b6991788 100644
--- a/src/datanode/src/heartbeat/handler.rs
+++ b/src/datanode/src/heartbeat/handler.rs
@@ -153,6 +153,7 @@ mod tests {
     use mito2::engine::MITO_ENGINE_NAME;
     use mito2::test_util::{CreateRequestBuilder, TestEnv};
     use store_api::path_utils::region_dir;
+    use store_api::region_engine::RegionRole;
     use store_api::region_request::{RegionCloseRequest, RegionRequest};
     use store_api::storage::RegionId;
     use tokio::sync::mpsc::{self, Receiver};
@@ -213,6 +214,7 @@ mod tests {
         let instruction = Instruction::DowngradeRegion(DowngradeRegion {
             region_id: RegionId::new(2048, 1),
             flush_timeout: Some(Duration::from_secs(1)),
+            reject_write: false,
         });
         assert!(heartbeat_handler
             .is_acceptable(&heartbeat_env.create_handler_ctx((meta.clone(), instruction))));
@@ -295,7 +297,9 @@ mod tests {
             }
 
             assert_matches!(
-                region_server.set_writable(region_id, true).unwrap_err(),
+                region_server
+                    .set_region_role(region_id, RegionRole::Leader)
+                    .unwrap_err(),
                 error::Error::RegionNotFound { .. }
             );
         }
@@ -411,6 +415,7 @@ mod tests {
             let instruction = Instruction::DowngradeRegion(DowngradeRegion {
                 region_id,
                 flush_timeout: Some(Duration::from_secs(1)),
+                reject_write: false,
             });
 
             let mut ctx = heartbeat_env.create_handler_ctx((meta, instruction));
@@ -433,6 +438,7 @@ mod tests {
         let instruction = Instruction::DowngradeRegion(DowngradeRegion {
             region_id: RegionId::new(2048, 1),
             flush_timeout: Some(Duration::from_secs(1)),
+            reject_write: false,
         });
         let mut ctx = heartbeat_env.create_handler_ctx((meta, instruction));
         let control = heartbeat_handler.handle(&mut ctx).await.unwrap();
diff --git a/src/datanode/src/heartbeat/handler/downgrade_region.rs b/src/datanode/src/heartbeat/handler/downgrade_region.rs
index ac11792803..fd85c75ba2 100644
--- a/src/datanode/src/heartbeat/handler/downgrade_region.rs
+++ b/src/datanode/src/heartbeat/handler/downgrade_region.rs
@@ -16,7 +16,7 @@ use common_meta::instruction::{DowngradeRegion, DowngradeRegionReply, Instructio
 use common_telemetry::tracing::info;
 use common_telemetry::warn;
 use futures_util::future::BoxFuture;
-use store_api::region_engine::SetReadonlyResponse;
+use store_api::region_engine::{SetRegionRoleStateResponse, SettableRegionRoleState};
 use store_api::region_request::{RegionFlushRequest, RegionRequest};
 use store_api::storage::RegionId;
 
@@ -24,16 +24,20 @@ use crate::heartbeat::handler::HandlerContext;
 use crate::heartbeat::task_tracker::WaitResult;
 
 impl HandlerContext {
-    async fn set_readonly_gracefully(&self, region_id: RegionId) -> InstructionReply {
-        match self.region_server.set_readonly_gracefully(region_id).await {
-            Ok(SetReadonlyResponse::Success { last_entry_id }) => {
+    async fn downgrade_to_follower_gracefully(&self, region_id: RegionId) -> InstructionReply {
+        match self
+            .region_server
+            .set_region_role_state_gracefully(region_id, SettableRegionRoleState::Follower)
+            .await
+        {
+            Ok(SetRegionRoleStateResponse::Success { last_entry_id }) => {
                 InstructionReply::DowngradeRegion(DowngradeRegionReply {
                     last_entry_id,
                     exists: true,
                     error: None,
                 })
             }
-            Ok(SetReadonlyResponse::NotFound) => {
+            Ok(SetRegionRoleStateResponse::NotFound) => {
                 InstructionReply::DowngradeRegion(DowngradeRegionReply {
                     last_entry_id: None,
                     exists: false,
@@ -53,10 +57,12 @@ impl HandlerContext {
         DowngradeRegion {
             region_id,
             flush_timeout,
+            reject_write,
         }: DowngradeRegion,
     ) -> BoxFuture<'static, InstructionReply> {
         Box::pin(async move {
-            let Some(writable) = self.region_server.is_writable(region_id) else {
+            let Some(writable) = self.region_server.is_region_leader(region_id) else {
+                warn!("Region: {region_id} is not found");
                 return InstructionReply::DowngradeRegion(DowngradeRegionReply {
                     last_entry_id: None,
                     exists: false,
@@ -64,61 +70,89 @@ impl HandlerContext {
                 });
             };
 
+            let region_server_moved = self.region_server.clone();
+
             // Ignores flush request
             if !writable {
-                return self.set_readonly_gracefully(region_id).await;
+                return self.downgrade_to_follower_gracefully(region_id).await;
             }
 
-            let region_server_moved = self.region_server.clone();
-            if let Some(flush_timeout) = flush_timeout {
-                let register_result = self
-                    .downgrade_tasks
-                    .try_register(
+            // If flush_timeout is not set, directly convert region to follower.
+            let Some(flush_timeout) = flush_timeout else {
+                return self.downgrade_to_follower_gracefully(region_id).await;
+            };
+
+            if reject_write {
+                // Sets region to downgrading, the downgrading region will reject all write requests.
+                match self
+                    .region_server
+                    .set_region_role_state_gracefully(
                         region_id,
-                        Box::pin(async move {
-                            info!("Flush region: {region_id} before downgrading region");
-                            region_server_moved
-                                .handle_request(
-                                    region_id,
-                                    RegionRequest::Flush(RegionFlushRequest {
-                                        row_group_size: None,
-                                    }),
-                                )
-                                .await?;
-
-                            Ok(())
-                        }),
+                        SettableRegionRoleState::DowngradingLeader,
                     )
-                    .await;
-
-                if register_result.is_busy() {
-                    warn!("Another flush task is running for the region: {region_id}");
-                }
-
-                let mut watcher = register_result.into_watcher();
-                let result = self.catchup_tasks.wait(&mut watcher, flush_timeout).await;
-
-                match result {
-                    WaitResult::Timeout => {
-                        InstructionReply::DowngradeRegion(DowngradeRegionReply {
+                    .await
+                {
+                    Ok(SetRegionRoleStateResponse::Success { .. }) => {}
+                    Ok(SetRegionRoleStateResponse::NotFound) => {
+                        warn!("Region: {region_id} is not found");
+                        return InstructionReply::DowngradeRegion(DowngradeRegionReply {
                             last_entry_id: None,
-                            exists: true,
-                            error: Some(format!(
-                                "Flush region: {region_id} before downgrading region is timeout"
-                            )),
-                        })
+                            exists: false,
+                            error: None,
+                        });
                     }
-                    WaitResult::Finish(Ok(_)) => self.set_readonly_gracefully(region_id).await,
-                    WaitResult::Finish(Err(err)) => {
-                        InstructionReply::DowngradeRegion(DowngradeRegionReply {
+                    Err(err) => {
+                        warn!(err; "Failed to convert region to downgrading leader");
+                        return InstructionReply::DowngradeRegion(DowngradeRegionReply {
                             last_entry_id: None,
                             exists: true,
                             error: Some(format!("{err:?}")),
-                        })
+                        });
                     }
                 }
-            } else {
-                self.set_readonly_gracefully(region_id).await
+            }
+
+            let register_result = self
+                .downgrade_tasks
+                .try_register(
+                    region_id,
+                    Box::pin(async move {
+                        info!("Flush region: {region_id} before converting region to follower");
+                        region_server_moved
+                            .handle_request(
+                                region_id,
+                                RegionRequest::Flush(RegionFlushRequest {
+                                    row_group_size: None,
+                                }),
+                            )
+                            .await?;
+
+                        Ok(())
+                    }),
+                )
+                .await;
+
+            if register_result.is_busy() {
+                warn!("Another flush task is running for the region: {region_id}");
+            }
+
+            let mut watcher = register_result.into_watcher();
+            let result = self.catchup_tasks.wait(&mut watcher, flush_timeout).await;
+
+            match result {
+                WaitResult::Timeout => InstructionReply::DowngradeRegion(DowngradeRegionReply {
+                    last_entry_id: None,
+                    exists: true,
+                    error: Some(format!("Flush region: {region_id} is timeout")),
+                }),
+                WaitResult::Finish(Ok(_)) => self.downgrade_to_follower_gracefully(region_id).await,
+                WaitResult::Finish(Err(err)) => {
+                    InstructionReply::DowngradeRegion(DowngradeRegionReply {
+                        last_entry_id: None,
+                        exists: true,
+                        error: Some(format!("{err:?}")),
+                    })
+                }
             }
         })
     }
@@ -131,7 +165,7 @@ mod tests {
 
     use common_meta::instruction::{DowngradeRegion, InstructionReply};
     use mito2::engine::MITO_ENGINE_NAME;
-    use store_api::region_engine::{RegionRole, SetReadonlyResponse};
+    use store_api::region_engine::{RegionRole, SetRegionRoleStateResponse};
     use store_api::region_request::RegionRequest;
     use store_api::storage::RegionId;
     use tokio::time::Instant;
@@ -155,6 +189,7 @@ mod tests {
                 .handle_downgrade_region_instruction(DowngradeRegion {
                     region_id,
                     flush_timeout,
+                    reject_write: false,
                 })
                 .await;
             assert_matches!(reply, InstructionReply::DowngradeRegion(_));
@@ -182,8 +217,9 @@ mod tests {
 
                     Ok(0)
                 }));
-                region_engine.handle_set_readonly_gracefully_mock_fn =
-                    Some(Box::new(|_| Ok(SetReadonlyResponse::success(Some(1024)))))
+                region_engine.handle_set_readonly_gracefully_mock_fn = Some(Box::new(|_| {
+                    Ok(SetRegionRoleStateResponse::success(Some(1024)))
+                }))
             });
         mock_region_server.register_test_region(region_id, mock_engine);
         let handler_context = HandlerContext::new_for_test(mock_region_server);
@@ -195,6 +231,7 @@ mod tests {
                 .handle_downgrade_region_instruction(DowngradeRegion {
                     region_id,
                     flush_timeout,
+                    reject_write: false,
                 })
                 .await;
             assert_matches!(reply, InstructionReply::DowngradeRegion(_));
@@ -215,8 +252,9 @@ mod tests {
             MockRegionEngine::with_custom_apply_fn(MITO_ENGINE_NAME, |region_engine| {
                 region_engine.mock_role = Some(Some(RegionRole::Leader));
                 region_engine.handle_request_delay = Some(Duration::from_secs(100));
-                region_engine.handle_set_readonly_gracefully_mock_fn =
-                    Some(Box::new(|_| Ok(SetReadonlyResponse::success(Some(1024)))))
+                region_engine.handle_set_readonly_gracefully_mock_fn = Some(Box::new(|_| {
+                    Ok(SetRegionRoleStateResponse::success(Some(1024)))
+                }))
             });
         mock_region_server.register_test_region(region_id, mock_engine);
         let handler_context = HandlerContext::new_for_test(mock_region_server);
@@ -227,6 +265,7 @@ mod tests {
             .handle_downgrade_region_instruction(DowngradeRegion {
                 region_id,
                 flush_timeout: Some(flush_timeout),
+                reject_write: false,
             })
             .await;
         assert_matches!(reply, InstructionReply::DowngradeRegion(_));
@@ -246,8 +285,9 @@ mod tests {
             MockRegionEngine::with_custom_apply_fn(MITO_ENGINE_NAME, |region_engine| {
                 region_engine.mock_role = Some(Some(RegionRole::Leader));
                 region_engine.handle_request_delay = Some(Duration::from_millis(300));
-                region_engine.handle_set_readonly_gracefully_mock_fn =
-                    Some(Box::new(|_| Ok(SetReadonlyResponse::success(Some(1024)))))
+                region_engine.handle_set_readonly_gracefully_mock_fn = Some(Box::new(|_| {
+                    Ok(SetRegionRoleStateResponse::success(Some(1024)))
+                }))
             });
         mock_region_server.register_test_region(region_id, mock_engine);
         let handler_context = HandlerContext::new_for_test(mock_region_server);
@@ -263,6 +303,7 @@ mod tests {
                 .handle_downgrade_region_instruction(DowngradeRegion {
                     region_id,
                     flush_timeout,
+                    reject_write: false,
                 })
                 .await;
             assert_matches!(reply, InstructionReply::DowngradeRegion(_));
@@ -277,6 +318,7 @@ mod tests {
             .handle_downgrade_region_instruction(DowngradeRegion {
                 region_id,
                 flush_timeout: Some(Duration::from_millis(500)),
+                reject_write: false,
             })
             .await;
         assert_matches!(reply, InstructionReply::DowngradeRegion(_));
@@ -304,8 +346,9 @@ mod tests {
                     }
                     .fail()
                 }));
-                region_engine.handle_set_readonly_gracefully_mock_fn =
-                    Some(Box::new(|_| Ok(SetReadonlyResponse::success(Some(1024)))))
+                region_engine.handle_set_readonly_gracefully_mock_fn = Some(Box::new(|_| {
+                    Ok(SetRegionRoleStateResponse::success(Some(1024)))
+                }))
             });
         mock_region_server.register_test_region(region_id, mock_engine);
         let handler_context = HandlerContext::new_for_test(mock_region_server);
@@ -321,6 +364,7 @@ mod tests {
                 .handle_downgrade_region_instruction(DowngradeRegion {
                     region_id,
                     flush_timeout,
+                    reject_write: false,
                 })
                 .await;
             assert_matches!(reply, InstructionReply::DowngradeRegion(_));
@@ -335,6 +379,7 @@ mod tests {
             .handle_downgrade_region_instruction(DowngradeRegion {
                 region_id,
                 flush_timeout: Some(Duration::from_millis(500)),
+                reject_write: false,
             })
             .await;
         assert_matches!(reply, InstructionReply::DowngradeRegion(_));
@@ -356,7 +401,7 @@ mod tests {
             MockRegionEngine::with_custom_apply_fn(MITO_ENGINE_NAME, |region_engine| {
                 region_engine.mock_role = Some(Some(RegionRole::Leader));
                 region_engine.handle_set_readonly_gracefully_mock_fn =
-                    Some(Box::new(|_| Ok(SetReadonlyResponse::NotFound)));
+                    Some(Box::new(|_| Ok(SetRegionRoleStateResponse::NotFound)));
             });
         mock_region_server.register_test_region(region_id, mock_engine);
         let handler_context = HandlerContext::new_for_test(mock_region_server);
@@ -365,6 +410,7 @@ mod tests {
             .handle_downgrade_region_instruction(DowngradeRegion {
                 region_id,
                 flush_timeout: None,
+                reject_write: false,
             })
             .await;
         assert_matches!(reply, InstructionReply::DowngradeRegion(_));
@@ -396,6 +442,7 @@ mod tests {
             .handle_downgrade_region_instruction(DowngradeRegion {
                 region_id,
                 flush_timeout: None,
+                reject_write: false,
             })
             .await;
         assert_matches!(reply, InstructionReply::DowngradeRegion(_));
diff --git a/src/datanode/src/heartbeat/handler/upgrade_region.rs b/src/datanode/src/heartbeat/handler/upgrade_region.rs
index 0d1ef0476c..9acb3da9c3 100644
--- a/src/datanode/src/heartbeat/handler/upgrade_region.rs
+++ b/src/datanode/src/heartbeat/handler/upgrade_region.rs
@@ -31,7 +31,7 @@ impl HandlerContext {
         }: UpgradeRegion,
     ) -> BoxFuture<'static, InstructionReply> {
         Box::pin(async move {
-            let Some(writable) = self.region_server.is_writable(region_id) else {
+            let Some(writable) = self.region_server.is_region_leader(region_id) else {
                 return InstructionReply::UpgradeRegion(UpgradeRegionReply {
                     ready: false,
                     exists: false,
diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs
index aa80f52a5c..0bac53e4d6 100644
--- a/src/datanode/src/region_server.rs
+++ b/src/datanode/src/region_server.rs
@@ -54,7 +54,10 @@ use snafu::{ensure, OptionExt, ResultExt};
 use store_api::metric_engine_consts::{
     FILE_ENGINE_NAME, LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME,
 };
-use store_api::region_engine::{RegionEngineRef, RegionRole, RegionStatistic, SetReadonlyResponse};
+use store_api::region_engine::{
+    RegionEngineRef, RegionRole, RegionStatistic, SetRegionRoleStateResponse,
+    SettableRegionRoleState,
+};
 use store_api::region_request::{
     AffectedRows, RegionCloseRequest, RegionOpenRequest, RegionRequest,
 };
@@ -274,37 +277,47 @@ impl RegionServer {
             .collect()
     }
 
-    pub fn is_writable(&self, region_id: RegionId) -> Option<bool> {
-        // TODO(weny): Finds a better way.
+    pub fn is_region_leader(&self, region_id: RegionId) -> Option<bool> {
         self.inner.region_map.get(&region_id).and_then(|engine| {
             engine.role(region_id).map(|role| match role {
                 RegionRole::Follower => false,
                 RegionRole::Leader => true,
+                RegionRole::DowngradingLeader => true,
             })
         })
     }
 
-    pub fn set_writable(&self, region_id: RegionId, writable: bool) -> Result<()> {
+    pub fn set_region_role(&self, region_id: RegionId, role: RegionRole) -> Result<()> {
         let engine = self
             .inner
             .region_map
             .get(&region_id)
             .with_context(|| RegionNotFoundSnafu { region_id })?;
         engine
-            .set_writable(region_id, writable)
+            .set_region_role(region_id, role)
             .with_context(|_| HandleRegionRequestSnafu { region_id })
     }
 
-    pub async fn set_readonly_gracefully(
+    /// Set region role state gracefully.
+    ///
+    /// For [SettableRegionRoleState::Follower]:
+    /// After the call returns, the engine ensures that
+    /// no **further** write or flush operations will succeed in this region.
+    ///
+    /// For [SettableRegionRoleState::DowngradingLeader]:
+    /// After the call returns, the engine ensures that
+    /// no **further** write operations will succeed in this region.
+    pub async fn set_region_role_state_gracefully(
         &self,
         region_id: RegionId,
-    ) -> Result<SetReadonlyResponse> {
+        state: SettableRegionRoleState,
+    ) -> Result<SetRegionRoleStateResponse> {
         match self.inner.region_map.get(&region_id) {
             Some(engine) => Ok(engine
-                .set_readonly_gracefully(region_id)
+                .set_region_role_state_gracefully(region_id, state)
                 .await
                 .with_context(|_| HandleRegionRequestSnafu { region_id })?),
-            None => Ok(SetReadonlyResponse::NotFound),
+            None => Ok(SetRegionRoleStateResponse::NotFound),
         }
     }
 
@@ -842,7 +855,7 @@ impl RegionServerInner {
                 info!("Region {region_id} is deregistered from engine {engine_type}");
                 self.region_map
                     .remove(&region_id)
-                    .map(|(id, engine)| engine.set_writable(id, false));
+                    .map(|(id, engine)| engine.set_region_role(id, RegionRole::Follower));
                 self.event_listener.on_region_deregistered(region_id);
             }
             RegionChange::Catchup => {
diff --git a/src/datanode/src/tests.rs b/src/datanode/src/tests.rs
index 35f513bc83..2acc66a592 100644
--- a/src/datanode/src/tests.rs
+++ b/src/datanode/src/tests.rs
@@ -32,7 +32,8 @@ use query::{QueryEngine, QueryEngineContext};
 use session::context::QueryContextRef;
 use store_api::metadata::RegionMetadataRef;
 use store_api::region_engine::{
-    RegionEngine, RegionRole, RegionScannerRef, RegionStatistic, SetReadonlyResponse,
+    RegionEngine, RegionRole, RegionScannerRef, RegionStatistic, SetRegionRoleStateResponse,
+    SettableRegionRoleState,
 };
 use store_api::region_request::{AffectedRows, RegionRequest};
 use store_api::storage::{RegionId, ScanRequest};
@@ -106,7 +107,7 @@ pub type MockRequestHandler =
     Box<dyn Fn(RegionId, RegionRequest) -> Result<AffectedRows, Error> + Send + Sync>;
 
 pub type MockSetReadonlyGracefullyHandler =
-    Box<dyn Fn(RegionId) -> Result<SetReadonlyResponse, Error> + Send + Sync>;
+    Box<dyn Fn(RegionId) -> Result<SetRegionRoleStateResponse, Error> + Send + Sync>;
 
 pub struct MockRegionEngine {
     sender: Sender<(RegionId, RegionRequest)>,
@@ -220,14 +221,15 @@ impl RegionEngine for MockRegionEngine {
         Ok(())
     }
 
-    fn set_writable(&self, _region_id: RegionId, _writable: bool) -> Result<(), BoxedError> {
+    fn set_region_role(&self, _region_id: RegionId, _role: RegionRole) -> Result<(), BoxedError> {
         Ok(())
     }
 
-    async fn set_readonly_gracefully(
+    async fn set_region_role_state_gracefully(
         &self,
         region_id: RegionId,
-    ) -> Result<SetReadonlyResponse, BoxedError> {
+        _region_role_state: SettableRegionRoleState,
+    ) -> Result<SetRegionRoleStateResponse, BoxedError> {
         if let Some(mock_fn) = &self.handle_set_readonly_gracefully_mock_fn {
             return mock_fn(region_id).map_err(BoxedError::new);
         };
diff --git a/src/file-engine/src/engine.rs b/src/file-engine/src/engine.rs
index 32e1a1d58d..e6313f4322 100644
--- a/src/file-engine/src/engine.rs
+++ b/src/file-engine/src/engine.rs
@@ -26,8 +26,8 @@ use object_store::ObjectStore;
 use snafu::{ensure, OptionExt};
 use store_api::metadata::RegionMetadataRef;
 use store_api::region_engine::{
-    RegionEngine, RegionRole, RegionScannerRef, RegionStatistic, SetReadonlyResponse,
-    SinglePartitionScanner,
+    RegionEngine, RegionRole, RegionScannerRef, RegionStatistic, SetRegionRoleStateResponse,
+    SettableRegionRoleState, SinglePartitionScanner,
 };
 use store_api::region_request::{
     AffectedRows, RegionCloseRequest, RegionCreateRequest, RegionDropRequest, RegionOpenRequest,
@@ -113,22 +113,23 @@ impl RegionEngine for FileRegionEngine {
         None
     }
 
-    fn set_writable(&self, region_id: RegionId, writable: bool) -> Result<(), BoxedError> {
+    fn set_region_role(&self, region_id: RegionId, role: RegionRole) -> Result<(), BoxedError> {
         self.inner
-            .set_writable(region_id, writable)
+            .set_region_role(region_id, role)
             .map_err(BoxedError::new)
     }
 
-    async fn set_readonly_gracefully(
+    async fn set_region_role_state_gracefully(
         &self,
         region_id: RegionId,
-    ) -> Result<SetReadonlyResponse, BoxedError> {
+        _region_role_state: SettableRegionRoleState,
+    ) -> Result<SetRegionRoleStateResponse, BoxedError> {
         let exists = self.inner.get_region(region_id).await.is_some();
 
         if exists {
-            Ok(SetReadonlyResponse::success(None))
+            Ok(SetRegionRoleStateResponse::success(None))
         } else {
-            Ok(SetReadonlyResponse::NotFound)
+            Ok(SetRegionRoleStateResponse::NotFound)
         }
     }
 
@@ -189,7 +190,7 @@ impl EngineInner {
         Ok(())
     }
 
-    fn set_writable(&self, _region_id: RegionId, _writable: bool) -> EngineResult<()> {
+    fn set_region_role(&self, _region_id: RegionId, _region_role: RegionRole) -> EngineResult<()> {
         // TODO(zhongzc): Improve the semantics and implementation of this API.
         Ok(())
     }
diff --git a/src/meta-srv/src/handler/region_lease_handler.rs b/src/meta-srv/src/handler/region_lease_handler.rs
index 06cf818d23..de491da371 100644
--- a/src/meta-srv/src/handler/region_lease_handler.rs
+++ b/src/meta-srv/src/handler/region_lease_handler.rs
@@ -111,7 +111,7 @@ mod test {
     use common_meta::kv_backend::memory::MemoryKvBackend;
     use common_meta::peer::Peer;
     use common_meta::region_keeper::MemoryRegionKeeper;
-    use common_meta::rpc::router::{Region, RegionRoute, RegionStatus};
+    use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
     use store_api::region_engine::RegionRole;
     use store_api::storage::RegionId;
 
@@ -297,7 +297,7 @@ mod test {
                 region: Region::new_test(region_id),
                 leader_peer: Some(peer.clone()),
                 follower_peers: vec![follower_peer.clone()],
-                leader_status: Some(RegionStatus::Downgraded),
+                leader_state: Some(LeaderState::Downgrading),
                 leader_down_since: Some(1),
             },
             RegionRoute {
@@ -352,7 +352,7 @@ mod test {
         assert_region_lease(
             acc,
             vec![
-                GrantedRegion::new(region_id, RegionRole::Follower),
+                GrantedRegion::new(region_id, RegionRole::DowngradingLeader),
                 GrantedRegion::new(another_region_id, RegionRole::Leader),
             ],
         );
diff --git a/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs b/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs
index 836ca4c532..ec5114b9eb 100644
--- a/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs
+++ b/src/meta-srv/src/procedure/region_migration/downgrade_leader_region.rs
@@ -22,8 +22,10 @@ use common_meta::instruction::{
 };
 use common_procedure::Status;
 use common_telemetry::{error, info, warn};
+use common_wal::options::WalOptions;
 use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt};
+use store_api::storage::RegionId;
 use tokio::time::{sleep, Instant};
 
 use super::update_metadata::UpdateMetadata;
@@ -95,15 +97,32 @@ impl DowngradeLeaderRegion {
         &self,
         ctx: &Context,
         flush_timeout: Duration,
+        reject_write: bool,
     ) -> Instruction {
         let pc = &ctx.persistent_ctx;
         let region_id = pc.region_id;
         Instruction::DowngradeRegion(DowngradeRegion {
             region_id,
             flush_timeout: Some(flush_timeout),
+            reject_write,
         })
     }
 
+    async fn should_reject_write(ctx: &mut Context, region_id: RegionId) -> Result<bool> {
+        let datanode_table_value = ctx.get_from_peer_datanode_table_value().await?;
+        if let Some(wal_option) = datanode_table_value
+            .region_info
+            .region_wal_options
+            .get(&region_id.region_number())
+        {
+            let options: WalOptions = serde_json::from_str(wal_option)
+                .with_context(|_| error::DeserializeFromJsonSnafu { input: wal_option })?;
+            return Ok(matches!(options, WalOptions::RaftEngine));
+        }
+
+        Ok(true)
+    }
+
     /// Tries to downgrade a leader region.
     ///
     /// Retry:
@@ -118,16 +137,17 @@ impl DowngradeLeaderRegion {
     /// - [ExceededDeadline](error::Error::ExceededDeadline)
     /// - Invalid JSON.
     async fn downgrade_region(&self, ctx: &mut Context) -> Result<()> {
-        let pc = &ctx.persistent_ctx;
-        let region_id = pc.region_id;
-        let leader = &pc.from_peer;
+        let region_id = ctx.persistent_ctx.region_id;
         let operation_timeout =
             ctx.next_operation_timeout()
                 .context(error::ExceededDeadlineSnafu {
                     operation: "Downgrade region",
                 })?;
-        let downgrade_instruction = self.build_downgrade_region_instruction(ctx, operation_timeout);
+        let reject_write = Self::should_reject_write(ctx, region_id).await?;
+        let downgrade_instruction =
+            self.build_downgrade_region_instruction(ctx, operation_timeout, reject_write);
 
+        let leader = &ctx.persistent_ctx.from_peer;
         let msg = MailboxMessage::json_message(
             &format!("Downgrade leader region: {}", region_id),
             &format!("Meta@{}", ctx.server_addr()),
@@ -240,8 +260,13 @@ impl DowngradeLeaderRegion {
 #[cfg(test)]
 mod tests {
     use std::assert_matches::assert_matches;
+    use std::collections::HashMap;
 
+    use common_meta::key::table_route::TableRouteValue;
+    use common_meta::key::test_utils::new_test_table_info;
     use common_meta::peer::Peer;
+    use common_meta::rpc::router::{Region, RegionRoute};
+    use common_wal::options::KafkaWalOptions;
     use store_api::storage::RegionId;
     use tokio::time::Instant;
 
@@ -264,19 +289,73 @@ mod tests {
         }
     }
 
+    async fn prepare_table_metadata(ctx: &Context, wal_options: HashMap<u32, String>) {
+        let table_info =
+            new_test_table_info(ctx.persistent_ctx.region_id.table_id(), vec![1]).into();
+        let region_routes = vec![RegionRoute {
+            region: Region::new_test(ctx.persistent_ctx.region_id),
+            leader_peer: Some(ctx.persistent_ctx.from_peer.clone()),
+            follower_peers: vec![ctx.persistent_ctx.to_peer.clone()],
+            ..Default::default()
+        }];
+        ctx.table_metadata_manager
+            .create_table_metadata(
+                table_info,
+                TableRouteValue::physical(region_routes),
+                wal_options,
+            )
+            .await
+            .unwrap();
+    }
+
     #[tokio::test]
     async fn test_datanode_is_unreachable() {
         let state = DowngradeLeaderRegion::default();
         let persistent_context = new_persistent_context();
         let env = TestingEnv::new();
         let mut ctx = env.context_factory().new_context(persistent_context);
-
+        prepare_table_metadata(&ctx, HashMap::default()).await;
         let err = state.downgrade_region(&mut ctx).await.unwrap_err();
 
         assert_matches!(err, Error::PusherNotFound { .. });
         assert!(!err.is_retryable());
     }
 
+    #[tokio::test]
+    async fn test_should_reject_writes() {
+        let persistent_context = new_persistent_context();
+        let region_id = persistent_context.region_id;
+        let env = TestingEnv::new();
+        let mut ctx = env.context_factory().new_context(persistent_context);
+        let wal_options =
+            HashMap::from([(1, serde_json::to_string(&WalOptions::RaftEngine).unwrap())]);
+        prepare_table_metadata(&ctx, wal_options).await;
+
+        let reject_write = DowngradeLeaderRegion::should_reject_write(&mut ctx, region_id)
+            .await
+            .unwrap();
+        assert!(reject_write);
+
+        // Remote WAL
+        let persistent_context = new_persistent_context();
+        let region_id = persistent_context.region_id;
+        let env = TestingEnv::new();
+        let mut ctx = env.context_factory().new_context(persistent_context);
+        let wal_options = HashMap::from([(
+            1,
+            serde_json::to_string(&WalOptions::Kafka(KafkaWalOptions {
+                topic: "my_topic".to_string(),
+            }))
+            .unwrap(),
+        )]);
+        prepare_table_metadata(&ctx, wal_options).await;
+
+        let reject_write = DowngradeLeaderRegion::should_reject_write(&mut ctx, region_id)
+            .await
+            .unwrap();
+        assert!(!reject_write);
+    }
+
     #[tokio::test]
     async fn test_pusher_dropped() {
         let state = DowngradeLeaderRegion::default();
@@ -285,6 +364,7 @@ mod tests {
 
         let mut env = TestingEnv::new();
         let mut ctx = env.context_factory().new_context(persistent_context);
+        prepare_table_metadata(&ctx, HashMap::default()).await;
         let mailbox_ctx = env.mailbox_context();
 
         let (tx, rx) = tokio::sync::mpsc::channel(1);
@@ -307,6 +387,7 @@ mod tests {
         let persistent_context = new_persistent_context();
         let env = TestingEnv::new();
         let mut ctx = env.context_factory().new_context(persistent_context);
+        prepare_table_metadata(&ctx, HashMap::default()).await;
         ctx.volatile_ctx.operations_elapsed = ctx.persistent_ctx.timeout + Duration::from_secs(1);
 
         let err = state.downgrade_region(&mut ctx).await.unwrap_err();
@@ -330,6 +411,7 @@ mod tests {
 
         let mut env = TestingEnv::new();
         let mut ctx = env.context_factory().new_context(persistent_context);
+        prepare_table_metadata(&ctx, HashMap::default()).await;
         let mailbox_ctx = env.mailbox_context();
         let mailbox = mailbox_ctx.mailbox().clone();
 
@@ -356,6 +438,7 @@ mod tests {
 
         let mut env = TestingEnv::new();
         let mut ctx = env.context_factory().new_context(persistent_context);
+        prepare_table_metadata(&ctx, HashMap::default()).await;
         let mailbox_ctx = env.mailbox_context();
         let mailbox = mailbox_ctx.mailbox().clone();
 
@@ -383,6 +466,7 @@ mod tests {
 
         let mut env = TestingEnv::new();
         let mut ctx = env.context_factory().new_context(persistent_context);
+        prepare_table_metadata(&ctx, HashMap::default()).await;
         let mailbox_ctx = env.mailbox_context();
         let mailbox = mailbox_ctx.mailbox().clone();
 
@@ -416,6 +500,7 @@ mod tests {
 
         let mut env = TestingEnv::new();
         let mut ctx = env.context_factory().new_context(persistent_context);
+        prepare_table_metadata(&ctx, HashMap::default()).await;
         let mailbox_ctx = env.mailbox_context();
         let mailbox = mailbox_ctx.mailbox().clone();
 
@@ -508,6 +593,7 @@ mod tests {
 
         let mut env = TestingEnv::new();
         let mut ctx = env.context_factory().new_context(persistent_context);
+        prepare_table_metadata(&ctx, HashMap::default()).await;
         let mailbox_ctx = env.mailbox_context();
         let mailbox = mailbox_ctx.mailbox().clone();
 
diff --git a/src/meta-srv/src/procedure/region_migration/manager.rs b/src/meta-srv/src/procedure/region_migration/manager.rs
index bb3eff80c0..01ea887ca9 100644
--- a/src/meta-srv/src/procedure/region_migration/manager.rs
+++ b/src/meta-srv/src/procedure/region_migration/manager.rs
@@ -246,7 +246,7 @@ impl RegionMigrationManager {
         region_route: &RegionRoute,
         task: &RegionMigrationProcedureTask,
     ) -> Result<bool> {
-        if region_route.is_leader_downgraded() {
+        if region_route.is_leader_downgrading() {
             return Ok(false);
         }
 
diff --git a/src/meta-srv/src/procedure/region_migration/test_util.rs b/src/meta-srv/src/procedure/region_migration/test_util.rs
index 65e33ab3d9..cb3b5a3dc3 100644
--- a/src/meta-srv/src/procedure/region_migration/test_util.rs
+++ b/src/meta-srv/src/procedure/region_migration/test_util.rs
@@ -449,7 +449,7 @@ impl ProcedureMigrationTestSuite {
             .find(|route| route.region.id == region_id)
             .unwrap();
 
-        assert!(!region_route.is_leader_downgraded());
+        assert!(!region_route.is_leader_downgrading());
         assert_eq!(
             region_route.leader_peer.as_ref().unwrap().id,
             expected_leader_id
diff --git a/src/meta-srv/src/procedure/region_migration/update_metadata/downgrade_leader_region.rs b/src/meta-srv/src/procedure/region_migration/update_metadata/downgrade_leader_region.rs
index 3b3f6a6c0c..d8bad44871 100644
--- a/src/meta-srv/src/procedure/region_migration/update_metadata/downgrade_leader_region.rs
+++ b/src/meta-srv/src/procedure/region_migration/update_metadata/downgrade_leader_region.rs
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use common_error::ext::BoxedError;
-use common_meta::rpc::router::RegionStatus;
+use common_meta::rpc::router::LeaderState;
 use snafu::ResultExt;
 
 use crate::error::{self, Result};
@@ -53,7 +53,7 @@ impl UpdateMetadata {
                         .as_ref()
                         .is_some_and(|leader_peer| leader_peer.id == from_peer_id)
                 {
-                    Some(Some(RegionStatus::Downgraded))
+                    Some(Some(LeaderState::Downgrading))
                 } else {
                     None
                 }
@@ -81,7 +81,7 @@ mod tests {
 
     use common_meta::key::test_utils::new_test_table_info;
     use common_meta::peer::Peer;
-    use common_meta::rpc::router::{Region, RegionRoute, RegionStatus};
+    use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
     use store_api::storage::RegionId;
 
     use crate::error::Error;
@@ -155,7 +155,7 @@ mod tests {
         table_metadata_manager
             .update_leader_region_status(table_id, &original_table_route, |route| {
                 if route.region.id == RegionId::new(1024, 2) {
-                    Some(Some(RegionStatus::Downgraded))
+                    Some(Some(LeaderState::Downgrading))
                 } else {
                     None
                 }
@@ -210,7 +210,7 @@ mod tests {
 
         // It should remain unchanged.
         assert_eq!(latest_table_route.version().unwrap(), 0);
-        assert!(!latest_table_route.region_routes().unwrap()[0].is_leader_downgraded());
+        assert!(!latest_table_route.region_routes().unwrap()[0].is_leader_downgrading());
         assert!(ctx.volatile_ctx.table_route.is_none());
     }
 
@@ -251,7 +251,7 @@ mod tests {
             .unwrap()
             .unwrap();
 
-        assert!(latest_table_route.region_routes().unwrap()[0].is_leader_downgraded());
+        assert!(latest_table_route.region_routes().unwrap()[0].is_leader_downgrading());
         assert!(ctx.volatile_ctx.table_route.is_none());
     }
 }
diff --git a/src/meta-srv/src/procedure/region_migration/update_metadata/rollback_downgraded_region.rs b/src/meta-srv/src/procedure/region_migration/update_metadata/rollback_downgraded_region.rs
index 4e6f20ef19..0d568ab7b0 100644
--- a/src/meta-srv/src/procedure/region_migration/update_metadata/rollback_downgraded_region.rs
+++ b/src/meta-srv/src/procedure/region_migration/update_metadata/rollback_downgraded_region.rs
@@ -65,7 +65,7 @@ mod tests {
 
     use common_meta::key::test_utils::new_test_table_info;
     use common_meta::peer::Peer;
-    use common_meta::rpc::router::{Region, RegionRoute, RegionStatus};
+    use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
     use store_api::storage::RegionId;
 
     use crate::error::Error;
@@ -110,13 +110,13 @@ mod tests {
             RegionRoute {
                 region: Region::new_test(RegionId::new(1024, 1)),
                 leader_peer: Some(from_peer.clone()),
-                leader_status: Some(RegionStatus::Downgraded),
+                leader_state: Some(LeaderState::Downgrading),
                 ..Default::default()
             },
             RegionRoute {
                 region: Region::new_test(RegionId::new(1024, 2)),
                 leader_peer: Some(Peer::empty(4)),
-                leader_status: Some(RegionStatus::Downgraded),
+                leader_state: Some(LeaderState::Downgrading),
                 ..Default::default()
             },
             RegionRoute {
@@ -128,8 +128,8 @@ mod tests {
 
         let expected_region_routes = {
             let mut region_routes = region_routes.clone();
-            region_routes[0].leader_status = None;
-            region_routes[1].leader_status = None;
+            region_routes[0].leader_state = None;
+            region_routes[1].leader_state = None;
             region_routes
         };
 
@@ -207,13 +207,13 @@ mod tests {
             RegionRoute {
                 region: Region::new_test(RegionId::new(1024, 1)),
                 leader_peer: Some(from_peer.clone()),
-                leader_status: Some(RegionStatus::Downgraded),
+                leader_state: Some(LeaderState::Downgrading),
                 ..Default::default()
             },
             RegionRoute {
                 region: Region::new_test(RegionId::new(1024, 2)),
                 leader_peer: Some(Peer::empty(4)),
-                leader_status: Some(RegionStatus::Downgraded),
+                leader_state: Some(LeaderState::Downgrading),
                 ..Default::default()
             },
             RegionRoute {
@@ -225,7 +225,7 @@ mod tests {
 
         let expected_region_routes = {
             let mut region_routes = region_routes.clone();
-            region_routes[0].leader_status = None;
+            region_routes[0].leader_state = None;
             region_routes
         };
 
diff --git a/src/meta-srv/src/procedure/region_migration/update_metadata/upgrade_candidate_region.rs b/src/meta-srv/src/procedure/region_migration/update_metadata/upgrade_candidate_region.rs
index 75f93f760e..b710a0e1f3 100644
--- a/src/meta-srv/src/procedure/region_migration/update_metadata/upgrade_candidate_region.rs
+++ b/src/meta-srv/src/procedure/region_migration/update_metadata/upgrade_candidate_region.rs
@@ -43,7 +43,7 @@ impl UpdateMetadata {
             .context(error::RegionRouteNotFoundSnafu { region_id })?;
 
         // Removes downgraded status.
-        region_route.set_leader_status(None);
+        region_route.set_leader_state(None);
 
         let candidate = &ctx.persistent_ctx.to_peer;
         let expected_old_leader = &ctx.persistent_ctx.from_peer;
@@ -106,7 +106,7 @@ impl UpdateMetadata {
 
         if leader_peer.id == candidate_peer_id {
             ensure!(
-                !region_route.is_leader_downgraded(),
+                !region_route.is_leader_downgrading(),
                 error::UnexpectedSnafu {
                     violated: format!("Unexpected intermediate state is found during the update metadata for upgrading region {region_id}"),
                 }
@@ -190,7 +190,7 @@ mod tests {
     use common_meta::key::test_utils::new_test_table_info;
     use common_meta::peer::Peer;
     use common_meta::region_keeper::MemoryRegionKeeper;
-    use common_meta::rpc::router::{Region, RegionRoute, RegionStatus};
+    use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
     use common_time::util::current_time_millis;
     use store_api::storage::RegionId;
 
@@ -286,7 +286,7 @@ mod tests {
             region: Region::new_test(RegionId::new(1024, 1)),
             leader_peer: Some(Peer::empty(1)),
             follower_peers: vec![Peer::empty(2), Peer::empty(3)],
-            leader_status: Some(RegionStatus::Downgraded),
+            leader_state: Some(LeaderState::Downgrading),
             leader_down_since: Some(current_time_millis()),
         }];
 
@@ -298,7 +298,7 @@ mod tests {
             .await
             .unwrap();
 
-        assert!(!new_region_routes[0].is_leader_downgraded());
+        assert!(!new_region_routes[0].is_leader_downgrading());
         assert!(new_region_routes[0].leader_down_since.is_none());
         assert_eq!(new_region_routes[0].follower_peers, vec![Peer::empty(3)]);
         assert_eq!(new_region_routes[0].leader_peer.as_ref().unwrap().id, 2);
@@ -319,13 +319,13 @@ mod tests {
                 region: Region::new_test(RegionId::new(table_id, 1)),
                 leader_peer: Some(Peer::empty(1)),
                 follower_peers: vec![Peer::empty(5), Peer::empty(3)],
-                leader_status: Some(RegionStatus::Downgraded),
+                leader_state: Some(LeaderState::Downgrading),
                 leader_down_since: Some(current_time_millis()),
             },
             RegionRoute {
                 region: Region::new_test(RegionId::new(table_id, 2)),
                 leader_peer: Some(Peer::empty(4)),
-                leader_status: Some(RegionStatus::Downgraded),
+                leader_state: Some(LeaderState::Downgrading),
                 ..Default::default()
             },
         ];
@@ -382,7 +382,7 @@ mod tests {
             region: Region::new_test(RegionId::new(1024, 1)),
             leader_peer: Some(leader_peer),
             follower_peers: vec![Peer::empty(2), Peer::empty(3)],
-            leader_status: None,
+            leader_state: None,
             leader_down_since: None,
         }];
 
@@ -406,7 +406,7 @@ mod tests {
             region: Region::new_test(RegionId::new(1024, 1)),
             leader_peer: Some(candidate_peer),
             follower_peers: vec![Peer::empty(2), Peer::empty(3)],
-            leader_status: None,
+            leader_state: None,
             leader_down_since: None,
         }];
 
@@ -430,7 +430,7 @@ mod tests {
             region: Region::new_test(RegionId::new(1024, 1)),
             leader_peer: Some(candidate_peer),
             follower_peers: vec![Peer::empty(2), Peer::empty(3)],
-            leader_status: Some(RegionStatus::Downgraded),
+            leader_state: Some(LeaderState::Downgrading),
             leader_down_since: None,
         }];
 
@@ -455,7 +455,7 @@ mod tests {
         let region_routes = vec![RegionRoute {
             region: Region::new_test(RegionId::new(table_id, 1)),
             leader_peer: Some(Peer::empty(1)),
-            leader_status: Some(RegionStatus::Downgraded),
+            leader_state: Some(LeaderState::Downgrading),
             ..Default::default()
         }];
 
@@ -485,7 +485,7 @@ mod tests {
         assert!(ctx.volatile_ctx.table_route.is_none());
         assert!(ctx.volatile_ctx.opening_region_guard.is_none());
         assert_eq!(region_routes.len(), 1);
-        assert!(!region_routes[0].is_leader_downgraded());
+        assert!(!region_routes[0].is_leader_downgrading());
         assert!(region_routes[0].follower_peers.is_empty());
         assert_eq!(region_routes[0].leader_peer.as_ref().unwrap().id, 2);
     }
diff --git a/src/meta-srv/src/region/lease_keeper.rs b/src/meta-srv/src/region/lease_keeper.rs
index a1065d4cbb..194f3710c8 100644
--- a/src/meta-srv/src/region/lease_keeper.rs
+++ b/src/meta-srv/src/region/lease_keeper.rs
@@ -62,8 +62,8 @@ fn renew_region_lease_via_region_route(
     // If it's a leader region on this datanode.
     if let Some(leader) = &region_route.leader_peer {
         if leader.id == datanode_id {
-            let region_role = if region_route.is_leader_downgraded() {
-                RegionRole::Follower
+            let region_role = if region_route.is_leader_downgrading() {
+                RegionRole::DowngradingLeader
             } else {
                 RegionRole::Leader
             };
@@ -220,7 +220,7 @@ mod tests {
     use common_meta::kv_backend::memory::MemoryKvBackend;
     use common_meta::peer::Peer;
     use common_meta::region_keeper::MemoryRegionKeeper;
-    use common_meta::rpc::router::{Region, RegionRouteBuilder, RegionStatus};
+    use common_meta::rpc::router::{LeaderState, Region, RegionRouteBuilder};
     use store_api::region_engine::RegionRole;
     use store_api::storage::RegionId;
     use table::metadata::RawTableInfo;
@@ -265,11 +265,11 @@ mod tests {
             Some((region_id, RegionRole::Follower))
         );
 
-        region_route.leader_status = Some(RegionStatus::Downgraded);
+        region_route.leader_state = Some(LeaderState::Downgrading);
         // The downgraded leader region on the datanode.
         assert_eq!(
             renew_region_lease_via_region_route(&region_route, leader_peer_id, region_id),
-            Some((region_id, RegionRole::Follower))
+            Some((region_id, RegionRole::DowngradingLeader))
         );
     }
 
@@ -492,7 +492,7 @@ mod tests {
             .region(Region::new_test(region_id))
             .leader_peer(Peer::empty(leader_peer_id))
             .follower_peers(vec![Peer::empty(follower_peer_id)])
-            .leader_status(RegionStatus::Downgraded)
+            .leader_state(LeaderState::Downgrading)
             .build()
             .unwrap();
 
diff --git a/src/meta-srv/src/test_util.rs b/src/meta-srv/src/test_util.rs
index 576d1aa923..6b9ccc99a0 100644
--- a/src/meta-srv/src/test_util.rs
+++ b/src/meta-srv/src/test_util.rs
@@ -36,7 +36,7 @@ pub(crate) fn new_region_route(region_id: u64, peers: &[Peer], leader_node: u64)
         region,
         leader_peer,
         follower_peers: vec![],
-        leader_status: None,
+        leader_state: None,
         leader_down_since: None,
     }
 }
diff --git a/src/metric-engine/src/engine.rs b/src/metric-engine/src/engine.rs
index 358da1d216..42948aa6cd 100644
--- a/src/metric-engine/src/engine.rs
+++ b/src/metric-engine/src/engine.rs
@@ -37,7 +37,8 @@ use snafu::ResultExt;
 use store_api::metadata::RegionMetadataRef;
 use store_api::metric_engine_consts::METRIC_ENGINE_NAME;
 use store_api::region_engine::{
-    RegionEngine, RegionRole, RegionScannerRef, RegionStatistic, SetReadonlyResponse,
+    RegionEngine, RegionRole, RegionScannerRef, RegionStatistic, SetRegionRoleStateResponse,
+    SettableRegionRoleState,
 };
 use store_api::region_request::RegionRequest;
 use store_api::storage::{RegionId, ScanRequest};
@@ -201,14 +202,14 @@ impl RegionEngine for MetricEngine {
         Ok(())
     }
 
-    fn set_writable(&self, region_id: RegionId, writable: bool) -> Result<(), BoxedError> {
+    fn set_region_role(&self, region_id: RegionId, role: RegionRole) -> Result<(), BoxedError> {
         // ignore the region not found error
         for x in [
             utils::to_metadata_region_id(region_id),
             utils::to_data_region_id(region_id),
             region_id,
         ] {
-            if let Err(e) = self.inner.mito.set_writable(x, writable)
+            if let Err(e) = self.inner.mito.set_region_role(x, role)
                 && e.status_code() != StatusCode::RegionNotFound
             {
                 return Err(e);
@@ -217,11 +218,15 @@ impl RegionEngine for MetricEngine {
         Ok(())
     }
 
-    async fn set_readonly_gracefully(
+    async fn set_region_role_state_gracefully(
         &self,
         region_id: RegionId,
-    ) -> std::result::Result<SetReadonlyResponse, BoxedError> {
-        self.inner.mito.set_readonly_gracefully(region_id).await
+        region_role_state: SettableRegionRoleState,
+    ) -> std::result::Result<SetRegionRoleStateResponse, BoxedError> {
+        self.inner
+            .mito
+            .set_region_role_state_gracefully(region_id, region_role_state)
+            .await
     }
 
     /// Returns the physical region role.
diff --git a/src/mito2/src/compaction/compactor.rs b/src/mito2/src/compaction/compactor.rs
index 12b9dd5fef..d919633ba9 100644
--- a/src/mito2/src/compaction/compactor.rs
+++ b/src/mito2/src/compaction/compactor.rs
@@ -39,8 +39,7 @@ use crate::read::Source;
 use crate::region::opener::new_manifest_dir;
 use crate::region::options::RegionOptions;
 use crate::region::version::{VersionBuilder, VersionRef};
-use crate::region::ManifestContext;
-use crate::region::RegionState::Writable;
+use crate::region::{ManifestContext, RegionLeaderState, RegionRoleState};
 use crate::schedule::scheduler::LocalScheduler;
 use crate::sst::file::{FileMeta, IndexType};
 use crate::sst::file_purger::LocalFilePurger;
@@ -129,7 +128,10 @@ pub async fn open_compaction_region(
 
     let manifest = manifest_manager.manifest();
     let region_metadata = manifest.metadata.clone();
-    let manifest_ctx = Arc::new(ManifestContext::new(manifest_manager, Writable));
+    let manifest_ctx = Arc::new(ManifestContext::new(
+        manifest_manager,
+        RegionRoleState::Leader(RegionLeaderState::Writable),
+    ));
 
     let file_purger = {
         let purge_scheduler = Arc::new(LocalScheduler::new(mito_config.max_background_jobs));
@@ -379,7 +381,7 @@ impl Compactor for DefaultCompactor {
         // TODO: We might leak files if we fail to update manifest. We can add a cleanup task to remove them later.
         compaction_region
             .manifest_ctx
-            .update_manifest(Writable, action_list)
+            .update_manifest(RegionLeaderState::Writable, action_list)
             .await?;
 
         Ok(edit)
diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs
index e9177d40bf..ed8cc92909 100644
--- a/src/mito2/src/engine.rs
+++ b/src/mito2/src/engine.rs
@@ -53,7 +53,7 @@ mod prune_test;
 #[cfg(test)]
 mod row_selector_test;
 #[cfg(test)]
-mod set_readonly_test;
+mod set_role_state_test;
 #[cfg(test)]
 mod truncate_test;
 
@@ -77,7 +77,7 @@ use store_api::logstore::LogStore;
 use store_api::metadata::RegionMetadataRef;
 use store_api::region_engine::{
     BatchResponses, RegionEngine, RegionRole, RegionScannerRef, RegionStatistic,
-    SetReadonlyResponse,
+    SetRegionRoleStateResponse, SettableRegionRoleState,
 };
 use store_api::region_request::{AffectedRows, RegionOpenRequest, RegionRequest};
 use store_api::storage::{RegionId, ScanRequest};
@@ -436,22 +436,27 @@ impl EngineInner {
         Ok(scan_region)
     }
 
-    /// Set writable mode for a region.
-    fn set_writable(&self, region_id: RegionId, writable: bool) -> Result<()> {
+    /// Converts the [`RegionRole`].
+    fn set_region_role(&self, region_id: RegionId, role: RegionRole) -> Result<()> {
         let region = self
             .workers
             .get_region(region_id)
             .context(RegionNotFoundSnafu { region_id })?;
 
-        region.set_writable(writable);
+        region.set_role(role);
         Ok(())
     }
 
     /// Sets read-only for a region and ensures no more writes in the region after it returns.
-    async fn set_readonly_gracefully(&self, region_id: RegionId) -> Result<SetReadonlyResponse> {
+    async fn set_region_role_state_gracefully(
+        &self,
+        region_id: RegionId,
+        region_role_state: SettableRegionRoleState,
+    ) -> Result<SetRegionRoleStateResponse> {
         // Notes: It acquires the mutable ownership to ensure no other threads,
         // Therefore, we submit it to the worker.
-        let (request, receiver) = WorkerRequest::new_set_readonly_gracefully(region_id);
+        let (request, receiver) =
+            WorkerRequest::new_set_readonly_gracefully(region_id, region_role_state);
         self.workers.submit_to_worker(region_id, request).await?;
 
         receiver.await.context(RecvSnafu)
@@ -459,7 +464,7 @@ impl EngineInner {
 
     fn role(&self, region_id: RegionId) -> Option<RegionRole> {
         self.workers.get_region(region_id).map(|region| {
-            if region.is_readonly() {
+            if region.is_follower() {
                 RegionRole::Follower
             } else {
                 RegionRole::Leader
@@ -547,22 +552,23 @@ impl RegionEngine for MitoEngine {
         self.get_region_statistic(region_id)
     }
 
-    fn set_writable(&self, region_id: RegionId, writable: bool) -> Result<(), BoxedError> {
+    fn set_region_role(&self, region_id: RegionId, role: RegionRole) -> Result<(), BoxedError> {
         self.inner
-            .set_writable(region_id, writable)
+            .set_region_role(region_id, role)
             .map_err(BoxedError::new)
     }
 
-    async fn set_readonly_gracefully(
+    async fn set_region_role_state_gracefully(
         &self,
         region_id: RegionId,
-    ) -> Result<SetReadonlyResponse, BoxedError> {
+        region_role_state: SettableRegionRoleState,
+    ) -> Result<SetRegionRoleStateResponse, BoxedError> {
         let _timer = HANDLE_REQUEST_ELAPSED
-            .with_label_values(&["set_readonly_gracefully"])
+            .with_label_values(&["set_region_role_state_gracefully"])
             .start_timer();
 
         self.inner
-            .set_readonly_gracefully(region_id)
+            .set_region_role_state_gracefully(region_id, region_role_state)
             .await
             .map_err(BoxedError::new)
     }
diff --git a/src/mito2/src/engine/alter_test.rs b/src/mito2/src/engine/alter_test.rs
index b48dc2ccfb..2e75bf19fa 100644
--- a/src/mito2/src/engine/alter_test.rs
+++ b/src/mito2/src/engine/alter_test.rs
@@ -24,7 +24,7 @@ use common_recordbatch::RecordBatches;
 use datatypes::prelude::ConcreteDataType;
 use datatypes::schema::ColumnSchema;
 use store_api::metadata::ColumnMetadata;
-use store_api::region_engine::RegionEngine;
+use store_api::region_engine::{RegionEngine, RegionRole};
 use store_api::region_request::{
     AddColumn, AddColumnLocation, AlterKind, RegionAlterRequest, RegionOpenRequest, RegionRequest,
 };
@@ -213,8 +213,10 @@ async fn test_put_after_alter() {
         )
         .await
         .unwrap();
-    // Set writable.
-    engine.set_writable(region_id, true).unwrap();
+    // Convert region to leader.
+    engine
+        .set_region_role(region_id, RegionRole::Leader)
+        .unwrap();
 
     // Put with old schema.
     let rows = Rows {
diff --git a/src/mito2/src/engine/catchup_test.rs b/src/mito2/src/engine/catchup_test.rs
index 5f4dd3b15a..a9de0d6008 100644
--- a/src/mito2/src/engine/catchup_test.rs
+++ b/src/mito2/src/engine/catchup_test.rs
@@ -22,7 +22,7 @@ use common_recordbatch::RecordBatches;
 use common_wal::options::{KafkaWalOptions, WalOptions, WAL_OPTIONS_KEY};
 use rstest::rstest;
 use rstest_reuse::{self, apply};
-use store_api::region_engine::{RegionEngine, SetReadonlyResponse};
+use store_api::region_engine::{RegionEngine, RegionRole, SetRegionRoleStateResponse};
 use store_api::region_request::{RegionCatchupRequest, RegionOpenRequest, RegionRequest};
 use store_api::storage::{RegionId, ScanRequest};
 
@@ -34,8 +34,8 @@ use crate::test_util::{
 };
 use crate::wal::EntryId;
 
-fn get_last_entry_id(resp: SetReadonlyResponse) -> Option<EntryId> {
-    if let SetReadonlyResponse::Success { last_entry_id } = resp {
+fn get_last_entry_id(resp: SetRegionRoleStateResponse) -> Option<EntryId> {
+    if let SetRegionRoleStateResponse::Success { last_entry_id } = resp {
         last_entry_id
     } else {
         unreachable!();
@@ -45,6 +45,8 @@ fn get_last_entry_id(resp: SetReadonlyResponse) -> Option<EntryId> {
 #[apply(single_kafka_log_store_factory)]
 
 async fn test_catchup_with_last_entry_id(factory: Option<LogStoreFactory>) {
+    use store_api::region_engine::SettableRegionRoleState;
+
     common_telemetry::init_default_ut_logging();
     let Some(factory) = factory else {
         return;
@@ -102,7 +104,7 @@ async fn test_catchup_with_last_entry_id(factory: Option<LogStoreFactory>) {
     put_rows(&leader_engine, region_id, rows).await;
 
     let resp = leader_engine
-        .set_readonly_gracefully(region_id)
+        .set_region_role_state_gracefully(region_id, SettableRegionRoleState::Follower)
         .await
         .unwrap();
 
@@ -159,6 +161,8 @@ async fn test_catchup_with_last_entry_id(factory: Option<LogStoreFactory>) {
 
 #[apply(single_kafka_log_store_factory)]
 async fn test_catchup_with_incorrect_last_entry_id(factory: Option<LogStoreFactory>) {
+    use store_api::region_engine::SettableRegionRoleState;
+
     common_telemetry::init_default_ut_logging();
     let Some(factory) = factory else {
         return;
@@ -217,7 +221,7 @@ async fn test_catchup_with_incorrect_last_entry_id(factory: Option<LogStoreFacto
     put_rows(&leader_engine, region_id, rows).await;
 
     let resp = leader_engine
-        .set_readonly_gracefully(region_id)
+        .set_region_role_state_gracefully(region_id, SettableRegionRoleState::Follower)
         .await
         .unwrap();
 
@@ -243,7 +247,7 @@ async fn test_catchup_with_incorrect_last_entry_id(factory: Option<LogStoreFacto
     assert_matches!(err, error::Error::UnexpectedReplay { .. });
 
     // It should ignore requests to writable regions.
-    region.set_writable(true);
+    region.set_role(RegionRole::Leader);
     let resp = follower_engine
         .handle_request(
             region_id,
diff --git a/src/mito2/src/engine/compaction_test.rs b/src/mito2/src/engine/compaction_test.rs
index e19f95088c..7951331b20 100644
--- a/src/mito2/src/engine/compaction_test.rs
+++ b/src/mito2/src/engine/compaction_test.rs
@@ -19,7 +19,7 @@ use api::v1::{ColumnSchema, Rows};
 use common_recordbatch::{RecordBatches, SendableRecordBatchStream};
 use datatypes::prelude::ScalarVector;
 use datatypes::vectors::TimestampMillisecondVector;
-use store_api::region_engine::RegionEngine;
+use store_api::region_engine::{RegionEngine, RegionRole};
 use store_api::region_request::{
     RegionCompactRequest, RegionDeleteRequest, RegionFlushRequest, RegionRequest,
 };
@@ -302,8 +302,10 @@ async fn test_readonly_during_compaction() {
     // Waits until the engine receives compaction finished request.
     listener.wait_handle_finished().await;
 
-    // Sets the region to read only mode.
-    engine.set_writable(region_id, false).unwrap();
+    // Converts region to follower.
+    engine
+        .set_region_role(region_id, RegionRole::Follower)
+        .unwrap();
     // Wakes up the listener.
     listener.wake();
 
diff --git a/src/mito2/src/engine/open_test.rs b/src/mito2/src/engine/open_test.rs
index 9e9a9e3486..c7ad47535c 100644
--- a/src/mito2/src/engine/open_test.rs
+++ b/src/mito2/src/engine/open_test.rs
@@ -53,7 +53,9 @@ async fn test_engine_open_empty() {
         .await
         .unwrap_err();
     assert_eq!(StatusCode::RegionNotFound, err.status_code());
-    let err = engine.set_writable(region_id, true).unwrap_err();
+    let err = engine
+        .set_region_role(region_id, RegionRole::Leader)
+        .unwrap_err();
     assert_eq!(StatusCode::RegionNotFound, err.status_code());
     let role = engine.role(region_id);
     assert_eq!(role, None);
@@ -134,8 +136,10 @@ async fn test_engine_open_readonly() {
     assert_eq!(StatusCode::RegionNotReady, err.status_code());
 
     assert_eq!(Some(RegionRole::Follower), engine.role(region_id));
-    // Set writable and write.
-    engine.set_writable(region_id, true).unwrap();
+    // Converts region to leader.
+    engine
+        .set_region_role(region_id, RegionRole::Leader)
+        .unwrap();
     assert_eq!(Some(RegionRole::Leader), engine.role(region_id));
 
     put_rows(&engine, region_id, rows).await;
diff --git a/src/mito2/src/engine/set_readonly_test.rs b/src/mito2/src/engine/set_readonly_test.rs
deleted file mode 100644
index 9de3f0a832..0000000000
--- a/src/mito2/src/engine/set_readonly_test.rs
+++ /dev/null
@@ -1,98 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use api::v1::Rows;
-use common_error::ext::ErrorExt;
-use common_error::status_code::StatusCode;
-use store_api::region_engine::{RegionEngine, SetReadonlyResponse};
-use store_api::region_request::{RegionPutRequest, RegionRequest};
-use store_api::storage::RegionId;
-
-use crate::config::MitoConfig;
-use crate::test_util::{build_rows, put_rows, rows_schema, CreateRequestBuilder, TestEnv};
-
-#[tokio::test]
-async fn test_set_readonly_gracefully() {
-    let mut env = TestEnv::new();
-    let engine = env.create_engine(MitoConfig::default()).await;
-
-    let region_id = RegionId::new(1, 1);
-    let request = CreateRequestBuilder::new().build();
-
-    let column_schemas = rows_schema(&request);
-    engine
-        .handle_request(region_id, RegionRequest::Create(request))
-        .await
-        .unwrap();
-
-    let result = engine.set_readonly_gracefully(region_id).await.unwrap();
-    assert_eq!(
-        SetReadonlyResponse::Success {
-            last_entry_id: Some(0)
-        },
-        result
-    );
-
-    // set readonly again.
-    let result = engine.set_readonly_gracefully(region_id).await.unwrap();
-    assert_eq!(
-        SetReadonlyResponse::Success {
-            last_entry_id: Some(0)
-        },
-        result
-    );
-
-    let rows = Rows {
-        schema: column_schemas,
-        rows: build_rows(0, 3),
-    };
-
-    let error = engine
-        .handle_request(
-            region_id,
-            RegionRequest::Put(RegionPutRequest { rows: rows.clone() }),
-        )
-        .await
-        .unwrap_err();
-
-    assert_eq!(error.status_code(), StatusCode::RegionNotReady);
-
-    engine.set_writable(region_id, true).unwrap();
-
-    put_rows(&engine, region_id, rows).await;
-
-    let result = engine.set_readonly_gracefully(region_id).await.unwrap();
-
-    assert_eq!(
-        SetReadonlyResponse::Success {
-            last_entry_id: Some(1)
-        },
-        result
-    );
-}
-
-#[tokio::test]
-async fn test_set_readonly_gracefully_not_exist() {
-    let mut env = TestEnv::new();
-    let engine = env.create_engine(MitoConfig::default()).await;
-
-    let non_exist_region_id = RegionId::new(1, 1);
-
-    // For fast-path.
-    let result = engine
-        .set_readonly_gracefully(non_exist_region_id)
-        .await
-        .unwrap();
-    assert_eq!(SetReadonlyResponse::NotFound, result);
-}
diff --git a/src/mito2/src/engine/set_role_state_test.rs b/src/mito2/src/engine/set_role_state_test.rs
new file mode 100644
index 0000000000..1d7a46f436
--- /dev/null
+++ b/src/mito2/src/engine/set_role_state_test.rs
@@ -0,0 +1,159 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use api::v1::Rows;
+use common_error::ext::ErrorExt;
+use common_error::status_code::StatusCode;
+use store_api::region_engine::{
+    RegionEngine, RegionRole, SetRegionRoleStateResponse, SettableRegionRoleState,
+};
+use store_api::region_request::{RegionPutRequest, RegionRequest};
+use store_api::storage::RegionId;
+
+use crate::config::MitoConfig;
+use crate::test_util::{build_rows, put_rows, rows_schema, CreateRequestBuilder, TestEnv};
+
+#[tokio::test]
+async fn test_set_role_state_gracefully() {
+    let settable_role_states = [
+        SettableRegionRoleState::Follower,
+        SettableRegionRoleState::DowngradingLeader,
+    ];
+    for settable_role_state in settable_role_states {
+        let mut env = TestEnv::new();
+        let engine = env.create_engine(MitoConfig::default()).await;
+
+        let region_id = RegionId::new(1, 1);
+        let request = CreateRequestBuilder::new().build();
+
+        let column_schemas = rows_schema(&request);
+        engine
+            .handle_request(region_id, RegionRequest::Create(request))
+            .await
+            .unwrap();
+
+        let result = engine
+            .set_region_role_state_gracefully(region_id, settable_role_state)
+            .await
+            .unwrap();
+        assert_eq!(
+            SetRegionRoleStateResponse::Success {
+                last_entry_id: Some(0)
+            },
+            result
+        );
+
+        // set Follower again.
+        let result = engine
+            .set_region_role_state_gracefully(region_id, settable_role_state)
+            .await
+            .unwrap();
+        assert_eq!(
+            SetRegionRoleStateResponse::Success {
+                last_entry_id: Some(0)
+            },
+            result
+        );
+
+        let rows = Rows {
+            schema: column_schemas,
+            rows: build_rows(0, 3),
+        };
+
+        let error = engine
+            .handle_request(
+                region_id,
+                RegionRequest::Put(RegionPutRequest { rows: rows.clone() }),
+            )
+            .await
+            .unwrap_err();
+
+        assert_eq!(error.status_code(), StatusCode::RegionNotReady);
+
+        engine
+            .set_region_role(region_id, RegionRole::Leader)
+            .unwrap();
+
+        put_rows(&engine, region_id, rows).await;
+
+        let result = engine
+            .set_region_role_state_gracefully(region_id, settable_role_state)
+            .await
+            .unwrap();
+
+        assert_eq!(
+            SetRegionRoleStateResponse::Success {
+                last_entry_id: Some(1)
+            },
+            result
+        );
+    }
+}
+
+#[tokio::test]
+async fn test_set_role_state_gracefully_not_exist() {
+    let mut env = TestEnv::new();
+    let engine = env.create_engine(MitoConfig::default()).await;
+
+    let non_exist_region_id = RegionId::new(1, 1);
+
+    // For fast-path.
+    let result = engine
+        .set_region_role_state_gracefully(non_exist_region_id, SettableRegionRoleState::Follower)
+        .await
+        .unwrap();
+    assert_eq!(SetRegionRoleStateResponse::NotFound, result);
+}
+
+#[tokio::test]
+async fn test_write_downgrading_region() {
+    let mut env = TestEnv::with_prefix("write-to-downgrading-region");
+    let engine = env.create_engine(MitoConfig::default()).await;
+
+    let region_id = RegionId::new(1, 1);
+    let request = CreateRequestBuilder::new().build();
+
+    let column_schemas = rows_schema(&request);
+    engine
+        .handle_request(region_id, RegionRequest::Create(request))
+        .await
+        .unwrap();
+
+    let rows = Rows {
+        schema: column_schemas.clone(),
+        rows: build_rows(0, 42),
+    };
+    put_rows(&engine, region_id, rows).await;
+
+    let result = engine
+        .set_region_role_state_gracefully(region_id, SettableRegionRoleState::DowngradingLeader)
+        .await
+        .unwrap();
+    assert_eq!(
+        SetRegionRoleStateResponse::Success {
+            last_entry_id: Some(1)
+        },
+        result
+    );
+
+    let rows = Rows {
+        schema: column_schemas,
+        rows: build_rows(0, 42),
+    };
+    let err = engine
+        .handle_request(region_id, RegionRequest::Put(RegionPutRequest { rows }))
+        .await
+        .unwrap_err();
+    assert_eq!(err.status_code(), StatusCode::RegionNotReady)
+}
diff --git a/src/mito2/src/error.rs b/src/mito2/src/error.rs
index 2d045e8c57..8aa799cbb9 100644
--- a/src/mito2/src/error.rs
+++ b/src/mito2/src/error.rs
@@ -32,7 +32,7 @@ use store_api::manifest::ManifestVersion;
 use store_api::storage::RegionId;
 
 use crate::cache::file_cache::FileType;
-use crate::region::RegionState;
+use crate::region::{RegionLeaderState, RegionRoleState};
 use crate::schedule::remote_job_scheduler::JobId;
 use crate::sst::file::FileId;
 use crate::worker::WorkerId;
@@ -483,10 +483,22 @@ pub enum Error {
     },
 
     #[snafu(display("Region {} is in {:?} state, expect: {:?}", region_id, state, expect))]
-    RegionState {
+    RegionLeaderState {
         region_id: RegionId,
-        state: RegionState,
-        expect: RegionState,
+        state: RegionRoleState,
+        expect: RegionLeaderState,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display(
+        "Region {} is in {:?} state, expect: Leader or Leader(Downgrading)",
+        region_id,
+        state
+    ))]
+    FlushableRegionState {
+        region_id: RegionId,
+        state: RegionRoleState,
         #[snafu(implicit)]
         location: Location,
     },
@@ -954,7 +966,8 @@ impl ErrorExt for Error {
             CompactRegion { source, .. } => source.status_code(),
             CompatReader { .. } => StatusCode::Unexpected,
             InvalidRegionRequest { source, .. } => source.status_code(),
-            RegionState { .. } => StatusCode::RegionNotReady,
+            RegionLeaderState { .. } => StatusCode::RegionNotReady,
+            &FlushableRegionState { .. } => StatusCode::RegionNotReady,
             JsonOptions { .. } => StatusCode::InvalidArguments,
             EmptyRegionDir { .. } | EmptyManifestDir { .. } => StatusCode::RegionNotFound,
             ArrowReader { .. } => StatusCode::StorageUnavailable,
diff --git a/src/mito2/src/flush.rs b/src/mito2/src/flush.rs
index 05561b6080..9606e92d04 100644
--- a/src/mito2/src/flush.rs
+++ b/src/mito2/src/flush.rs
@@ -36,7 +36,7 @@ use crate::metrics::{FLUSH_BYTES_TOTAL, FLUSH_ELAPSED, FLUSH_ERRORS_TOTAL, FLUSH
 use crate::read::Source;
 use crate::region::options::IndexOptions;
 use crate::region::version::{VersionControlData, VersionControlRef};
-use crate::region::{ManifestContextRef, RegionState};
+use crate::region::{ManifestContextRef, RegionLeaderState};
 use crate::request::{
     BackgroundNotify, FlushFailed, FlushFinished, OptionOutputTx, OutputTx, SenderDdlRequest,
     SenderWriteRequest, WorkerRequest,
@@ -195,6 +195,8 @@ pub enum FlushReason {
     Alter,
     /// Flush periodically.
     Periodically,
+    /// Flush memtable during downgrading state.
+    Downgrading,
 }
 
 impl FlushReason {
@@ -407,11 +409,23 @@ impl RegionFlushTask {
         info!("Applying {edit:?} to region {}", self.region_id);
 
         let action_list = RegionMetaActionList::with_action(RegionMetaAction::Edit(edit.clone()));
+
+        let expected_state = if matches!(self.reason, FlushReason::Downgrading) {
+            RegionLeaderState::Downgrading
+        } else {
+            RegionLeaderState::Writable
+        };
         // We will leak files if the manifest update fails, but we ignore them for simplicity. We can
         // add a cleanup job to remove them later.
-        self.manifest_ctx
-            .update_manifest(RegionState::Writable, action_list)
+        let version = self
+            .manifest_ctx
+            .update_manifest(expected_state, action_list)
             .await?;
+        info!(
+            "Successfully update manifest version to {version}, region: {}, reason: {}",
+            self.region_id,
+            self.reason.as_str()
+        );
 
         Ok(edit)
     }
diff --git a/src/mito2/src/region.rs b/src/mito2/src/region.rs
index 8fc9095ae5..b05daf3da0 100644
--- a/src/mito2/src/region.rs
+++ b/src/mito2/src/region.rs
@@ -27,12 +27,16 @@ use common_telemetry::{error, info, warn};
 use crossbeam_utils::atomic::AtomicCell;
 use snafu::{ensure, OptionExt};
 use store_api::logstore::provider::Provider;
+use store_api::manifest::ManifestVersion;
 use store_api::metadata::RegionMetadataRef;
-use store_api::region_engine::RegionStatistic;
+use store_api::region_engine::{RegionRole, RegionStatistic, SettableRegionRoleState};
 use store_api::storage::RegionId;
 
 use crate::access_layer::AccessLayerRef;
-use crate::error::{RegionNotFoundSnafu, RegionStateSnafu, RegionTruncatedSnafu, Result};
+use crate::error::{
+    FlushableRegionStateSnafu, RegionLeaderStateSnafu, RegionNotFoundSnafu, RegionTruncatedSnafu,
+    Result,
+};
 use crate::manifest::action::{RegionMetaAction, RegionMetaActionList};
 use crate::manifest::manager::RegionManifestManager;
 use crate::memtable::MemtableBuilderRef;
@@ -59,11 +63,8 @@ impl RegionUsage {
     }
 }
 
-/// State of the region.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum RegionState {
-    /// The region is opened but is still read-only.
-    ReadOnly,
+pub enum RegionLeaderState {
     /// The region is opened and is writable.
     Writable,
     /// The region is altering.
@@ -74,6 +75,14 @@ pub enum RegionState {
     Truncating,
     /// The region is handling a region edit.
     Editing,
+    /// The region is stepping down.
+    Downgrading,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RegionRoleState {
+    Leader(RegionLeaderState),
+    Follower,
 }
 
 /// Metadata and runtime status of a region.
@@ -172,83 +181,91 @@ impl MitoRegion {
 
     /// Returns whether the region is writable.
     pub(crate) fn is_writable(&self) -> bool {
-        self.manifest_ctx.state.load() == RegionState::Writable
+        self.manifest_ctx.state.load() == RegionRoleState::Leader(RegionLeaderState::Writable)
+    }
+
+    /// Returns whether the region is flushable.
+    pub(crate) fn is_flushable(&self) -> bool {
+        matches!(
+            self.manifest_ctx.state.load(),
+            RegionRoleState::Leader(RegionLeaderState::Writable)
+                | RegionRoleState::Leader(RegionLeaderState::Downgrading)
+        )
+    }
+
+    /// Returns whether the region is downgrading.
+    pub(crate) fn is_downgrading(&self) -> bool {
+        matches!(
+            self.manifest_ctx.state.load(),
+            RegionRoleState::Leader(RegionLeaderState::Downgrading)
+        )
     }
 
     /// Returns whether the region is readonly.
-    pub(crate) fn is_readonly(&self) -> bool {
-        self.manifest_ctx.state.load() == RegionState::ReadOnly
+    pub(crate) fn is_follower(&self) -> bool {
+        self.manifest_ctx.state.load() == RegionRoleState::Follower
     }
 
     /// Returns the state of the region.
-    pub(crate) fn state(&self) -> RegionState {
+    pub(crate) fn state(&self) -> RegionRoleState {
         self.manifest_ctx.state.load()
     }
 
-    /// Sets the writable state.
-    pub(crate) fn set_writable(&self, writable: bool) {
-        if writable {
-            // Only sets the region to writable if it is read only.
-            // This prevents others updating the manifest.
-            match self
-                .manifest_ctx
-                .state
-                .compare_exchange(RegionState::ReadOnly, RegionState::Writable)
-            {
-                Ok(state) => info!(
-                    "Set region {} to writable, previous state: {:?}",
-                    self.region_id, state
-                ),
-                Err(state) => {
-                    if state != RegionState::Writable {
-                        warn!(
-                            "Failed to set region {} to writable, current state: {:?}",
-                            self.region_id, state
-                        )
-                    }
-                }
-            }
-        } else {
-            self.manifest_ctx.state.store(RegionState::ReadOnly);
-        }
+    /// Sets the region role state.
+    pub(crate) fn set_role(&self, next_role: RegionRole) {
+        self.manifest_ctx.set_role(next_role, self.region_id);
     }
 
     /// Sets the altering state.
     /// You should call this method in the worker loop.
     pub(crate) fn set_altering(&self) -> Result<()> {
-        self.compare_exchange_state(RegionState::Writable, RegionState::Altering)
+        self.compare_exchange_state(
+            RegionLeaderState::Writable,
+            RegionRoleState::Leader(RegionLeaderState::Altering),
+        )
     }
 
     /// Sets the dropping state.
     /// You should call this method in the worker loop.
     pub(crate) fn set_dropping(&self) -> Result<()> {
-        self.compare_exchange_state(RegionState::Writable, RegionState::Dropping)
+        self.compare_exchange_state(
+            RegionLeaderState::Writable,
+            RegionRoleState::Leader(RegionLeaderState::Dropping),
+        )
     }
 
     /// Sets the truncating state.
     /// You should call this method in the worker loop.
     pub(crate) fn set_truncating(&self) -> Result<()> {
-        self.compare_exchange_state(RegionState::Writable, RegionState::Truncating)
+        self.compare_exchange_state(
+            RegionLeaderState::Writable,
+            RegionRoleState::Leader(RegionLeaderState::Truncating),
+        )
     }
 
     /// Sets the editing state.
     /// You should call this method in the worker loop.
     pub(crate) fn set_editing(&self) -> Result<()> {
-        self.compare_exchange_state(RegionState::Writable, RegionState::Editing)
+        self.compare_exchange_state(
+            RegionLeaderState::Writable,
+            RegionRoleState::Leader(RegionLeaderState::Editing),
+        )
     }
 
     /// Sets the region to readonly gracefully. This acquires the manifest write lock.
-    pub(crate) async fn set_readonly_gracefully(&self) {
+    pub(crate) async fn set_role_state_gracefully(&self, state: SettableRegionRoleState) {
         let _manager = self.manifest_ctx.manifest_manager.write().await;
         // We acquires the write lock of the manifest manager to ensure that no one is updating the manifest.
         // Then we change the state.
-        self.set_writable(false);
+        self.set_role(state.into());
     }
 
-    /// Switches the region state to `RegionState::Writable` if the current state is `expect`.
+    /// Switches the region state to `RegionRoleState::Leader(RegionLeaderState::Writable)` if the current state is `expect`.
     /// Otherwise, logs an error.
-    pub(crate) fn switch_state_to_writable(&self, expect: RegionState) {
-        if let Err(e) = self.compare_exchange_state(expect, RegionState::Writable) {
+    pub(crate) fn switch_state_to_writable(&self, expect: RegionLeaderState) {
+        if let Err(e) = self
+            .compare_exchange_state(expect, RegionRoleState::Leader(RegionLeaderState::Writable))
+        {
             error!(e; "failed to switch region state to writable, expect state is {:?}", expect);
         }
     }
@@ -280,12 +297,16 @@ impl MitoRegion {
 
     /// Sets the state of the region to given state if the current state equals to
     /// the expected.
-    fn compare_exchange_state(&self, expect: RegionState, state: RegionState) -> Result<()> {
+    fn compare_exchange_state(
+        &self,
+        expect: RegionLeaderState,
+        state: RegionRoleState,
+    ) -> Result<()> {
         self.manifest_ctx
             .state
-            .compare_exchange(expect, state)
+            .compare_exchange(RegionRoleState::Leader(expect), state)
             .map_err(|actual| {
-                RegionStateSnafu {
+                RegionLeaderStateSnafu {
                     region_id: self.region_id,
                     state: actual,
                     expect,
@@ -303,17 +324,25 @@ pub(crate) struct ManifestContext {
     manifest_manager: tokio::sync::RwLock<RegionManifestManager>,
     /// The state of the region. The region checks the state before updating
     /// manifest.
-    state: AtomicCell<RegionState>,
+    state: AtomicCell<RegionRoleState>,
 }
 
 impl ManifestContext {
-    pub(crate) fn new(manager: RegionManifestManager, state: RegionState) -> Self {
+    pub(crate) fn new(manager: RegionManifestManager, state: RegionRoleState) -> Self {
         ManifestContext {
             manifest_manager: tokio::sync::RwLock::new(manager),
             state: AtomicCell::new(state),
         }
     }
 
+    pub(crate) async fn manifest_version(&self) -> ManifestVersion {
+        self.manifest_manager
+            .read()
+            .await
+            .manifest()
+            .manifest_version
+    }
+
     pub(crate) async fn has_update(&self) -> Result<bool> {
         self.manifest_manager.read().await.has_update().await
     }
@@ -321,9 +350,9 @@ impl ManifestContext {
     /// Updates the manifest if current state is `expect_state`.
     pub(crate) async fn update_manifest(
         &self,
-        expect_state: RegionState,
+        expect_state: RegionLeaderState,
         action_list: RegionMetaActionList,
-    ) -> Result<()> {
+    ) -> Result<ManifestVersion> {
         // Acquires the write lock of the manifest manager.
         let mut manager = self.manifest_manager.write().await;
         // Gets current manifest.
@@ -332,8 +361,8 @@ impl ManifestContext {
         // after `set_readonly_gracefully()` is called.
         let current_state = self.state.load();
         ensure!(
-            current_state == expect_state,
-            RegionStateSnafu {
+            current_state == RegionRoleState::Leader(expect_state),
+            RegionLeaderStateSnafu {
                 region_id: manifest.metadata.region_id,
                 state: current_state,
                 expect: expect_state,
@@ -376,18 +405,92 @@ impl ManifestContext {
         }
 
         // Now we can update the manifest.
-        manager.update(action_list).await.inspect_err(
+        let version = manager.update(action_list).await.inspect_err(
             |e| error!(e; "Failed to update manifest, region_id: {}", manifest.metadata.region_id),
         )?;
 
-        if self.state.load() == RegionState::ReadOnly {
+        if self.state.load() == RegionRoleState::Follower {
             warn!(
-                "Region {} becomes read-only while updating manifest which may cause inconsistency",
+                "Region {} becomes follower while updating manifest which may cause inconsistency, manifest version: {version}",
                 manifest.metadata.region_id
             );
         }
 
-        Ok(())
+        Ok(version)
+    }
+
+    /// Sets the [`RegionRole`].
+    ///
+    /// ```
+    ///     +------------------------------------------+    
+    ///     |                      +-----------------+ |    
+    ///     |                      |                 | |    
+    /// +---+------+       +-------+-----+        +--v-v---+
+    /// | Follower |       | Downgrading |        | Leader |
+    /// +---^-^----+       +-----+-^-----+        +--+-+---+
+    ///     | |                  | |                 | |    
+    ///     | +------------------+ +-----------------+ |    
+    ///     +------------------------------------------+    
+    ///
+    /// Transition:
+    /// - Follower -> Leader
+    /// - Downgrading Leader -> Leader
+    /// - Leader -> Follower
+    /// - Downgrading Leader -> Follower
+    /// - Leader -> Downgrading Leader
+    ///
+    /// ```
+    pub(crate) fn set_role(&self, next_role: RegionRole, region_id: RegionId) {
+        match next_role {
+            RegionRole::Follower => {
+                self.state.store(RegionRoleState::Follower);
+            }
+            RegionRole::Leader => {
+                match self.state.fetch_update(|state| {
+                    if matches!(
+                        state,
+                        RegionRoleState::Follower
+                            | RegionRoleState::Leader(RegionLeaderState::Downgrading)
+                    ) {
+                        Some(RegionRoleState::Leader(RegionLeaderState::Writable))
+                    } else {
+                        None
+                    }
+                }) {
+                    Ok(state) => info!(
+                        "Convert region {} to leader, previous role state: {:?}",
+                        region_id, state
+                    ),
+                    Err(state) => {
+                        if state != RegionRoleState::Leader(RegionLeaderState::Writable) {
+                            warn!(
+                                "Failed to convert region {} to leader, current role state: {:?}",
+                                region_id, state
+                            )
+                        }
+                    }
+                }
+            }
+            RegionRole::DowngradingLeader => {
+                match self.state.compare_exchange(
+                    RegionRoleState::Leader(RegionLeaderState::Writable),
+                    RegionRoleState::Leader(RegionLeaderState::Downgrading),
+                ) {
+                    Ok(state) => info!(
+                        "Convert region {} to downgrading region, previous role state: {:?}",
+                        region_id, state
+                    ),
+                    Err(state) => {
+                        if state != RegionRoleState::Leader(RegionLeaderState::Downgrading) {
+                            warn!(
+                                "Failed to convert region {} to downgrading leader, current role state: {:?}",
+                                region_id, state
+                            )
+                        }
+                    }
+                }
+            }
+        }
     }
 }
 
@@ -434,10 +537,10 @@ impl RegionMap {
             .context(RegionNotFoundSnafu { region_id })?;
         ensure!(
             region.is_writable(),
-            RegionStateSnafu {
+            RegionLeaderStateSnafu {
                 region_id,
                 state: region.state(),
-                expect: RegionState::Writable,
+                expect: RegionLeaderState::Writable,
             }
         );
         Ok(region)
@@ -460,6 +563,40 @@ impl RegionMap {
         }
     }
 
+    /// Gets flushable region by region id.
+    ///
+    /// Returns error if the region does not exist or is not operable.
+    fn flushable_region(&self, region_id: RegionId) -> Result<MitoRegionRef> {
+        let region = self
+            .get_region(region_id)
+            .context(RegionNotFoundSnafu { region_id })?;
+        ensure!(
+            region.is_flushable(),
+            FlushableRegionStateSnafu {
+                region_id,
+                state: region.state(),
+            }
+        );
+        Ok(region)
+    }
+
+    /// Gets flushable region by region id.
+    ///
+    /// Calls the callback if the region does not exist or is not operable.
+    pub(crate) fn flushable_region_or<F: OnFailure>(
+        &self,
+        region_id: RegionId,
+        cb: &mut F,
+    ) -> Option<MitoRegionRef> {
+        match self.flushable_region(region_id) {
+            Ok(region) => Some(region),
+            Err(e) => {
+                cb.on_failure(e);
+                None
+            }
+        }
+    }
+
     /// Remove region by id.
     pub(crate) fn remove_region(&self, region_id: RegionId) {
         let mut regions = self.regions.write().unwrap();
@@ -548,12 +685,70 @@ impl ManifestStats {
 
 #[cfg(test)]
 mod tests {
-    use crossbeam_utils::atomic::AtomicCell;
+    use std::sync::Arc;
 
-    use crate::region::RegionState;
+    use crossbeam_utils::atomic::AtomicCell;
+    use store_api::region_engine::RegionRole;
+    use store_api::storage::RegionId;
+
+    use crate::region::{RegionLeaderState, RegionRoleState};
+    use crate::test_util::scheduler_util::SchedulerEnv;
+    use crate::test_util::version_util::VersionControlBuilder;
 
     #[test]
     fn test_region_state_lock_free() {
-        assert!(AtomicCell::<RegionState>::is_lock_free());
+        assert!(AtomicCell::<RegionRoleState>::is_lock_free());
+    }
+
+    #[tokio::test]
+    async fn test_set_region_state() {
+        let env = SchedulerEnv::new().await;
+        let builder = VersionControlBuilder::new();
+        let version_control = Arc::new(builder.build());
+        let manifest_ctx = env
+            .mock_manifest_context(version_control.current().version.metadata.clone())
+            .await;
+
+        let region_id = RegionId::new(1024, 0);
+        // Leader -> Follower
+        manifest_ctx.set_role(RegionRole::Follower, region_id);
+        assert_eq!(manifest_ctx.state.load(), RegionRoleState::Follower);
+
+        // Follower -> Leader
+        manifest_ctx.set_role(RegionRole::Leader, region_id);
+        assert_eq!(
+            manifest_ctx.state.load(),
+            RegionRoleState::Leader(RegionLeaderState::Writable)
+        );
+
+        // Leader -> Downgrading Leader
+        manifest_ctx.set_role(RegionRole::DowngradingLeader, region_id);
+        assert_eq!(
+            manifest_ctx.state.load(),
+            RegionRoleState::Leader(RegionLeaderState::Downgrading)
+        );
+
+        // Downgrading Leader -> Follower
+        manifest_ctx.set_role(RegionRole::Follower, region_id);
+        assert_eq!(manifest_ctx.state.load(), RegionRoleState::Follower);
+
+        // Can't downgrade from follower (Follower -> Downgrading Leader)
+        manifest_ctx.set_role(RegionRole::DowngradingLeader, region_id);
+        assert_eq!(manifest_ctx.state.load(), RegionRoleState::Follower);
+
+        // Set region role too Downgrading Leader
+        manifest_ctx.set_role(RegionRole::Leader, region_id);
+        manifest_ctx.set_role(RegionRole::DowngradingLeader, region_id);
+        assert_eq!(
+            manifest_ctx.state.load(),
+            RegionRoleState::Leader(RegionLeaderState::Downgrading)
+        );
+
+        // Downgrading Leader -> Leader
+        manifest_ctx.set_role(RegionRole::Leader, region_id);
+        assert_eq!(
+            manifest_ctx.state.load(),
+            RegionRoleState::Leader(RegionLeaderState::Writable)
+        );
     }
 }
diff --git a/src/mito2/src/region/opener.rs b/src/mito2/src/region/opener.rs
index 64272a183b..b2a76490cc 100644
--- a/src/mito2/src/region/opener.rs
+++ b/src/mito2/src/region/opener.rs
@@ -28,6 +28,7 @@ use snafu::{ensure, OptionExt};
 use store_api::logstore::provider::Provider;
 use store_api::logstore::LogStore;
 use store_api::metadata::{ColumnMetadata, RegionMetadata};
+use store_api::region_engine::RegionRole;
 use store_api::storage::{ColumnId, RegionId};
 
 use crate::access_layer::AccessLayer;
@@ -42,7 +43,9 @@ use crate::memtable::time_partition::TimePartitions;
 use crate::memtable::MemtableBuilderProvider;
 use crate::region::options::RegionOptions;
 use crate::region::version::{VersionBuilder, VersionControl, VersionControlRef};
-use crate::region::{ManifestContext, ManifestStats, MitoRegion, RegionState};
+use crate::region::{
+    ManifestContext, ManifestStats, MitoRegion, RegionLeaderState, RegionRoleState,
+};
 use crate::region_write_ctx::RegionWriteCtx;
 use crate::request::OptionOutputTx;
 use crate::schedule::scheduler::SchedulerRef;
@@ -169,8 +172,8 @@ impl RegionOpener {
                     &expect.column_metadatas,
                     &expect.primary_key,
                 )?;
-                // To keep consistence with Create behavior, set the opened Region writable.
-                region.set_writable(true);
+                // To keep consistence with Create behavior, set the opened Region to RegionRole::Leader.
+                region.set_role(RegionRole::Leader);
                 return Ok(region);
             }
             Ok(None) => {
@@ -235,7 +238,7 @@ impl RegionOpener {
             // Region is writable after it is created.
             manifest_ctx: Arc::new(ManifestContext::new(
                 manifest_manager,
-                RegionState::Writable,
+                RegionRoleState::Leader(RegionLeaderState::Writable),
             )),
             file_purger: Arc::new(LocalFilePurger::new(
                 self.purge_scheduler,
@@ -362,9 +365,10 @@ impl RegionOpener {
         let version_control = Arc::new(VersionControl::new(version));
         if !self.skip_wal_replay {
             info!(
-                "Start replaying memtable at flushed_entry_id + 1 {} for region {}",
+                "Start replaying memtable at flushed_entry_id + 1: {} for region {}, manifest version: {}",
                 flushed_entry_id + 1,
-                region_id
+                region_id,
+                manifest.manifest_version
             );
             replay_memtable(
                 &provider,
@@ -377,7 +381,10 @@ impl RegionOpener {
             )
             .await?;
         } else {
-            info!("Skip the WAL replay for region: {}", region_id);
+            info!(
+                "Skip the WAL replay for region: {}, manifest version: {}",
+                region_id, manifest.manifest_version
+            );
         }
         let now = self.time_provider.current_time_millis();
 
@@ -388,7 +395,7 @@ impl RegionOpener {
             // Region is always opened in read only mode.
             manifest_ctx: Arc::new(ManifestContext::new(
                 manifest_manager,
-                RegionState::ReadOnly,
+                RegionRoleState::Follower,
             )),
             file_purger,
             provider: provider.clone(),
diff --git a/src/mito2/src/request.rs b/src/mito2/src/request.rs
index d88bc994e9..1e4c6b8dc9 100644
--- a/src/mito2/src/request.rs
+++ b/src/mito2/src/request.rs
@@ -31,7 +31,7 @@ use prost::Message;
 use smallvec::SmallVec;
 use snafu::{ensure, OptionExt, ResultExt};
 use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef};
-use store_api::region_engine::SetReadonlyResponse;
+use store_api::region_engine::{SetRegionRoleStateResponse, SettableRegionRoleState};
 use store_api::region_request::{
     AffectedRows, RegionAlterRequest, RegionCatchupRequest, RegionCloseRequest,
     RegionCompactRequest, RegionCreateRequest, RegionDropRequest, RegionFlushRequest,
@@ -483,11 +483,13 @@ pub(crate) enum WorkerRequest {
     },
 
     /// The internal commands.
-    SetReadonlyGracefully {
+    SetRegionRoleStateGracefully {
         /// Id of the region to send.
         region_id: RegionId,
+        /// The [SettableRegionRoleState].
+        region_role_state: SettableRegionRoleState,
         /// The sender of [SetReadonlyResponse].
-        sender: Sender<SetReadonlyResponse>,
+        sender: Sender<SetRegionRoleStateResponse>,
     },
 
     /// Notify a worker to stop.
@@ -587,11 +589,16 @@ impl WorkerRequest {
 
     pub(crate) fn new_set_readonly_gracefully(
         region_id: RegionId,
-    ) -> (WorkerRequest, Receiver<SetReadonlyResponse>) {
+        region_role_state: SettableRegionRoleState,
+    ) -> (WorkerRequest, Receiver<SetRegionRoleStateResponse>) {
         let (sender, receiver) = oneshot::channel();
 
         (
-            WorkerRequest::SetReadonlyGracefully { region_id, sender },
+            WorkerRequest::SetRegionRoleStateGracefully {
+                region_id,
+                region_role_state,
+                sender,
+            },
             receiver,
         )
     }
diff --git a/src/mito2/src/test_util.rs b/src/mito2/src/test_util.rs
index cd449e53fa..0bd85747c0 100644
--- a/src/mito2/src/test_util.rs
+++ b/src/mito2/src/test_util.rs
@@ -52,7 +52,7 @@ use rskafka::client::{Client, ClientBuilder};
 use rskafka::record::Record;
 use rstest_reuse::template;
 use store_api::metadata::{ColumnMetadata, RegionMetadataRef};
-use store_api::region_engine::RegionEngine;
+use store_api::region_engine::{RegionEngine, RegionRole};
 use store_api::region_request::{
     RegionCloseRequest, RegionCreateRequest, RegionDeleteRequest, RegionFlushRequest,
     RegionOpenRequest, RegionPutRequest, RegionRequest,
@@ -1114,6 +1114,8 @@ pub async fn reopen_region(
         .unwrap();
 
     if writable {
-        engine.set_writable(region_id, true).unwrap();
+        engine
+            .set_region_role(region_id, RegionRole::Leader)
+            .unwrap();
     }
 }
diff --git a/src/mito2/src/test_util/scheduler_util.rs b/src/mito2/src/test_util/scheduler_util.rs
index c1b85279de..ba777b157f 100644
--- a/src/mito2/src/test_util/scheduler_util.rs
+++ b/src/mito2/src/test_util/scheduler_util.rs
@@ -31,7 +31,7 @@ use crate::config::MitoConfig;
 use crate::error::Result;
 use crate::flush::FlushScheduler;
 use crate::manifest::manager::{RegionManifestManager, RegionManifestOptions};
-use crate::region::{ManifestContext, ManifestContextRef, RegionState};
+use crate::region::{ManifestContext, ManifestContextRef, RegionLeaderState, RegionRoleState};
 use crate::request::WorkerRequest;
 use crate::schedule::scheduler::{Job, LocalScheduler, Scheduler, SchedulerRef};
 use crate::sst::index::intermediate::IntermediateManager;
@@ -124,7 +124,7 @@ impl SchedulerEnv {
             )
             .await
             .unwrap(),
-            RegionState::Writable,
+            RegionRoleState::Leader(RegionLeaderState::Writable),
         ))
     }
 
diff --git a/src/mito2/src/worker.rs b/src/mito2/src/worker.rs
index c2fbc80982..e790ed08c1 100644
--- a/src/mito2/src/worker.rs
+++ b/src/mito2/src/worker.rs
@@ -39,7 +39,7 @@ use prometheus::IntGauge;
 use rand::{thread_rng, Rng};
 use snafu::{ensure, ResultExt};
 use store_api::logstore::LogStore;
-use store_api::region_engine::SetReadonlyResponse;
+use store_api::region_engine::{SetRegionRoleStateResponse, SettableRegionRoleState};
 use store_api::storage::RegionId;
 use tokio::sync::mpsc::{Receiver, Sender};
 use tokio::sync::{mpsc, oneshot, watch, Mutex};
@@ -734,8 +734,13 @@ impl<S: LogStore> RegionWorkerLoop<S> {
                     // For background notify, we handle it directly.
                     self.handle_background_notify(region_id, notify).await;
                 }
-                WorkerRequest::SetReadonlyGracefully { region_id, sender } => {
-                    self.set_readonly_gracefully(region_id, sender).await;
+                WorkerRequest::SetRegionRoleStateGracefully {
+                    region_id,
+                    region_role_state,
+                    sender,
+                } => {
+                    self.set_role_state_gracefully(region_id, region_role_state, sender)
+                        .await;
                 }
                 WorkerRequest::EditRegion(request) => {
                     self.handle_region_edit(request).await;
@@ -834,22 +839,23 @@ impl<S: LogStore> RegionWorkerLoop<S> {
         }
     }
 
-    /// Handles `set_readonly_gracefully`.
-    async fn set_readonly_gracefully(
+    /// Handles `set_region_role_gracefully`.
+    async fn set_role_state_gracefully(
         &mut self,
         region_id: RegionId,
-        sender: oneshot::Sender<SetReadonlyResponse>,
+        region_role_state: SettableRegionRoleState,
+        sender: oneshot::Sender<SetRegionRoleStateResponse>,
     ) {
         if let Some(region) = self.regions.get_region(region_id) {
             // We need to do this in background as we need the manifest lock.
             common_runtime::spawn_global(async move {
-                region.set_readonly_gracefully().await;
+                region.set_role_state_gracefully(region_role_state).await;
 
                 let last_entry_id = region.version_control.current().last_entry_id;
-                let _ = sender.send(SetReadonlyResponse::success(Some(last_entry_id)));
+                let _ = sender.send(SetRegionRoleStateResponse::success(Some(last_entry_id)));
             });
         } else {
-            let _ = sender.send(SetReadonlyResponse::NotFound);
+            let _ = sender.send(SetRegionRoleStateResponse::NotFound);
         }
     }
 }
diff --git a/src/mito2/src/worker/handle_catchup.rs b/src/mito2/src/worker/handle_catchup.rs
index 505c994d36..cacd563ed7 100644
--- a/src/mito2/src/worker/handle_catchup.rs
+++ b/src/mito2/src/worker/handle_catchup.rs
@@ -20,6 +20,7 @@ use common_telemetry::info;
 use common_telemetry::tracing::warn;
 use snafu::ensure;
 use store_api::logstore::LogStore;
+use store_api::region_engine::RegionRole;
 use store_api::region_request::{AffectedRows, RegionCatchupRequest};
 use store_api::storage::RegionId;
 use tokio::time::Instant;
@@ -47,7 +48,8 @@ impl<S: LogStore> RegionWorkerLoop<S> {
 
         // Utilizes the short circuit evaluation.
         let region = if !is_mutable_empty || region.manifest_ctx.has_update().await? {
-            info!("Reopening the region: {region_id}, empty mutable: {is_mutable_empty}");
+            let manifest_version = region.manifest_ctx.manifest_version().await;
+            info!("Reopening the region: {region_id}, empty mutable: {is_mutable_empty}, manifest version: {manifest_version}");
             let reopened_region = Arc::new(
                 RegionOpener::new(
                     region_id,
@@ -112,7 +114,7 @@ impl<S: LogStore> RegionWorkerLoop<S> {
         }
 
         if request.set_writable {
-            region.set_writable(true);
+            region.set_role(RegionRole::Leader);
         }
 
         Ok(0)
diff --git a/src/mito2/src/worker/handle_drop.rs b/src/mito2/src/worker/handle_drop.rs
index 51b42acb40..a569f22360 100644
--- a/src/mito2/src/worker/handle_drop.rs
+++ b/src/mito2/src/worker/handle_drop.rs
@@ -28,7 +28,7 @@ use store_api::storage::RegionId;
 use tokio::time::sleep;
 
 use crate::error::{OpenDalSnafu, Result};
-use crate::region::{RegionMapRef, RegionState};
+use crate::region::{RegionLeaderState, RegionMapRef};
 use crate::worker::{RegionWorkerLoop, DROPPING_MARKER_FILE};
 
 const GC_TASK_INTERVAL_SEC: u64 = 5 * 60; // 5 minutes
@@ -62,7 +62,7 @@ where
 
                 // Sets the state back to writable. It's possible that the marker file has been written.
                 // We set the state back to writable so we can retry the drop operation.
-                region.switch_state_to_writable(RegionState::Dropping);
+                region.switch_state_to_writable(RegionLeaderState::Dropping);
             })?;
 
         region.stop().await;
diff --git a/src/mito2/src/worker/handle_flush.rs b/src/mito2/src/worker/handle_flush.rs
index 14a70225bb..b2bc5fd2e8 100644
--- a/src/mito2/src/worker/handle_flush.rs
+++ b/src/mito2/src/worker/handle_flush.rs
@@ -36,16 +36,18 @@ impl<S> RegionWorkerLoop<S> {
         request: RegionFlushRequest,
         mut sender: OptionOutputTx,
     ) {
-        let Some(region) = self.regions.writable_region_or(region_id, &mut sender) else {
+        let Some(region) = self.regions.flushable_region_or(region_id, &mut sender) else {
             return;
         };
 
-        let mut task = self.new_flush_task(
-            &region,
-            FlushReason::Manual,
-            request.row_group_size,
-            self.config.clone(),
-        );
+        let reason = if region.is_downgrading() {
+            FlushReason::Downgrading
+        } else {
+            FlushReason::Manual
+        };
+
+        let mut task =
+            self.new_flush_task(&region, reason, request.row_group_size, self.config.clone());
         task.push_sender(sender);
         if let Err(e) =
             self.flush_scheduler
diff --git a/src/mito2/src/worker/handle_manifest.rs b/src/mito2/src/worker/handle_manifest.rs
index de5f4e563d..e97b30afec 100644
--- a/src/mito2/src/worker/handle_manifest.rs
+++ b/src/mito2/src/worker/handle_manifest.rs
@@ -27,7 +27,7 @@ use crate::error::{RegionBusySnafu, RegionNotFoundSnafu, Result};
 use crate::manifest::action::{
     RegionChange, RegionEdit, RegionMetaAction, RegionMetaActionList, RegionTruncate,
 };
-use crate::region::{MitoRegionRef, RegionState};
+use crate::region::{MitoRegionRef, RegionLeaderState, RegionRoleState};
 use crate::request::{
     BackgroundNotify, OptionOutputTx, RegionChangeResult, RegionEditRequest, RegionEditResult,
     TruncateResult, WorkerRequest,
@@ -84,7 +84,7 @@ impl<S> RegionWorkerLoop<S> {
         };
 
         if !region.is_writable() {
-            if region.state() == RegionState::Editing {
+            if region.state() == RegionRoleState::Leader(RegionLeaderState::Editing) {
                 self.region_edit_queues
                     .entry(region_id)
                     .or_insert_with(|| RegionEditQueue::new(region_id))
@@ -159,7 +159,7 @@ impl<S> RegionWorkerLoop<S> {
         }
 
         // Sets the region as writable.
-        region.switch_state_to_writable(RegionState::Editing);
+        region.switch_state_to_writable(RegionLeaderState::Editing);
 
         let _ = edit_result.sender.send(edit_result.result);
 
@@ -199,8 +199,9 @@ impl<S> RegionWorkerLoop<S> {
                 RegionMetaActionList::with_action(RegionMetaAction::Truncate(truncate.clone()));
 
             let result = manifest_ctx
-                .update_manifest(RegionState::Truncating, action_list)
-                .await;
+                .update_manifest(RegionLeaderState::Truncating, action_list)
+                .await
+                .map(|_| ());
 
             // Sends the result back to the request sender.
             let truncate_result = TruncateResult {
@@ -241,8 +242,9 @@ impl<S> RegionWorkerLoop<S> {
 
             let result = region
                 .manifest_ctx
-                .update_manifest(RegionState::Altering, action_list)
-                .await;
+                .update_manifest(RegionLeaderState::Altering, action_list)
+                .await
+                .map(|_| ());
             let notify = WorkerRequest::Background {
                 region_id: region.region_id,
                 notify: BackgroundNotify::RegionChange(RegionChangeResult {
@@ -291,7 +293,7 @@ impl<S> RegionWorkerLoop<S> {
         }
 
         // Sets the region as writable.
-        region.switch_state_to_writable(RegionState::Altering);
+        region.switch_state_to_writable(RegionLeaderState::Altering);
 
         change_result.sender.send(change_result.result.map(|_| 0));
     }
@@ -338,6 +340,7 @@ async fn edit_region(
     let action_list = RegionMetaActionList::with_action(RegionMetaAction::Edit(edit));
     region
         .manifest_ctx
-        .update_manifest(RegionState::Editing, action_list)
+        .update_manifest(RegionLeaderState::Editing, action_list)
         .await
+        .map(|_| ())
 }
diff --git a/src/mito2/src/worker/handle_truncate.rs b/src/mito2/src/worker/handle_truncate.rs
index da5b74e511..863b1961a3 100644
--- a/src/mito2/src/worker/handle_truncate.rs
+++ b/src/mito2/src/worker/handle_truncate.rs
@@ -20,7 +20,7 @@ use store_api::storage::RegionId;
 
 use crate::error::RegionNotFoundSnafu;
 use crate::manifest::action::RegionTruncate;
-use crate::region::RegionState;
+use crate::region::RegionLeaderState;
 use crate::request::{OptionOutputTx, TruncateResult};
 use crate::worker::RegionWorkerLoop;
 
@@ -63,7 +63,7 @@ impl<S: LogStore> RegionWorkerLoop<S> {
         };
 
         // We are already in the worker loop so we can set the state first.
-        region.switch_state_to_writable(RegionState::Truncating);
+        region.switch_state_to_writable(RegionLeaderState::Truncating);
 
         match truncate_result.result {
             Ok(()) => {
diff --git a/src/operator/src/tests/partition_manager.rs b/src/operator/src/tests/partition_manager.rs
index bc79079036..2f1ffeffb5 100644
--- a/src/operator/src/tests/partition_manager.rs
+++ b/src/operator/src/tests/partition_manager.rs
@@ -142,7 +142,7 @@ pub(crate) async fn create_partition_rule_manager(
                     },
                     leader_peer: Some(Peer::new(3, "")),
                     follower_peers: vec![],
-                    leader_status: None,
+                    leader_state: None,
                     leader_down_since: None,
                 },
                 RegionRoute {
@@ -173,7 +173,7 @@ pub(crate) async fn create_partition_rule_manager(
                     },
                     leader_peer: Some(Peer::new(2, "")),
                     follower_peers: vec![],
-                    leader_status: None,
+                    leader_state: None,
                     leader_down_since: None,
                 },
                 RegionRoute {
@@ -196,7 +196,7 @@ pub(crate) async fn create_partition_rule_manager(
                     },
                     leader_peer: Some(Peer::new(1, "")),
                     follower_peers: vec![],
-                    leader_status: None,
+                    leader_state: None,
                     leader_down_since: None,
                 },
             ]),
diff --git a/src/query/src/optimizer/test_util.rs b/src/query/src/optimizer/test_util.rs
index 6b14a5af52..58e014afff 100644
--- a/src/query/src/optimizer/test_util.rs
+++ b/src/query/src/optimizer/test_util.rs
@@ -28,7 +28,8 @@ use store_api::metadata::{
     ColumnMetadata, RegionMetadata, RegionMetadataBuilder, RegionMetadataRef,
 };
 use store_api::region_engine::{
-    RegionEngine, RegionRole, RegionScannerRef, RegionStatistic, SetReadonlyResponse,
+    RegionEngine, RegionRole, RegionScannerRef, RegionStatistic, SetRegionRoleStateResponse,
+    SettableRegionRoleState,
 };
 use store_api::region_request::RegionRequest;
 use store_api::storage::{ConcreteDataType, RegionId, ScanRequest};
@@ -89,14 +90,15 @@ impl RegionEngine for MetaRegionEngine {
         Ok(())
     }
 
-    fn set_writable(&self, _region_id: RegionId, _writable: bool) -> Result<(), BoxedError> {
+    fn set_region_role(&self, _region_id: RegionId, _role: RegionRole) -> Result<(), BoxedError> {
         unimplemented!()
     }
 
-    async fn set_readonly_gracefully(
+    async fn set_region_role_state_gracefully(
         &self,
         _region_id: RegionId,
-    ) -> Result<SetReadonlyResponse, BoxedError> {
+        _region_role_state: SettableRegionRoleState,
+    ) -> Result<SetRegionRoleStateResponse, BoxedError> {
         unimplemented!()
     }
 
diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs
index 483d3cc1ad..850b9ad3e2 100644
--- a/src/store-api/src/region_engine.rs
+++ b/src/store-api/src/region_engine.rs
@@ -36,9 +36,32 @@ use crate::metadata::RegionMetadataRef;
 use crate::region_request::{RegionOpenRequest, RegionRequest};
 use crate::storage::{RegionId, ScanRequest};
 
-/// The result of setting readonly for the region.
+/// The settable region role state.
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum SettableRegionRoleState {
+    Follower,
+    DowngradingLeader,
+}
+
+impl From<SettableRegionRoleState> for RegionRole {
+    fn from(value: SettableRegionRoleState) -> Self {
+        match value {
+            SettableRegionRoleState::Follower => RegionRole::Follower,
+            SettableRegionRoleState::DowngradingLeader => RegionRole::DowngradingLeader,
+        }
+    }
+}
+
+/// The request to set region role state.
 #[derive(Debug, PartialEq, Eq)]
-pub enum SetReadonlyResponse {
+pub struct SetRegionRoleStateRequest {
+    region_id: RegionId,
+    region_role_state: SettableRegionRoleState,
+}
+
+/// The response of setting region role state.
+#[derive(Debug, PartialEq, Eq)]
+pub enum SetRegionRoleStateResponse {
     Success {
         /// Returns `last_entry_id` of the region if available(e.g., It's not available in file engine).
         last_entry_id: Option<entry::Id>,
@@ -46,8 +69,8 @@ pub enum SetReadonlyResponse {
     NotFound,
 }
 
-impl SetReadonlyResponse {
-    /// Returns a [SetReadonlyResponse::Success] with the `last_entry_id`.
+impl SetRegionRoleStateResponse {
+    /// Returns a [SetRegionRoleStateResponse::Success] with the `last_entry_id`.
     pub fn success(last_entry_id: Option<entry::Id>) -> Self {
         Self::Success { last_entry_id }
     }
@@ -58,6 +81,7 @@ pub struct GrantedRegion {
     pub region_id: RegionId,
     pub region_role: RegionRole,
 }
+
 impl GrantedRegion {
     pub fn new(region_id: RegionId, region_role: RegionRole) -> Self {
         Self {
@@ -85,12 +109,18 @@ impl From<PbGrantedRegion> for GrantedRegion {
     }
 }
 
+/// The role of the region.
+/// TODO(weny): rename it to `RegionRoleState`
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
 pub enum RegionRole {
     // Readonly region(mito2)
     Follower,
     // Writable region(mito2), Readonly region(file).
     Leader,
+    // Leader is downgrading to follower.
+    //
+    // This state is used to prevent new write requests.
+    DowngradingLeader,
 }
 
 impl Display for RegionRole {
@@ -98,6 +128,7 @@ impl Display for RegionRole {
         match self {
             RegionRole::Follower => write!(f, "Follower"),
             RegionRole::Leader => write!(f, "Leader"),
+            RegionRole::DowngradingLeader => write!(f, "Leader(Downgrading)"),
         }
     }
 }
@@ -113,6 +144,7 @@ impl From<RegionRole> for PbRegionRole {
         match value {
             RegionRole::Follower => PbRegionRole::Follower,
             RegionRole::Leader => PbRegionRole::Leader,
+            RegionRole::DowngradingLeader => PbRegionRole::DowngradingLeader,
         }
     }
 }
@@ -122,6 +154,7 @@ impl From<PbRegionRole> for RegionRole {
         match value {
             PbRegionRole::Leader => RegionRole::Leader,
             PbRegionRole::Follower => RegionRole::Follower,
+            PbRegionRole::DowngradingLeader => RegionRole::DowngradingLeader,
         }
     }
 }
@@ -331,20 +364,21 @@ pub trait RegionEngine: Send + Sync {
     /// Stops the engine
     async fn stop(&self) -> Result<(), BoxedError>;
 
-    /// Sets writable mode for a region.
+    /// Sets [RegionRole] for a region.
     ///
     /// The engine checks whether the region is writable before writing to the region. Setting
     /// the region as readonly doesn't guarantee that write operations in progress will not
     /// take effect.
-    fn set_writable(&self, region_id: RegionId, writable: bool) -> Result<(), BoxedError>;
+    fn set_region_role(&self, region_id: RegionId, role: RegionRole) -> Result<(), BoxedError>;
 
-    /// Sets readonly for a region gracefully.
+    /// Sets region role state gracefully.
     ///
     /// After the call returns, the engine ensures no more write operations will succeed in the region.
-    async fn set_readonly_gracefully(
+    async fn set_region_role_state_gracefully(
         &self,
         region_id: RegionId,
-    ) -> Result<SetReadonlyResponse, BoxedError>;
+        region_role_state: SettableRegionRoleState,
+    ) -> Result<SetRegionRoleStateResponse, BoxedError>;
 
     /// Indicates region role.
     ///

From c1e8084af646acac341ca114c7d499cdabb0d5c1 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Mon, 30 Sep 2024 16:53:59 +0800
Subject: [PATCH 051/128] feat: add `add_handler_after`, `add_handler_before`,
 `replace_handler` (#4788)

* feat: add `add_handler_after`, `add_handler_before`, `replace_handler`

* chore: apply suggestions from CR

* test: add more tests

* feat: use `Vec` instead of `LinkedList`

* Update src/meta-srv/src/lib.rs

Co-authored-by: Yingwen <realevenyag@gmail.com>

---------

Co-authored-by: Yingwen <realevenyag@gmail.com>
---
 src/meta-srv/src/error.rs           |  10 +-
 src/meta-srv/src/handler.rs         | 411 +++++++++++++++++++++++-----
 src/meta-srv/src/metasrv/builder.rs |   4 +-
 3 files changed, 360 insertions(+), 65 deletions(-)

diff --git a/src/meta-srv/src/error.rs b/src/meta-srv/src/error.rs
index 7c8d17cbe0..4c3d974c0a 100644
--- a/src/meta-srv/src/error.rs
+++ b/src/meta-srv/src/error.rs
@@ -733,6 +733,13 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Handler not found: {}", name))]
+    HandlerNotFound {
+        name: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 impl Error {
@@ -803,7 +810,8 @@ impl ErrorExt for Error {
             | Error::InitExportMetricsTask { .. }
             | Error::ProcedureNotFound { .. }
             | Error::TooManyPartitions { .. }
-            | Error::TomlFormat { .. } => StatusCode::InvalidArguments,
+            | Error::TomlFormat { .. }
+            | Error::HandlerNotFound { .. } => StatusCode::InvalidArguments,
             Error::LeaseKeyFromUtf8 { .. }
             | Error::LeaseValueFromUtf8 { .. }
             | Error::InvalidRegionKeyFromUtf8 { .. }
diff --git a/src/meta-srv/src/handler.rs b/src/meta-srv/src/handler.rs
index 5363b6c548..9d82920c08 100644
--- a/src/meta-srv/src/handler.rs
+++ b/src/meta-srv/src/handler.rs
@@ -232,17 +232,6 @@ pub struct HeartbeatHandlerGroup {
 }
 
 impl HeartbeatHandlerGroup {
-    pub(crate) fn new(pushers: Pushers) -> Self {
-        Self {
-            handlers: vec![],
-            pushers,
-        }
-    }
-
-    fn add_handler(&mut self, handler: impl HeartbeatHandler + 'static) {
-        self.handlers.push(NameCachedHandler::new(handler));
-    }
-
     /// Registers the heartbeat response [`Pusher`] with the given key to the group.
     pub async fn register_pusher(&self, key: impl AsRef<str>, pusher: Pusher) {
         let key = key.as_ref();
@@ -458,25 +447,34 @@ pub struct HeartbeatHandlerGroupBuilder {
     region_failure_handler: Option<RegionFailureHandler>,
 
     /// The handler to handle region lease.
-    region_lease_handler: RegionLeaseHandler,
+    region_lease_handler: Option<RegionLeaseHandler>,
 
     /// The plugins.
     plugins: Option<Plugins>,
 
     /// The heartbeat response pushers.
     pushers: Pushers,
+
+    /// The group of heartbeat handlers.
+    handlers: Vec<NameCachedHandler>,
 }
 
 impl HeartbeatHandlerGroupBuilder {
-    pub fn new(pushers: Pushers, region_lease_handler: RegionLeaseHandler) -> Self {
+    pub fn new(pushers: Pushers) -> Self {
         Self {
             region_failure_handler: None,
-            region_lease_handler,
+            region_lease_handler: None,
             plugins: None,
             pushers,
+            handlers: vec![],
         }
     }
 
+    pub fn with_region_lease_handler(mut self, handler: Option<RegionLeaseHandler>) -> Self {
+        self.region_lease_handler = handler;
+        self
+    }
+
     /// Sets the [`RegionFailureHandler`].
     pub fn with_region_failure_handler(mut self, handler: Option<RegionFailureHandler>) -> Self {
         self.region_failure_handler = handler;
@@ -489,10 +487,10 @@ impl HeartbeatHandlerGroupBuilder {
         self
     }
 
-    /// Builds the group of heartbeat handlers.
-    pub fn build(self) -> HeartbeatHandlerGroup {
+    /// Adds the default handlers.
+    pub fn add_default_handlers(mut self) -> Self {
         // Extract the `PublishHeartbeatHandler` from the plugins.
-        let publish_heartbeat_handler = if let Some(plugins) = self.plugins {
+        let publish_heartbeat_handler = if let Some(plugins) = self.plugins.as_ref() {
             plugins
                 .get::<PublisherRef>()
                 .map(|publish| PublishHeartbeatHandler::new(publish.clone()))
@@ -500,39 +498,94 @@ impl HeartbeatHandlerGroupBuilder {
             None
         };
 
-        // TODO(weny): Considers classifying handlers
-        // to make it easier for upper layers to customize handler groups.
-        let mut group = HeartbeatHandlerGroup::new(self.pushers);
-        group.add_handler(ResponseHeaderHandler);
+        self.add_handler_last(ResponseHeaderHandler);
         // `KeepLeaseHandler` should preferably be in front of `CheckLeaderHandler`,
         // because even if the current meta-server node is no longer the leader it can
         // still help the datanode to keep lease.
-        group.add_handler(DatanodeKeepLeaseHandler);
-        group.add_handler(FlownodeKeepLeaseHandler);
-        group.add_handler(CheckLeaderHandler);
-        group.add_handler(OnLeaderStartHandler);
-        group.add_handler(ExtractStatHandler);
-        group.add_handler(CollectDatanodeClusterInfoHandler);
-        group.add_handler(CollectFrontendClusterInfoHandler);
-        group.add_handler(CollectFlownodeClusterInfoHandler);
-        group.add_handler(MailboxHandler);
-        group.add_handler(self.region_lease_handler);
-        group.add_handler(FilterInactiveRegionStatsHandler);
-        if let Some(region_failure_handler) = self.region_failure_handler {
-            group.add_handler(region_failure_handler);
+        self.add_handler_last(DatanodeKeepLeaseHandler);
+        self.add_handler_last(FlownodeKeepLeaseHandler);
+        self.add_handler_last(CheckLeaderHandler);
+        self.add_handler_last(OnLeaderStartHandler);
+        self.add_handler_last(ExtractStatHandler);
+        self.add_handler_last(CollectDatanodeClusterInfoHandler);
+        self.add_handler_last(CollectFrontendClusterInfoHandler);
+        self.add_handler_last(CollectFlownodeClusterInfoHandler);
+        self.add_handler_last(MailboxHandler);
+        if let Some(region_lease_handler) = self.region_lease_handler.take() {
+            self.add_handler_last(region_lease_handler);
+        }
+        self.add_handler_last(FilterInactiveRegionStatsHandler);
+        if let Some(region_failure_handler) = self.region_failure_handler.take() {
+            self.add_handler_last(region_failure_handler);
         }
         if let Some(publish_heartbeat_handler) = publish_heartbeat_handler {
-            group.add_handler(publish_heartbeat_handler);
+            self.add_handler_last(publish_heartbeat_handler);
         }
-        group.add_handler(CollectStatsHandler::default());
+        self.add_handler_last(CollectStatsHandler::default());
 
-        group
+        self
+    }
+
+    /// Builds the group of heartbeat handlers.
+    pub fn build(self) -> HeartbeatHandlerGroup {
+        HeartbeatHandlerGroup {
+            handlers: self.handlers.into_iter().collect(),
+            pushers: self.pushers,
+        }
+    }
+
+    /// Adds the handler after the specified handler.
+    pub fn add_handler_after(
+        &mut self,
+        target: &'static str,
+        handler: impl HeartbeatHandler + 'static,
+    ) -> Result<()> {
+        if let Some(pos) = self.handlers.iter().position(|x| x.name == target) {
+            self.handlers
+                .insert(pos + 1, NameCachedHandler::new(handler));
+            return Ok(());
+        }
+
+        error::HandlerNotFoundSnafu { name: target }.fail()
+    }
+
+    /// Adds the handler before the specified handler.
+    pub fn add_handler_before(
+        &mut self,
+        target: &'static str,
+        handler: impl HeartbeatHandler + 'static,
+    ) -> Result<()> {
+        if let Some(pos) = self.handlers.iter().position(|x| x.name == target) {
+            self.handlers.insert(pos, NameCachedHandler::new(handler));
+            return Ok(());
+        }
+
+        error::HandlerNotFoundSnafu { name: target }.fail()
+    }
+
+    /// Replaces the handler with the specified name.
+    pub fn replace_handler(
+        &mut self,
+        target: &'static str,
+        handler: impl HeartbeatHandler + 'static,
+    ) -> Result<()> {
+        if let Some(pos) = self.handlers.iter().position(|x| x.name == target) {
+            self.handlers[pos] = NameCachedHandler::new(handler);
+            return Ok(());
+        }
+
+        error::HandlerNotFoundSnafu { name: target }.fail()
+    }
+
+    fn add_handler_last(&mut self, handler: impl HeartbeatHandler + 'static) {
+        self.handlers.push(NameCachedHandler::new(handler));
     }
 }
 
 #[cfg(test)]
 mod tests {
 
+    use std::assert_matches::assert_matches;
     use std::sync::Arc;
     use std::time::Duration;
 
@@ -541,17 +594,9 @@ mod tests {
     use common_meta::sequence::SequenceBuilder;
     use tokio::sync::mpsc;
 
-    use crate::handler::check_leader_handler::CheckLeaderHandler;
-    use crate::handler::collect_cluster_info_handler::{
-        CollectDatanodeClusterInfoHandler, CollectFlownodeClusterInfoHandler,
-        CollectFrontendClusterInfoHandler,
-    };
+    use super::{HeartbeatHandlerGroupBuilder, Pushers};
+    use crate::error;
     use crate::handler::collect_stats_handler::CollectStatsHandler;
-    use crate::handler::extract_stat_handler::ExtractStatHandler;
-    use crate::handler::filter_inactive_region_stats::FilterInactiveRegionStatsHandler;
-    use crate::handler::keep_lease_handler::{DatanodeKeepLeaseHandler, FlownodeKeepLeaseHandler};
-    use crate::handler::mailbox_handler::MailboxHandler;
-    use crate::handler::on_leader_start_handler::OnLeaderStartHandler;
     use crate::handler::response_header_handler::ResponseHeaderHandler;
     use crate::handler::{HeartbeatHandlerGroup, HeartbeatMailbox, Pusher};
     use crate::service::mailbox::{Channel, MailboxReceiver, MailboxRef};
@@ -621,24 +666,13 @@ mod tests {
         (mailbox, receiver)
     }
 
-    #[tokio::test]
-    async fn test_handler_name() {
-        let mut group = HeartbeatHandlerGroup::default();
-        group.add_handler(ResponseHeaderHandler);
-        group.add_handler(DatanodeKeepLeaseHandler);
-        group.add_handler(FlownodeKeepLeaseHandler);
-        group.add_handler(CheckLeaderHandler);
-        group.add_handler(OnLeaderStartHandler);
-        group.add_handler(ExtractStatHandler);
-        group.add_handler(CollectDatanodeClusterInfoHandler);
-        group.add_handler(CollectFrontendClusterInfoHandler);
-        group.add_handler(CollectFlownodeClusterInfoHandler);
-        group.add_handler(MailboxHandler);
-        group.add_handler(FilterInactiveRegionStatsHandler);
-        group.add_handler(CollectStatsHandler::default());
+    #[test]
+    fn test_handler_group_builder() {
+        let group = HeartbeatHandlerGroupBuilder::new(Pushers::default())
+            .add_default_handlers()
+            .build();
 
         let handlers = group.handlers;
-
         assert_eq!(12, handlers.len());
 
         let names = [
@@ -660,4 +694,255 @@ mod tests {
             assert_eq!(handler.name, name);
         }
     }
+
+    #[test]
+    fn test_handler_group_builder_add_before() {
+        let mut builder =
+            HeartbeatHandlerGroupBuilder::new(Pushers::default()).add_default_handlers();
+        builder
+            .add_handler_before(
+                "FilterInactiveRegionStatsHandler",
+                CollectStatsHandler::default(),
+            )
+            .unwrap();
+
+        let group = builder.build();
+        let handlers = group.handlers;
+        assert_eq!(13, handlers.len());
+
+        let names = [
+            "ResponseHeaderHandler",
+            "DatanodeKeepLeaseHandler",
+            "FlownodeKeepLeaseHandler",
+            "CheckLeaderHandler",
+            "OnLeaderStartHandler",
+            "ExtractStatHandler",
+            "CollectDatanodeClusterInfoHandler",
+            "CollectFrontendClusterInfoHandler",
+            "CollectFlownodeClusterInfoHandler",
+            "MailboxHandler",
+            "CollectStatsHandler",
+            "FilterInactiveRegionStatsHandler",
+            "CollectStatsHandler",
+        ];
+
+        for (handler, name) in handlers.iter().zip(names.into_iter()) {
+            assert_eq!(handler.name, name);
+        }
+    }
+
+    #[test]
+    fn test_handler_group_builder_add_before_first() {
+        let mut builder =
+            HeartbeatHandlerGroupBuilder::new(Pushers::default()).add_default_handlers();
+        builder
+            .add_handler_before("ResponseHeaderHandler", CollectStatsHandler::default())
+            .unwrap();
+
+        let group = builder.build();
+        let handlers = group.handlers;
+        assert_eq!(13, handlers.len());
+
+        let names = [
+            "CollectStatsHandler",
+            "ResponseHeaderHandler",
+            "DatanodeKeepLeaseHandler",
+            "FlownodeKeepLeaseHandler",
+            "CheckLeaderHandler",
+            "OnLeaderStartHandler",
+            "ExtractStatHandler",
+            "CollectDatanodeClusterInfoHandler",
+            "CollectFrontendClusterInfoHandler",
+            "CollectFlownodeClusterInfoHandler",
+            "MailboxHandler",
+            "FilterInactiveRegionStatsHandler",
+            "CollectStatsHandler",
+        ];
+
+        for (handler, name) in handlers.iter().zip(names.into_iter()) {
+            assert_eq!(handler.name, name);
+        }
+    }
+
+    #[test]
+    fn test_handler_group_builder_add_after() {
+        let mut builder =
+            HeartbeatHandlerGroupBuilder::new(Pushers::default()).add_default_handlers();
+        builder
+            .add_handler_after("MailboxHandler", CollectStatsHandler::default())
+            .unwrap();
+
+        let group = builder.build();
+        let handlers = group.handlers;
+        assert_eq!(13, handlers.len());
+
+        let names = [
+            "ResponseHeaderHandler",
+            "DatanodeKeepLeaseHandler",
+            "FlownodeKeepLeaseHandler",
+            "CheckLeaderHandler",
+            "OnLeaderStartHandler",
+            "ExtractStatHandler",
+            "CollectDatanodeClusterInfoHandler",
+            "CollectFrontendClusterInfoHandler",
+            "CollectFlownodeClusterInfoHandler",
+            "MailboxHandler",
+            "CollectStatsHandler",
+            "FilterInactiveRegionStatsHandler",
+            "CollectStatsHandler",
+        ];
+
+        for (handler, name) in handlers.iter().zip(names.into_iter()) {
+            assert_eq!(handler.name, name);
+        }
+    }
+
+    #[test]
+    fn test_handler_group_builder_add_after_last() {
+        let mut builder =
+            HeartbeatHandlerGroupBuilder::new(Pushers::default()).add_default_handlers();
+        builder
+            .add_handler_after("CollectStatsHandler", ResponseHeaderHandler)
+            .unwrap();
+
+        let group = builder.build();
+        let handlers = group.handlers;
+        assert_eq!(13, handlers.len());
+
+        let names = [
+            "ResponseHeaderHandler",
+            "DatanodeKeepLeaseHandler",
+            "FlownodeKeepLeaseHandler",
+            "CheckLeaderHandler",
+            "OnLeaderStartHandler",
+            "ExtractStatHandler",
+            "CollectDatanodeClusterInfoHandler",
+            "CollectFrontendClusterInfoHandler",
+            "CollectFlownodeClusterInfoHandler",
+            "MailboxHandler",
+            "FilterInactiveRegionStatsHandler",
+            "CollectStatsHandler",
+            "ResponseHeaderHandler",
+        ];
+
+        for (handler, name) in handlers.iter().zip(names.into_iter()) {
+            assert_eq!(handler.name, name);
+        }
+    }
+
+    #[test]
+    fn test_handler_group_builder_replace() {
+        let mut builder =
+            HeartbeatHandlerGroupBuilder::new(Pushers::default()).add_default_handlers();
+        builder
+            .replace_handler("MailboxHandler", CollectStatsHandler::default())
+            .unwrap();
+
+        let group = builder.build();
+        let handlers = group.handlers;
+        assert_eq!(12, handlers.len());
+
+        let names = [
+            "ResponseHeaderHandler",
+            "DatanodeKeepLeaseHandler",
+            "FlownodeKeepLeaseHandler",
+            "CheckLeaderHandler",
+            "OnLeaderStartHandler",
+            "ExtractStatHandler",
+            "CollectDatanodeClusterInfoHandler",
+            "CollectFrontendClusterInfoHandler",
+            "CollectFlownodeClusterInfoHandler",
+            "CollectStatsHandler",
+            "FilterInactiveRegionStatsHandler",
+            "CollectStatsHandler",
+        ];
+
+        for (handler, name) in handlers.iter().zip(names.into_iter()) {
+            assert_eq!(handler.name, name);
+        }
+    }
+
+    #[test]
+    fn test_handler_group_builder_replace_last() {
+        let mut builder =
+            HeartbeatHandlerGroupBuilder::new(Pushers::default()).add_default_handlers();
+        builder
+            .replace_handler("CollectStatsHandler", ResponseHeaderHandler)
+            .unwrap();
+
+        let group = builder.build();
+        let handlers = group.handlers;
+        assert_eq!(12, handlers.len());
+
+        let names = [
+            "ResponseHeaderHandler",
+            "DatanodeKeepLeaseHandler",
+            "FlownodeKeepLeaseHandler",
+            "CheckLeaderHandler",
+            "OnLeaderStartHandler",
+            "ExtractStatHandler",
+            "CollectDatanodeClusterInfoHandler",
+            "CollectFrontendClusterInfoHandler",
+            "CollectFlownodeClusterInfoHandler",
+            "MailboxHandler",
+            "FilterInactiveRegionStatsHandler",
+            "ResponseHeaderHandler",
+        ];
+
+        for (handler, name) in handlers.iter().zip(names.into_iter()) {
+            assert_eq!(handler.name, name);
+        }
+    }
+
+    #[test]
+    fn test_handler_group_builder_replace_first() {
+        let mut builder =
+            HeartbeatHandlerGroupBuilder::new(Pushers::default()).add_default_handlers();
+        builder
+            .replace_handler("ResponseHeaderHandler", CollectStatsHandler::default())
+            .unwrap();
+
+        let group = builder.build();
+        let handlers = group.handlers;
+        assert_eq!(12, handlers.len());
+
+        let names = [
+            "CollectStatsHandler",
+            "DatanodeKeepLeaseHandler",
+            "FlownodeKeepLeaseHandler",
+            "CheckLeaderHandler",
+            "OnLeaderStartHandler",
+            "ExtractStatHandler",
+            "CollectDatanodeClusterInfoHandler",
+            "CollectFrontendClusterInfoHandler",
+            "CollectFlownodeClusterInfoHandler",
+            "MailboxHandler",
+            "FilterInactiveRegionStatsHandler",
+            "CollectStatsHandler",
+        ];
+
+        for (handler, name) in handlers.iter().zip(names.into_iter()) {
+            assert_eq!(handler.name, name);
+        }
+    }
+
+    #[test]
+    fn test_handler_group_builder_handler_not_found() {
+        let mut builder =
+            HeartbeatHandlerGroupBuilder::new(Pushers::default()).add_default_handlers();
+        let err = builder
+            .add_handler_before("NotExists", CollectStatsHandler::default())
+            .unwrap_err();
+        assert_matches!(err, error::Error::HandlerNotFound { .. });
+
+        let err = builder
+            .add_handler_after("NotExists", CollectStatsHandler::default())
+            .unwrap_err();
+        assert_matches!(err, error::Error::HandlerNotFound { .. });
+
+        let err = builder
+            .replace_handler("NotExists", CollectStatsHandler::default())
+            .unwrap_err();
+        assert_matches!(err, error::Error::HandlerNotFound { .. });
+    }
 }
diff --git a/src/meta-srv/src/metasrv/builder.rs b/src/meta-srv/src/metasrv/builder.rs
index 662de433ab..88d7e5bd6d 100644
--- a/src/meta-srv/src/metasrv/builder.rs
+++ b/src/meta-srv/src/metasrv/builder.rs
@@ -358,9 +358,11 @@ impl MetasrvBuilder {
                     memory_region_keeper.clone(),
                 );
 
-                HeartbeatHandlerGroupBuilder::new(pushers, region_lease_handler)
+                HeartbeatHandlerGroupBuilder::new(pushers)
                     .with_plugins(plugins.clone())
                     .with_region_failure_handler(region_failover_handler)
+                    .with_region_lease_handler(Some(region_lease_handler))
+                    .add_default_handlers()
                     .build()
             }
         };

From caf5f2c7a5650d0cf86f2fb8e151fc265580e59f Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Tue, 1 Oct 2024 11:35:04 +0800
Subject: [PATCH 052/128] docs: add TM to logos (#4789)

---
 docs/logo-text-padding-dark.png | Bin 25777 -> 36537 bytes
 docs/logo-text-padding.png      | Bin 18102 -> 25396 bytes
 2 files changed, 0 insertions(+), 0 deletions(-)

diff --git a/docs/logo-text-padding-dark.png b/docs/logo-text-padding-dark.png
index fe0c976ce1cbdaf09904c7caa0e1bdd7bdbd87eb..51bc012a83fa28d8688c3f56db3e81513349eb22 100644
GIT binary patch
literal 36537
zcmdqI^;;WC^FADcdkYkoQlPj7EyatwyO*HB-6@n7D9|FsA-G#{Pm8;|Ly;DOThVWN
zp6BzN^PY45g7=qPgln@qvpX|8_dT-_YAUk0Smam$0038B?v(}rfV_+N`x*lc@yVkN
z`2het1<1dW(DF9f&%jKk(>}jg6%Z+-1AbxtqEslW_J~MA{89H4A~kgv2`y1+2~>3=
z^@7PT75pfN$eaxz7E2`7K_pW}@QtY!e~JozTI5rVK7OSg|8sCjuZd<uT**P6+sNrz
zsLDZ`$DVGYR@-IfTW2?*3Jg;aiE#}jJ($q>e_yuK0XzO^qW}2+e+B_K#_pc~IruNX
z9797ff;N1WK4SjooWF;K99;f&ng9JVg#YOpunm{vjq2BbktRJD+r0z*e}n%jRV@+{
z$u7kCu|L+o>R|-E5&4^H|Ct7gi_s|V6dJXr`LBAc0hbT|y3C*2K*V9Fz7$3%S#SQb
zU4M#3vKxs1=itA{WP=Rk7+YL>c185(;J@s8qxhxYza2!nIK?C9kQFqNuZ91r2Mo1&
z75OjA{He`Ca4_~6e<Ow{>R*=mQ#8a8QvW&lFEaiA{Ror!fMCx@!G*xx{#~dS;EFqV
z8NillK9mM9k~zs5rk)%!K{Z1MU>A^W-tVTjtGG7KA5V{>`m&z2R{qP8=RGq6n<6={
z;YbQh>$zA)Y+M58P+_M42Yj<md0$<~3U$D}TioZ1r{H=Fm=M53(00WOsSycgdM4Dr
zHyI<+-iCdqv+HB}FNY*24IAP>Jvz}7S6nhdUQ+?EZiLXk$pV^1Up?%dt|B$!!xjMb
zz$)ci)4L#eBn=jn{9&;Iu!043TI#YJj|o79G0NSuPKO~S&$l48a#9DC-G~50ao1b{
z4S>ez?^3qCo*^q(+=qMrU+4eB7?cUQYMhV+0kBwr5z5*n;2w8P-PRmn4e-Vd_Sg*d
z$Ak4g8xLR&t^>S$EdDsyXp3Q+q1{wfV^dy9jQ{%fjm^>>W7GHpAHk{Y$5x)<@52u9
zyUAHdM-whlXyhBSfE8tpDD;%%L#Xweademe1)l#IVhBADTyO<}L+z6gLbnU~%+3#q
zVy-;`#76YS3yNjrc^2!YrS#|DX#s?h1e3qx_lok?oYF6BQe9RDnP9F-jdNW2&>yO#
z?lo4gpg?J0@z2kH`PK?@jg=VQTyYyi-<dpd+W8m7*&lP&oYP&NSIWF;;P@=h;_3J1
zEtb0n52&fULuiHU;a*Vvu2}CoXFBh#(}wZkYIyql6x&bYj{B7#ybgQ4TGSj*=@WxF
z2;D+5F=c4F{w18nVI-$4&4>ZSb1wYoA0~6eSA`>0S2-i)A9(s7C>Y0jJ=YIWq0%AM
z%U7}+VXmh1#Y20z!$aRAg9w7*IaEQz+NZipE&JH&-x_J-2IG1#@0L(~nVA1w8$vEn
zqGWv3FOf+rE!Cz_j`cR~;1;=iFY=IztE1tK6MOrfEY{*hbh-QD@&4qLg&?DYQ<rBi
z7>YIbTTnW%&pscb(EM9qT1XWq4-uV=_&~8)2+*>CTU9seB%$Y&9Un#xR0l+V55?{A
z#tQ}^y_DiU0@R~b#bTe;w+HyU{!6U-_#}i9r8#bhiYm&Up*Y?c2z_<Z{9_veKH*Oi
zd<YtMY<bl>P!Ff*ZMnho)gf8z{Wo*DVE7m+E+$GmJ3wAZ0O9G5?_gh`df@M`zTPX%
zPM(h`Nuf@Bvq;BL?HL(2*!^Z1)%RvaR21|tZr2wnt^&Nv7Nls28+fwOlu0wqXR4sV
zaYQ&YGyqs3n^g}I4nFK+Sgy@I0E(1G!8iX5s{d0?{Dz=U3L1Y~nu1$GNfE5PQS(3&
z+yJ<u?H}iU1}=EzyaV)Z-<Zk`;*4*lWqIuN`4HN07_bTYm+Ls>1Lk+8u`J?<Q#Xr-
z^Xf>jHL_snclJ$`Yd<7PWHTwiA>ha(4!=o!^h)x^`F|4pdCiWkptut4x<bT|Vc7uw
z_snw1_9S;$iT<D(<;~>Nq@SxBt`4(B<=AHz?M^RTJ-3D-%cy&1bTJmW3qXfpa^fkR
z{Rk&DWNM^6eJqI$iPH5|5s^l}Zl2o~oS7FVc>xoDU(+2Yq!q=N?MoI__s2o)fF#oQ
zNP=XrUb*|pA^Mdhz#NMLePJHLtbGvX71I>3V4t%EcNqSL<q)gxpqhgpZbDRr5iYGx
zELo0LKKbp#;hS}xM*_0dBm(h8sq4+-<o5<9BSH0C6<!|y2pdzF-`U1yQI1>y=*W7c
zbRFCyyA>I(-I+9`cRzw?q)!?wle)de?$97|(AEZay-ogoJ<vX8=K+97vzk4)dLghz
zcI?QBAc%i$mg}||a0~9oI81KCy!!ONxb)wwAQ>O!glP8|7`!yA%@28(oJ7!-m{SH<
zATPW+WkkMu6g>K=brJR2zp>Qy#k>4~iz)IW0#Xx7D~bfBASIP|0~T{>7)Fj=Te)3N
zwyat?qa*@w{%(rfoFV#;b|=)bfJY^AMU_<Xw;kTOHL5G$0C<eQ29Fa0&;#yQ+`d|S
za+mG&Jj;~rl#pqqH|U<{X>{}BGm>9R;_GCnOKi;ZXMI>PnLM5UyP1yRp_~x!zK{jQ
zN8Mlo>LW&dtc+-k=a-XM7F#SPjG+ykRNTF8T%<`C7MzbN8?;MbSbAJ<dxKM`O!~J2
zA+V$wy@)7LckIJ{hu1WHe~WV=G0)Aw3(KOfjXpFg<Yvfd2ro+TJATOOH_rHO`aA+2
zz3BkIk$&SJ?UKb@dSea<lvGiQgMeuwK(I=hhK@(#Z@()`#ADg;>fEqHiv{}A#G6L>
z;YceE;hk3<iRrEQ!n2!VIBRKqoj%BPSKJa->h6E{9jGE^^1SsXy`v>&#prAWfB(7~
z{bo-i7VJj;-qzI=<X1RQgf!>7WK9$G?Y%0EDM`RS2sA1Al_FOIve*4^f!kdX-zw^E
zA`dr$GCodyJBoheU`@V4(ksqXYNnftBv8~U0f88)wZ}Zb6)*z-<_6Fx%;Xf_pVHO*
z<cfg-Lcx~uGkc#H=kvJpxQSFbvPHiRDXn*Ap&jMeFt(z@GL8>^UCIw^1-SablWB+&
ztqfW3*X#hY@wdhk>5^hzA=C;io1U(gV#V*1$d5#kGn0;wGDMexaIU7_07qU1lASZS
z{9S)DYQpyfRsE3LV?bs;l@nkQiFC3w67|7VtL3)klXyEnOVEW=+(4_SzPPGDxLtA(
zCd><PO9rFb>~2V5HoQ?+_o-t9G(PpzH|6KAO0jJ)$`S#sKy<e+t9|Z-Gn0BgW8WE1
zV&3p34Ws&!8<5qN6T#n(mkjog<JK$6NR7;zQ*(9h*s;2lFeau|Ce4iw{?$gHi-`UC
zQo?%RhYuJ9R88kKI@4-2)pP)_+UyeJZx_8CF)fQy6PCsp#>5mfo^?2a3wWkW0`A=b
zvaj+>IT?P*-KMmrt>UfCAMdmQp8LIArQjrI!fquPtowd&01P1Rz-1(cqoP_cRp^UG
z$sSxuiC}D$@sX@eZ93-Zz?S~u(2OWU4z$5ZNP0xh3RrLN@cOsnQGEMIT@5>4`?LA&
z<2^+7O$WHa$6{4qS%Vd*y`BORO%!>yX0D0*?KC=NvRcHQ-kO-LDvW2ohLNG*VeH3g
z*Qk>_@>>WYEyDz6-v>=NCruw6Qg39+l9*|iU3PkVvu|W4cA@3kzbVRABkh)%3GF>6
zrM8Ckps{U~OtUels|mfS{*IwPEMS`+MCNP(VfAmmttsZFmCv|y`+KxV7Z*mrtIW-5
zArC`Cr*4IGKxr@U_l#TRQi)j@^gH@2>G94&XJh@XmuT`V^JkLz96qeMcC>vA3olYn
z<SKoWUJ8MOEcw>lAaA>zclv_$G~fPuTz{khAF&Z&tH})YGclxt{74=!fRp^Lm~#Fh
z<O(U6Rc7{^(<1Gu&o6l$wz95MeOQ!w=c$B4LT^8-5q+exP^GmprPnR4mMfaTKV_Hw
zE&K77ux&Mj$DmUdsa13{n{O!>Z5pe07AyRalWd(zh?1^`mNWePw*0*{+0g(KZRmZ^
zEXmMML9eRd&YPrTLw(%cf4Pdc-ymQRPO)y}g$DfR_S*IxE2*g1H4_K>s*Dxqz8r<o
z>z3+Z0c3CE1QC}`CDm`vUyIgvPh9UACtU)#{vjxg?J+`7q+L)$l?SyEHVuDgOV4zd
zy~k_JV@@-oEav4UgOYn4#yF3MlV^s05=iLe&aaIWXiKG*EKpk=&B!W8?8Iq$|FN-L
zhQDF6+;v_K1cc>}4AV5Ev@5%*A`|^G5-RvGCwwI?V_6U93Js_Sh2)x_8+#;j+e}lV
zR_6tg77}WuH>dprm_X@%<J)x}T=x1TLtGMow*z+P2amST;e@&|YZO*hpTBxj@x8LL
z-?KuZ`JnKPDpkGb=CA^Z^5iSaA)M)vsQ%woZ~lzf$LAaYWIDS)I-lb7TCYt65{3}!
z>bR&2V4e0~4N5QCAvxzDIRVvdHcoCUv0X%PI61X_G!-|!t^Yn*I*bj0YBLGGYz))r
z)wJ8+|4cAu6wu|V`Km0?)r<dtOapN#^O+5>t%TdW^)-4BWzSp^;p?<lC0*ZQkJ((#
z_%pltFDh;#K+FepY71LnYDeABXKq$@G{{}AVV#A=oveE6VsGXh7_5xU?CXcH4<B6t
zt6j>&qcO_F`t$wYKE{WJKYaAAl6fiKX}tF`_@_}<7ykKtNMwKPmrbX{>#&E}b#?NU
z1+j0~hNPV>FaE&`xH7B(MIKdJ<c%Z4zg?hurc4Y4Q((asuS;*WYb?XgSPvoCGmW^y
zGcwijNW#SLsc`gg)*{%OqgU}dmD$Y5g42eMX8D3yPh(C+At1x6qh>**E^dIYB&pJ5
zk2s|+`L?a{#iirl(=5RdBg{r|e3;>nI*7zADU29txX_*lYppYv59f2z>H72D<m$!l
zs!R%EOWmm|C#1Q-U}Ge<p=&|VheEfOo?z3DSHEoV0~3?1gMSv~29FYYyVmT0yM7>d
zzO0|WUm5*}m3|~axIgA=^r(7d^;MzOvXm%sGofGsaGQPLg@xgssF`W$`cu*i)M|Cm
z>0oxOLYgroJjyq?k{#i_`qWZFfcq9S6<Qaq8q8~)E4kuG%T^r2&$>Omu76A8pxOwk
zgyIpUk;7t5cox!Jqx=|2IZ`W~BFWL$?hCe=BYr?-SIO0}XT3U5cPhAY%C0f5HC^FD
zQbaey!3gqEV*whke_0TUY>S*zWTw~Or+<o$nBF?rfO#Pva7zgzOkJxe+wc8ca%)2v
zQ0T7i+VD-vn=m35=_Owj_m0^Ibh-47FTphf0@bPA$J%k_=vQdL#;N47g1NjymjctX
z!b1Nr2%hI_zyNZnR9k)B;|`Q6E5g=8u1*PLI<1G#Wvxx+EoM}dV6=!n@b<*SvD?qG
z<a5RH@JaN~rvl~FbR2${DNMG@6seSZR~m_o0`s#i|FQrbVgZ7I6XJ`;Z>f!6ieJ6%
zI@gQ1QxI^=OJvnDafmfm>lDWI<_uOgBr6@hx&LN&B~C2arct{Ikh3XlD;$1LhZg?p
z2fy{!<Ug{8OhP$U@HQVFgGtM*Ga1gPazIXa7CUJ-s~95HlV27%Fyq2I-(Y{LVoQ6_
zHs3j_Eyj>|Ghu^7!@{`<DOa+nvOy;l4AL;Z{75Tn--X18bBazI-4^|i2=x>naSXjt
zo6k4^&T@p%j;27QO211U(TP{9zDZwA{Kz9z0du+Y+%H;Jkx@ijPt4Oz?uBU_E{s^D
z{b+*~0cA<9A8xR^G8Myr%_iKnHvMw}Lc{`=AF88jJu4;`q;n0|mAA57tQKPIIK3_;
zaTy5&31Dx)J$*%^K}bwZ%a(w$PvgnMIBp;;!P?pn1DUG{-6;ok6qzi8Z3-LZ^q#Ev
zr>4qObN@YW@m>T^fQ#fnEr#qwmu5<T&=`M-Dg1{ih%X{83B0E(9(U$vVtyLNw4EgN
zR9_8Eky(!Q?GP8vs1=ea@ThAlqf@vn;iwErAg=LiaL1=BLsi_MN_H4BtL|Dz|JLZp
z^88HWjmpdIPrw;d<T<De8`?2FFbY?X60UZLVHSGCoad+y+8Dxpe-CaW2=|5P(zz?2
zZzT3q%+*Jd2>2;aQuZzWCtAVw5i|h1Bn@d2YXD|&uS#k+9MF!tX7S}$_r|;!NG#_+
zn>U0GH0B)J)G!W`qdJ9L{j8_+mI6su;IOjob{>P73QdUQ+V||Zj6#7GOr(L$X1KQ-
zjTAmXt_?@TsX^KA;ZI6OffsMgE1wUJGf<t4qHf?0H)r%dyX!4pUXIAmH@{$19uYH@
zh9!XU=dfV>$5{$QjzBZ>+rsX2k4-=0O!6;5$R7*8WdIWV%t&FlfD-zx{`KwO98F((
zaEg5LprJ~KF0JR=fr7YOOF2uY&HvIjh=5_tJ~rsL_$JfvSdr9I6*cbaqJUy;`rQL<
z+ef_x`!s3c2~!lP-j6besbHEX!PvSLFsrRaZ!aC>cQJI_Msy~MYkYuu3^>WyfjOWz
zq2%_p_#(41OTdLcX*)^V$MTGvAY2$yK1)Ydj_NEgAE^6M484d^b1Lg4%fu0+OJU=7
z6i|V7v}bdh6?f}53p*!7VAH20z~hQvy-2NX5y-n<KbGmmreB#JZv0p1r%ysEh<8J!
zHTN&YSu9}MxqPkA)^viH;<7Ju7c#h}BhC%K$blqfA9~n0cf>T6pBMI=FQ1$2>jUt(
zH=+SE{TpiZqV2AVtqm!9580x^V>qU<uE$R_hPFoDn7?D7Yo(LU@>!f3BJRc{!hdfh
zmc<?Qiy&dk^o{X4R?^2?c<bAJM#SEVb0G3wWxy)xQ4TS_Gf)4fA!TQaa7YZD8F&?N
z)XJZ6h5RWg>i&DS7bl>tVVU${In39_x;`_{$C;}p^jUzjBekV9*LDCv=M{GP0+&9X
z_*Pye(vff+7N+Z9viL04S+?k6Cr*dlriF@>hZWMO-SY^&e49!wq3REw<dtPejOl7y
zsyyv9JX8k>Ipq&b&P7YepvansaU0yq3`!6U&ku#lHvFz1w2<C~kBtufTqcrg&#L#>
zRMPX%!-)+p9CCX6sfW(0+L&C6#bUn?`%uDq-GM4*fe8N`Me<gZu|C&{+p*&PjK7KU
z;!SXhG1^);7598`e1dKs@pYpARUsfy;zSJY$pOAVgC3YK+7@t0Gl1{$f@$LYizZc|
zR6C1_z`NREhjqHWjZqmQO(UYF=tGkd4Ipiz_Vqp!sc4)ly4MM}9h%XCCQM%Ovxg2_
z`p>}!Ya%Cl7fG~(!&wU=^aRHOokI=%d>dEQmoK#$I~Nt{Zk0+zsa6D6*n@&@CDS6%
zU*X9mB&*A(i^^!=8~u?Z!-#-Ft!E|aKV#@0wi`3V8y@XChlu{-`Ej?d03ZQgk;}uI
zN1L{e4EacQ@-jTxdHhrJJm;2E>Bka*0aZ5EkG)l~z;)!+515@(Sn5?|AEW|`U{&;u
zlkT_?StP-9Y095xCaOXEMk`A*aDDW{(Dg<^AHAB~2f|$Mz#m|fEHJ>T@L^Wu<7`OW
z!rO&A`uanf?ec7@7j+pa5cIWB^)L0a!!E^bwHTF=o|mfhzjAA-+S(AY<TNQbg;^jx
zPSr8vSf6ESTbnHQN2gFoJp<P$%C+~%0{l*AKhf)#kx8)6v$0;fDdBlQ|LPFKtcr-|
zEaur)>3Rz6^Te8?IJ+)!+vG%E68-8qk$@{A94Ey|Cyhnnp`-f=jCS6vrLsZ8g>4aR
zXM*<vb!+^0_@tS-<k=e^-avlLT!b0mCq!Q!f6aLBlobB!WbJ}#W^5?I<!*|pV0W0^
z*JtQF<$(a^_j2*X$P`ViozMm!K{)Rm7P_SlYs#CXP0opy5!$c<PxYX&JGU)1ZWhI@
zr+xGIko0%JfDiC|mpJ97Z!M?<yDTBphtQI|1-uadsr{Qf?fopBr}$6cl*lV78-kl@
zEGFCWxqN2=gig!fxvkq4BcdF#B#|O<9V+l%9($2qs8D!5n$4P3@x6M6dsL#4m^Abv
zj}qSo{36JTcLtdm(so5@Ju@+OeUQ$n3W)Qn<Bjw31gb!D5$HKi@gROuK3&|EtFSsi
zk8w-~rTG;!mis~ROl;u$e*5XLWe7)wYrw<D+F>KMz}o2~W@jE_hA~!YKh9tf@_f;J
z2Ua-MB>04yWg!@eu|RO-s=#Z;&U+JcusDqJbY;MBZqs+BOQ_n=1GM8ufi>4s`=KP4
z$|2E&0!1C`0I<}4fjoCHCtReIP2~hGn6OcBuXXFq8aiWylJ6ZNNb8#4DXTjf63CDQ
zCIU$KDuRE*-oJc8I*0I2xD0d<<(f3G?H!H0>%4<etPdg?cvO-z@jzGn;l>dc!#!G@
z)5((JvMh|C74pbimdWjMP(izdSFsA07>)r{p58}0J{^A|*ywK`=#i!B`vs!qN3=MD
zqsl=_?bKzND;?GQt?ChKStt1c_XaI8HA|7!Tk|<Pz509dqIN&BL`p^y3buu>Mq81M
zTyP`|Hcq0dz2iY|Yn#=%G|6ndQ5?7>Y)w2-!F_fd3|2_+($Qf!TA&OcAg3P3WYAei
z&s818bSFXrEk5U{NtkVYP<GvHMZ0v#5Ep*Ds}WM62#p(HWX1V7bV#L#{*xf2LbDAI
z*8cr=6h1ZJ)o736n{Si#-j1%Fj)*v7^a3BGeS4)gZej(~)ofOo?MQm6vfxR|*ovm*
z*{@Xgo2jg#Yzg*me>@s$&iIIePpI)r->Udp68K6R>B{OsuHLEwjbu@!Y&zb<VZMtp
zNT`|LM6W%jWRmnX;}b}l`kE?p*w4^hR0l(oo|q50+#vQICb~QOH-O<RnE_2_wS*~o
zNBiHWRm+S7ieE$I2*?}8p9@4j5A8R(=;ybN=tN!Z#Z6(irV%F3qBJ91!g%%}YL!mI
z$G)J6xmga19c5SC5&W7DH`mc_@d>c)OPvU+$k?Ed&Cs!!jQt#k^6jPBcqa2Pd3kMl
z;t87X)ca1fa36_*z)Akuk)sSROH<ahM|yYC)@^J(8Rj?<i&5&ZZd>Ca>r_pjPa^Gw
z=t<wI{1$vBkAB=gE?~*!*Q&)|D48DT0vV~1XU7$LB&5vOe`<hT=6rb;K*e{w|NQ1@
zu#_B&e#CEN43T_VK}{>sE*5MZ?Wa<ys^Jo)oRT7f4jWDTNjW1A49IBpn*LU>QJyZ|
zj=<rwze*_6mTKI5Vt08$I5JCO&e-A_cKj|qw|AFm$8PpZtF&6eiwWK&lFXxFzZc6Z
zE3FWg$ytYLB(6H0UyM5*DRj2<`9_o}8&Xw9-0P;S=hl_G$p5l7nt$~$Z9EX=0MDd0
z0(45)_HuWsBh%e_+lcv)oh`jtMfNr@%NoxFYGo;x{9iACCxYAC%9(r&2DkKVv@R}l
zCyE~g(2GqxCK+M;fwE1D1kzM$?5aU&3iELZpLEcMa-#h-hxp8A(C=*Mr_KJlYE&_k
zLFB=@x#QoK$N({P#a@v^W!R|AbIdN6ZgL9vP0@5*hp5*RCX3ZFeSG<kT(@wqM#K|2
zv2BHCz*RP;j34Wt|6JCA-6Icrd!$+_I||R)tLF01ec=Fyj#MXZqMo(A4UaANN6BJ(
zX#2%!WGuM)f%A<w1jgMfpl{39>TD(y>ZP(V;n$33zKY(dt4KE`DYuxG!W>#nke?p|
zce;tU4}7C2e<85C?1#u!KZe8TXFcwoGcP^wH+ZsJ5>i3$a_vOz$=5IPe~|-*dY!=P
z`bl9j&Y!o`emWm$eATXWf~)V6&bb+e8HXpjs4)IL@+1<bsdjs!Ihg<_->;*S_h)my
ztp-0ZlV-NO4hvG5ck(zLu-L0nPnVF4!i0NMIe~JtD0)z>cpC;t4;@YCUWH$#$zsn{
zx3VpXe3`H}eZTkRol%@Yi1$LA`8{9Wu~8j2s5>=?n<clt$7nb^I@V9&dumA>&2KmQ
zE)EUZ^W7acAlQBJK83k|FEATjY2FdDKVIYFcv}TZhXC$b7hjki)P6yz2R3p%tHnt~
zU9`fz8j{6@v08>WM&zR)E#u_$Ip0R&{{WirRLp)@_{o%qmumE6h`kOEK$9u*KA+O@
zdf-?Soc0-Uv3`2qO77km&PO@kk&a+X$G~0_nCY{oTZU4xLbcH%<2sW4U6Nj@9U3Jk
z=h>U1o-N)xw~IobNmJpP1DoC^y@L29qF`MDx%Fo10Y>bQK&IeRDg5M|j3uKiNsUn)
z0Y<LF-%=kFJV-_b>nGsuxrS7*C|xVxUA8{(pk|sSJL6K8EvITnUd%ILN)*Yp_~AmY
zzOp2uJR48=0FUuIVgxQS@ZXz;CbF$f-whTiZ52a`Y({4W!;vD(_vkgj*ORlHYhhr8
zy2OSWK0oT7Ete&W)Kk%?$|<c1;{)RbiZ*0*iMi_E>Rb~W@>0eeYNIVjf(W>Sh8!p>
zLu0d(_8UIk8lh)}sxD&=2_XWr&~-rjvoQ}5_jY{@_wE)`3LgTiyP?+i-@j%vZot#U
z3lW^-$`RJRxK&E-SkvbdPLC#WlC{Yky)%6GYcGb<U!#U0?X|P~rPY}|V!ZeJ2sr+A
zzV+SpJ~CL4Z~844r4G3~C)z-jq-u4$y(os6M2Tg+?xsC<*Iud%t}fLLYV3vpGvV=F
zuC?;M^DVA9t7{<<<W3eT(cjw$gCBW>vgjm$7Ghl*ogyN{VoZDZ!BG{1B+>|fDnL%b
zs1X7B$54<!exd+&EU+y*w!3oAPy)?!#aa2L8b?)E`oZ|~>i=@S-X5Y9R&=V`sXiy~
zImWVAXBd5SF15XSTAv``h43es_=c{XcBruItW32()dR&uyHRbqFTm;;&*R-p)lNdm
zOP<%a#}E|*hqNgik^y#6$SOZ1sKcU7vGW$Z@T+_4Bw+}0H`b0Kx@x-hfDJKaPTh!;
z+Qd4fU^g4&tv<*;wJ9(*+A$*Iu32HX1HZ_a+A#%qJ{PWO4`2+rXc|!k!r8_wKLzp!
z=kh=~E7L;d0~tcA{V^EFxDnA7-M9oPE)n{9lGioF<ee)5fTURoW?c^}hM2?}wm%W*
ztn%sWe&%BIz$HdGp#wpS^27{5mRgjp;0!YA2o{-W_R_7dH#uA5DQC(T)JZK+h`hu{
zrha-PuEj4|s*8<T<^+=g@GF7zBI}8ly|);&wJ?6ZWt{sDg`GhZrO3x>mK0*8_59%#
ztCMx?4<Ay2cLzD4f;qfHZiMKWZ_&mNnoP(i!EIS@Q#`t86BH%$MTug5qf&7!Al1{c
zEuR8A;$oIsrBG<r_X#4K)?&hBl0Wp4X&XII<=1^NkLsIoECz6xwrdn0o3=ZZIn{vb
z`ogWAN^Z6uB7i*)??tOdZBh?kz7!`sAh&B|%bWun4)t{LX`}&qOQr_mE{Fm*W_gVy
z*6?Eq1S}NUoV03gzn437j-&2^I#mD{DA}Y}?H{&t%|+);yIUqA&$4qeEMdY@I}&S*
zciQb-wRdJ&?D@3rsE3#NELmhPJWb$3`xwVV_O~2lm)<-thw>GZuWVn0yaK{G#yo)F
z8RV(XiBwEQU%?UG>=IH(73dy<O~!Bke$wYh-XDZ^`JSR#+*ElV5LW^lpXL`s^&PM6
zY!2#6<XfXd)@~Qbga4)-qt3%Xi{Z7V=NiGYcpr{7#xCpxYWqAJfC=svp}cnmwxv`j
zhoM*BzR5bzb2rqPU|Bd<Lrja-wb7U)RJh)u?Ho6av#ewm5E=To88%o$Zn?LPFIlxa
zb@&FO_K!v0`rf~(le{9ao3rhAy=TsYP(y|sWC0!WpPXo8Z9hc&=xWWVvDH4!7qj}F
z;1wRnSwYD7Ndx!w^5O}O4~q_i!IZTdB-Vz5KJguF7*jAr9GLNRtXHLbs8-L9Jh%`&
zGqc2;QuZVqcCj*qIV6m@Lp6;xAQf#1N;|YafdPD*s&P0gVTvABFN%&^33nA6>KUhe
zomm0ppBd+OZE&Lo2bsA)+eigm7=TcEGs%vkLs(2R1dkfg2w-IrRupdt5{J1ijOj%U
z^pLY~ls|q3$Jp8bX4a(5L@HE0z=lmpBZrk6<+h#^?|xG|6-u9Y4;LV%-V64Rw!gj9
z;<XyojmUfj>YPWWd-<Z;91nM>>`~OIu`8+QYq_Sd2O=1Q_+5KjMZ~+f_EObxWcK!?
z3!K~g@FC9HVP(LIss`ug@0%*`>6kpQ?f%x_yrqdVShwdPV54zz+9{QoWb5rEUtEW3
z$hWi?AwKd?8AD8vf2PrvCznjyfEWY?)LGFL$1g?b#Zq(H0+UuAZ9N+3RFuVz>rezG
zxj@bhn=sYz$pVo8ZT3G`Pin2`PqE;`ZYwD<%aKI`?$7jbb$-<3nyE=9zXGlKA#0-@
z)Ka{QRHHndBtSQrUhAN*-=irY#(u-5BFZ@4B(3(YIQkyBb8G>BL$h~90&Xx8*=?au
zc9huPpwF6A@{(J5M`2uLd_=~EhTz=={wX#s+>#TGArHGc=z~tRG&yJRM+vEg<auF$
z|Fb+u55zGR(GjukW)7+_ir%&W8>DMkh?PT$L7A7|*K2FUh}hp}P%%9J)cZ&J$c*Wp
zDxT%<3oanJ9R?JYbW&wTJ=?-v8{QEKNk_SEqc=P!G9F$TI-}AdE$CWy6c=H0yU~A?
zFlqd7gX9puea9pl68JPll>5}3(F%Oxt2I)Q^xc2bNk!v$*Vz746-uJaP;i#7#J>R>
zbJ&n-^9+8x-eTW{uFyGEqW!VBYZhy57DekhYdJG_can^~V14V$fF;F+(BY|e4PLlR
zI`Z1F@&{Yq)~28^lJE<WKgxf(V<Uxu2~zv2`7BRVX<jGh7!$&A1_NG>mxZ?>h~}LL
zT#$nq*kHPqyZ3eA<gm`S*1Q{bCkcu}SE@%o);8<U-Obq&syzEgqD=Y{&*Zr+f7=SL
zJaiJ(H@*<QoGw25Xcr|!bh(=g1o5UK^691ou94OFe-%{?*M0mGC4?goJaCF*RcabX
zH%j8}_iNFIVQer85|wEng6#GFrdv#=;hRo$TwZj?F-w3aA5IJU%7<CUW?Ih|xA^|N
zRU!!HO`Ft&a3RpBr3DL?typqvLKpx#%;|DvC7F`G-b*9(M-36VDP8Ay8`|bc8ZnAO
zQyRRJ`1#nH7Cz~1$s}P|{Gx>$1b!YuW+ch+J)NR<yv@JBGp)bMHH@9w$>UP6DgI}a
zU7_JLssW$z!)2UXk^pp?@aSFzn(@1ER#>|3<^r52+|>#Z%?wKs!H1zEwF>%4PR1~i
z+%L<cH6?jPiqi;df?5DD3O5TMdU@N3iaLan=zefH0vhb5RJ3#<H7hm#S%oYSn((Z!
z52gHk8~Kbln(`}e$}}7nlz%A@RH|;u(C}vCkZb$Gjm?b5%2Qk$(`06QK$7+oy%9G=
zbfgJKj018H<spyTpS@O5^n=upyBM0K--un8dRwFlUVjuCntIMs{9F~AfelChgWZ<e
zPhsGi2Yt}uPx10I0Z@dS$RncCl@H9n5dB>xOL4!!5ZUhn=3_aU9t744`GtshW2w`n
z+yV5x1U;LT8;SIxt-iu)&{CorR}7|Or<n`$xUaNj`ZTRnJ!!hYHiW@FLvL#}#BIT6
zE95a{TreANS6S1Jz}4luvJ86C5{6ax2X${+I092PY85>$sI8ku9;J;3hql#|=)OhG
zi{^gOp|xcw$>_52Y_U)+_zO5(4;R%z)1YVlV~>>d%_k&IEL(fjmvpr&zJ1=&2&dhl
zTfg1b>BZcwd|MDIf6oYo94K2fcqPJE3+3nw^{V1?=y|L^X%$@fK*04}_ai_2E@|Sh
z6#J%!l(wEWL=Y^o#XJxNX1}Z7{!%}~zUgp}4{T&KIs0WunZvyV=TjYa5CI7H9>+&!
zFR^SFbjcco#iI`fG-n=TkDmkX$>ApNGv-E$o5Zs;Zwc|JTe4Vw6qHyNO+=n}@7_<e
z<@p|A9gYX&iM@Sqn|&E;YhjfpWC+qUs@l#MVh2{a5Dhjv-WGDTz1vvDV2@;#&kCdo
z6ken@98NcsqXA8V-;#c~lZgxo%+>z-df!jf6Ote$m#u37?bwqqksL^$_b1qWV*}pc
z_;i1vAAD^Tr;xKa_CgujOF0@DsJIgE?x^g)a-OoZgP=<|zts6*oyVzMe|54!`OI0I
zL;5Ud(^!ka*O1QEC_n{~bx&7OHS?5>X?AJ5@I-xF--BTITUb4j9hTPkvL@eyo5b;@
za8@T`XH4)|d&kCG{{cgrgR_rCTGO_2OmM44L7SS8=g4e)MRLuJ7GkS6Ats6W)o6IA
zKE|}{%|So$^IV4<C)!RaJ_A9Wk_zxev*4m_t{pA`he4lS+~uHkNx${d+)#qHHd%3^
zsRXyD?=eM9e6A6a!C2qakP_$oz-~!K$PR!s3so3iT^pRUM2jWT*lH+=Kcpo8foSGz
zX%3kw@1*Z_y{~NtCCp0QbNMiC3gkR9jCx&0m*ou8yKSQkzSolBm_DsHNk%=N$S)D`
zr|vg25TOH2ep?E0;b9(@kMg!vmcQZUY?Ja&$#bo659^LVzeKCjy`}gyG)2KpII6W%
zu_||W=Uh)RyptwQBV`YE9lv)VT*<UOkh<e%%~FxtQwyoUEu&$`FeJ=k9sP8I=3YKb
z5t0ce;fwaWQOgQ|;S{1R({`Qbc6!?tIaBY~G)y`>j?Gy{zaKrr-#q%*{Z?ypdrpvp
zs*uFFsS||^cRDc~J~;TxS?=9e6XCE|HgB5N%Jvx9>y>5fudJc0zY_E9l36wTgodY{
zyPoTi_I@h=@OVu382&DwJ%yOd5v)A4zc#lO@WMRzM%iUMsBb?w9*w6)87T~Wji1rO
zD$gQhJ`rjZ#VgwdtW+<!8A9<*I2CVcHXv0GjDUWO%R5b8cZ2W#XemRl^_+C&1Kqf_
zPG(KJtO#iK7=bk9pK6<^sk|*4?CTSLwN5e6e|)Ob(E=`@8hOfD<4saUhBWiO$%rxS
z7*Frz{nD_K6~z<s_<Xpy#>&&l)D&YR@zwD%rB%y__n5@Gtz#Ca=F0C+QnsI~``exk
zQP4AUMr#o>ZuvUxuO~{6d_3+=YRHZ|46LDgDO=D)?Mqyo8XZViC-`Z`k1L^WC8{<g
zdsRN-e9d1r8$Ee@sTL#IW8bqtf5}i#fZcho0pt=|*>7os<lq-~;-5PHLJPX*^>C={
zYC8pI9e-m4YASP09$3N=azeQ_%gT`Hyg}w}XClxL!?Q(H(Z$}JXB58zZ44uK;`lyh
z`h$HevtGli@N$IZt5$44#|&0L%2&TD0iAo2ob02okQOJZn(huIc<fj}$Xb}UU@Kg)
z;b)**vn;rL>ET&lIn&VlkO{6tW}fli)x0YNmA-iWIU;)RK-qf)hQY8wz%Km;COaDl
zpxD;m7#aGGRX9PoPDwcGtbh#9DPntPGPSTUq~t@dLL(s05OFJR)S-vpy~O6ZH*)<o
zFN_BinWIN<FKg6~By}QO%QiNw)XyD@3SGjj`E+WwpZepzg{j1))XnC~Vsb#5AJh~x
zh$_hHM<`THF5Q~;oHqZ4&wiX|xiv9K>Iv4F#&SY8LsYHcysb_${aBq}skQ3odK0?Y
zy<KU;icMFmk%1V7rfV+_mGolig%IAuN6d`5hpW9FIB_Sgwl>Eb@fI>3Svw5wJ=@M1
zOLp?$0ONQSw}sKos%LMvY|V2TiX~DH^Q<!mO!`l0ZKQHjq=sjy&bW!e^{V?4Ib0uK
z?$@X{rVxXzYw@1!ZY1^s$F1c?WUeHrXJ5o~Bf?uHk@9)xB?k0l^&ERnklwRJqpVkn
z`JS=sz_<~pD!9$h62mf%<_FKEU&)v|3c{_>Zv>8FQ3ahe5>$DuMC+0s7kEG6{k|3G
zKnk}BXfwO|9`|_Vlf?*OYQfT$&MLYrO?Ui21CJaEqNby#cPEkP5FVQ-L)z6bVXp}D
zMLT{?XQD=4;Hm1$4f3<!k4YT7a(TH?5$*Zz<b^Y6v*-BpYNQvtN<`QUnYzU%=%E>?
zmtxhLAJx}CQ&zF8z&+l<04mG*dNeYIp}^HVOL3Cp=j;Yb5PQptis+|S)aH~QO?k_z
z-K!(@%m?c8&JS@HFhmmsK^iE;5zJzFoqr|33Ach2v=R5#nhEFnHKj1uZf^*_lLYnJ
z)W;GZxd*K-?*q!uqL#1fp3<6g#Kt%AvM=ESuiU9uP*07ad*4!0P|ROc&K$8;yj|-R
z@D=-kh+;;d%Wz8!M;~x47imJDuy+@py&xxwFJ(|g7^6MIvA~)z0?x4&XNc0UETBWC
zv0ZXLT{%2BZ%$WNOAG|m+s00VEZ<e|tga&OyZR?)N1>MP7_KZ56FGi0M_m(4BTY;7
zR+;4eh@HO?r^>EfO!a0;hk2nehia`eRG7dzA)M?xS+%>ppJX}u0LdeQL7y9&Cx&ms
z=y1*#DtgRxM_r2*rr{&qL;Hp5IH?;8CtT(2is^-cDWjG=*_J(ynf9`!<r3IJ*_+Sn
z(%3_&u^GB225~vd^NDwlkLiaJ!>TeeA1kEieGjG?p2!EOB6e~?UCShL{_JJ4e{9)%
z*CMuPk;iYmUKl|IZcBzE#ADsWigJ`#g=i-3!;9Lwx(DU%{avj=lQ|u)qAf;tzMXNL
zv&20Jq@fGc+?^TnqDK+P7c%3QdA%Oar9f$hZ5zGVpN=l+b1xEK6;;tobvG@zGs?Co
zL7A?bQQ<UIHVvB#xYm_GCS5TlUnir5uu4xJgcA7E8;~dc_?nY3q+LfE$`;uhsQJ+M
zDyjctOlXvY0P#=!k+JW#w4EixQcqRpJV~t!JQ`w2G!w>@<qIVDJ4Ea)#*YQU7{p45
zlsP#gkra7>e&@O=-m5mBa(6!aZrgUp^4Au{+-%)xgQ*I&#lZV3Wl5h}O)g?-i$A10
zBB6k#plK;4U5P|R`d;%|KoOPp%YI5rM&!Qd{OTqlvmdqQTR*eohv2QTs^gseQs?Ov
zsD1f$AMKF<Jk0JfXdJP1uXy)>A_{=wNG5{51&J)|8>a*}OPV+Z2jL5)iJ06#&J*9~
zZ!-&93lci$YEu<=8#NbFQZ@#}sC?3>rp})J^3w8t<2jiGu1i~y1{#}(4JhWMv+6I-
z%CT_o*3M)D?~0g)w)e{<kmj=TSlm3yhp|IMwQ4PKFbEK|G&S;Ia`IepYajnE^V-Nq
z@;L>Y`qwB?QUnd%n=D|aB1bd&XF&uJa@}L7X<r*_@q1IJy_8QP3&;HfK?|Xwg97<}
z>ePR<WGrCfwLPe?X%}bIyrS5z>+JK5%X=i;q=h@^Y|Lz6^CqE1QG1)f(hh7eg?X}z
zA@oZgLNs}K#^&`rEm=y|qu&tPTTw=^lO3f$d5lf4TVv$<JZg_EwpbuO!3M!lJlY+=
zhc4IYHt<&azhHeO-y#@+9gpyEli+#c-IIa1r&>R1n|K2bJ!yp;zzS>hx;JAx-Ych{
z-(?*@w$%E&HwLf`{3%SC>5?q1c#8(|+PR~C;w?WL>s}r@GuG^rnJMUsOB+t;$eBd=
zuP<XPt~OKO!u1z}uu)%&8a|Vj+p2o^_C%ALm!w*R*X!eT)q%fW3~W_q<03lUz>W09
z#&>QOwzkQ4v=H<tPZZs9!FO|5q@HZiVM)ZrhZid!M$t|%QlvD}f=H`I#rZM}-6QP_
zHdosrPYmDf&m=We^`4jMtL6|BSUyB)B(k_9*hooJa6Rqw$mr6!>ojlCia@8Head1*
zv_Jj{ywnJ9FQ-qi+wi7ZWl%*pE|{IZ{^wmB)7a+eRp3a<{AuHg0`N*A&fJF)oGgI{
zG!~wc=w~?%Cg3W_4Dz*gx&r<Lw$b(@c5b0f<!lYB@j{>X6Jt2bW1CL_=EMkoHr#VU
z+wIzmEJmMZU?>4ah1S<6NnqIb=Sj2QCt_^_vc^G_=Gsr09V~BC>e}Aqoop*r;}Pxp
zu)kiC`I3>9t-ne*86zNu;yZo@Q;Uo10XH`De*5^!wdidAN0o2S@~ay13u--4K7RY=
zm_(Bntgtvv2E)i#VDfznz0v6I>ylO0c=XZQZV??=AZ;e5r7YFYy7K6A6n|Pt18sxL
zhzeu1^6u=MyFS1)Sn10<1m;7d#pX}Ufzu@M$&-UTa4)-%blWJIOhX9P>9_SU$ec9N
zR^x^?C8oNxUKA>h0PiRPheN?KQizZi;DS3TlfS@|EK?zsPUI8FW_y7lgmDHg<Vm~x
z4)nw5uN-k#BiY5}>~+9Cn{O46jLA1bgR^Kk&Bjp6vrk!DfC~nm4h)VL9}Zd2eyRxT
zP}Uc~bVuT;Ns_r#r{<UeJr~=f524=aRGaU7seqZOs)21`49U2*thvtVKK{yr6qgJQ
zgB4zvP7OHa<TI6@k6oiWBtK(q<;f0rHkGh&NzItc7QOkP-$wn@Xl|Lbn<GE1czWR}
z1<QgEF^6P3B|`yr<c?CjM@IPawd~bm9wt7-QnfY_qqN^)mor*z9W^B-r41miW(#|c
zcfF9eT>HEy{nGe^VAln1Epqkc{a(5{B7`kjCDxc%SQyl_%8ooMb{HrwS43?Vj4r7V
z>915pM|7xn8Wu02`u;qeXY;y3C|=Uw04|`9rLpsYHqorDs^f+3@TH;DBPwH=9Rjb+
zDND8!^6z54SAcG}e^Tj2B-ik7QU3z#gr=r^bJUigtsFodV05w7IfYNTNQPM$W*O$s
z)}!v3<~;JVE*^Mwd2-@e4<7COxU0IftV~E}!n}YvP9JJf{JSKRuV9BdZ<f`T@!8Pl
zJyhtGzR822L1*UHmU*2ZYg9RY-8F~r`$s;`wc*i{J=FVkZE}57FS}xxwL1xNwId6z
zhLs7?`F<BZdCaWAh_-kq<wFAg<o7x=4fMrH9t}jlpZh$4d*&r8<Q7a~xq}v;u{0FD
zRz2;HE-R*gXUsO{u{yL%un%lo7Z^kJl@U`D!po(6LQ=~Uc(w?Y?ttFPlv7$0->}?*
zFTpIC?rGw)eB2=IM=#hn_vjAud!-q=^0=St2_GVM{hR)oNlpeFnFq`hDA;Dbm%-b(
zAs6G8GMO!G0?lgdyI>}Il+Xm?wZAm06p0SwiDo3nRlJrhdNT7$adX8HR)IWSC0OG9
zNs>WkMn3)3J?;p*w$H0wbC$LXWUfa4sVD{Q=LgTS+U91O8+1mSslRRR<+Fr-sn)sP
zx8@4#Dtjo=vQoQvP*VXH?6>($<%*`|{j_h%r~eo)KI<jx5~Dv)6hduHd(RMD2=@5M
zPbPalf&Yx`voL+8dlI(0x|)w<qHcoyk)0c&5>$bB4Efyj8f|hcs+~Z&_@$*?RQ#H)
z0O&O`?fI44oWx<|XbQ9Xywd4UjgS@Wr}1{Q@r~`t-?!rRcbLY`jctDVJsMS)FF##i
z&-$=V6nFa7GJmVmyBGV6SAJF|VtAnQR+peI+r%&Tu>@;ncFw~T1k&~-cE<93d`$`A
zY*IK`dU>4&Z@{>wl`l=NxT!(+(W*-4UC(urng?zJrh9kRhZ=CWHO_~lRQi!?|00`8
z9y0@yL1r-%*gNzRE#vi9rDqGOUxR(une_@WXs=|82Zjk5Q*uI5mQkO^?jQNZaHhpu
zC>XEuGKN&_FN}^nIzU0}jj{IjsnU`dDBW5|%2wB+kQeh4q69u}{}p_A%#eb(-dh=r
zq1NZo!(Qo|@&y(C7GPC$C|B-F*Y@vzJ?6Qlq*i@8ch_cnG9es-;DX_1o%lMH?OW<N
znv9be`_D;Hen%deo8i8J_Ug({YMWPHm_T(S3%aOibKoKOEHl{5Z~66VH8Gi(<#M7c
zU@}{aOzX>IEZs+8gvFj)Hx8zxS_*XK#+aS&NJJwCW1FH~-O|lm2m}(bCl2gN=pWH@
z?9ptu(-OY;qi_0~uTaa!<pZ(LB}+oDmk=qH^qL3~gyv_Zg?V3qI&<m)XxT{~oQuE7
zz15+~z-Pb@iswoaA6+<zHZHF{-TfSgc=%_W(qD6HoXIw}ifc9yB{87sbh+(Yy%LpY
zgh_}X8th&jGBzz_%>Iriy{Nt$0;TNayUPyPu0MrGlxm^~zt~M~^g~?Gekn`-VgxG@
z2>q%;INT`8(aHzTP9gf(^l+C3>@&jts<;6%)7W}SjzN;ILPB2^#@lhk&jl$`k!EqV
ziZgzn<$QOQ@9=ow9X$B|dI9KLYP76em3V>`Dg#Kns^yV?y0Na2ArCSM<h?2-QWmDM
z^;|m0U9Y--9LVl+#DwvX*InOlgNt}%6#4PO5;F0+o`!9qfsA;Pr`Z{GCe2a8{A4M|
z+l2x7_Eldx0{##VHfT6z;zEjM@p!_gdnn-&rVH0cAgY(XTr7JX8*gjsX+CNdXf!;>
z_)b5TmMq`NQHsDVa13!iMM6S5ut<au=^;v7C~dDtNPFr;XGyr^$@R5cp92W+fMY%f
zhG5axjcGRsW{(qjO%U%O;*WH2&76of8v^?T>_BP+JDOf3d;@qTQ!Y%biG2UEEAqXY
zO!dMfu9!cVM>KH58GNx|Q@za^fjh~Ecu1L~bP$&t<<~qvtZdIP_W5!#0k^X$Wn_uQ
zLlooe>7^rbAeS00A>FuR!xvvM(Ao(~FF8Ij{061KpSDn_XE|5fK?9A@i|=<gdx#kN
zkXu<lL<iQ6JUEfKz(xA*+S$NnUWn=gZ0DwQ3eP|1k4C={6^A0TDxjyymk&Woc77J)
zCTbW%zSQIeTw1l(i!8chuWEo7a+;nM<H2B0ME#r)Keb&VA}?;ViBAHGculmwz;XEt
z$`h}=X4Akykfuu8^wQRXV~p|9_2d44GfqS{xFW<c7T>N1KmDbx-g=lxJIM{In*_CQ
zu0~6Z=#q8jIzt7*YvCJIXD>I#-pO-h4z)$0HpdZ(rRBA*E?EGn9_X}w^B8l?$B_kn
zWyXj~?UaB#CE=A(@c{G-Sl#HxnP`?;!~QBQp9LRB(6hwY2Y^qEHOKmDNyTelFE0ow
z97^K`xQw2u(wCb#&o)_{)?1xd<o!=kMIQIvwSyvPDQ-$7Yx1bz{;^5&RQ-~5lpl@&
zLtSaCK8*Dm@eGF+62E|W+2wo5SZBX+s++G|G~LCm<HZevE7*ADXhM#!=C<(&cb9ix
zc)8S4lr6`GZvz9cG6UpIwh=cHWD_XzBWhYSYcNgH@c_5&`;b!qTCETvk<6@U;p~J`
z*S7w+qvOVnE(!J-b(`2bEe{132BN^anX7b~pNrkVwqh(cW^b?8lOrU%((xFIY)@Xp
zQV<}P^9PGD&R$dX`zIkO!eV+QCsTRcn`IjcJq4V;n@Q4U&QOI!OT}7+@q2PccSQ&6
zjpcmvK(?m!A@+?|UM2aBfqjB9lp<V1tnmvy{#wNBpYU;Zl4QJ9CX$ceeADJC`!q!y
zuKHQ#swvOYrM2}-F(Wy4(QBr$En{b`4T(ts?-yjBhdA%XH2oUIN8ecd<W6`hZ$iLg
zebqJ9HtD4{RA}NFgl>ic3kvq_R|sfG#MEhdtN_c)C@G)7&nkImBASSD{8OgQ2&$S*
zSINfiYyWA%c&V_%XF(e}D57<F*IeArT(hXxq^k9WS;*Yc^VFB3fAsk+UXj1nhMzc^
z=EQjuhr-k^KLD21RQSh0Mr|Dhqwgo>SQMl0gBVX$w7eMlySVxtpX@fTfcbGyAXWKI
zA_ceM?#1F_NjR0a324rR4H@`A<8K1T=0vI9<*m_>rJ669wQS=<NA)3}dc>~DUn8ow
zqvZ4jgu}rCI9kPx*WvZQn<pNZm{YmpzLTBp@!Z-gdW+tGhv+QcLBhZyk@K;O&8Q%@
z9^ihH)*_BClV~M{=?<r?s})S*qHS{?Fno1cd_qqD+&aqmAkk2Qv)gIp*y_%Zo??5{
z205fc8Z1I{wltnaAgMsPrz)i4nVjF9c$}o~;aJzV0SvfK>o7)X=22oDPf{4>pkU2-
zGRt-l<*LF|c3=h=5aT9!K3*&xa3^Z>_#V%7+V6eJkNeLU+R~U-M6*B>lMCFq%1qm%
zK1F5h_mLq_YFd>J?G5O!{1YTZirjUL2u>3}$>#j5M&DAib3I{cnc?!U97Mey4Tci5
z*dB?|5)=>ynzpDx2PrPf>9;lx_#?#MH`?3VB<3*d2p_-1@tze=jF9i#NZk+!OXrQ4
zJ;!IX{!00?Zin=7{VCSVnhu5XGX7dZKe#mNhO9h%V>Fgidoqa}lX5X_O9?iVFy_>g
z6!jcmjJ!K*bQ>?+q;ILatv_m7vH>62TRVm`a{<X$cY?zs_g7lV7lt4daghrl%gNLA
zetK&h<rKP*KuD`0<N8O^NJ}tQ?1U4{aFZaJ%rcc{Nfp4v_6_y=EN{mEy4TF~V?U{f
zu`A87q<XKO`0E+fpeBl{=>OyDEr8-$o;OgO;7JJX354LXxDyEO?#|*ai%SR)f(Ca;
zupq(R6C4&;oZ#-TxXU}azkC0$-m5ybMFD%}%=FhiJ>6gTxsOF`^<~~Q`Rx=$_<bY#
zhaZ#o>`z*&^H~m%f5XR@+dLfnENVYodrfMv)=rQ<&Snxu*O}tG$?TmD_h*!wT3e=I
zox?X&($8gSQ=&WA4d23&+>Z+q7Vi@rCmnBvRA=i!QErfW?ne*hMGE0DwF6IhH|fiS
zxZNOBlP5aO0Jhx~`N(=>oNGrlL4T#L?~gB*GAHZD?chi!C;6VYtwlkHvBpwO8ciP5
z06C7NEw|c(K?yDrFa8aaq1S$P64ghGRy{h;@SB7B#WSa}#azgzgH(DPlE`PFmx=EQ
z4H9}Y&Tr_yH<!TEi$|HU+*uywbo|+5brgA*QxiNVX~FMZ)AJ?knwwf*Z(K5(5}9Re
zA0yp)Qy<)j*nE_Ro-rR`Qg|cEKJd;$?TNe_5oY-s)z5G?70)hf1<k=)F+4LN@9)!7
z=MpJ~syLturtB@e9AGt~(0g(jwIyVhj5M3@r}5d+!p)OjoZL{TL+`dunvfUZty;P$
zL#^UZKfW_>5;A$b%9}UzfL%nXVzjLHwc4tkW?T&m>gA?1)byPT8N@m%aJ^X)VuxbI
zlj5>eFa8V)<A{8^Q|)s@lOk<FEj|Q}A2DH(@i9L~L-tFq=BRZ7(-f--9yH4+)V2j}
zsGs~CfhHBLY}pAXV^6ZD7$mWOocV}$YGehk%(En2Wv8V%Kp4BZp?$CVrsYr9^KNYE
z?Y<1hi_Hx6>z)ktzhT%CaZm^8(^JBawfbyfPww%i1>(a%D~0Tta^K{7k%2os3g_%3
zUG@9RiT;fb_asLiNmB|78rEN%Q1V~Km9u)JfQ3K_Lyvaj=NR^P-``E^wcG*?80wL!
zW0T7it2RgJNH)Hb_&9?Sh(0Zh)!^#COAvA?^mcC&OJS=2&0bwA{MI+y;}7mFvAo0Q
z_KzDsb$@5!nS0cWE&ryRsd@WI;Yt(b_0Y+VZ%2ZAIcH=cti?F?em@D|<v5t=T<!t!
z5iOZAuL5g5Zy%>M+TaX%O;$|nZN<4nF(tg-7FkEGjG_Z1VWPR0zCKN@6hV7)a#3=h
z8Qaw{{$yu#sS>E)4uXw46SnYs3<n9WcJe(LL{pfuJn@QVnE@`^29s=(=O++1Nc~ji
zL=A1POp$Ky3(d?Qz`UDKwOKReo5_(BV6(<~7Kly^0$w#aZ-tX(6q^6VT@A&6U^<0p
zjAF{Z)AZ4G_*eCWUs6T@mvURFMm@Vcr~QoW?ac%;ItAGfG?7?0ZKlS+q9Zv!enQcX
zzb(AhpThmuXF?a&nCu#KOFxz(+TBC7U{)Y%RRWYHx34>L673|kyPT=yE~}Ne?nedc
z_%-U%0)-%P)PvV?v2g~1>l|db+epHoxi7kpFN5IwbU%NM9M9(wHM)TW_ez2jos}+n
z^q2nFAHV)Fh&4!CK6_qvaQ|*vA7i&rX9UTx<+I#Jw*W1APH%FMFNAzOy^qF|uGqpw
zcG+QBoJ#r~a5Bl#iC8Z5EbJ>J3RO5HY{TBxbh?%pbK6YJzoXlA1F5&?LTKycZ*C%J
zH{Xi_Mf9d(^ZDdV#W%Hma(jtez(1VIdox+s%#YH>Hyh+KNqTfJi`yYOxypzWVYz>s
z0+$ljeS9AUYA1ByH?YEThqyq9uIlggECm@r6C?W8%+~24rMOlqxZzwOBa@z#&(1%r
z8_I9z;@%n~U`wXOc2LCY(0$<_UN2a4j&GW&<^@^iOe0BZ6&{Sm<~UYnqTcF`nT2t(
zZv*{z(g!6JTM)+9f|F($Io{4fU`3zXg*;)gblyIZGc`+#(DG3)M1?Nf8Ly#%wBn4k
ziZljFp?8z)J=Zy3JTK`#Kk|te=vdecPM4#Yl&~k3rv2GOFiKq&2e<b9omxJ1)holn
z{oKk@jO@3Dtguk!<*dbpMgOU+$uXqGgznHDTc7yGM9~a~=>_Pk1VQuoQo=$(OVozY
zs+TPz$ieL#15-5IHSb`W^Nq$W`QKzy)-24@OKtbSmW(*H4urC+@84R0wUK7u<U5#<
z(TS(W4+~2J0^;w%Y9M#Zb`NmTWx>E{Y6shBOKzqZ#c}0gokx6fofA*FZe!Wi`{b8B
z@o^+pw*B44wJbo*8h-al`Q%d9$J>xNi8tf9v_06{i-bb)^dR32x^_lR$O&ea|Fel?
z7?ELBh6LNF^Mh`DASc@Kh(FEt^{gSlOK&A~`13OlwfVD8THVGSV=bEBsmziY>oZ9<
zCVIqc72Lk1a0~0=_GCO!Jd|4jS}ICp4tXR)ra`th3G!?)9b(`@l=m~Weq(QORU$l(
z>Ow@B0Ahd8WidA1W%91Uu-mJhW=F<R5b)b@$<1?C=(Wil>H@WmYA@=o<Q@=Fo0oO`
zI#oB!K(ao*BQo~h>@67F5&d#b7?ki!3x~6SusO{#>t6mwl!xA1JM7GnPotOhHvOt_
zB2?AARE!rCYGCuVmA=ca<b!^jvcfZM7vPX|Y$aoRQda5!oLHWu>S&1G8uqAs)m!T2
zIns>~Oz551{#Gy#@G|G4r+-&}ddZjHA=i__Q|p@JFlT;2S{E@B7q&O(ahw>p;U?Qh
zu~##bJ~f2W#7!->QWm#Qj5G$tMTV)F<&wm>y&5P3fk(aMC;%pS{A7qv-S9^-ZNe2{
z`WZQ#R;w6<<AJz(hPeZhr1XI~irTSNO}Ms81Ab#sDtx1GLgd`-<xZex@}|~$=tO2|
z8p~(flpzV~{rudPxO<7bCV&+zLysOTvOWS%_q8Z<Q1a-|R8qj81yzy1cW)<K>_b<$
z#nI*zNMaot)EJ=Nw=k1+6J&iEcRMtgt)R9fieCmCVqCvjJS?;XZ>SfV-iN*8d5F|C
z?8l{)#v&?dj`OgLi+Laa?r8489v~z9&VSy}UL!ZPq0<>@D$Gl`U#Bfmy%tcZQLs10
zQ(HqcuXi<_oAtSyfDaiBy<~tRP{5?X`ipE9$NpBnY+I4WrgXyA+|HSLL*0dz5GKV$
z!U$c6pL}8_TSB)AxbUSNV_kBcKfmkE6tbySSr@=Hx3A&Ha_@0F`cPeL0R)=h&GF$)
z53!wW-^&KotLL_9VapQrm%j7RP!MR7PE~7<`S5x5$@eNi8yel7?`%u9R7-<BRNWYb
zi4s-V&&i<*@9BoNCs2Uct!vJVuWo;f(N68QL`zD-{pAM!u32z-<uh+R3mbL9NUlbB
zuCiEJoB&;ZR-KTnuF(o}@19@ql3gH5mHKP!F4Te-XmoQx>L-GXNAaIyp4_41TfGA7
zZ%lQ|%$nXNYe7C(OmoF3BG1RQSOrkBCyW)h83lRjfAMLQwJ3vcI<Yf-(0M?utz4WJ
zG&9gNpr}ZHV2a<ah0gURtF9**muN>_XOkHg{@v>=jy7&$2_Iaj*H_|wa4Cv3(TuT8
z4XeBD(9}BlZNqA!#GQpMqeVIlESaPLpf3sEZE@ts7qBp9<S{A|uNmdl1HN=va_6QE
z*fH-sFdnBSpf-E5x0Ow3c=g2z*`10)s>DRmI|Y&9RWL9W4S#_`HP$)9bs<YMF=o@m
zyhpnjTbN}P*9tm77i_E4`tqwvxr0%&NxNTBg#oRDeOXwq=Pl6Kfa_>n#MgFsPjK#=
zXA2@5$q!CNs_r!d(OKP@x42o5zB<s_ALK5L_)-NZG?ATtzqvA9zmQE6gDuo4a_!7@
z6d@bqxH#lKb7N`!i65|csK1Tkzxdh>LKVta=Xs{QA3fupjrh{+aFP6VYuE4dn|@mp
zwsw*W^svPVajmu-T2A*O4vOan#wlD*W3}ailf%Vw;*$j2MJDPpP=i?^R}6wZKcU{)
z=ULPGd6B3-lmsL(v<$1FzksMMCge_C>Z<7dsL$p{+}zZ$R1nUa%efC!!#8{m{P%v(
zcZ|`FZ)uy+40u&GFJ%wUy$eQJ!%%x(Eb5kUS`-OSIYPb#Wu-7<m7?k}q#NXHB2n00
z;q&>)5*X9O?-#Qk)(zWsTOl4$`5-u<N4->L#8WrI2V_1lrF&slQ{TN<RfapeD!8M*
zXAOij^~>q(!LEx2yrp)$vAwD{c^Jz><S4h=--$=Mt<;PZ+>W!aF$Tg4GX2o~2O~YF
z*)GzObS4x4TajUrM(#j}((L)6=yQH+dkO{D5$`~;QRd=1fT2K>U(8dBRB`4e`)KAm
zPmbzSctg_*H2a#%6{1_WWlE@0Aai}u-;|dzj2>K`@rG8sH{?N6=&fX;))zv7w^G_B
zd8ZDog1fArJvZxo*%~G+K<6n^ja*GM&i@d~I`Jgtl&sAgNm2d0R^K!pq|O%}O=<!Z
zas8k!Yd*|QpW;6F5pLZDmd@4UP&HG91(+&kZSz!S1i~@^6GCG#oB}Zapc>zZu{@9o
zOu@Usu>ki+8pBLzH3sA!?kmr`aGnKu&2LIigC3l1!?RzD4OO7aK5xlNdPuhD>JcZ}
zVTHY^nzunRTn&!k|5SbZ(gVgfkvz1o7HkwVU8B^0J;xBz!f0Mr{tTpayp=@DN)$}V
zcDd^>0Wa#afZEtMvc>zEE15T3R8l-4Y46r_nVJ@OVTSdwk$hW_CYv*<pm-maR7z8+
z9hlT1M+#D^ENt$MAc??fG|d&@?pjyqBi^f_N;l2P=Z7BOgs74fiAq%&H3T;<W0MgD
z0lg;{HoTUiuYGM&m7y;sHC*5N1=BwS=MXhHJu&+?xC;Z>@Q&;a?cN+0pbOd+tv(}*
z1z1m|g>7+aH+@>Jm3Q?@0I>C?Q!m~c^8@4{sU$FTu>1{~Z%&gKBzP$p=|K{qMvI5D
zlx>|k>dY>FG<^%Q$%()=a(%5HUjY^hnCR-+oJuaWH<}1hSUyRiExPbItfe@Yg-dKB
z_2+S={}$wZQ!{@&c<r;rWAHb6F4+Ig%m?TMqGN7?v-+xK5jaG@^^T1zmF;qTc%wl9
z`y2Z_iJ2pCO_`R|s~KMbiYOUnlYper+Gt&8o|>HQ{FUrc%0=#k(25{Gn!lB+*{m!u
zHGJCTslmE_8>62J#aXvUWNm(XAKdCTsbR}<r>#u_HX!xO$7DuNh4i~&TC!wEw_uU4
zI=CD=M{7&BM7FN+jT4Bi6&I%4i_Ze-n!30~;g0w_=*Tk6-V5DRIb&TVfECnp;TPyB
zvT|TM=#96dV1p?$&hg}RCZ<PEEK;Dg+T{)^(=PJ=ZGDy<zAvcKyh^G^dmH%jFqb!K
zl#DD_jjL&}-|}tu2%F|ja$K0L`fgo=nPyqz`~v>9S#f%syi$zIj%(=>%gf!43}1kP
z)(O=>T#cWWN*llH_`32ZE?+L^XMN_U;}^7bFBfY!Q!hGp`Ri2s{AC`k$2}&U{F6<E
zSYb6yEBgudfADFG;?4HIPQriJ0#UdNG7~~Gm|?DRVA|t**S-NNkA%pUG@CaY75sF1
zS*O^f=8FSJhTA}cFL9^c2g@+y7`!;4pBKhSBc$KfQGL9f^kHsTN3kyneiT~9XX%%m
ze$JP=Z5_AM4o+R$0x2$sOWAoDc;8HV#Sb_P`PVo0<#N)}EKpHIFF=PhOGbR-fFh#e
zALKKu;?Xru?y-nDG}KkR@Tf4H=wC;IRLq(+Lydw*eS*VC%5!$qPSI9h2d0q6afLDt
zhYxHKTCDt_X=bO{&zsyIgxAgMGN1PtE|3|06_(kf)OFxL$Q`YVygN~Sr>TOQ(CN#|
zp&SbAv9!BL!p9@FFndB2AIg)SpE=hHvCahiQ^=J-m8#%(wV{X%Z$*QWCj^x`<eqQJ
zRo>LFq}7KJnfDB=!t!^V1LaI7<gZ`Ve99csN&5ak!<cds)%^=cJ4K3k!%Sq>s%Y%^
zv72Km_b){`wCA|c#Z5bGOf5+oZ5XlhpHc~x%!5*iJB!JqfYQ|u$U`?7QN4C8reU`S
zVl|ane~>Dl;u*VMA{FwfgVc?JV5r%z-c&gH@C_)jgcMHA1A6l&`4Nb`Cj@WRhCKSk
zi#lwrMPG2#AR!!Hjz%tDY3V3TK%`yG(gpzgJwQtrzugTzSRa#DlQ(^t@b-}`9*5{}
zXp#1>==B2u!yRAF*pxIciInnn0g}FLsNL8oeU){J{y@^X?%K#vs1ZUE7^OtCBS2Im
zCY+3WRBJ@B8@AIreYS?`_``ae3|6UR=VwD{)DeX9zS>Uv_JDxhfYeJ?nN_pPsDGTc
z4!II3&&vlWFsI_f>!w6~sr$rMs~mZFR7q+ilGD?4t#NzcuA|dB_pZ`|bTpjne)xd#
z(7k9k;rmZHw?nzp=2!l6-))%XH0x<6W}mJ%jGohZG?}FnB?E^m2)(}X4$!M}nbkfr
za>6J<#owFm5fJ;c^GMlt&vK=CXaIuAAeJZA*$Jne`f5|;oyOa&<R0M)!*RlPTZgfW
zeOkOdDZn)wm(=^L5)Bu!m%~kz9sM3Nv?em_f2xSUHcGrCng<hYbMi<6m?%JE*B3F9
z#C22V;E$+6L}r+$+}L!Wds3;C*|Z1JP>O~sHp8lxkp<cLD`z(@W4Z@9E+kPywH_=W
zvr7gwA>ZrapjN_73tXcGJ06FRks28qiJXKsgr3aR+Z9-!;?8oLe1u$SSGlC`6R>aX
zy#*&<$@kQ`zHoYFoCa>MnVN<s9!PV9>Sla8D%_Y<`^0~kYefgE`*F2VX6k<Jh1v17
za{T?|HC$twW9JQRq}wmXfM54SLD3+gfG$HSiarR8e*BW+d>Y=aJ1f3S)d|td*=;2=
zos%qBTv?+`diPUu$?AdMP<1zAn0~uD?VLFs59#3yVJVZND_q9Y@a2Iw4Gidur%d`%
zK5IAZ+v@D&3$(U=pxc%k!bGw}s^8hz@xtQ^TXeQDz;nkSPx_6)`M+d>cyu2(DdQX~
z_?qi;*7&8Ob;oPDT*zr5nc41YVD28vsGZ<HIlcD__|>zfop@*^8nU$pP$&<Qv#$(N
z7vbAubA&sHoGun9;kjmUr-vTjc3SfY@!jMW(nAY^>nJn_87>sw$~{=#KDB|oQ5jM$
z!Th>7bXFILy0`*;l0>W{HdOL}KCsI9>j3Npoe5EH)e+n?8+7(9@TK|EH~viAD~;F@
zK|l#JHrZ3WBcFLhNW+Fl@QJRRkol-jSml>Z4wby=6$5Q2u2PAR9u9iyr`x!HXc+;5
zM_6XjBym5}_~L1h=y#MTHk-qw>8&F(pFlYOHjlf1nA82kP6zve8mXyD{q2d{kG_Jd
zH5L<p@!Y9X^t`e11#R{(YP;Z=Pnr9E*6O=8+g-;Axo$LQKt%|ny(6LJZg#Xq5GLHq
z@RfXNPij3kXr4AkaGAA8wmoX2wuRY&6{ZQ6->hGFlr|}}2-Jn=?tg~Ud-vLRJv|))
zoy9FIJsy|Ur$7qC&z`T5ChzVABa##Lpgw=4B92e@A~^R2*C3%p6e5vDb9nj@<*SHi
z>5^g^DB6>s8YLfdC225aH4Lt6l(g~WEs}cJF+qkNYEHursk=YhefO94^flq$+1}kn
z9T!)NCC%`wEO_QEh7Vy=*0LGcI9_ZyD#&1=Nc4Hu*4jwt@AE=aDDEE+ijj6Kzsr;c
z^`hTL$`4dlDvh-v8H#2b;?qHCixV5ZwPZ>#@4!I4^!8UGCXvGj4Ggst9E$FtV)Tc^
zzh<T#)H^R}u3&jT_91BHTkDZC>)u6+f{O&6g|}UFPpOD-4Ev0pfGRKHODu;#+m^A#
z^EV4Ft-9+0X8mBPJQDOH_CD>hAZ47G;RjTFa}f%4F0fDQ47O$zJuEG)Wx_u>gIjk)
za@l<WTgx*)jpNk%2)fwt)3@>1(QA(Kb%I;L49c7Gkw)3CZown=1Ub;f@5At*$1!^!
z*l?5*?B-@6FH85uLe@+bBZ8+4$r;x0ft8{WGaf1uFGm-85trR)r5fe=@iw?VK4Vkd
zsdc`w;+wN=+itn7sa|HxXU&<cT<I!1Eu@tW<_p;NaC*_w;W@rLBv45>VbY(ZvTVAo
zu{vORcrl!B2)+7e0>$oCY-i_>OvRsDd<SJNErm9}qJKa2UHHn~g(XI>e+jsBZ1=xl
zSinU#Ob&rkJT=erExBVy(f^4!Y^Aurnr5*_ned4~Y`N_v5-60;bqU{d;d8t|ax5lr
zHoQ?<mp2}rlNm^(y6-_mnARpTy*p*8+3v%!e)HpVi!tuXPw`NFrtTJ~^h`+av7?V8
zvjG40z~vAp`6M0KbIty6+aT9!bq{)fD!yDrQZCD+C~6B{aQ_44e7SQFl9N$U$GO7i
zv6w|^kn=^IK9^ZN)zOOQ$|(lGjIjE`tkraMtpZY;M?46_-H1Y{{i2Z9E?9fP%vZ2b
zC`>Ee96Q=(*R|ykrR^x<df|2pdpXZ&5wk3)*<wI(q)<DySuEFz0BeT>lri}ZvA-ox
zh8N?!YtMVAsf);rimYVTh6Yvo?xD?*k2jwgyFsh3vNfvGV7%)g;G^yVQHe61mb00b
zc>l+(KN6u*o|gfyI0$RsSYQ8wrBW>53LPn>92wUS2*YI&tR^R7{QL^3jy$1|36bya
zw#GjqUld9V3|{9ACK~XLPK%GA3M}#j;shC?*^uNQXO}PJXoYxN(@O-8L=OwkuO@FF
z%}-Y3FG$74CZ-t~j*<m>)$lAXl5JZ3#|3zZ>(40A9pek!?iCwa5<D6qI?lzpAr1k0
zxw`96Y_5&0N|z54_2^v1eRp#_2;++D;{?cpi&^wl@RE*+69JX<j@TNOd0r^<yAp4D
zAP#D^VpoW`#o#mNnUTf;mQ>cAd3JvK(pZC;%ScAz$YYsXM?COzKMUS<O(gCTnJ)3U
z1z!v$DV`uVLl;(0z`eA;IgQpFGcxN!-A8qOVOhL%<;c*%(JPjQ9V|#u&cRU#L|Jq<
z-R>RL+&~CqhLEk>Nur#uvO|ZAt20f4z<lGrKD7muE88-@>!K88F`AIMKhOD#A*)dk
zj?Ky1tr7<5dFz|yImhWS;!U_SHe^K@yfy*?9mPYUE#JWgcFaJ6{B}A=BZ6yvVA!ZG
zm}jBP5F+|qY;GB=_4V^iGxvH9xm;!XgfSnF?_#5`RTk6{JbCxa`Gm7rfmN9ih&A_n
zjxdz&UsP}o0Sb>D!Q$GMf||L~e<XHHudEncGUlL39(bR4!IPej8Hc37O1}Zql8h?F
zdWnRq)70zuOgHUod>}!mDvjl8qm489UD)R*YpCy>)G9s`QXR7Ka(z(8GlFz+bLA#j
zNqk2MLY!fxS4`M(=(jJZElw4BYvDwgsf3dj$-4j@O5ED<<||?cV$8#PSmSCBQ{MqS
zjpf8U{3h?S$pgG`Yc?sesPTc%`Qo}*W<XL&6|%iGD~M3mzY=h0-zC@&-GX9|AtGfR
z-4Hbl2Fm#xRq)pu+U1<iTKTqDKYQ6=V;pD>eu|qiSa^W?ghfn|)IEK_-5pgNp7Xp)
zLAb5}Z<;-3ov;y?RnsMi<-{l>J&?RTQi7Zt>ihH;YCqBhY4fy@#=GXE@ytXZwPo`}
zi!Xa+oe>-Hn+;AX&?q|HFEpc1-AUXXNCqDr3BB4h9m@$H0f{%^&Ae$uw`!Mz1+bR7
zCy7z*UUh`g?0o1CKGW8Ux|ztBD?t>^Ux_x+5Z6L5>iBN9jLQg(4n-6lAU|0wBj?q?
z=}TtvgjgFNGV&Uns$-j!#1wjO@KWHr4PGJMWvX&WZ93V0CHfs3qCUW1PD<J^!|jk-
zw2j=J=ljOWzkhopu!P{?l}{T@{!<($Nz+cK`!FXqd5>Vl!s5NAbpg!v!uJw;sf%hl
zA+?^d)(CSmz#1&{B6C7OEEm*{id^4)i_Mb(@vr%`%nO|vsRwaGM60t;;eFsg{(Y8y
zAC*M75#Ss(%K11$i#Ox%T_{f}2paCV!S(Vo1ySQIhUVb#&~s635{&a*cUoAEX}szd
zvL}pCb#>{O(n4;S&9aFmPW$hIEv$()mrk^$i@P_i{=ViypTxtjcf1g!HT5?_)dzzY
z5OAxNUhaIAM#5wD)}-DSi<D0LGZRAXsrrpMU~(I0bf^4LT4c=28(TKR<*BBjOX-_;
ztMu)I^sn-r*2s3$ONQl_+23vR6K6l1yl`vvclO)TP&qV;k~(g=Fd?mEj|~|{zs^Ul
z(9z{Uv&5=ijqm6|SdzxA*2Ad~w)=_N!${0><4b<6qtX?lj2*2%KcV+QRi;Uw%6*WE
zE9E-$otT%8r^g1`Qx3Ry|7{iX;7L1#D}@))OSKGboB`TIZUR2HO`AAnh4Fhqo*Hh`
zQ5<BrGvJ9GAl9zZ;i%o$>J9ytF)n!TM7*!ET*O4ozzUIxC3WJt$}v~*GCvxG!goga
z^Ugc?HHkBL37Od!vdFA;^Ox-T`O;Bt&*>%qf>l*0rR-yFwmJBoa;SU3zwZ`KD?E1@
z!2@l2aUH@eFeH;}#WC4hJHS<E905wjMimHIEjWpq$oN|9(1dX+hoIk+X8gPQd~W?B
z%?8m3zkY^es#z{K;Z|l~*V9*}A2W5e|5p_|QS%htQvzwHV3s(z6CIt2FkOOToB8Z#
z&Rh@~j!gRKjeIDhlCg{SWszPDBJtr>UR%)Fs#n1C{N2zc4vnhOuxS?fsy;!Ez2EM^
zE4990{V+xyO4o<ZRUWn9!{7E-xyf5y9e6~ehPMdYOjM}Z9JT7B13QQiddQ6DyvOBY
zpDhiWYgor5Abud8{qi(q;snh|vxGIkiUa4M#5gU*9-qawG2FETM-65YtcLKHU#&2s
zzAk^2qmI0PoVGS}L^Oe|H7gZ7LgI#;#=2{)<T^)Cpk_X~`vPH9evmJhGKBUNm#+fG
zRk!nssg%5~b-sW2<4W|rB06)hi2n^OjMr-IXklZ^#*VPpE}$s=Plyc<r8m*o*(qL8
z?GW?@T=@<cXaiq1)iMb+>q8$@$?YTc8^K(*23M`QZ`$8Sgim-N`8O^rb4&bSTJ6Bm
zj4BsbjQ;3Q`zc3CFiCM}CdB4v>2!}L`PJE}+VVG@u2=+TZLWxiUhGo(`cBchH&Zhk
z2;uT0?haHqi^OS`W>lml#Ks084*46~Q=!H%Zd4xIn055NXu$|Yhx}Jer|^d%##vL5
zkjo~)kQ{?)cQZzIBE%!uINjm5=L!_BVY`ZwUkNr!$T{I8g9g7696@&<e&0shsv6y?
z=V>@DG^Py`3x1;iP`u#nSR<B`iO*ZX`qK(C_qx@)94EYAJ}jGV!QR_9V~yRfXe4F#
z`^Vu$5|^wDM_m%plym#lx;Kx!!c08em&0Q5MNWh207J!9Cf8}CdyypQOHNN&6s4ur
zRE>$>NLd|Mna4MUs^9fngXNv0Xv?u8leU2gVzjUjs%UOiy*m1jc9N)cFe}>L<y)+^
z>VvRTLweYd{C!golGpg}mFL4{Ul=_}nwgmA!0Vr8J5$C~T3mm;WIJ8yA88lFY!p!n
zIJXILxhF+6P+?h1*c3cih5hPf(T3bv*$QrbaAb(hN6gVx{e?k8Kc8T3&^BqTQOa#J
zoyz&9+p%1$60$>avKk-Q6~z0A%OH>eM`@mD*66$0krL4y<ZVfr9If-0G}OM2Ky~<t
zY}1|bRLJ_hI$84d-NjCi`|kRpX>%uU-Mda~;cQ})uo=>=ms=V2efUO>_5PJp*XU;e
zs<MXOAf&!Y`F$3=gLm(+<v*BpQ=L+EoMn@R%L_8n#i+lHpyajQ!2m65#fjzkVgO&+
z#&`H=+4glnsVU;Z8==+5-Eb#~0s^n-Tl4Z4j-JvtGqN(bC#Li>m@b+5isA#CA<e6&
zn*G=?YAiV_tzspt@|jFgkhY2Q4~~7aQm&h-*_TqRFviR5xE^~HTC&bzXO!)A&<aZk
zfjMTT37OFvTMbfs0`=|9q{wv3x*%%TrhZwMCfqmo;x400{sHyJ+U&<w9rR>0W!w6X
zF+<7G>l7|ZIeCdbyhtm>YN`B>?$!%l+NH(O%om80q&MUeW+a`#ZCYAU`~%xluX)v7
z|JWI!kEdl#;N0)<v2jq)!brrzcT3(~&RIUGcTf%Q^b0Ajyrn_jJU-&C;5}X<g)@WV
z@Zd^e1(liJ8$tyuny<kGsXpIHxn{_po#(pLSHWWUk=6=aM6k#+S^_!N1y8;`0~nlf
z@nSi+)|9QVcUz4Nop2eSNtM<!!6@xt*TMsz*Zum|;NCFqcK+n78HvYfW@fiBg+rNK
z-iO=O!jt)7<|_I;C0G$h`vTOfpp^b66Y8yXN5`+9FZkYcXYXWm0=rIwdEvR)Q>$AG
zT!oGqZ*3zeT)w@Cq}+B@<Z80%tS^XuWF~!`9<}Rm-YNW~>Ei~+ivFSO!CIKGYNxm}
z`NGG$huS*!E;@X3KK^qJ-|FBo#HUY!JD6m)qJXBWJD}()%-lRtmdwhBkUC1xK(>V)
z3(@VYCeL`F%oD6$b1fCRI$J>lU){fmZF?<dF@V>+o-8nhLV9<(d$|1W#kV8XDrIw(
zB`z32SR;DX1KXzT&16(ik-U#DcN9ATZxQ_!YM}*oULlDPr;p?O81wo&jzjZ<m2BY8
zDOLq(96T~+31Z9Gi;{^F*<=sGHiitaA#`8klcU}k3u{<0D4g-~8;qV>%ji5h`ms!}
z>v6O0M9E9)&fkiceeJp$R@KAY5X_o{D`bkG{5_yAOW)0Ye_BTOt7d>h_-fg=s%vBL
zn}+}tZ0%kdCXF*BT<l@*r`33;8q>T|n^{A-E(QtnRmU-o8G2P0$o845|0@TODw(iG
z1U(@eFQ_-=vt<@7jGqJYMCq2=mu|65u!Y3~FXJ9w6z1o~I3Fgw$+#H({X#ZbIIm?%
zHtcoq(kEaCg$6A=^o)s1#a$W1hl9{Ph@mHMpi!!$V-)1Gs!0ALNm#h7g4#?#j%Tc?
zAF4F)Yuj=u3}`o*#U(9g6oX{*sng$U6)91~x#gs`rTD|Ig&j`POta1)EOe&cmfR|4
zIen>=G#x}=Gmr75-^!`0pMF(&?b7pg*T}cq5#(a=o&})k8XR>!h?t>eqlC`pMNXUc
zah2nKawjZ`YJtl;MG!-L*148hEYs8ue}YfSa}|`@kx>N<2LoSj`<6-&rSIR8AcTN<
zyo-FNi)C^}qr<eHUvhw&7kYUhRzW;yU2~kCo9;{ij#n9iMFgp@ynb0t_`A=~%?U?c
zjx^HH7)b-QTjj$kh~!RSp<~+tz*Xz*TRz+b|LK9*+I*YSXtG1S-3tg*8YV5;@w2hE
z1rw8!*NhdS<{{-hJQ_7fIWC#FbqxSM``QYoDzK2o!a-L}ysk*$Xdfcy6}L9iwxA_^
zwmYorTT;w-Fs1gzuy!d@fiF=iEfsoQl$&+EEryCLnC$&26P50kF>c|#(W1^xwYL!r
zAx<YH=xsC>6Bw;S(9{v8uO8`Q+q8>WS4|=**1xl~Bd(T4xoQG4?RH-7C{Ntjf$SJk
zO&(C<S!>19R5|@J0gdlW63d~8YXQAQn<ziwY9oqr+g#k`=iQ=(S^2$UEwf5oxq8bR
zs?URQUi6}U09p+#ft~8r<pq}$?0n+oTp{XmF{AG9yM3#I*%lcv*51DIOPb7PCK<hc
z)JzjZPLb~GZ`+cG93VD)@+b8*ckeMLlb+kk{m*Ih<67Tn$MdSFGAf@}%PuRLE{y5x
zslQyzamT627;Z$5hj@Olvzx{ggse+Jt{2i<KqZ`SdXM7f#YaXv-z|KssDpBKE4|vu
zloo!?j_SjPVzg>_ZVrv|4z>cIj7BPDAL!68tB0T5C;9b$dt1G&N*m++?KvH68*SqV
zS#14*m8-PdPXzw+au}fjolCy&ZLkOqNPV_P4rjss8!e3NlZfUh)Kv|n)pmRZbmg^T
zJ6SA2U3lqZoc7w)vghNOEgsti(Ohrqrg+i$81hB_%<){3-66UxWo0be1xvts^n`l=
z2fKh<`>;y_f%D!<>Q{C9$TbXh%tzHKMzVGRF0h|2cJ(_~LFG##6itM79lZO#MB=6f
zpIh!PZeY~UHED@=-gM;DAIL(QP<)4=)5_>r?g7Re^*u~4IS&$TOQBKPSzBBf=V3*E
zhM)((z-8$V!UF!Sf^sir%Dl7!Q^cGep`hoa>xua`qjD2;b(F{F>Ih0QE<wwjHL%Iz
zg?@|MS*M0n%hx@tR^zWg_Uz_eUNzHnu$;Hu@9v^LQ)R~2k<1Tt$mfO>>*2|-H;cVH
z6H3{)4OTjSLuYlH{|1@JRH7=ZZN$J%!T*It7hPZqqFb}d2n@C!N{VY#H61`WIe`7Y
zsL09H#eI_#_F|CCoc3>@s~1nzn5;1g?;R%ENo@}0a3Bi}z2se9bzwxI&GpiwvYD^M
zxizYUB1!|G-uYS!i@RGR(J?cH)7IPE-Bi<%^gl%7#6Crs?jyoJl#Q~pKN)UzC?OjC
z5zet#?r7+iagtl?$~{1IEnq~ppvofpuVxqGpI4#}Ek3no<Hp0$7KyoHAcYKJ@4lp`
zqxc}d;N?t>NvbkXYoH|B5w&`*5Avn;sHB)PnZY`rLT+!}TKSFc|4ZOFD~TYIAkM*}
zvmz+OCTeXq&(W)EkYoKJr+Zqa56v&GJkvwmKw&|}ham0gIu8gs{-!-TNa(W8%v=tY
zH}TSkesee`#no$?PE}*=54(GyuIU{ygn^J&h9n4l-Bi~38DABA<&QqOY`)hpdAo;g
z<N?GM2O*5p14vm2-%kP?vI87qHG(#ZzLtKk!);cY=t9aW=DkT<VKBAbI?|uym3EAA
z<tv(Yj0*d!+3{j{f$0lbNA*#|ap6~`$HAk9@i`9-sU&!nAAykBi_vh%1r-iba;^#<
zmb;^ubB)ZJU&40bKB~aBfSPMIIVIVxcZ4s5Uv9VuJ?biXZPsbn<zS`JAEZ=}HW-uW
zm$LK7wSAMRH6N*)<Jcwoo&Hx%eb>y^;7`i&BX7ZLxNy~TV-D(B3WsV#ke`V&H1$1g
z|M&I)!gZs-I-O1<t8xybtR0!i*R5w7)JmOgF`0H;)<zY4BTz;grSB7{0=kWY>I{!|
zM4OKfL<mkiARl=Y-B?Wb(^XlOp$=h?xF@6+YnA)U&2w&G`_1;lIM1yJE5d3L6eig9
zPI|Hn9WLsPmiHmz;8Co+s1)m<>f8nQLJAdrN}~!?7`xWIdzb1~h!t1U>no9ab;qe=
zE-lK1bo=0sKl_HugnY)`tMiJxFR;5ccBBL!L5Z|v)@s6w$Lq9p#gW{|jU3&s5fr|8
zrZmlOcwQlim_wy!Za$Dc+p6V_{uHwIyZhs6WC%g)l*t17GvL6SV@{?X{b{Z!oH2Q;
ztq~TW&a!=^@S?vA0siHrNyY=)!8)o4rF6&94m?$Rw~rRA^j6h}*@zA!=1&~YP~zM=
z7MC*Lbrz{T$usu=qTa<a97-ol)U><QUuSB<{5b-wkU--^C;m{HKHK+5Z5XT==Q^Y6
zd6g|4<OsY7IRqV^j<)Pc$2sYLI{GM7@i5)nLF_|V6CNUzDcbR;!f#<W*&{k)o3-Cp
z=uNY?1%zo}O$Z!s+`EECv9%7eTnL@qJC}EGrEp?>Z9CB~-b`h&>pTXurVKUpLQq@1
zq(v`7S5?q)i*pj&DnqGK7_lL#>uBhYd3v4Po7|pG<RZg^<wk3Flb9=uV~=1<E5P%g
ze|4M=slX8BCu6yZq-dlG6EoCa3^E+Lnyq*l#g?ME@PK;3WxtJVB9`e`ok48U4<(M^
z#os3k7QyISR4$ih5n)63E?X}rQVwYvxQvJs+*T(W70d0QAnR~iHyVu)gwt-O-Cd7X
z-y7JU3Y|Qx<sp1Tu6UaU!}b6)BPdZ!`7-~*@=eM(1+OmFnO<Z^L|^(J3R$T!38XCe
z`Z=R{+*r5E_m`aDmDN%D;(1dKM|9`!X$2HNs=L^s!VB3icC6rA^V%~Z#tQDc*r2`~
z+-Nd*7Rvl_P>w}R{ul*=-*RUC@uK^&(6^)guJXFqo*<{j-%jWzOnmk45%qtceB$Ss
z#Pn)J5L)5B9}E_3U|Fl@TjM3x(QvL^j&-whmS21CY2`w5DSF1UF1$`1Ill402aYS#
zlt|iA{T6~2SM&ZW7F7ir^Y9<o8g6)oRLBKBbf{)2>5wN_33k*j_y=*87kJwt<0#y{
z-TibE^wp9Ymi9a5C{AJCOYXb4rFw)o;?n|@vcK>qO+Q*`T$KDxuXJfwf|%2L6(r%C
zH}$g2DXX}Dp!qBJD7+l&1`la+KhRMG+|tg19O}|IE)3>>qAS66fBjO__u&@#<Jn0u
zZ94kQy?kgVsAHajs9Ps598>e4ZH?7j^3;Tk;#eFKg`z#%N<CrJ>hR+wJV51l`&OHt
zcZ8EZ^V)i@d;n)lg{{{IZ|HP)v51;0nw+_Fn@Wq_m_X}_Ki+IJ3^Pk>WO4WAq;Q9;
zS{-GM7L4Y&O7zqRjbd$0)91x>fA0oCllPO}2$(+dBK)@BOLY}H!C#{LxCF)l(*QY_
z404MBCfaiq48`SH`$pNvkx}Fc*YU0(wr>s%w?hw)2qK1~Uq^6q%M=0iF~vBarV@uN
za|~*AHi~-tr)*6rE#<T~Pmut0G%G!sY&~OJFr<F4IO!HB<O@Vzd5^GNnD(WLU^+{M
zx5Ah)tuoTp_H5D}4#R~WKfUce?z8Onts<c~R&HFMgas5Px*@YNDy=~Iu2%3bxt*r8
zlI}H8gCOu5iInamwrOXX%v!Gli~Qx-pw+=t(Gn-(lG%oM>}ve|#=CC_OIRz%JlTG9
z;yot*uWW@loFdtC$SoQ$&lR_>?hw3UQ(uD?&fX0$hRv+nJXbE#PW@xJIt6A+D-g4)
zIBLfWlHVF&3SQQQbI8+6$uPC%?HsjAiKkrA{j8nJFGp!cxm7?h+Z4z17s(qg;tZPU
z_A7dJ_*#~qk}(T)1!U7>yvrFp(&>a9-xoc(zh$@Xy7$}jo7j^RLXO#&Wi8(E1ixnj
zZA!`*?yOqw@ax{xt8Via9%=s8wDb@*TG>My-qt{C?3_saYi1j(!xHVKu^eNA&v;#A
zH^T`cA$Yl8B-XZGv*BHje-AMxg<?_KJ_%`XWHVo7Hr-Ku_|@elcRZenu;R!ZXyE<&
zs*Uz4W+GJ@D4vv+I%?-@R({C#p*06brsCz<k%!*U4@UuvotiRK0rb5iY!x}k6=;u*
z`Ch@&!QD!90-E}ZTAQ%SM8m%DttIuuPj$*%6d(Lt0deJ<i1}-h<%A5=zpGh{w%v8T
zC?kH}REw$_j^%E&Zz^Yw*4Hx4oTKXnB#+M087>HxM;mFR;3&H9kz{*Re;=KyIy7!j
zaR=SxO8nG9h|y7vpt|sE+$hD?A(<Ov-VE45y&oRr9)RoC&jvD14axhdRt0rz?rKM}
zSzZxfd3h{oSZ!|KCHH)Y&UN<fNrjnDhi`^Jh|gNCYMTV#sDs9Nm1lOV<yNOzeJcZy
z1`<`GzTYis9gP=on?Rl<5TH$2g&dgti!a@*#ukTCI6f$!J8egy4_F`ES6O|!VJmw0
z&Ch@7`RrWrb}dq{0xy#o^-uwyhL<B%?OYu!rChb3?_LsM@?d$?gJAT^yWudAop1mj
z!Rtd=?QUUKi<|k^j7^(4rmfRi7f66I-34}rP3CrnuI;Zpo|8S9gn`ltg`|z6D7sTC
z2HfLYAFd-PGGVhI?^EpO_BSL7taeo+%Tm2^&E3#P&<CRoZq27QbA{-KYTJ$;&oqIU
z8N4R%(a-y@Hn1goiRmB~X)UciF3Oho0s`KJf51`>9CV}=X{tdLKb=-$XKIiaEC}Hx
zHd5yM>0n>A+?+dd<NFOW=OPXz#>~Di+y_)MS?S7y&I8MqWFhXyj#g@J#O*fRWt8L%
zPjYkpev@|3&R#2hT1RHcq+B#WeattA=m7Dmxi*7NCPii!j_Z@mJ$0Q!DH({fKAvO?
z7?<BT#r8XRm|{Llr97PW$v4P;RLxyX<vO*;G*`t3J^$P-z}wt-xlm?``CV%<nOBlc
zLuj)Rt@CO)ip?NRPZ0IC5I=(RopVjdi{+-1s&~I6ceh6U*6$%6S{cL`y;Eug5Ub<y
z?HTy66JF3V12(0=Ksejleas5hTK>C09r19z_vM2m&z&Jg_yX1m@cxwi%Bo>?Imb?a
z-rK9Qo72}r2x;lB5MO+LcAhV`y&#=(9bkkYaUZ@VW#xG|D2aHN(ibI|TxAdJF`jk{
z5Lt8H3){0Wl@8Wv>N$hduV;43t89y;j>ar49;C-@A}fkk8f)A6<P7}0=)|&oF^2pa
z9L;8fNY5n2x4cRQWsdLr5X}9mmI}Wzn5+j*WF-kMC`9+9*gfVwTgN@v=rG>gRMP3J
z)ues>Jc~TZ>mx~lTz=<Vb#`+!Tk$JxKeWa(rQjYfL<0!WEtC-Btj@(PwFoh=9p&~F
zfDzQ@e%FcAC$C$9o3z#K@CC2;A~(7>PaaOm`p{<jO><JaVjfBAQIzk?G6D5za>>gH
z_ig#~(;HWB0;$phGnQkq{RW>lx3fE<qv5hO$M)Rb;BQqCLC>IyxmKt?68R~91<0jl
z!n9#zM53V*$}KZHYWPF{IOSnjW<zG&d)FyjjrL|p#!z`j$ME?*!~-~YCbnIp1U)}N
zdT)~VE<NFD#b5)mvXa>!=Z+kcut@VINMNq*EYkp3q3l<2<+IP7tuOmS#B#I|_6$%Y
z#s0adwNZTZa*Yw$mW+@>c45MxS@LU7hZivluL(zu_C`ziSs_l6eM&=(>$k!X1Aw7g
zItU?#uhtE}8<dvw;y`Wkesipx4pDCk@!M1BAwCqYk50Z<#Py>Ct90t&0NMR7{%oG%
zCDcf-VX7#!FhScUE}a+85&vF0K1jkWu%_nCf_C8CmNOaBFC;i^OV-|OkL@IaCgnmt
z^!*v!y@@hr-Hd7$W8(S4@8Cc-zVbtmK}t8BUwJ#}I=qkd-HOSi^RqISa8AmOxB+5E
z7KFYsVtGO?Cb{*?LzEfJK?#D$ZQU1~R7o!oU-VNTVw^9qM4-_ky#}6$8cA3&cN<ke
zJ8&w9O3m4TLZ4ZXRi7sYLi=L(08+bwe%m`KZX@NJbt|Sl?4mtP`htb*X~Ego6zNKz
zD*K&dCq8IX&x*Ru1FK5W_=VuNAW>J~t=uhLmAvFbd3RrR{k<9ooKL=c@ICnB5yMM+
z>sRR>IhnXV{y3(zsw8n9mPD<H&bSh3L?!@Kq@MtIHUpM$v^W1fo3a>^un&sV|FZqR
z@J}|aoWj?YA0s%BT<n?tNz1$^iGd*NJZ}j;XJpqQ+U_*@fPKJa?AgMvWJ`yzBr{-4
zQLz4nZe;&P9$r@77qIDj`1K~e^_Bqfg*6*6+U2R1)Bn~1_;kaM6N|k*ro6Bg6~XG7
zHxuOwbFw$*+gh}X`dW<t$9dq2YlSK=kvw8<=U8rk@#Jf|xrg^p4u7TBWJg!#_U5(s
zjl1oXY}1v-@mxL;XeDy5ZS1u-5fE@yWF^HlsF9kese`j4En(FE0#6L(H4+e_6T;jS
z_A@<^^+WIchLIh&U$TGFtEFU|_iU$6{Rh~!NbpzwoJ#1?fw2^#goK@aS<<7F+@tnI
zBHdFQ_42*s&#;-Cy6IGTowoqMBmjW<T32yv{s%AwU<7r-;@0j4mRH9T$d@ve0S(%b
z*XyTQTCyrvyNvF>@H+}6m#$re?0GqT{))TB_lQ(FRlNaodwkMz0a|U*t{*)Yumkz|
zz&Y|N+M2+l3uFeN_=L0NhOzxCCPOJPB>yl}3W{Tj)y9!vI*jQ%npe(H80lTW$L0)8
z2DXzAAG!ohy+nhu^c>jeFYFN7S7Xk!V&*S?#_VQVwqM+0_>q!LwDj<H-G%ou%N_(@
z`xl2j`z&_!OeS~RQI?DPzuDLTv#DWz>Q|%6l?h+J0agoxZ-kh<&l@cd3nK6NH`Wy_
z?UMQJg^VLSnKYwgFse7eX39ALaANa0qW*~+T$2V_%Fem1ykTPMb%$`icpgkiM;*MB
zhgD${8T9NkaEU2l$mEVzv0JfS2$ygl)86Z1%)srYNR^l?>iS+z=V$@01RDdNlCxbV
zu~Ujx3rStq#3{UG^)!7oXUOh=*X9B=2k!&2!uc<@DfpZ$d%Qx${>rB7uWY<dxIO-z
zO+O6)z_*c{t&_$k+y=-i-oP-9Zsap^bCqegmSQca%4^)*&7NeBjiHL4K24<8&7lO5
znOuXXjtfpnTX=_YQ*$1QSzNNolx)8(0-dA|Y6<sBOMvH3c^)SQj}M*X`WI)p^1wQx
zGjGTYKoE}?36Fxc5%2x$!d#gcy1`_&Bt|<-s~^}~rzy}8XQ?`;5!L1Mo*7=l5u91D
zwY70(3cc0ZcM*WpG`qbJH{8i|WgA=bAnM<th+Y6geeapn_-`nzzeBOES$Dt<Kn4z-
z<-wYT1a6prNR@yPOF&$0bexZNXC#F6S9Tqe(N3o;@+GH9F|r`S4wj91GAGdT6}|H(
zLckvPeeHC3M)k{$1am%L-sSe&S0f>-*Mz+z(Vr%OczqS&oC<*OVAeN)<SemDcK?d)
zujIo2YH(E$&bLNN?P=u+yeL%%TUFkbnk_!Jo<?EVd%>#GWW3AvH9pT;5g|Nodt)XO
z-^t1(5<+DjFj@s-EJ09_N|#e<m^2~6RawRA5wp@0z;N^GjOFe_W!VZ{UWtK}kvEL;
z?kfip0oI!ZfMZ+TQu@z-Mh|+nkp*Z5l)yG$glVM)AtqYa!}esklU;8Si73&j>Vl<y
z%413z{T=~*By2s2MRDZB?v;xMsSR9@G@KCt-$p#}b=H&*Aus>g9G(Yf1IWxa*`;vI
zqsv4!c_v09j<2lEAzh%=S+dxB9BOXTTDmk*-@JTPLMnyhB~eOE-q?f5XJ#E8T{oA!
z2i(z}9)Q@9f%dT6znh&v5-<c76R+Co400oRXFUto#E-pB2UHWU8O-vX58sR^j|)EZ
zmGpzY6)479vy_(+9@m$_`x=^$N&szV?*JgYozme0O#Xix*z8|h5UDIl%iMux0w>H!
zFB5SLI|DMmR)Y<*_zZPvL*ShVc7xyF(C+IHr4im0vcoRW{7gQe5)nrQuUh;yz3fbg
z81BO4|7V50Ujvi=Z7X>-4!k*f5n<fy_Fd2rW|+KI)SqDnY!I(vW(~9<z!o_M)K%<!
zkUiart)pMMItv7PMJH_44*?vw^ZuQ+k@2w7f3uDj(=)4{*vcC4(q=j7!a_B`Txlxy
zG-({8$SJ*8wfmc0)tGlwH;<xrf~+TfSoz`8+yuM_{vomD@%%UHbqn5t<5Pxfi!E?N
z;EmBTxsa1JTibu17#0jHin>wlY58M#593J4dShY+*bN_{+@nw14*$C0RQR$K`&Q=X
zK!4536hh3W^=HKO%0Ex6NSmoTBP0m!w9p(%W&eVi9su(uR@pA#5d9C#fNqUi|ME&V
zVqSTVo@_CRn(@FNRJzjWx%z&AeYvPlh>mR`nuyd{ERpHa8A0EQ>xncdvV=QR*T^o`
zaxy#}*yMIx0f-v5ccy3m3sHAsU~RC84i>hr*Opm&a}fj4O(L;d%qA4ZF69wKKdo<X
zU2k?LH}aQ?xe*=iV;%Wo3tmlC`u+@h!1m%NXG5&C>^-ySzstbpvvgi~WIAh~3<eZW
z8&Ev>-saTCe|BIUumcUI1A{GI#X3DrINI<_(!A7VWD^$hb@nJ^ujeu0D&1X2*UNf{
z*iz_15%dl0_eaGnKrIdYH;xY9SomRC*x|wn`h|&yVB<H77&YO#g&<_X!dD)`xL@y9
zi1m14<b62^49Qg(nUG#Tm+D-Eqg86^?B=Q!rT!rhr_+@ASM<R16awbZY5fgHYqzlt
zY8jexhNa6h_QeHhqMuz&aNo}9p5h<X|A)T+gv#(2>81rzawDBKulCro!jA-thv*kh
z($d!@fb5TnGq7{~rv(4{a(cj0pEcI1>eXK%<(i}jORSjkiRdl{oef>hy+QiVr}@8;
z+Qb0o2EDY^Cjzd{v;w>^yMZRv02eHD_ME&CH4LlX2Qwwdv*LlZ0ePzP-&OsezY2U(
zBLicRh$+s|y8XCPT=C+~MCdz`od>$!y;?J_KujqwThR;QsLWFMe^=puxA2Jp^i?Uv
z&#Tke2qDI!PZc3MW5V<;g%&}~ep?V5i@V=u)ED9GTAC^R|L^Rc2RqULdU=-Aie%L4
zfl%>|FNSBu1DAQ=CeDg%$p>ME|M=qjy0KQ&JTip<9*+M1e_+nP-^Q3eAQkqFfNQa=
zcsP@zbs-`Ih_)>0+5RzY9JXsLv~KiS74S1=7}r}X9{j&``ri+RN@7TDAw;P~`wbW7
zO^E2fE2)Co5D&DJSL;nE3oqgjJDj_sofmWrK9h~K4hIP1t*fN`|8DUb_7xDSpz9wn
z)%l0J+mGQ?zm7D!TtV2W=CT|A&&2NYKe)*traq^L_rPI>cZ>zonX?~ClpkK3_an9P
zjGWJnb*rHH5|TT?Bm*$fZ#=$W&k^yjx<|fkW4s;CedCtQ{_748Zd^07<M-dSaN?sg
zjmLjC9N#Fg|5|26zpcisb<m6mtFmqbi<9lLJFS52U$g`z(D)MQ-mf80l}Pq>IPig5
zMhSu@EEt*?fu+WEV{p?R&Ht95=4Z3(1p)Nb1WHy+vFDL3fU%!60MiSfn4>eg+d!@b
z7Te5dc7V7OK#9{bf`!)+JrMvc26~8R)I*~&G@9ffIhUL~^pC%}>D1|MhE`$>K;Y@>
K=d#Wzp$P!J?*b42

literal 25777
zcmeEt_dDBN{C9{wTYGEOsM^#jf>v8wOYPW2jS$q{lv1nq9<9-;y<-QpM^x<uLG6ky
zwkP*}f4|rBA3Q%j=eiQfxjs40d7t+>ulMUjztVn5Nyb73001b}R8@2V0N^I>^92bJ
z?p<6TvW5FY3RN}s0020;|9RoLjC+D`oy_l_8@<<awR`Vv<!%e`_VyNbaB=dmv4YwP
zySm$FW8_%?fO`Nn73DWRIe!;?1I(T;^dBH{vfoEE{&mtw&eoMu_PUe!EsE<NmxeJN
zU8)u#{T%WGll$-5H_C3xO0PbfDJc`GJWmU|7iMX!o>k;rn^K{KMM)qylb=>VBsLZ$
zyoa&ZerqRw0W$V>S*W!uyYuVV28O%;HU6saDBzv{`9iqK^<Se`#%qcCx2Y1o3K9OV
zQx}h}Am+c`H-P{9sM7y`^#7p*wuck|{1uIHOa{^^4VrlkD^FhB)nhc(Bzl2oWc0xb
z6e2d#B<5---qzs5;M!=iau{!}@=q)BVs?Tw_uu1pM1E5K#ns4-NbCFDJE8Q`NYEzK
zGxW>z$G1djjAHWC@~#vM(}(Xtb}yW$t4a>3|I-cT0=fdjnnu#esm`C%HMY`%tpFro
zmA`+CWH`HTzbK;9=G>ovT4#H`At~Y6fB#UOU#Uu5FKhNuWMFjw%PgT$1<J-GLL2C9
z+Oy!>V__=i+k^tr{HuATZ^*_4P+E~ABQJQFhE<$qiJ8nWV)Wl<;MX;4X|Qxt6AQHB
z8pr4VG?M3ah<P+cNnz-M7@O6yrDd)=HNUOczBl>IB-VW)6LI9#Ow9eS%Idn{##J??
z-LXRtPrJv4Y7Hyo(HN6lNi~Aqyr_fwzA_g5J0yiF?l~oYS|sZ*KWyqyd5uG-*~o_h
zCc`nLUm@YsiCX<1W8op(T(cI{W0{-j=4aqd@t2`rXD3gee13&4*IrNY7yZvriVFbk
z6z;~S_$ByqDgJ`=E_n@CT$nX^o>ba@#z{2!25A2<Hh4HV@Oh`qSomh$1_ph_zL9$)
zxR?6BW88nJBoYcvb7SamwY;VA&XrEvsIY1bor53MEbH6&_Vl%7inX;Db?lX6@foP3
z1UDsPhp78L^L;ujx0NW*tSNV}f{@!##^OnJ@zj{M|Gdqc5vuRCfz_XI_-UY;{*&Y@
zvT@*(U<m^Yl!icBGD-eb1)~rCa5$WHdjD<P=Wiw!c0u2%kSt85oHe!-rZ%&L!((v2
zFBhhL$$k4LgLOXD1cA~2j+F8N|5&Ny?VLztK3*Zv(<-!I425?ksrcdLqx(m`NFSC$
zp1%%UAgz%+rfIUtRKm&uij~;9yya1)z<+-ql@g18oTA0v|KvP_6aOW_yP)U@MOGMp
z?UkFZbQR&|14(ISafD=TSwuKgF`Hm>U6R)A>W_91V*%vfok>i39g_DIoE8`=pc<Tw
z1sVlqT69-bh#VRWLDDt-fpXbVBEXM8r=oe-+K;w1WQ9UovSaIiyT$Vrpg_nNZ?}2X
zmE$V3M4+iY+Cg~rNj_Z?v7VVSUrqv{q$rJ%8SjpU_v<=e5VE@YXz)=N&zAfrLp9t+
zO-u@AuP@tKmZfC<txb!G9NQSuv#>Q+hZvNf@-|BELj*qpPxv1<6ZHI97cQ~-6=V9Z
ziTsGfKjwNLy9!|SpcmzeSkDtrYh2`TacTI;UvH@IZsQ<&*WRS(S5fl84fX{3huLgI
z>x<BT>qXNQFcys7v8~Msx25TIb=Z46K8+mp1d54Tn46}Fd5+sHYbd>WV-sm#%{GpD
zZvUk4U)8%y`V!CY=|78YQw9^V>Auz518kJH5-EZ(uhWV|>$Aj*6DuG;&!=m>6Sdme
zO{wHsvs?9;GyYrZJ+g7yj3@D)hrU4{t!B8gqT|*M7yWpf#{R@j5^a2e2oTvah|QZ!
zCr~)`i#ahf4F<dNTR2l5MfvtKt@lUz_8s<(`@T<K)eLw^OKdXy9G=0IWo#{Ik&POi
za){ITS0>yLvT=pAz6qts)==j!5F;O{FdIJa1Eo*TE;%8z-^+rdD^}%^CbLh(1AacG
zt-jj}VGU;1_!gZ(_}*kV)-QwOA_rT4!rTF6s$l9Me@o`vXDw6c%fMJ9EA^k`>K`g$
zKRdmZk5mM*GR~@S$S87pXgg7p>#?x_{!uci8d|aZXiIN$IlP|V7<-;WI4uK4BGUhe
znS-Q3qM$uAH~OqNbd(Iz!&1^PvQq!JpXXmaN+KfMyncr`ybJUO?$En;;!O3mLsgcA
ze0<klx4rX-ieP?MwL#V{V;4TP@vtW|%mJs_#dBv7h0GU03L7jvYIGR|i0<b?3jv0<
zv;Vo!q=!lt5vOj>4%RnB<z|+&GgCb?zft&$pm;N_ks7!%>}WMr&=p-J2IlNXP0aVm
zIY^=mS1@FjqhT!7U=*sSiN5IHh^<=uDx`y^S#UMPKa`RT7Dx&;@nBTcv=M}uiapFi
zq{l8J;RX)%#zaT4%R`G4jE!SOa&bh++Soz(D4dE=(()|mNKjkLM({tA>D3MCz-Nzh
zwx+}a3&~;El1`bk6QkF?@r95FkyDgwOir>5@R~PY<)LM9nbQ+JJ-t%XhNl>DbPS=C
zSZQ-iX>a`h1PLRu@$}zPS(1gynj#PHW-c@P%~knIDLjv}VtcS{-C?!-)^fpm{s$C0
zm+|{I$)tzz7@#ES3f$5VNo!)~=1^nvUwGojS&}UnCu$ENZ>zZn2cv17Q){xDBOAW0
zXgns(%Frrnne=W%yCEw%I5|P4tRXr5lf!p~VMYq@VDOA5HZg(nGG8G^?$egVf0Ady
z%}YE3{J;?JB?A=e`tk)^kS<>g*=|RBA|!VC8@!(kQaCg!0S(%FTGG%la`lvT&~}a-
zMrfq4T(S2ea)Ec$`o)*P|FR{zXLx@39YPZu4A33VbFqL@oC_x3YJ#vm_?jUF<iq!j
zs83A7x()6c0Jn!_;y0pYTAGtP>F1R!N-pJ8Gu9U2f6aTwD(ZaN$2*>@6m6H|IY(Vm
z1(1^EbK|l&{xtmeAS2Iq+5zsCUx#G}3IUL>JQk{*yOe*exGM3a9{mZ@X(zJl7yZqY
z*RY=>CZ8UCt^@9GTkIV%?YNpNmV0$O&1PUqCsxbxqBoM4VwK{t@jLuRq8Eg09?q1C
zOt$ZY-y1%C`9RLNCl^Ls#eMUnBU^{@LYn?=;wM9DwkQF^7HU!J=Gz-_C@b^<9mp7`
zBf3#|oCVm_?F&9NM1ZLa(^3y3Iq9Eyiq~doYSV`bN!nz`B7KJhcZ1^uv6E9COIwIT
z^DOpy=C9uqg=%oq_Uf;di%2cpK_n+G$3(cu?U7I1YEJB8BIDPMA2UvD0Bl3V;3a{I
zt0A5t%Yll8Ay%-Ni_1W48)+NWNLPiCCmsTS0j4i_H}MYD&)qcS<k3YZ{x`e?Le|8n
zr~z@J_aTc?p)DgZs07<|6=@b1Q*XMDY}j{)O4jNRc6uW%fc`w~-t>nIH*{T7?J><|
zC(jqvYak-*aewJpsU)BetKB|ni4b%kQN0MVX4#pac}6z?3N@%H#or}VFDLny2f&E^
zjG+4E5@MSB^15<}2Cas^kKThTz=L%iO;w00{m}hzH$qe^!5ITo?Z*RL7GxZX$6nB6
z?;rv7>PB*m^as>7e2h@L9FAMPjcQIiHptEwh3dPGG~gmvmTvZQ2MLwez}Dx`B1twv
z3avsjn=`k)ciO^#%7g!Pt*#*tyC5E1!!LO~k4z?R94C#y_`TPmyARDe2x7=*_kQx+
z`WLBOe^T9LBDmr>dy}}UNytmD$d>rSLqhZmD%>dAH&Ag8=n*;-k{PI&3-|bp(VkG^
z#a9%;-ZNM+vlMgbdMh;!l*hXQdULDlN?8Jr<9R`LLfAyhnC6=I*cX3IRB39!Y2hJG
zs`kZheiRk<-UsP)^JMF7f)CQ+zhJwe4mowVdZ6{{HN?o}fTTNFq=OnpEV7l?yM{U~
z^qkji1Ivg160bF}tuKHi7w!qChacH;#6-RMXaTU$Dy^tm1_i4SCAEg=r2F3?{NzS^
z9o(}!)JmPC_%aR77;5oPuMa5<2xC8!s4i&hDWwF8E9EzWdKf8J)tugGLXoC$;Zd{q
zyt^VZKh=zcx#q|)ht}jxc3dTz+vSyYP{stCeL3HF@koPDOE))kzNfBvS~?S~9>&6W
z@jCC5*I?@|!W#32sjTEVpZp87Y>8C>zF?YfF>V$xE-rzj(`S2JsG7VDU1z<b34M={
zjy8Ewb{Gqcv}xO%^oOL9#dIqyxThFK@5TR049{lDL^KE7jh2t6e922SM#(#)LN%Q5
zaNQdMlNNXWJ<)rAiXjJ>GdcJ*%|#d?f{R{tZl;N34Lr;oBXPDzMbee6M0|KyzLvrL
z>v-N5e7j}p5Ik^%|Na$kDr-(S^4$HNaOm&%jvHOdDiaa}o2t#DU9buamu{{&O#Vqa
zUKQGFjo$z2dCpz9bV%?rucw#r9`wgdh^exxtED}yST6V6Z%TV<#8(A1MM&hs<TV<>
zeqQ!far3$~JySneNm`?EQ=C<gCcB<!Ky-JLR%e*)Qx9p0WYl9@LDrwl9KdO8#~{@7
zNRUeP3H(tl6_jJ&_J{ujCI~I}5Lp`~)4yWi{&bei=g})9S87R`!MNc%CycZmSs;i&
zHB^Una5b$YId{ClguY333}VvxJHh^4`fhq+dHoz`cIyDoZdqkwObp4G4Vv#fW7MSO
z+*62f15hEdpTF)BA8&K|W&aNi>z#ucTt2PllJz*-(0OF}bAf!`nn$24fuk1D*%RT!
zA;a%Ci0}nBi|X{`dgrfElV8Kj>LGXe+u1{xRvPVQRwOK%*`l&Xu82|-hrd61a|!h2
zs_Gn#hQCZg(HM7RJyyaDxHo!={CqkRZF362to(eM^hH0p&L1YwnNbnCL+%K#y}v($
zlBrd5aoSoo7u2IUJuP(Dk&s_jNOg8XTcN+f-a;F=x_R9E$P|=fDBkvvWPAE`hCpIe
z&jSe38*a1<N=Ix;=^upT1ElfGQhf#Wt>DgXuB?dpO!Ft9uk)IW&>o{^S0FXTU1M!g
zI)ggap7!(JfG;NzP|y5tE@LJKG=2SPqg=xm)1vrHp|&!2pJ|OCr>o)JVI<}~6Sg*X
zapm7Gi?kUcSPcEsM3|cB#oX>VTYAye{5H-VIcVR&Nj>-)QnG5N*={8BtvF`7XthkK
zAIPXXXFu}WT?mRl_1gNm%0wMMw;X=zc=o$*@f2)8tC-7Af<M=CWT?9pfY{P0A<VEp
z%NfPKbu+Q<AdzNB-teIw%FA4&4|@K`c(C!i_gmT7Zc1+CQ)){wP3EQNJ1W_hJD!U3
zysr$Uv)M)eI#AM9JI7d6;Ah7O$vdHgoP6(vUGjN<GhP1?$cD_ySW;G2o5LWeZL6l4
z1y9?p9MiOpgcBoN3!W%)igh&9IU;_=!g-#{)}w&@HJLDXw_9PXlFHlEX`)lS(|-0=
ze%>#e9I#sTFGi?G*WDq3QExtlvJ{A%$Y5l@3RrRid6b9^HLjDDyfgb>*AS_M<gpH~
zzde(_2|W@x&k#y-7IQ_^7P_hZAmU?Nq;Zldv1KsiX>*MbUI@LAbv<Bab>mJ844L}<
z&YS0k5T*JvZZvl*;9Y}cbr5e)g^`z<UsVnGNwb_Wb3?qK{X93buaKdn_t9FBO&MS{
zR8gsR6(IQ*1mzYsj*fa0F9gLqw+}Tt-}m?9u6(QW1%!t(MU5K_uJC$4tNIcBCM=Yr
zAY7pTl)hWxB}EZ{9YuCgNtYM%ql7XR&>qcN9c@5K?SffhSc<_N^7QS`?7Jr*=&;CW
zoEC{K2j1G?Walrm^f<C7)H4QB1)w$=re4*8731=k7>Dl_q4zT>46QOVj?JB9ON+BP
z{jrj`ph_-J=Dg~w0ivxrG0v6GeeY~rPb)0Lxrr3BLDQkXx>KvX1+ab~&r^&cA2B>_
zH+BwLBp}r&<dyh%7%=lGB^>{KKmHQ&7+gWG$f1C66L9-8kaI)?tMo`~lrK7aVqn^c
z>Y#nGdsi*YBjn-AA%56Vlk-pWUZRcNIOk-iKNs@yHWW)=1E?uDB_dhHMaHLo4U|Gy
z#mdJcP5g~AI7b<)QJ;^^ZD7^yp$cu6Eg#KUJ>+kAB*o9=?fKXZnRnyL=t1M*9?*Pm
z#;1orD7bV}8bS-9q+#m-Z{U*MT9+@2IoIFQBV#-yL*mfe5Iwih_ko%Hy5x=h?CmLQ
zlK~+dtn>S~-ojEY5u>L55BReF%za&o7Nj+K;`?wpkaHmJ0djG`HE!9B;^S-C;9;?8
zsInc21TtHuWt2{;a@v1$)U`X28?kS0&lNN!6@Rxwy8d`yG_9=NUsN+^;->*=zdqGR
z{3bo6m39{COzwUbm7+VQXoE})u{mL7ckc4>jkEB?Q?v*kQYS^pK*e$BF3!;d@6Wls
zvjE&iOoo2d4Qshq5^qC}2JEp?FiLx$yGCb=$M42knRHtTRi`1ew$}$gVDVfoy@fRd
z#3W*u0gswW(0zUGwk3)EPgs)j`95~s3#52SU!ByUW;PO`<|2Bc*{yj0HJvFhADdq3
zE+sh5&c|?F^ZSq7J6x$Qcd8jNBx;P6lgrKUeziOqNuf^C$;#eTE1JYmCq*#7R4S#w
zqRtmI_q1eb3Gg^z#QT8Sa_L7?{bgg|7S-g2O(n0fF3g<PO+`Z*%Lm&4vKxcU84q01
z{_xLF=O2bcoPPqF1vkv)h28KK7>lS^HQ<42n7aKba2n#mT+x?T<k{E3v}g0Rv1;Na
zH1nib%O^O<vlP12Go{IxxZ<p{)mX{rCS=w4)k0^M#)V6l9TY0pqGyN;HqD+;xymA)
zQKC&f7GvskXxB`=L`%kw|3Qwo%p6Q8;6{*76Z6MA8iR++ZH0%8889`%2U-n-0Q-@+
zbNl<IJ*M{6G8X+^eZ0c`(5=bQYx_Uj3+I2D<VC7tP~q>2=P1(eXiKzQOTc;Hd21X|
zs;R7M|IDU_smYXAi@adP`&G2+)?eUNIV$`+9v?nys{ftTuXe!Wwq@3{DsjXh^_ulk
zBkq`1BXw{(oB!DRDCPrJ)RS#z()Rp;XvB8@7o>Qje1_A?k(#S>?jWnwdL3FZ8sGVj
z$f*VWUUtDj$0A%zZLB37PJ+tx;Q4T5NDL-+24g~_RPRWK-(vk01^4v}_=-<eilo?p
z^(p>$^j#kAkGfNT)t!9%PY(UWMy3zzXOl7gBpe%9m4ISy<&JLRU}fgf01<y?)p!Q7
zYianKdu*-~6S^TJPk9Awio441gOCQuy~;1MX)b;+JeToyV<X%CNb`@^bDtTa`u$?u
zd3VP@a7iU@@6kn6>>9~ZqXt^{hsVd~%@yP*rSq}DPpU>sgf1Fiav75Ngy|DQzfjIL
z;pF4~Qtvt@vGn^OK;cOK(t2+E=n%UtsECEgrQ2EniFujET77%0_&jSLb}3ZKgCn=#
zFm7x`Hx~t#<y{=Q&bJ44#S|&+E!J2ZBb41bi#w(&WVOy(xihY2wHnn`rb|-(jarit
z?Cj>bQ46mbPjmXirxt#!W_H!ZjZ~+5F2;Cfj>3tdVwq-dy5d&ll_ekGBW}O+|JuN}
z);K;y5&R-&`%q>9j*5VZyRnzXsj4`};Z+h$<9yqXSt7XokV2HYW%-GDY&lD_U{kn7
zm?$}8!+W<{L<0_%kYM#CW7?BraFQ)A&Snj)BHnyw$;j$PEc$3LP@tOW>_MajrNvJ(
zzRrO#E@WXKo9PQNVpBtgu1jn+S?7Sa2e^#qwEYY2bZ#5DZ`6H*J*tphnucFwM$P%~
z0%ENjb0>Q`X<~UGra)>v=4jv&;3fhYAx7>)o6a6A>~c8~)sg=iHu$Xq+RXOo%bP(i
zlu^?9lRM0tr`*>`b`L_m^;~GdCFkTJ34Z3mtiry(>hSuwbcHbZATx~en`k*Vfh{#)
zxyZ_P8TETbV8?5Wa*F9F;ig_+t#Z!t^Q*?P^&>H@3WbqVht|UAp7N8WMZ4wow92;)
z*Yo`~#nH{wMiPU8!@fOCK}=Ieb^ic|T=T}V;e(R<{1kl4>RNPqnBAdumyXDjThX0&
zZ~2@u;%^NyzV&(6jjCuYmR$=Pnlw$7SWF)lreSJ%Vq>6P=Tx@aiqt3|Dol>+;PGH*
zia2~7CQ^SLvheWQ>qr~@W@3&=Kz#g-UI-P&<46Lge}dRwijF>n4fZbKN#i9Dv4pEi
zlHM)oM`N$r$;y@)^j<J1WW5h0KC|lfe@eK?h>ove;Oe$DTDo(_jWL!u#_SjuHTlCF
zr@$e3PAlAkiige9-JB*Z@)~Z;nw5$Dp%D3xhaF<hV!jL_e&#wj5RW-SdzrLsu`7vH
zJYPdQaH_S$``1O-S}9#R<)#0qSv^nl{q!|3O^U5(WFa47#Sk=ex@+w-P`71xxZs@g
zmQm?bM|>Hlv7^XpM?fW!xmNS8WM~!I?$xt&r!4Y6A)L!zIZ>+aQw#+nuA<c(E&^9h
zXz4@&@sJNB;~TnH+P`coh~K((EsVkYLxuPeZj0CuEw$4Cx<^Wicg`T{5C~yT!Z4U~
zA`&hTdinibgE!HF`}e)#K4|k^13P5^!!4SPVuPD>3^6P&?#>w!L9oGa<_WSp!flhq
z5Sq=>T{_{h_kkMPl8k!VM2<!F(I92`_7+@0)3+vV@624T`C}(g$IrY&`ElApAX59+
zlaw3HKf^G=zBriiTs<yAt6g*?Tqa%1@2iA>4&k;ik#UEIGK|av40s}dTjj21)doLG
zpF3ytE@8n=mbBocfqr%Qw5iZi9!wpb9op+jWY8%}9(qY%Rf2!q=;j=&ar$R9_3B;F
z9<nhmBgOKePKDg37#c8wrl+`Xs#<$Hj^xx8v+{j9?_O6@)4H%r>5eTqc<x70tR#3b
z@nq>o;6+`Vz0fG>#o0R-w%ZLwu3J(z+EgB8q%WL>XWL25A~<_FsUU@`AkGb`+0?&F
zr@8~{TY@6%<ALTnrCK<6vQ$t6k8vgM!70Jbo9>QIg1<OZRqw?gg|Vl6!NJT}zy#ge
zxqRL&FZoQv;<GA5Lo@H`&)S$h8S5U0euW{Lr5z9Bj=Z;P!OXwotXgrd=}U-zxXh97
zA}+}zH`3<zy(2x)-2gGI$=ajLK7sO_@x80_z12T{xc*|_KMs(ula<(^nvG{6{MteR
zR$*-)7A)IMw-%}aRl{qixvo%=hLVVe5>zq&GgUt<z<<>sL{4Me;zKsucl{+^lvt5i
z=zIl>q`uQw27x`7_3g;^^kE!9;*5>Gx7)Y9UR;J|L!r3u+L5r_ml!F+BeZ1jXmsJK
zs6CM{ZdocRSBxdy7@hz{e!rCbm8AGBRHATT&)bFKa-%O1g8AF|7FFb8zBAyS5TWDy
zxLw(wmJ<@%AJ~@|Cc!^SfMkcoUhkz8RNb_Z{-xH$@Aw$Z)r<|{hlvb*3g|3ymcb}U
zlVQF6n+XD4Lga+UC$V)@!lt)SuxBV!`e-m{U&>`-L)MbFGN5oz#i+uBy{S8S_mVmM
zvOVr+J0cy;@nNxxN8gXAX{wrhDCzR)qx7x{5F|8v()nk5$BUJ`^F!FoFFDw~HGb6T
z(zx7y-$|U4yeNYaE7LUACX^6{h)Rd5ea!=Me34J?zE9D8O_dh(BEC291@&4F_I#pq
zR33$?+_(tJ>(W<Ee1CP9vE1SVQjKxK1<nUmd+#)ggwvxt_fz59c=P3tBxwtpBt^Mt
z&mk64qOHDC31?ur;>45KWsi<kQPEy$R^ROVLyr$VXwv${_kS-6Yd<NKR~u|=c<*tW
zLd5%1*vDL?tHNUI-Jj{f1r)~PU`_#*I($eFB#X~w-4QL9JXyt5DIJ9O<aJ))L#>bw
zQsH-0yECw8n!Sa4Ja<-;T1LC$T=kXxD6VKSG|xUF+~@Jqqy0=R<xt^Ex_bA29k5xf
zx-pw6#`~YmzMD#~n(&8T!wQMLaoCphUkWy|zo7!>`l@ST($LPj){v#-MX@`j!r>vn
zi=b1kIjj&2a4v>&+9X5yURhIi@qC;A=Bmj`H~zwP35Fr4?A6-TkLiNn;<c`V9(WhQ
zbiYfu<tGOBT!S82O|)eR0pf`{0K)z4AD_y`%~OnbX?fxN8LpyK_hV8s#0B+iOf=ug
z2G%Pb)p*yUPj^o?%HFRPHnn4NZ^)e@EjO@gZ)df3&@3mTh}&3JYkS#ZclE3&x;g*t
zMzM&a6B;wk!Ega~7Wy4~0HXHr$zFbwU%umcw`s%1THc_n!pZ`r4te)2Y{RJ*Y6$56
z-1gR_6r6CG4etE}%)Sgw4h<$^>uEvfZY|9ypYik}31sjX(Gg?sJ13dYP-g<AKmW@G
zK$BcKsa_hf8(O<Ab=+RlVVseYu&Mg-yq^`v-`401^2EF)!Jr5mwQsQ_^RUkTyRe1(
z(yK|Fiy8-e3k_~OY=IO)wN>=KVclNDj}Mk^PZyy1v<?>}F46--SM*p8t>HIDR@y0&
zZpowlpXDD~q;>70XE~(5_zFO&T+(0YvaW*sJ1NTbe`dfH3Ne#d52<}`u=~W4Zl>Hz
zahS*0OEfRv#K<d9$KHmI<$b6e#^X@?krPe(JeYE%E6~fY_YUDKb(8;}Or_m)z5Drh
zLF#6XO)HbyA$e9JOZ|6O8+ZL`#6Di;2{uA2T}vw(56W|E2F2w~V4iit9p=tZ5F{{C
zgssm79<LHBokqF)(SqBTyoz~KY9H!4qR`8m@3i%&^py8hI<55};B#GMpXN^orB=kr
zPQ@T44wa#^tnXiARiXSFNxovRg{#f}$@2LrLENSNd)nRyEqL)Dw2IJ~6ZL55;EfBk
zW3mCJpXo4sb(=@5-9IHlrEo*BMcjW~Q61Y=5fvO!EpanEPuPTy&Rs23HO1dp@lpqW
zB-eflnvLaI*nKbH=p-uq`ANh|cv>oREYkRh5WI`ALk;<TdD_;I%#^Jw_e||}>DSVw
z<@=#PVa`q45L!3ooh<%SAf_40B@DlEv*6tGMK!Th{@wB>W!(H5MqZTHS24f9{yo1a
za@8w4m*R0sLFau~=PwnM_u@HB&9jP-@|@Fqg||OgCj&NsUS#Yide@3tCvcqEkoZ<I
zrI{pS(_YuqsVKg!C<+{~W&UVoityT~Y>dj5ui`OzN<P?Oly3wPv&eZzE`aBOJ7jr3
z78grjJB{e^CGRm~ayN>vbt&7Pm^;dimD{+4kk8n0nGf1#CJrCA^T$OX#+yd0TN+3a
z7e-^wtUZySbPnu84N>r4WWAe(bCC6(^X&<_p$6;$SMxKHI+KP;J-F2yL?)UOZ9ub*
zCj>LzW-YxMj3}bOgudr>IV_^<I<JJK&VGbaixl5<d<VvNkB(dDobbk-GhsG|>ADFw
zhLL#KabFRa21CdG3z+64V`31rqx=cVeyj(UiqxN@@_t;LiOcC~8{yYfl{|m<3KX9~
zX-n!tIbUa?1AMaqid{U)3ZX21l_E8*A+AKux39WEt*IK{v90*W?$q}vaV6d^!u0D}
zXo9P?<m$GczgT3pc2FWAXr@uJrc#V;6)ClEPDFJAx$V5jQZn;Tqg`(s^_Ye}*f&m6
zM7fgI_eE~tkYR)^IWlW*VzeqWpATkhWcPW=*e&FHy2Yw#&ouN^_JRL(Hh0~zl#jyk
z17KtOohC_FeN$<HIKucoo`=KAhgem5BWmxlou{AVdNPVS1wyxAb;)ZC%e--UC6D{%
zCf*adeA8}L?+d_h$_kR2I`L1YXHB^D#~62mr2BcCkTyKG*BUUyC+EQz$6p>mJE@l<
zQXcPq%$ENPXD8-wq72Dj-VRA$#l9vN<^gY2-wp7IElWH5Ya*n=pM*51&mtKwq(H7V
zVo!#RFei{1ULT3xnp@_k0e5jyyWH5dY<tgxoV^o`2I*>#bwUoweTnQbudXiG?v&1-
zESX*2<f~X!(2XVic*5FQY4&Bs6RoUK0rWL?;d|vT*O}*gyz-IuyvD0za*^VZ)p)<T
z(Mxzp>NTN%m>nwjSeKJ#14>k`*{S4XvX39y&M9&BxNYcger+?Y3Y=U`V#?Dv@3bez
zgTR(dzklQ>b=3ohRa?`DGdL2sT<#q%W%e)g9;u7D!Ne(Hq+z;&FXFp*CoUA$GE+7Y
zV=&TGhN0Uzy*O>IT<&NAccTKYVgAXtR+2m)D8BM8w!IEhagyo@2hB7{ijp<;)>-sT
zEKKwOJ9b&lA`uM$q8ssMx#ifruJ@1KTB3abog?dNl)dgZBQHvt;&e_6Q1d-cS^1tU
zwV4Eo@trri3eEUh?8xH%_M>h`V_RzS<YdW?3A@|v^ZRn8Q-*3l`;6d;vhm0juO{%z
zmgu;rvmEJHh}M=D(Y8=i*G?l$sGz++niDp1scUA?!{cJwIbw4L<RmJ72}pm%j7WUu
z7QG5$h2`hE_3K|H45Yqf7am;N<6o%BG7)2J(i8Y_4c#j=v4ZxGKmFeO!7utvH?h;N
za$6^a24i3Nb?uWG+l&zVS`t>bT-TMxhbz#lVPe5XskzJy9KC0a1vX(a_8@=;o-5HV
zKc1)M)mKye(Z43TcAI0h)r3=InZdHgzICx|p>K5j(x|SGfA*37!0zj%=b4rQgRfA`
zD>q|7C00Nb!JAFLY6q4G|9gt<D4=nu2eg^%(Jw!kTK=<Pg*$40=upi&?yQ8U5CLW{
z=G2B^J5d)~|9++2V0v$`t#iTYvsq$ke#2;i>(sx}XNJ|P)=a`1dKEGsME9D~p|bQu
z|7fdN9^TirgK6fVu57J4ZF6n6%(gb?NqY@OL5rx}&e2ugXS_%F(w1kn-kiNiIJ|v#
z^~#G?(6)g7R>`k0lQM|Y5yucoJ6Kxput6FGZ5&jT^=|WNHxNwe_CSm6p89-Psa4Rp
zD7kBg9X_mu2_4@zulvG`mPav<tLttd2Z)!|XQx)YhE4R<%3P4Eyyd?IwKCGaz;cNq
zCcze|zujyxKg1HA0bQ)EhTW-)J{h4@oJdkX&+t&Ly4jyT`<}8*VXJ?F=Z7TN-p?_X
z>MQHm*X_W&2z%{$wXo6MQN9maYXLB1N0G}{5u%#~)6um;>GLeS>)7U+8)i*D@1r(j
zH?tXf4bd&j>mv$^itIhcjEpB7SPbVLJG<K7Pxg5c_yLNqT6Htn-R69rA@Lb=3@>~!
zH=5nHM~uOquzT&gh8M-P`tAiScU?Rch4i4bp{ezz_ev>WJ^xsQA$S6;XMK_g<^LPu
z??vLthB;N*HAwwIJuZPr?v+rEj^GQdbi$Fzj2e%cr{Uop#>;N@QSG2VlvjR-g67B0
z>8HH$byBbG&gv`FBA1SBC$ZKKf&f9DtD%U?M#wT~PmHNEI%816m)U2bZcZ!kQpfUX
zh!mcA*Hmii&@bgP#Cn_?^}RLu-&rNemq9-*h+v<freV*AY!y|jP2JWy$a7Mk%B5Ng
zfvNDSCK{e<X3(!dLZ@(Zo`<53;1C!+=2a@WIdE^HPqJbgcszTNPb{9t{G;&PIo_3D
zeApuJvcI-zuee5|V})mCz{jR}z@Kf|IX(KS{})P)F-_6e93)74&Lho|x_tL)7~*T|
zP7L?qtq@f<;C?pRviQN(;R%040)9Rjs{bLH>QS(Om+tR}2&%7+jgOlRWW~c{KD-C@
zLV6VOlXk7k@eAF$ba>b^zrVfDAhHej=YyCC<F$M`%M34j`(B_r7+qr?eRv;F@=iha
zK(MK2)q#LM1JB%7=PLsz#8T{(Se^iVq`gVY*~>7oD<*7#ev~haeq$$gO&84o_605p
z?hMN{w*R{QTP}8kL{-?M-EnM%>O4|V>@AM;(Ye5{@-*$$M!yTmWF;N%+gyLBoKGcD
zBN<9ZRs}Bd`e{*-)KA;Js*4T0MB8%WfHJth;wN%C#{-2Lq5a=Dv`_t5>1$z4hkEY5
zn*!=}<I~`$6=@=S3h%+YK>gcgn=1*&F{fSYctk|aL9oxR*~o9ujF5RZ#>Y-yTCDSQ
zS8?X2J#<4kDJy<0WI4w8+rP-#S@$28c-(m!y)G;zkwygGMS0USvOd4Nac(Z}!{NQs
zJP)C{lB-y@2x1gxSoQu3H?i~e>I_1@xbeg&lt0hV<@e|k2@Xq%^>!b^&s90oRavWl
zQhDRENygA%=lxzq$q`Y?ZPCUQnO#kmdQ?oq7LlZKak#{@;|dOua_XXni)gcw`~WEt
z_nm>?F8eO&zU%X;8bUY(CN{NI)?ek@Xa8iz?6)%vzuj}n^^RK`Ul#@~qDCQnov;<O
zkGnKN+WVkg^JfW6==069+P@2na1PK+-o>LM!K=BtEr>ANF4op*A1Q=K0b$)uKbSJw
ztEG;d(p5mRu4u~PoNR|Ipy=^aUaonTD?Y@JSz*{uw<Jxf*)~-89kpLs+*QUwcdLy<
zYTjh!e<>6KJxwwFEnIagnH+i$uu{Q*f9xq=<-gea{MKW4k!QU8_Q#d!fncYQWZ+4E
z`<N}(cko4kv%9qP`6ASS&m*f<mLys5w>O<z{a!<#Px_kyL8cWffccVmb8^Z*k6F5H
z1!+fDm$%oz@pP&?NEGKXd~wOyh+MQys$9(W4?{3%q|)<0;&;`4ACOiqP!44|H>FGi
z#)Po<e!w5#%}<hD1P=pWuI9@*Dt*5JEL=dYC8d5fM%U!c_!4g>o>0|doEn14vijUp
zTyNo54_89cwDZ)V2{tvOW;v0|j<=>V1JXM!(bx75D^3JxlMY6t(JUuOv0qL;Q%!>;
z(<hkD;4gNj?eX?Sn?_)laL4P?b>lse&)oN|Pguk3R>=<>_I2zY=@NSNig@@ETz%da
z{+2e$7@9k*5JiT?ua;SplK#T5QgxgY8E<-%u&wyb{wj&lsD~9av+#Ccd^2!yM!;dV
zca_PRGW!g?V@gH^=;<J>RPfN{OYpVml#jttQ<lJOAL2q1K}{;mjl?Bb?}aD|osl?S
z0>e}Lsh+6o+5~OPN-;RC)8?{j#!*4ylsY)ykb!C=A2d@kvV%1XsZ`okw`G2@6*kob
z6`;C>^UV4Zr=RfvV*xBs(hRSjBOl$nyekGDIB`Z<A<Nn3EvaT}9Fnc6DM740zrIpi
zmcD)d8*urwlhh%BsF51Pg2no>_|W;V8FrkogV(p0VUJi%@sP_PzGrz6(r~O=Bdtfl
zAW%g4$CU3jbnh2d&$-UznhjQsL&{aAI1BuF5uXHIFTmL2lJdl@lkB<LJV*yntM9?t
z-_?dkia0bubTRW@>*2Gr=8XmcPaL#G98f#!l{RDe60~3pj)8bdY?R8L=&$3+70s1=
zd&ToQ05PfIdHKi+!D52;hSS3iA?+MkCe-Co89bfz$n0&11hm0aV2&J40d~)kg|os6
zO?S{<D*+!S^5_>>7{|{HX%gMf87pb}bK!!Z05oSj@}Q2|1sfY|oAq9(Ypwd6z0zdF
zDFDXOOeIWmEm&!H9u=~BMo~sJF7W|jdZ50<6cl~yz4s8jHLyohnKbMI<4(j9J9aX(
zskY!ffo-(ls-d@JXLilRuS*^8D`C}H7+5=<|77x%ueEmZfAT@>w8%RdFG(pNw6WdO
ziMIyj7WHGQfidwZV>r}hBz$Sd@P`S}nKt~Cvp=wl*Q~e^#zsQDvYl?Bl2I{rxjJ@v
zo~A08`Ypr)z5TZX?H2d|O0qB6%@YTn_J$%YipASMAsPHA1%#uYP&cvesey&|L{=AV
z?uVn-u9*$j|7NCMVP)ZFfr`5@Gtf(7mSOGdD=d)I`-20i%g&W7_w869vGl>Y0BUa?
zA2oMSAIM0!Sw^#B>`*ZCwUb<0xk{y1;Pn*V1&-2{;QHC|8vjkURmMR;Gt-Xrr7cC7
zKO;R>UVxLwM|D``Y7A>qs(K$ZutE4v?-2s;ga3XT+u4&Kf%UROL6_b}`giviBns{d
z&z7ecU)Z~aPH`H!SvH6-z$!L=Lj@c+5^p|@)t>%rcM(2m3wkyovV_%`P`orI_k+sK
z`N>x$xJkC(#vzw=KV|jdV31|ru)sE|0K)W<CP}!@3*b(V`q4lRO@R*dWe|>^#qsb3
z4tc;zqYd_ldz0ZC&^>Q<f76;f2o0cu((Pl69E9c~$KuNms&NbB0n4SpRh1wS+^MQ-
zTEg?$;$h{CXXI4eMxoIj&0gk{CBfwwL{`HDvvVoc{W01x>hY2{d9UaJB}ywa31u{^
z93vLQK{?i~fRC$T$2|N^*&ljy>#_+1AAQYbI-?YZcYc1${3BTu2JddV#&KPgZl%rw
z`>F1EgpzWXsvmuMi7R9Tt9q*3?ihf1kGW}IX#`RmZtsmhCZ4nxFkmZi4HX#8Xq*XE
zfF`>;Y<1@{cILlN5K@6BDw%$QOea}`>J0mwu&$I6u;?06n9VzpG;f*IZ!L8sV<#Gz
zseS>a1g%@OUeM<QuyO*e^J|VRU*29HL`VCI>8;tL>23go*LP2K&4fUn2t@^sK=%3J
zxtts<Dh1v!yrY^t)qx_Ff7kO?8xBEuf5jk&yveo{HiZD%aL9eChUfcthzwrrI6_$>
z2sbpY@4|$OYsv>qVP=xO@*X*=8PS=kJlhKLN*3*byCuvyA}7Kw;v=np%f1zztPm&N
z>8fW?%h`nv76p1ZQZh}k7sOC2Sn7A)iB^duey69tN@)2=xCXHe8|lgKsff>|$caY^
z`$0lFZhz0{45)SfR=w?N1XJfTG)-aCL+3l0u+r34f@^A@_ytY{ukE?8iKhg!wZd?o
zFo8Zc)Z9gVocZAGZ8ad!6M!Xn=UvJe>RN<aPkX3pl6j|<iP$d;W*gVX4xFpRRdTs;
zvq{i=@6zSeD_Tq1sJwn{P*Y(qE(=)qtBm!}3nrfKf2dmJexD|V;zik}8omE?i_vW#
zM9u{cNev3ocSYWSL&WOu1|bh<z!P5|pQjgT&OC=5b>6=w@m`nyP~TLcu^goQgw}?d
z*3Pym{h0S_rFv<uZN;V$+O8=7{Q*5n<@vkLUY->Vy=D-ffLWAo6WRDJ(geO|e6BJd
zqK7j{qGvSDjIq~YiLkNG(b$9GsH~bBJOi_Kte4M{@wT#F_AII?xK#BEXy8OSL$kT#
z6(hRwrE84;!=t$gw&kIXqS9<=DEbfsxAoB(_JcWNiXYR;l&J&;#)#UVz2R(j2j^Rx
zI&?IpU9bx1bf-<)6b08iJ#Xy;CAxD0HgsFK%Z2!xzF{vv#gUHVsvWltc*3gGubaPb
zzwrzSLKO;jdB2!DBi!WO3l{$Amj`Sj#eYw+a|UMcPqP6=9no758yN^qezN`kcd_9I
zTjricmx9U-;5K#uO~VzZ?khO+YTk^%q3Ubu#q!bPZf-`CMBfjkt8Ehcd76zc+9=vm
zto;*;XQy?kzs*itqvF3ju6UR2KA)VSLE=&W&h|z-%wCzg^@~m7a2c_bdh_V>O^S;l
zXUc$1T&yB!*zc9&OgO4fJn=UgojJ1WJG%S$RP%<x*5|=qexJR(Fmn6RTom6$Qs;R<
zMo<MrprrUUdHX+A#I<^BWT_{Sa1JVUUbY33GH<+C0?)^9Fmv)FO)huUH}5Chs3U`6
zZTJ%i(c0aUXtT^#H|n;VXgNcOVYjQag(Hy*PeDp$|5P$Zwsm$-C?q7fWVVERW=}%-
zX>T5_MPhybncvVZrFqw_$S&X-gi>+jSLAag98)c=XmCRiO6WOOF}%gw3wxdCN0QxN
zJkJh90j`R1#G`K}B;!ln50skvs!lf;5C^9tFRbOg7@2xZK#)ozGmnLjUY)Lovq$o8
zb_C54w!UOdx{d17;~B>A8Gdouk@M`DmWG`Ep}H^XoY|Oj%wzk3^ag>g^!gwA#pVio
zy;Gth+%m*rY)+XR1*5uT`tB~zg})tBTeia<kZcs8Fm||NQqyBY1m2=rvjVj2%7IL*
z&z7=l?MU@EpBtiR%2^&qe{RqVks`k~=X^*3<;Td{!KYQLwuLYNqs9HUF@cP{cZ1>f
z*?IOXI@@pWiXt~mk_#G(J$dv3v~~v=5))@E8AO)tAIe~Dd?k9ParMwAuIsKY(U)?J
zO@TG9v>YglB2%_iKN7m&N=?J$zg%rXq<Avx6@#DtB~tf~CD(Ov^`)HbmVjQoKi=)!
zWh>Cak?WlZ;vZq}4V7LPj>lQtd#ix(!uoO;&8)U?3y4b`T%s>kvhO=KT=RtuF;&J3
zee*w6oa}Z**8TlGk-G1k-t~v2`Oca3*GnQ%)?$`|&QEe5F{vq{p<*z0)-SvOM>QSN
zOFzmC-M6fM0~Z#oE8BCCLHm1L*Yr9TI;)LDmjS`&6HcX9PYtzIpOx~7#T2WJ(Eq-?
z?hJgQlwC|U9!zv4P~12T*K!H`w4dB$kAC!wd$3ba;)kM=x1>biS=AMboz2{@RrNi<
zLN~>#VYl_@1uy8+OPX}Gx{SnOcg~wkRcUe2_6NfU$*s*DdlZ+Cv>e{H!xD#IH|MIX
z93##Yo@HdBdY~Hywo^~n@Qe~;9)hCYMBd!mLI;DMDzX&^GBka@`7M}U@Ae6zX7ytk
zuYsixCK&jl(rBLa*N-`a519jMa(LFkHy_FX$&Cwct=CWgwBa>r_$s0sm39s5O=cI5
z!Vu4i`ut{qO^ej4JN0pPT8Nvy6X#?Z1ZZ)lFWPY4pa}}84Bcwx1qCm^i>%!IiZWuE
z&)$rp%zY@M^T_;nk@?|fM+(w(yIN^?I;)cA#OBN9pD<>lgG-aEN3L|GZ>mytPpXon
zbz{^Ij9<ka`=5{B$D0Qe$kP-oxWn?P%SJwki&-M+D{y;EsgPId6RG83xuC`&kUcHl
z<HL~ko%JUJ9C&Re{XOrJQRHA))tT}oK0D|-f1$^WT?=I^<bMgi#p|gwa><pBI&T(C
zz!S0ZL6}U(qaVC|Cq5)*%+ymB=u13tEKQQ65Uhc|y_io=70@3)(fL$muRxJHWSprh
zA5fxC-twxd2$jb%Tg-5h4fY9{xLhbDj;9>QJV@!_(p2?D)1h|OVSwGegfjT6A4$dZ
zh1-VXt8EsCk^rZxq!wGW+Mbk<{~6f(pS=01XTt=T?fLk^KE6IJ`C(hhiYU0Yx~GQC
z39VFCXWUzJQ#DAyaSu%Cm^!*+Q98EuO{}hxxI9&$_nB|=4zk3|O(`^XlB>PaP;-fM
zOa;7{R39-<Y3#DZrCFJmNHPAQPoJ}3=H-n@)U3p_izF-gm<{JRG0pwmd;-aD70U!m
zPT8%SKMyMiRzFui%`ps0hZe`5;#K94_qI?p)v_LL6aZF(@(Gf?o~svf?gkpmUc0EM
z7-;7wx_BSVh0a$_0Rr!0<o!sHgpN&L^{*MQy*8U;ZSbdHtzrE`AuAmBr?>e9M^Cj=
z@Qlfskty(*iT}k?ZIU!Nxu$-l>H2t}QE<s=_fXDe2{kI>ZtO)6k8K8pGzo$29FKbz
zjHavm{4d91m3m;|qR}f9l^hq9<H08(WE-Ya_o6##U9jDU6SH~X?X2CYKkYQfV&w#Z
z<?8bSmV4h&d8$0)JAS?D2h7G>@jS+<ycT<(?{VtWw~(Ik3sY-nfwJ8F^}pc2pMPj|
zw!SR+%naEDeLg?m+gwtQhknU?;?=cE;kqMew^l3v!J@C{B9>fA`xwR$a<CBKVfVo4
zE^kfmk2ZUoQ-`hPWg4uhIJ*{G)Cu``Ar$XI60OjcJgE&AgBRn?3#TfCnBmv^2Zu-*
z^2U8E^o2!>=RUi&DE4_sT_aSRb1m>W(WEGPgr0R1@8X4$@GE4KF4Deqi74;hHT_;f
z5z1Wu`vuOq_IaCrw?fABi_O-JZY=_%XeZSkgVzFu*s^gf(+)<R`aefDGWalYWzJr|
z(Y}BmY2h9K^44?nR&Fq9oH1+)^(qM9e7E}gGnYKA07JnN_h)|f)hqWLUAczubkzB4
zl@D^fZpp@r8MZvzt%0&i(y$Hts%=d|*!@?nsoc9{F918ZO6kg!N{Guf0z&0GQ`?S+
zI7#yl;Q`IC$aFV-Fa;qfVMjkfM41$d2g1}Jk*~*5qchbxpEk~7_7MRi!VMfbK+Cy>
zqoF2~*6;u&Ax}?u<42&hZmycOHg6ZVI{qGzn(&=;{pYGKN83<Yp3QG`=WGUj0J$Pm
z$&8pl;-f;u0KGpBGJaU-5u!ur=ufqut^_em^2V>16<5lpuad%lpM3Ag?QcVS4vvc#
zvk~4LQNvCdga>w_-UBEeG&ff5lCm3j{J~Mkv>KbU_gK%E()wu^cI?H*8!;?@=jUm;
zZG4|+;%)+jLINur_C=*OWv^ICTc0~>priz=a(U^kB?=U98-z!W-@vZh?Br(nHT~ge
z<1~?FGR<)v7bo7dcJt^RHIe5-TKS0rQgyhgCRYJQ@sORLCe~{Czujr2j?^_n6qSCD
z1_Y)ozm^YYaJm}$dU9U47cqTC{3|nxT&<dL$|NYqNi(n7+MO{We$T4EqwcpuisArp
z=Cgg>x^Ra#s<6v&Q=q$hYL+?S;snq~U;zoFJ+noFPVp;&M1&YV${m38DhU1z271-{
zWo!8;IQ6irZqipN?TL(1wx5ui(Ds*$<gbx9@ANhGj|@09^Hmx^kzzK8hne;JLOm@)
zzYT|={}a4#c)s(V<vl~Nx@e2WKEvTwjNCa>gKXJ?%rlNmF2|5yV_Kne++~O11S>tX
zrPDb(siCEi{Kw)2CYG#LnOa}3&n6#JhlBm)W|N#hg7WtzYKZZ@@eA>>da<wJW*_9F
zZ@nxor#3U)sSEH^#g!D(CW1SsffkhGmF60{F$^>%E*=g3jW{WOAvxoKl3D@btV~QV
zA58-r$;Mss@G=a)ZP&qq;o|J84S|7YNM9Vud0{Cc(0JyS)O=(5<rwG47kHI?WwFbh
z<}0rFZXAR41U|ht<FTL>8O~>Nn1>B0z!8>1O+!qD*+$2G!)GaHf5A%_q==?BTi=63
z8HXjLR<>k%0xKkluNe9fe&$UwijUvXG!?hF3ADZ>!r0^Q-xIy-^9X98RslrCZ~O|!
zUx<|(dmYC?F}^MYcfgw$N^R1Zo$*26Lre4gJsO4~4j;F_gcYT8tx4q`CqKeyc)HRC
zci>i-c;Y4-cuIy!AS@xo6za06Wo(N9#oerS2{!lb<vBSX^niQ_HUWL^?I;z={Qu<w
zNCri(3FiE=gSAYZc0#tx?S1On6D735Ad-Dh_{fW(_~lvaG1>epE^=sRl_KMOpHd>L
z29UY4Q!(<RiG7No#NRh)@(&ipNH!lj8QQbEsjoMJsKJBHarmcJr5GXHm7_tE2^3z9
z$i5bOiVj(ByRjUvmTJ8o%x!fjDo<ce+kGFz=xF^h|4seYXVHG0TdMBeg2bUW!~wu}
zg^%S3T>Dcw$~+&CNQWxk%a?WJRYI6zhMCrZJ@UeeBt85fWf#|<`pg={pba;nY<rd~
zL#-8#%7MrG`mu)XC&eApzXp=DDESy_lgu4)L~UoN+2J25dr-b}!wyOdG>bQ0wH%(Y
z<SqEsf_KGoNAAHpjTZI`2b4=(6^`w5AO0ME7e<ZNohGyhVzUPbeNap|Chnf96NzBL
z?l)U}f>3o`U+bvVjG9c7fJxXa!a9VzDoBb)n^5nf0=QZ4@II&>(7oR=9FJd8uzQqh
zCyH_Pc*Kp^dQ;FFcs=zb+@Hft<kcl`Yqpoz(JrzDcPWbjbI=x$e9G%};ONazwJI)q
znf1q%p<;^0wC8|(ga)$w!Jf6b3o97doRhqkoO!=((zM@{G+0=BQU~bI_wI7ES>xR?
z1czg*rI#=3sP*NBNrW`1&n}I@1L1oiIw{pl%5YL8K1`bte^Nw_VHoJ?rCEqnr<TL*
z=@do={SzLjiBCm6dAgf76Y*WJ#yCXkdD;oZ7T<H$SmYB0bdrgF0R`rAyC3GBnEtv2
z&4v<g8>2Zjr0RjHMJRGsQxk4ZjTvF~J7m^h?P*Xdo^TQ%wH@ACEDf8soHa=X^tWD9
zw=AQZeK8Kh;=Eh!#p8KvA4o8O+f)buShuqvWg>#R0VJ+@f`Pjm9yxawy=2#l?W?V`
z9~S91Q&yj^7-B=0(_l&WWzptnH<UpQCGaG<_xjQfU-7~9QG<X4yH9~iV8r|X)817E
zH2r;jx<-R^C=voA1StuFE)@ic4J4!mHc+}#y1SH;4yn;2Cylf)VPgU!Fhc2iroaF5
z`@er)KX3Nx?(BZgz31F>&pw~e7qD6T+7?1nz>d3*408~row8=YX@XM-n*5m(0W(Ly
zy(=96-TOerhdDeW0KIZ+nchyZj205?_U^+xT9#<xomsMN3O~9Q|H~&h_?<6wB6Ta>
z3tX$;HvdqLa`3XIfQTA^oG%Jqs@0Hc7~q7@`~oeFB<y8+i9=3G<RNWO%j+8KpigU(
zBK=wl5fOmJ=2a~2uSD|FKfF@W5u)Up5EhRyIEFKhL$P(qx8!zNc}~NNfV&x)y~C6u
zvFOPzw%_K3YSaRc3hNf!l8)_xq?qo`OQr%OAwD>MX&u{io7aY$IDi05sX%Z+?giW3
z7$aB&0TxJ(UFBF7MDr~9>7UDkU@QG=SqlPd9tc!(1eSR8N$XAtrReS>QqiRYV%SgO
zk}~W3X0nWKODB|k@h?}FsGP?8JmJkR_%%b?Jqdqb<KSkK!O0ey!x$3OHr`JDP-KDi
zhuFcLy^A|o-z|AqP@`#VvBbe**dAxEpD5Gn&8e?);$#``Tj(+X_Dzh7+fTVpoXF40
zWu<Psf0{ou(h8jJsRYiQxi6y|;5dYvdPF=+z^)YS$ogBri72FI{t*7&Cmz%oR;_!G
ztmkCf=iCaMSM`39g=1CGFBfFTtMM<si@D>cddMzfcF}jd+FWfc5_<i!HT>g5{!S~^
zH;YxZ_(NTF2j{|MKZoexS~S6Tv<;hb7@#5}kJpQwx#JM87SA18`Fa;UC`WL-ra+Lf
zy>mNE_iTMj3rVD04a@ya_y*F>&efk+`^47`LI~dTez<bY73~{zC4{v+Hm&&e0n37#
zhKX#>V>0}ly$@l<FLuJd4Or|~X-sW2J;cE9T7*1APXcA$@$TKjHjI9xc&2ZM3dv#!
zo`3FgeiO9;U-Z8|FM_W?7@R&9N$#!v)<*U}Es?FjW$@kycbtjSqxGKS-w$d++Q+{9
z=IA6xGJ)r>FB3Q_bs&a}w77Owb_b<9f!e)#HqNFC);gP^+j&tq=%sM>3>`G$@_zP@
zADq2UwKH{dkAU)QCrT0DV!U4GFCquhG+jOeQ5^KIalF`#Gu!S7se>V+6uw#)>Gmp6
zAr3cB+F226ZM;0_|9~5Z8mITlIpAO=+iZbJLNEl4ow#1$x*AgJI7PJ?2X(yZ%jU9b
z>&;HsJ?Lz7LG7Ynu^8KP$?gV=gn=MKWJZUrjA)N^4k8X+hj<^P<|p?+03-!+n=p3m
zeOG}TY^Y)5{AxFR#7u@h<MSS4g*$=zupJw=lciV1;_3SvGMk3m4PR^gop(BuuLH<x
z?rqaR04#jIOpI_&DvGzo4mT;FL?uEq;2<tgK+0Wk*ot><r@OU-eg4kfY5V%pB*8gI
z(Z;g~JLK0sLdDvTI6}6Y_1Bzt=L|FIFXzad<isQkrrmWf#0${ZOYO%eu;Kd&n1T%*
zHVWqtB|&AW(pjm^Az)+(*d_*8R#_LVYQYR?jUxC&Ts*16QmT1}zzu4a==PC9i;fW*
z9ztwMY>+tMA;_0i-1tz8>CWVo^jNOlwU>U`<cOT4o!j`R-%72K`57X2<HDX_a^n~r
z(9Qnomx+15Y;dEL8&Sk=7X7&f8?31{W?g(9ES)_;i&ciWxXL`BI<o?-P<<KQ#}XOL
z6a+81y>T3c8hna@(p=5<BMj+j%++sqSf2-QA*$IF2)+}Vw^O+wrez=qHtF=tzy*A`
zB<%QN0KSFp)eX~wg+ykoQYdx>X)Davs<4Aqkx8)I3TqmEG6E-h?`eSjJ7IzuoX{vl
zy1@wDjm-z$g?<cKsCd^)GMT%_4B+zt0kQyO3I6f&gJ289SCUnapMDLyfx9XxFD^fj
z9?MS^YuuYdU70&3*EgSC*@C7OEKtJuF9Xc@xvMll<EWCU(E$ljdmzaZ`*C;A$$7#?
z={*xnxL92?o#Na9XhPh&(oo?|Px91+QgghmCk4j_Ykb3`!A~z^EXVYipd~#LOi1{g
zYXbEmEo6D)hmKL5<ArmPCgzs}e#{-yg$BNfTzGxz#2l>OW&%c7{;(Tf{U+@6;f3rM
zfyi@)OQ!Q`=xnsQngVvcBSIKEPYy)n4S>Ssm&I8!-eV({k{Z{T(sP>WTP{yY{obfb
zm3ao;;_H9roKJjo`vTV!NIll^d&U*WeU64{7Tuw;{6&N2RN22g3ErjKF}Vnh>Ka&x
zr`-&J>%y#W>3JuPJz&n)xW+!->e<oX?B^r}1)o!bV$O4s<i}i#DN_KOE7tj7%qIPy
z7mKHgy7c%ezlxSO&84tZZlX&+B{Hcd+u*J654`*}ekTHfA0N$?lEH~xOC~W7nu0a+
z5z3FO>mk;lY&*$QR5cThk!tOftsIBG^;+}Br>!`&JwUydRplut`vE~i+?w2kc;Jb6
zn~V{!>!NJLxf6>sRly*J7I<xMLByY3Zfp%}ZHaN!r8EIoiw>mCMQDtp%ij;l;VIeC
zb@cTVFeKv`zP4w4VB!P4{Rn-%16UsEtPE(gYEThH`t6b-{OQpp$BXrI%E4-_Jo0;(
zuF1;7<As;}&yn5e7(G&+sf2(Kr<jG8h)+MEL!N;;WxFx*cGovuHqapfFuUrZ<SOfk
zv>Im5H#N3~HX+OM$t$Hy2W&mjPpGif7##v8_V(hO$fYTy_MXt1a*(eZ6-VL@byx3n
zvfUHk+oYy_eJV!&`?M%pC-B=s8`t_8kuiOMa2$xrn2J&&{llC%<HFVDJ*h=?^O##p
zNKK~&z!IAr1w#iFI|`Hl2<bXGcWCarQA9Ba(SB!q*5?FghE(2`ALP*HGY(zK=Y+XG
z49||L3^T=s$%(e3#y#_+<qMWqL})4pY6ZKju#$QtJW~@uaq^bX@(NR0SJVh>H%J&l
z%(4<AlUo4=gaziOjkOWC0BUUw*CL-C-SvUa#)RJAjjr!$k!4cgr^Ch$Ex+ZO5A#yQ
z8#C)juGqMV^e%B@!P~a70E%;Ezyk-_*r5J=BbFt1ISK@M<O<SlWhh%18V|F4(ZAE<
zQA=$;KxW#Djtvl@5x<-KlplBvTike}A;c}i`YQuQxGAtwdNwiWPu-$=&W~huA$Pd}
zF(;qDrtv{#3?Bh&Jf2A{kr<t?@`0UJXxBVm-42L2cS-vJB$j}Z(zR6ROyVnyP!T;y
zf%UbSk5k?EXbv@s<74LM0@yW6L)9{!@I><w>{+5+uPIZYASI#naAv`}m+p2aM+_MW
zuZ(L!-^krX1EICt9YUTp!Fh3(`lv4=GV0KpZk_zTvGm$^@`*(9?7g#*We_ptvTp)u
z!+m*X)oItuW#D)6G@vRexHa7X4}C@qB+93WJ^1{}H`#uRZzh==Gal!9P*%pcCa(gT
zDC9A84pAzJDPgfC^6zZQ4Ltpf8d`~8a_6j3d}1-^=I{G_uL8?W6zPopPGK!O(-t|d
zg#p2QAT$_BKqm9i>L!NuS($YyRJDXaGTpu5TMxrbs(J*fv)`BM!YI?rYLC}Nb$=E{
zsQF$HX1>(4{={)37fCxjU(mid%9Gt4Y3l8&dHA!Z8aO{p-RK<52PM@DE=9i>SDt;A
zEYcQr(4<gx$1vwTVKD_*cW-)Hd43kEIaxp9mHu`P!`o{}{lsL%teT30Z8iH?%JX36
zS%PBR2Vzfig;U@Jd*3Y)@9~1hJAR|B<TE0^G3`J<rJ3-EtTD4u%Cm#!FU7ZJs3U!|
z+r9V<boMhtowRQ_N|Z;NyblkO5~!GIfrtcDd^12T!$QmO0FWD1MP>I6wC@S>q=_Xs
zoi8wW-n41vkj?)2;Eb`P307GAe3iPGLNVT>v$M8>;de~CH#zLVk1eWPsp{bRM=w{V
zdVWd!P0R)lVvI7R>O4Ph;mLjsZ5qpK)U)tJ0-x&(Gl`LONtWjL=g50kWa7J#h&3fH
zk(npDc48mj`VJ*1c!A8iu&5QCWRfVA?ToGzqlKM6Of!wqRuS6tG@^Eb(AWgV$tOy*
z(e!dESLg{tcf)Py;^qm#oL`K2pgxTAtL3If8uMt=7!L%e2Jt2dHGxa4e4@~A0vCN9
z1o4@{+Yo9L>nEb?AkN9Ha9^1Hkh>!PiI#K0j$gvpYF!S-gSKNErpde^_t>|c@ZAWM
zfB(p~6sqHqVzH#R3)%FFpEAzNC2X)8CoPE?H4RFL$ghjbi=S`7Wga6j`Bu$85wD4B
zh?j^Bt}Eb!9l=}ZvuWJ6cy|Nlcs;wowsi}+p{6e&mST1JV!2Gfv8Jf8VVg&D-yJEv
zUaas^wS-ibtHnyNM*;7Gu%;_PWlIk187zaL=GClFdQ>nq1b+PcT550LNR2<D6@fRh
z2@}z?q?AlI*@+82Z*n!z&X>zl5WvGgXw;yu#Twz<l(}`&;%pM$HG;PC=RWhX7EZ3W
zjxA;7wwneM9X8h!EV1Ee+mLG!Mt$>v=IJ^(O%P%Es42;S55+>l;=&x~pcQgF`Z`KJ
zk?b4I1t86h=#G*TR6Zr(d1?Jb@mVrkx$ZPncx?6{>l53F&|VDgT&2~9grnj~mZRm;
z9e;yRkks5jm05uwc*df9D#UM`PAoG+peesPoT*lDiR?{~a}~O)`QA0xL0iJ-@0*{$
zpC<IyY_IlY)L-Nt9D7{7MCmK2*&T2ob2qS(`6AA~S}sIOwH0>hAIhR)+OvOwS=U3-
z4#8}nPLMmyo(2~c7<U*B>aCB?!*701YCm{mcndd?VjqSRfyX*l{0mQ^iaYaz9O?^0
zi&sZD(iysT#4BOt(H=D$54q<99F4#gN~&Mz85~&ZSM6J=iGM;@L$z;q7oi^qtnut<
zqjG+%%ljisbe#b^`guJ3ehr0t0uD?h8Noo-?f8roIhDSCfF%0)*!-OCE8<69@`g;7
zXgvG$0L1#V3QD8ZaA#+vB*-#97!>@Rg6p<~_tDVzA)EV!-Zwj|f;*odDgZdZ{cBHM
zHlP&JC(Uz5_i%LguI@^Ttq^8<9atK?6}}8>@R(iq217eWCbgE+t9m&1WmAT?tvi{H
z&6Sc~>Bu$_c=mJl^8<T#vhN^<Pb1BU!P7LybelARa8AM88=f~+eYjW>OmDeF4cutw
zpvhG}Wo~o);)i;MqOWKfR36fLLME(Qk^lNbBqn$#v}3tf&^hixXnKI~(e4-I&}Fr*
zA7WvWn3qyqt|>9ptw>I3iB1<()%9y7@aK6e2a@8g8E5;c;2_%w0^&bFYRwvpkT!n7
zHgbQ(x|LMBdSZ*|enhF)W^VjJwN8_aTj3#$ZlUenh`j!x-(jwI4<<(`AD_wP7IFn{
z_%*zPOQ}QU`~h+pZWJtBb`elkOTcPDa)zSWQbXPiVuf)srJ-z)az8gP)UODYAVMy}
zn^|`epFXT$_^r;iSW*XcgG2}YuV?y0f()7lV2hj^=r)zt46d@l#;6CufT%!Ez2sB4
z&U?=JOSQsdQEcYgF?lxzwmzd160onZZ12irCo)gm^%?TZrf9)wn1G42v$!N>b4Pdk
zm9ySPD$SEqg_ZR6?>b!ypu|rxUJPEPy8<4qBoU?h7qop*s?_x`ixYiYF4wSD`>&wy
zL5j-YgJHNKx_QDV^Bk)BEWXD-$w(`tc^L$wxpFESHAEF>M#|R)Qq(M7B37yFQsz+c
z_KnG-Opl1c&~iC=6VWcM9`!ETl;BceU&ques?r?-sCf~s!3TOdvc?52gmSnf@kOn#
z-lsDBSrHg$Om;y8<NWbXCbi;_de0i^*>jL-UCtsURrF0hWD8ny5xV$g4hk9%JNI<N
z7LTz6Fp9Pv@8Bsc<gZ7FZ=i&~%-kz@>V0to^v11=cg@|nSrR;eYF3LU5T)?0Z4fi=
ztw@|V$R@ykuvVzqy>QjM68aqjGaNR9NUms&Yi|(BU>qeIld4ulm^|nFfZ|!N_&3<w
zxMa|DMr$jiP==&=y4{t7X}A)shzwvAtIR7D6T|><Zm~_Os-cSsmV~G;;Bv<dJrX)b
z{*bWdMO^Z)AD67hN*BD?bJHPjGO4upcHBktaECD26h{&DQntPVdF+_G=)keoJ*M&P
zYU=<ehLwI^50B<Mt!AexAO=sLJu>ma&z9->sJbrm<DT7bDhv_wWHK24m;4i<=-x`N
z2B`B_j{)b6^XDDti#$&pVWD|_H~3usxfNHZXAl$87X@7P5AtR{_FX2Of4q~%Vokmk
z7<>ucV12pu(f5igL)I_+J7YB{X6aCz&1lq>AA2mMKa@`r-=FuiJI$*lsT(ghE(hF6
zKKs(dRS?3BmGtsNVPMu82`Lw<@3am`YPyC@;2Oy9zjb*7c@&dkn>4e3Z*f;U1kiqx
z&D`zLdwzfhY&Dh+v1*9|k4U$mPqftw$4w&Aoh&SdF<|8DwM6;|)7%4Dh1XJtNP(~X
zQi&hQTblgC%dHdEPcM%w@?JS0Lk}%s1g}HS_LoN>h97qkH=4A2P1-miiY-OIb&<{e
zyb^Fb=uT@_{5gT>M{}fJt~`W6V(lzBFmg&ML1~*8DR&k3mBP~x^#=J$Idz#BJE%yj
z$iBa@_UT5=ogZE(Nf@u4QFNv1C;FDh)1OyrZLyI{L$oCm^{1LW=YjzT-qtlZTyos-
zGR#au?|!a*<y||72t&~Uuef0EzO>C92qRYVfDGx|k(-W=|M-ov2Q<orR)<9m@|sdr
z_4=oG$7z4wz&NIM=&UuWpYs@~u&>;JS=$1j@`Y{Cw-%3gY}3XnhlnmK@>m;fX~j_V
z@$X+0`%#m%TLC7s*X0NEu7zz7Ap?6(nCWF?TG{NV$gq^Sc-rb4Ag(|))4JP&cbz-d
zAif>$>|l#}u|m)gkpjBkD!JY1_>dEud)DLspnyOP|Bx>BTc|wXl978xyR4*ZCk>9`
zZ=)X6o#O~Qzs#`W#2#O;xOk3XV5Bu}TN6b7%Iz4gRRI*g5(yKu)+i|QJit~@QTRq*
z&{MoVXn`Q;S5NnZ5Yn!nO-W-oqo`0hSz&3ljlb$T7MU?6(7XcIg)1%|{~{P#G#BHc
zD;px>#7O~ORwGiv?xflQ%{sYAU&)&=odv|{G)AzhNaoxhF<#7R_u>huAGZB_sXO&_
zI-${eI(m1n{l2WNo08vBO!K{z&B|BYy<(EyU2I3@QowMKq4Upk9PO#cN8SRma?Kr*
zgEo)xg_%cbC$`C)cyl_-SE?vOD>LF??*0XEL%C(93^3K;SVwJdwNh6o=+c`!BU9{V
zzO)0B?+5!L#mPOFDcLxXzuC1+11EN6$4u!2bou~%8Sc0xbW49sAJ)A?g53D7fqYVR
zSdgF$S(;kut3K<BJa6A!51kl#MhsOgmY97VLOq$kP7i2(#>|?}+L+$QFJkp};k-%O
z?cH6ZF42%<f!v^*ovWB9U+HB=Pvb{9_?aJ|ub#0*s4>a1waL?|D=$y`>4Fo8U2cq}
zlmUx8oeHs>u;RB?DK6x_^@thACdPMEcvRq^G3M&mo%zIqDFEX-HXp6Y^fAs>eMO?r
zG@jCT$Cfc@#bkss5Rb-Y<Ji4KCDg$gCxc}jXT$R)U)P^TRz78;*-F&qse5dA0mil_
z^uS+sHCdrscs-v-UoZ8KaIuEf%$pGEMD{UJ!GevTyvJttnR5IS`wY$}LRoU58IG?c
z#M6HGegUCsnYexd8@4E)GB)t}x#KG2S-8Ez6<q;23qS4AiWpg+0!@70lnMAtRk?%G
zZfkV1Wx@?`_E03!<Q7kw7Wp6LWL~u~T*>u3Clh62ODAp8jFc<Qc^<7fg}*q>@fs~!
zSbcP6No?&sATx?EY|3?|`qgQko_YU8>0JFmp^=g)`ruFSROIjc)ZdOq2vl!LDpL2E
z8y~^*GK81`I^3}6Uokt1%ZqTkI7Crc>@@s#r<va^r?(`g#cFr{cr>a1S;+Fo&Llt;
zdYDJ_^6Qg6+09rx6QfaocVBVYrU>nNKW3I^qbzhLQI&-qH)`shgV)(dSW6h;PTKCW
zHDkI9nql#eY0mAK_igKsnUbQ5h|F-xi1f<jXFep;b8u&tC!(v1q|K5XzE9NmJF`I3
zev9xxw5i)nA(vBYZ%d<HdyH$w^y26GLBkQ6CG6Ht>crJR;}?dc=+b{I6{U6|X9uhR
zOAr$A_IFMRK1+5laHS=)?|Hg{yM!5sBjrk1b_It^aQps=fi|;93GarpL)SMWz6Y<}
zmbcFRl!sdXs*?6=lAn=$a^Dm@;jlIf{931!QL#~uWMPY;hFj&xw-(h&Iar@^)fNA!
z7wRj@N1W*7Ge~fL*Jdpf(Pe_=@9jD`QGPU=8tW@T{zv-JU~#pJZbvs3eh?O}rwYE3
zQHURVv*%R~{m#acDUlnEerz$XkI*LL_JW(%xNNQt>65lw0hN7xT8ZbEMHKqNv&IV@
zY2InPon-RFZEa0Tz8^W0(%JXY1?v0?-;MazgHt!x9f5q;gipCAf=XPWg~7W7zG?T(
zbzb1NFjLOk%5TP6!AG(e`dW~HSiyB4fKlW3EDlv5kAD4s_VJxZE3pnmrv^P6PzwZ4
zB&-GH?$exeLvLUv+xF#zTeg%AQG}|LOV$Y-Gbi3BgUr}L<&}}RxPDi0I=QR*BDV$d
zRy+S!QM9^>P&)ma|4b4zc|ZGG+{A4-B8J22Lw&BX)>2*E(9WaFpxKKQ*Q|GFZN>*t
zv`ywH2g+_y7VNmko?~k0%O=UtIhVfKIfxp0A2?;m-Tc*~C=I^nA%8T(qdpKR1~uO+
z3RV}q;_$+URiNA*1PzB}`0Vz6vYk>!h#8OEg5#^4!&~K;yjU~iEMN49%MYg=P&MIt
zt}e43AHpoV3WnVE4DLp04E)#R`y(~Avj`Is4`B~VfEdPIO<SfrL86%^$3f$5?CNwE
zLW$`}yzXN#KcF4&Egxkkr7_Vc;gyyD)#^MB4NZS=SQ=;TWhr80En6OD34{djPe|zv
z@bMTv`X_QD=zr7x8Sx?VB>`!Mc&rp8r(P#(YGA6&3cw>1+-4wiTZfYyVF%mA^?P~-
z#qBShVtdZ&dPAE|I*i0S<nWjdPk=zg&+GI?X=rXem~`8qSA45#Ej3*<fZh<yqc8RE
zD$)qf!{Mn2XE>CP`j9%Jx%Xks$aslB^g@NEFHRvBMmh3<kLs0oDC5h7!kxT#27G*f
z1lp5Xv?gOL#hv3`1v9~g9qwpn>?X12Y<y%83AZPx0%u5lvg?_&MG2S0+n`l3P=nG?
zKgRkf4MY9^U6f8DTqMQ%(z2WWG-fhCu=wd>hI}3q<x>+?n#vcsl=W<?SiRsSlfo0x
z<5MrXaxbk(RTUN9yQ%Lp%EanBm=}Z#Z?(sv7ng&P*?A+w?*?`FL$ek?Tat`RKo}FF
zEoQg~Q*zp2)G^CQo=<%Lep&tfF8FK>?upOvrOp-K(<dV5;~NQ?!n+LU=z2<%_IQRk
z^lKodkZm4Lu(AP5ce!Gj&E(NP{MT6tlQSgozZG0aJ>uV3^hwUSMZQFIemw;yyx9&&
z&Uu(;^7(_{Nn1EWkn`=X0g*$(%r|K54)%Q*kN&s++AwyZl*hGr;@0{lC|3ZZ&FNVI
zV%K?QpSNsa>aWf`m^i5Ys7CZyA~Mj@+CkHj-dw=G1s?s&e+`JGlv8nJ9h}sM>)CXS
z0pQBUw)6A9-XJRxxTb_R;41<i3^U(Xr{4+PjqpPahD@ZsmJuF)Lqk^{@UJ5>z8K0&
z+8^GOAmPUcrJZ^ZTTt(j0>xNHa+e>CiWq%p#r3-Cyohga`3;_OowLh&CfOR};SsU@
zc}D=*NEcjs4YsmUui%OrT^GGFIP#6sX!<ueI=R0OPG<gG(?U62`Z)oB1WXp2gOgOi
zz5^M3U`x8^C8H~TUA7d6^+he`XYrsjuAjbJEVh0TCL>4rmdsA}_tMpFI$IKbF)*XC
zzMGzY_Z!kYruv6UF_4KM^ft$?4l17vfm4!BVk;!-<ZhCNl<nJ-Qb)_DQ#8<@^uP_l
z-aYg$1~_B?(TkS5^C;mFD7!D%!=f8ztqDt;bE|aCO5d^7ME3k0@h%kysbM*6vT4VY
zRlpk4+sdW0Q*6Pc+uz|jE^7ZM@jkfo=xNXh+rrYZEO(~HY0MBGu7i0OEaN32`KJy+
z?*C)yw*cP{@?`ct>bIS_uc$s+vNC%rT)vXp18}=^7sq@x{vE-H%X4%|*~LVPA53#(
zKZjzf*0$i|Py|CgLApRS;@e?*!PD_y9!UIuB+dJE$$fgq?iH_>oaWkP*`s2Z)CM@8
zC;0>k055oKS_8G1wK3Ehf9J@nnfl%+(}5~h#LJIvgpP%ps5wr)RJV@B{F72bF=fL~
zqkR|-{q=XEue0fYQ%FxI`jiJNQgXyv23m;W+Q>H}AXm1i+POW-bUXb+1%KS-_uVb(
zKNH5O>=qMs4(s5KiZxe>8jFQKmZ?FB3RjRHVI6~b+2;@b7IJlDUIV=Zz;AQn6Kv~$
z%FwaE;D1)YSwF<w+Kq~`s?rW-g4hc|&{}_G!y9=<NMdXq=LEGy@7(vw@mZd9g#9z#
zTkvl=hWlAfJTWZZ->1WU{7sCw5gI&yi;F+g(!~3li~i_AlfVCaqT|N#;{OelKel*-
z{vIrU9)RfC-^BALbK(6Bf<I>IAO35U;Qx*O@#z2Ar`I&HhsT=f%-=3>uN%SBRDGgS
J^VlNfe*mJ{2nYZG

diff --git a/docs/logo-text-padding.png b/docs/logo-text-padding.png
index ebe53a285856a1e3b302b1ee3cc106582923a07b..f99fdd3c5d3fa28f6af5a300d630fb267c357d3f 100755
GIT binary patch
literal 25396
zcmdqI^<UF}_dl)^lu!u?lai1Y6;WV-AkxweLrEp286%`bKnW?SF_7+#!N8zvL%K%8
z$Why9zE7|Bb$zba>vjDH-(R>rZ)1Br&*Pl)xS#upN0`P-g)3Aysm`4{cST9@xz@RJ
z7jVGyA;m@D?*m<jm2>CV&M7^ArsD<KNV|f8$U{lO(&t4f_s?r)&M-&2@llCpY{W&0
zbn3JR+Xjz|Da#5byt@1CF5;p?VE|~TM&?#t<Kv8Qqx)|HtK(nYZLF62{otNB<*RBL
zC9az4=)8}5zRUj8@ORRln@3M!X_c_e-R+4a8~FE18}z0VDYfzpRe~Z?N*|4({j51D
zWX^$T-=8C6J9qv+kKD>LloWemJIhP|_y^m$0G9J);up^U^A~Iy7eEwyH{C$Ge_!d(
zqHLHO7yfsho3t{B5yZ7T&|hEte=-F0UH{)z2H&3pp)RJ$F#bi(e?L;Vwf#-}e|?O&
zNJfhj#gP^KcU%AYr3~WX#s572_eG*??`2A#@o13$M+^V`3MftFe^<#nf1b?yig*Cu
z|LTtx_v!!X^8c+Oa|O_Z9fnfpf0g%M#^&<>u3{^5E&vkr5<&Gp$^%Sb@jpiU|FH=i
zxVD<lj+%)ZPW}^3BotoWX)YGN$0@Ogk=uBy4ojZ$oc1NgrXO1_fZ@jrJQKTck|gYK
zARU6dDt)q|?tq4aQI9{XTw?J;pteGU-KMDnHea-n|5LBD%CAV9Q$#qavAxD|>ZZ5S
zyCxg(njx{zvuOZwx|29zXLe-t=g>Uf88@u79>X8^;YaZ&SUh<mSZLm}`N;HWD^%J(
zeZPgLErH9q3A$?yYkLTvMY4jXLLz$p`MMeQt(hR2@f$X&G0zcy4r2|e`Rs(4P}|tL
ziChypBKe<rHq3sw<t-hgl;9DzFKApd${u=qF8?&Hu>!s(m7D4k4nE%crG1)%O!YrK
zc<g`b1KV^>w-bhTs;q~p`(&;`DBPHZ9?1Q}SKfzyzd$4T%E{~3t@ZS?gLIhCQiWUd
zL9bKISf$Ox`AfGk*UDY|6Z1|np;A?)Zwxd32$wuLV0Gi{l+^4reY_;L(I<hgQcHO}
zh9Y6@&^}8KVVV1Z)IzxW+Oz$3=^7{BHDlrtU^C*1y1%!w=pvQ>N_OISS3Ey>DE$j5
zMuDZgc|W4LOw?{z_28_a9lD%mvz+Ss5<LZ>2B~}fyf#UChG>X!KL1Z}2&Ttiw?-P5
zL{Z=g7m>t|@pi~yPi}L-Y<43iuAfnBrucFF^?n6b2eYH`5$}UIOVnw9l;7aBRnJLh
zh0>z4!_u>gp+fzMOHx|_z9TF6^uxhiZ=sE#6<&2{_{nPNX~D_H^r?gTb66}BWz+U&
zr_n;QlT|Fo_gCJhyXj8pfP29f6O}0T9_-aI{SyUqxY!CwojmE2u7YQC%*}7m&^}jU
zcPYZ|8`<GV^C`g><hzy0dOUN92-ue7v6N&(46Y*v+bD5BNd1}L-)SN0_)8mqL(UFI
z8enRAr%}G^7AW$7*;j4~EYK`pK!DnM*JWh)okjo|s92f*Y1r>>WR~tyD_ODLnFe!S
zaci~ui0)+HvlG&Up-q~Vu-|sX#%f_HR-at{^I5z9aXVIOdv<RyS851CD9|UEELdrh
zIH%N04$pRk&nl*^guVMCypJXb4XUz;#Ekape>(7e!Sm!{1E==n;jys&yah%apIrZ-
z^kIJ#k?ag%*GP7l*nGSQAM$Jabc4v>yu(in5%$UOG+vCCI5ayMG9v_gzwzGdSAZSO
zR{NVCEnVu?xk0~FJL#g5@onhh+ItzA4Z59Y|McCI_hgmCaCGWZ*L5tO0|b_cG8x|c
z(Xk!`GBrP@Ih~>L*2Gr2VV+$Y%g=?6OK=N5ik<+r>c3Y+pg(NC?fMC|;o!8kk&UJY
zO@RgK{;~WyvQ<UIM;X?U$Y%jUv&Y*}{*$MB>Su1#{dV+=hiCLSae6^vE7*9Vpz*Nr
zD@^^GOQHY#t?7?4=%Ds#_J0Hkgdv$0l^c0%!_j7g!^-dMBZTDq4K(f!0YDd|iAIAb
zezP9OFOwQvUxW+iI7?|hNTJv>@wWVLO!_~K2fUzD<i(uLvYuuvcbyK`Ho12}-0sYK
z%zqYpc?)Vfyg%X8jH>m7w4+wu)B-Rxh59y5i~D@jT<3p0RF)!(0cTw6SZxEGX<`!E
zrwzN8Beqf4S7cb}4eJy};xfy1kMU=xPDeABq~9c(`9X!xr50_EkN6F!iLqnk#1ka{
z=>`}9P(C^GIia?RAZXm=Q|~ZQQ+#Rp5B{9|6h0301t6)Cxp9X4=)5*9veb`Ih?*bY
z!!z>h?GY>fpv4P>n4Q;9ZB5jM2+S>N%mUWySC?ysR`5_O#uxo!J;PtHNxpOc*ZRy;
zg2Oe@pudIsWHq*GnZ$EvSW=VC2n6PURs*a23yy@=#x0~$S^V~AvGp5x|Fa_)=yV|c
zv<kHoU;5}y{ReW?*nDxtxXOF!#%_Dk%gRvJ^)nwE8?#Sq(^kLFKkU7lG{C%Gxt!E1
z*?4BmaavYyrMzVHH)eq7CM%%DRRm7EqQS&l&D*y;@F)`TU3a+OdU^YJuh{gYYY_TN
zq@y<c4_*;5ax`OW5E2qQwxaI4s!fM8Wyym&H2bHF$w?dr9tSuq%6nl?+D=U8e2J4y
zucYV5Dt|?aq=XEi?!7Ge>!xLLDRkH|<T`rW(r3HUwLm!ZJ=&LAeu`e%kQ;frY&hQF
zS?Vxa_5-^e={dSSRtcN)*`8}fNn1R_4`NcA%9GfdJtKgZ_F9UR+I=xdv^v4HS}XL3
z#eU@Z$6>q9tr0JC)~W7i?m6W?BGp~Dx}C)X?u?Y!e4<_Dd8awYZQ7SaC|Z*6+^o_^
zt?GhF#GtF<6ajr0(b#*5Y-e3<-qbsdP2%POW8Dc!ck0ird9(1JUi}L-3J{{pQ$|GO
zb5xr7Y_}45{C9YcXuFk9ZTyKgsE&2%{U0>mbkk_bT~*j{C47kbGO10h$=M7dfHS?B
zB<*Bp{QRJ<M6Ss5oyLIWUybvLT_am&WNayXq3KP3CcFXpTN}Q^a8zw}Bzn})d?ay%
zK08)F0)T!4E^qp5)~#y1Cn>7o!d;BSgHMYkO%ABBnk&6Eaj=H9Q7hkW<Z}tizd=2Y
zC4i3{iQB}Y8kkY0M?IB~Z2D5Mr*8F)=^o>6!JZ@<pP36D#4($RYw)dL-ve5lAZvO>
zg38zK&!}#tXNxa=B!sq1SuRxF`R}rb6tV&)+#mE@A2xQHV%}F9q><eE4sSiKNBcTD
zq3in;n->#|*8x9D`^`(Smk_B*xjP|*$BXcQU&_du9u0C0hmoN>{t6y4N0d5(7<spm
z5>wQ}{y$Nic&=Ny#A5IdOCtF-1JSaA)VK`RE?Vvim2n#Ldj2)kGjDR5ZgsY?gYvI-
z;--(5IqOi6GG9*f#5S$6`tJ1;t4n>0Yo#Y+tJ;sd%}y)Qe0DmCnarrX+Qj3JcOyqj
zKeC=~g%S_Y{@4V7x=c^IO))p`%S-HZ+$9~QkR2}drsR@EYWo>ZnC<^$_AWr0VR?>K
zdCFq>*x%n5jx^*jHNO+#?fk*yZ-OA{ue(bfsF)pUHOxm!*aM-Z#J1blcWt<sxYm5S
z2L2&H=hq|BzOMfvYwLtG?c@aQHumO_QlX+C0l+yVRKM^A>$f+Xbr0_nU{-G4+9a*i
zu_ooZ+~+>z{^TJD#n4n1!3xFL?;gZDTS<FGj%B#^UQK<Q<G;52{>I4}+<kwK;BTpa
zhL@hSAZ`U&!#t8s&_UKw8=|l`3WR|53h*lZ>fduJ1emPpXbL~hdeYB27Am!u;rrQp
zJHpJqcFH3KjEACKMRgLkQyh_<FQrf1U_YoKuf!(b1@)#&`&-_ipBORo_r)t#3n<xl
z%brF6tm$s)Sc9jNaNXp@^eNHT-~SIS@!u8mw`e}@ZeB0o2lgN|-D;>qoG95c)qg~K
zy?Y3&B^+TadalM?_dkC0KC?9v^>PFD#>Rj4S)ecJ<wL<2Sug2VbIlw)4C-6+2ZyqJ
z{(6>VfV{}M69NGH$$<39KodV_PifPkt#O5=k_>N@P5;wn`lCOV-T;Vk&B>o2mFC*_
z^lmo>bw)5dtM)uS09@v+y8q$8My=mkt^E|z-NJN@@%-r>yjQ62PHg(5eomhXfY8&@
z@>LCdC%Zicm<SE(x)+(cdb@Ffgpbz_6=GLxh?7<3l88=}fGys4qGxmL3#6=48eFB{
zap|vq>P;7Y-)|vC!M;86j>jK#GHF}nI32Z!+9=EmtbWUSSicOPh{zNAA!VyxCxmea
zpldxo8@FEZuG;}3wCdt&R&5)9@>}@iOD`Ne5h=c!$9uo(?BpBkNgir94gcs)wFdx&
zV_`h-CtOhi8)g7g+X%SCQYP#(OHIsI_wD+Vo@{h~JIZrx29Rc=a+sgK(PZyZlnwO?
zkS<biDX<vEbM<g(SnwGBvOzzTu+13jP8^bpUar=pW)zp&+!TDv#QzS4UOWGHpgg40
zK!{Sh7-m-6m~a@LIXzL@m)cE2uN&4xNuTb+qgh|l9|2iVE$VQnbTgiCg7<IMha~8<
z{kX(xf9Z}}G-`oMxEu|{V$@E>-k(6ov*}K(Cr10vT}v;!aP`Tn>1jqc$0OnfxEtA(
zc%PMFIf#@bYL@^860Ef;1xKQ|%2X^Zj>YIHl<P&Qw8Saa-moi$v#b3zeR{)VqTqTt
zn7S-S;dTcD`CskPLdF?hop=Pess|>`GHm$i;<wE5!gGe0GXQ)AYX$D4bE}Z90T~l7
z)c<&%l@w_@Fda&2v^y^oaj60xn&V9BY+wn1E8LsGS0qn+*N+jeoKLz?*<sW)qgwfm
zx!Sdqpf8p@<^FB#r1lbsvIC$q=M*gi{>Bk(RRFeJVP{Esy0*A2@RL4iSO0N)2#T`_
z&Sm;+k8{+5>rSV~Tz$Fnx7&a7`*WZnphl56BcKW09e*OU;bM_krt`g4C}&jG6n-f}
zcwur+D3WXZ@?ZCxLob71UVm!n>(tOV(|RG1Zo!`K*H>9ndvx8^7>Hk-*5At=8sz86
zkg*lSlx|zdFqg!Hg;3$sxBj5=;KP82y)PS(4RFd&f`1bV@f0X)L^geS)94pJws|Kq
z;)SK|mKddL*j~ZFFPda<Md8`EJ=|RYkfxfME}>V>nhir!lonWg_AOTnLujG=ai0%`
zJV^&OdTJ?0|A_Y#JOJmmWW0Ml)ubd5qW4+U6Cg%d^wo!9E+@v#-JF8=Za>Hs9;Zne
zacW<$AEF>#H>RRs(a#~(|D9(lssUvpnI|8fs5^*`+?W@yP;F)%zrpj)^jtE8Nl@~3
z{r7cp>ceKjG_TetKbfpt+>Nf_zgf&2(EGB606&$sVQ)1vMAk%ThGd_w7rlH%(r`G+
z$kL-H^N~5KNc2t&d?e`jFF_Pw^Iyr%lN~-9><-_xn0{5`rDnqX&aeYNWloI$OrT$!
zzuyP9A$G-Y3$7W(X_Pf(!UpJ8V!fu`ssByOuf&0BmWW$d_--ryVM_CZzM0r+lR%T7
zCLmP)%Ba=Jj90nIOgE8^1C@U_%=@ot7$Oa@&)vi7Pg@ZyzlC1^n@W@=gi3L^1W0dB
zdi~h3RGMy*+S}CLW&a)FJkR&H7qdwL($fx9%pHGP>oWd0!PgTS>ZF<hu{IM|C~NgE
z*WmbDMmz&pjMdzu_lx;*>e{GC*lvYQO1HUCId2R$o}rsydVbeYy!m?dz}fHhypYzC
z#lOGo97qHxNnN^!J$_cXaqHNC@v>uQ_UYQiQViCm-ga$rr-ZvcW$g*_LBwsEjr&KV
zE@z;2z%Q}Kl$F1hJmRG`KK#vU+<@6ncUkVOa+vC+c5C^qEw(aUG{3?^ITvJ2Q~{{H
z*<@QmXjearAEmqDP()59hbbwtTt4Fss{ZP~cu)b)zX~E)6u{%*;K!buCMi#Mmswj@
z3xD4EcW(7TruOvdoLK4x>lLfk`l6*2w6LF!!JE%Ca0_A}x%FXjx?Ge<Y^y=0kGLHH
zI1eWpfxe|l&+%~6lgbeSeo$zi9yfjqGMEg}$+9zR6j=GcQ2mDIJmt2rV|}Y*<MxY*
zatj6ZRGOS$G}4ny0tT;2{ZH0l%}W2y#rP=QAPD9wRQh_U%<j1~H*PDWVxpo;(#KpQ
zA|_Whxz0Dja>p(J5o#jAIwqq?z9COBG36waT23*R=Pn!dAaP>?#*}B;;GuQ^w%sXQ
zv&{be+(6oOsb}RRfe=mxfC(GpdCJ{tyAdK*n_UT?9R&a*%9G*W3{-<WppSZwUwtQ{
z`@x(EHE5(XykW#dZ)t4z&~a8lSoLnl?!|8{TO*E-8ekFWHjpu^NLhZq-&XP?pi2O<
zBoW>&_odZ1Og!$Ml(Q)RqBs&Ipuf9}wlTj=EwCtq_(isdMeff<ILx<w4!PHOvf)$<
zR7;qV2@(IUW3qtKm~isMw8XGsp+w>uI&zssNTr@5s@Z4E5+LIN+?vPeTZiW01b&|z
zg&r}u&2iAXVYS7mo?XdL?tT31Le%wsYq?!<VT2HB<~c_rF0W1UfCSp;C;kH|SW6=v
zfmqrVE1a`KEG6zB|8Rn}%MPG;kL`wwARf@h%ihaY)Te46`7;E?m`&b2dsE5Zn+T<~
z@_AU|C6%-Gp^wT<US%Z47OvB_Xw;tTV7OL4vEN9)RKYhg^^@OOD5eBJz^id;JM>RB
z3d}R>Db7mp3L{y1te5M9hZlbpMg7Q>kFakIXEYe@meMex35cYv!(~R;|M~z@sQNVH
zIvc`F#7c8gxP{52k$WpjKl<^ov=8Jf-e%ZywYS$&k-e22qXg6q_G7rD`VC&BGM5iV
ztzr$9L)veMH)&b6MzRlf?*&DcPc&imoyZGP{B!WhIm-#x6YnMR<F-z>#j6Fct&%wx
zXqRd*Qw&DPge&0zmK>I-S?ZYcIE?a<DS~FcGf-yu?Jd)APu;oK<!0ME)|g5A#jg0q
zyDC5NSu(=(h>fbk-blzo?j5<*Zu2Iu&q<SODR6TN!UGOdo2+Sn*ppVFdZfx%Gw5P`
zb$LQ4@r$0*qIbc^?th(xAhMTI2MaUr-(@U3s@I=+LQjU2ML2+b%ceQ+iTZ!7BGg_1
z`9>FFW5Xu`=(0zRGhIQZCBsygmz8+n6P~=JoRmJXST@0A&gVb?@z?ioIvTuWRCn*!
zi=T!8lUH3Yo(R!bH*b_|QtfcRE{T$hjg<l=nz&Gw;MF#tuU-xKNT&7Diwl$F^<rHh
zyg)SIG4<Z%<1FSQ?WR)hr*&2P>=}JZ*P9@?R2k9>hr}V1VOJ+U^X&AB9XD%#O$6@g
z4EFFPhyR%COhmZb6v^VOePnLntb#SV)IsT|&&eU7+QW+UVA*1Z>xCahRjiD%COdN<
zP=l<xcDp$6J8eK5T5IZ+x^iBvF2d$S<@oBfB0f%rbdidGRoS$y7*(vd>-=jc6m~f5
zhR1FdHD>ziSPC=}a)<(7?<JRKh9rEpU58ESB*L^<M^}wH(tvKs3dOyS#qD<1^vk~+
zp}Y12Icl9<xvwY&!(<Z4yxU9z7Ke+C^^+EBH$bK))oC+8F7ITey0&*2y7v1<0HmcF
z+G%xC*$U48^kf9GJ5Z|34`TJhtV?*NC|zHhKD-fs6>j12{c@!oeEJLwa~${XoHBhj
z8GgH4PbUjT`o=u9I&(u>cR}}&g}msd?9XIu2qfn-?P&UYb6Z87EY<1#HZd(G^)cGK
zD*-}Odzbxm+xb170r~jld*d9z-A8*-o;GAiPwBegdNL#rw74S3Dwo!uzJX@rQG;hh
zQXlt7a|Fdq5nO!9et6beLGXcu%d_(NzMOeOOFFAEN2jK~+K-4M-emuKu{oDM#mcXw
zy7s*_p?!jW_=HcGHK>BDo(14o7zsX!;qBil!h-Gtd}N1rJSY1lT$9KGYOdibWJ;BF
zJmeAjt(7ZiOCB<$*nJba2>Te<AyKcYb`u3CRFE)#5#F){uKnsu7+55SVrLRw>7T1U
z;GYk3Ckzp6QZ}$jo}`mANNc8Y3~b++{)~z|aNv>$kP>fGJIRD4KeuB{-Ho_~G1Mv7
zCfjrTat|F09p0}PeoN+^`+@GWnkr%qWa_1jzKi6<(5)QM*aK3GcyGEp2-_aSKarkC
zK55V8-AWQzG#k6_NY}^8GW(Qs(5yxPD;rdcu1Mx9gE!JicCM|K;I?g@w7(FOgw2{7
zbpvL?D@wa`i{G5oT?Lu4v2CY@xlrzD<`>CKms^CUd9UUxIVTib{O;0xLdF4bO6jW)
zha@xYK;UaDDPm4-*WR{Nj(Z1eR>PUgqxm@M$SQak%YR4lnUPQ%E@vlTcfx{h1Lv10
z!v$9{R`;mPqXQIZ{MHakslCK~J0x8UqU~$4yV*U>j?o1Lm8==p*i#w_2=Mz+(vJ-F
zzi7|~_?LQi!7Cf_VbLX~#FD-d2L8PDgd7G7zu#6h5ClxHywHhIZ^uj*BhpgO#PT@Z
zjP<pYWc0%f$gq3!Ju{~kck;a5=U+%dAl12N_|<NyU{MZmFg=t_BlY`}5dwz3o62p9
zCp&NEYc&SWX?BYu%O@eZOF<LMxGs>Oh~G*5e0P-SPh}*CK@IPc+awDj?;q>V3Yu|s
z7n{y$f@&0hv$&Ffi+$%Ht0ke{JicC5-)gsJYT2LGsOKnO-Mo{vwQ(N16H(hV8gJ^?
zA4)Hm75asPmFy=)a9v4gyd{$yB3H6}?oSm#_@)RYU;uzdO?8XNdnYbC7%hw<-R-Jc
zBOgn~7Bk^kUhB6M2tMP9*;KNb((yspo0MCLlCT5X%{tKPefAMgK@A2atnVB$G!Wk_
z{qWckVuu=;;ArGtbg@Mr{LWVCX8avfXjr|Ws4&ozMW)qK|B)TDxa7Cf$#bcG5O!|%
z%Swhk_A#LGD155{2dW!n>b9+%EZ%0s4{Y86FJc1`8M*cezosB5{KfGU0c1e(B}Hga
zd8TI-<J~`fd+Dd=f+OW)KVCIu*fQ!zu`uiI1y3q-^#MH*<JJu$XPWvvK?4jT&%YS)
z^6vO^Hu;vl>mbul+N$2=@Y=U~zY}NyuDin9YEY%JT|%`oA-*_^>uQI6s|HO~<{JFe
zlofgeZEfx%2%WQZk5Oreyt8{TgW5r5=zufXN{M7pQz`C^794TMBK+YY%y^KD?Xan|
za`8R0dOe5ns?Mbz-a331?lVB~1lcug0)$%r1a9@bcEv*d(#<<#ih(kT014fNHLTY>
zB|{Rh#op~-dVsxyH<z&%L!4jTvkc#e+ryJgCQ$jfZHEj}jMV`m$(Opa#X7-K$#4J1
z*2!oF=~j>a!jng)_|J7Ec9Q3(8uj)bK<MP_)I56&bPK%|m}8E~7BX|bNh#94f1fTh
zMUFILm_PaWgcNn9<jo}@F*lHgxDV;pyqo=5)+1L>i#w@Y8jHZPUN2_AnN7XPIay}!
zT=UrAH71b~(ubg=tCef|2PVi|yVB?T>reps9!Yq;lzx-qr=DnuI!nLQ>w$j+dzCWg
zJ25w;KoVT#*xZi9_%@!OlGJl^8e_MpP=x<!^xQf7p!=U$fRegQ_LvJ1kIDk=4C;%|
zJ*a)d6~Im57GT!kQEUp^s17-N)td~oy}khSBjy|kTr|?{wZM)!`ZVS-Y96pjd2h)*
zfM~E3GHRDLB1*sPk7{frH_kl9$@_}o$6SAr%+*OZzv!NA3Z9&7i3AQo4{Y2Uma0!e
zf-nm!mr5C@EHY?{OL@{wCe)uUGN8FScUI>g`JW!c8|pZ2uQl8Bd@UD#kQ?~;t^aX}
zX@lyS-aFnC+_bymx?r)KutsPO-J5V3W_`85YD7NCT2ae~D$Er+7R?6W3&q|Rd_+Qy
zV<3eV=W8xP+=Y#ei1@Dw;IJpNaK%Z;8kBC^&@!E{_5I8JM0f8WT~9K7Ccnow9GKqf
zZOK;U`?->F$NJ1uY2Os(d{I*GFcW@mwmMxZH<LkS>m}cc1!tnr3By!-8l}50;?Y>C
zcs}+Iqxdmba1niC^bDZ{0$(Wc>;+1Y_CR&xNRW2bHySlS!8R7ou6$RSUXcqE(Rbrw
z$~W>;q`;01fzL%&l`lXD|Gh#%);2Pq<ioi8@P-G2K*?IZJ80@|rj@DG#_AUPOgWs{
z*+IBrxsmU+l&UHr!Px6Y2E<Yw+0JO{PBiET!=|eJ&rCf(M1A}ElxK0ggc%pk$nPzb
ze(2c~=Sv}}SCUCWk3YlYW|9FXV>qSiEr|7DYxo?CR_+hVCM;$7^jmg!HX5>3RknDU
zsDo4+2=uTxL=2T!wJ#l}(=r4GK3#cGa*$M`-A09cTH>^2Z9mK676of)sekURxI^Jz
zdR@nFgLL|F_j>sc??kYTWbiWxi^!KZLaT?H;Mz&pUh+uz<_(qDcdqFzF}6k<dI-*X
zHr<%U%{;jHTjysEuODn?yns-wt0PuGX;LBQ$Z~ivWC2nI<YR$nAFkZuV_t*klej7N
zK9xU5e!7>hmGlahzpCS2UtEEJO-`W9d+ora+Y>o2n`iASVYg#uD+4k;1W-k+6vbEb
zi$qU~(1Z)4A>2{f##Y|Jdo=`W<KYiOmh_IBB2ibV-e$MhQ$;{KP!?3~zK@2OO!VbC
zVlROXfYkcnQ^)xL?rU!fWLParNO4Q_`Nhpaq&<Vu6i&=g+k<CfyYLVWW3rtt`|)XG
zE(}=D5#3}{N+Ee7eANG2E_Os$f}iy9T2BIGJc_}rc7d)o$X(wU>mBQaZ*2d)5r!0^
zJ_&`;y{NGjBk^7M)oI%VwCchxDe<itwMy-lm8%L-cu}%pP%Rxa0oHVHf@IJi`H3y9
z%x1T5->XnnhZf<~uHnX&5!V%|j&DhHUeB6IHGNr3k7LPLDj!_!sKJ~Jv7FBfSS~-L
zxjOi#K(X-Ko5qyfr6J?HrZaZ*9G09RigD^;j^tv(MtwoiGF6-#&^RCNx-z+D6z>8g
zS$?X{zBTE3tZMR@D7s*tQn=B5B+y=jU4skAHg8JbuH%VVEfweGs{xJB-kHcreG|QX
zi*aAGakb-W0Gt%8b3_ur<+Me^J2laBGX~;IUoE~g^XSlQP{T`JdQ0a)#$eCzA8PEM
z{cg~3(4V%v{D`j&ynKUNB=17#W6Hhtw|jkD;Zm&`dV4vPZrh5Cm%<b`1;>Ln%RRgS
zql3oze@m9^j&;wrxb`yW6jSEEnUN@E4pdNtP4?rF{r0=i*I9+DiNMC(AwMpkM6Ym_
z>W<c^?!5T?JnXKTPo>hq%(iBJpT`K;v3@ZD>+95fLj28-t9W3Ud)`gHtZ=$TLpnpj
z?+U4z>D9|8v%!#-?RBqTWwpAKCKP*hlBjH@V=Ie%+-~0TOEe<Tzs+DULX*e2V>R0o
z8jHPR`27Jv<<l!GqpA*UY^>Mt#665?Q<d0BW`xj=luOAmuqzDo^V(7vno)gwQ~wBP
z+W?m1FK9<i%|FJ|o!)#eKl#0oJ^;)7-R&Uaqg!cVL+;sPWXbjO;oj+qlGM@=d{`qa
z>IlX-?7Gok0<+v*k+RlZf0(#<*JEK7Q%pwhmVVG<V-F-E1L5;uL8jOG>`GBJwf?1f
zh|4#2lRpX>RozS0=xA8x%}xE1*h=seq)@sarF`R8qo3>4J2uxg!52Q{`29#QcX)V9
zv2pFoAGwqZ4bvx^nTiS!F`uYeRd3-I$-@T{A{P5n`E9Y>?S~f`R)R2|+A&vSWJZ2E
z>poMC7!;XU77$xyC}}oK+xrwDrz<edhiTQ0nCMOvRv$NG*W2T!azh100%c{6sIICJ
zHjFVR2E42XE1Db;5Sc*TwO@xuZu4;%S=3kan;!#>L{3{)zcRe_SqD0*AMWneTnzM0
z?}*o)O<b#TxXoMN(A`xG3+2X}2j`YbT70)ilg9VmpJnZO^~4NNR2a29#$NU&<YYM5
zMLd9^QZ4zt-I;Dr9Enc<%&yu@FDaTq4q^9`SFDDwPx_9d_KghMp9)aO;qSx`)>pc6
z1=)YaF6qeHnuJt!WHbpsgsO`)1)WRidE{fSO+~Ga?bk5c3#)%6vifwmL0t&RE01Wu
zkl!W3V&Ri`+ge+wv|{5Iq0b<zrl8gkVZ-4Ibm>JxTjMHZCFxV{VYGeZbh$w&c@S9V
zIW!Oo1zddC72V`<MbGKr8}>mseY3Bx*;m+?P2YvUCS%{E_iijsWJ73+6+J)oGE}1A
znnE*;5{vq!s4bd$(mS}0@YcE%eO=092irK6I*zce<kVFYqVq|d>~~cj4A&P_!pH-V
z@0XvA${BDYM8vm7h`EuD-1ME~Mmtr;^5C57(-{Pz*is-@{AtE=URUsI$R~C~HvM8F
z-Gl3;=I5P|<xiAeOTElXG>l6rDRyAS*}{}PJsm1zmwVkNHne*2J4~nDjZWH>d(q^d
zD+nb=gklQ%WoJ1e8QU(!!>EZyg?*bpg~#v+i)y*KAqy0X*t?;=2eMQ1D+qBfVuao8
zH|qY15@2dk77|q5H=^-&)z89Cjqog*cA!iFJMWzwS0Q7mi+`xj{_2IVVz+8X1ch5^
zd{bAsRu&jaP~pXBlVdvV2DYXn-l>Q*p$I}>ntMT}Xlr}Y0d1Ri?dsGcTN*EA&1+g>
zzNHJ>x`b0VsK{GqFM4XH0cT1>4a!;Z)Ld$isRMkW|Fs$ePQqioOj{jX-4UHkOy-y_
z!o`1m7f=i|Db82#`&#enf6%0<?kcR((VW$-X?OQ4vzp_0c6&9G^+k5N3Wuo>4$Da=
zsP87I70e0np8*>dI}&R%hw6UrWGH+bWm+<8RLvPer^x>51G)DYWj*Pxe#nIV^{MT8
zb9EVQN;ku&VvdsrHTC@^Bo|;e-$bpG1A&udMnaCLJx{gCQwZBb3drO;9{Ochu+Ur;
z6Qt<A%k7q;$!Os>2yVomHhKezZJ|Hc6AMSp&e0|E2)j%fbzK`l<mZURFJgKVJ0v{{
zx2S9hoj7RG4hsqBbWB<#vEGhj;g-KtId0lgfj87x5xn>$a~PIc0xOylOLfQ*1VKA3
zlF<Bi`*>bKMwftIYM+H-r-|KM+WSEuRM8XWGmKI)H<vhb&C5U+<n}dJS-32MAL)C?
zP&vyk%14p5o?*Tr+*9@zIWjocWSqyLc*r$ZP~#Aib<Dzehv(azG=respnL3OiB{H~
z9X^ejehy=k5BL$rfD(_~kH#_EjX+bj4OyQ{-@t1+sr{K6>Z-TFsJ(DQ1**nR=AGw7
z-$C7c<Ab)j>f#5^D9rBWFTE;fe3KLYlo<Uuviln{&97QII<<Xh>aI7~aWk~{p~PBF
z*)WVr(hsj!8MGW6zt$PQDxW8w<r4rKAI)myB~u-yq3;>*)9{dG;lNbWzUzCs%sC+*
zQ=u+5p6Gct3Nq!Wvu|D;s$ZX$3sw!^Hm$+3kJM@+LaQ1IB$HI2UGI&o#=qE_o3)dN
z(Y}PBlbo$`lKmIRO)kH+2$#$smzZj(eG3BX`c~8Cg?uW>)uY(M;HFqe7E$jnS9jT!
zLHoeM=HODTR2!Nova0J|nJ8Fr)#m)`b!S(a&&W-~GQ*vA>-Hw8%|fmrO{M}X_=N6c
zLRD@R1Bm+?E_|?r^7U98&npXqmG0s4bpq6(hhMrz2Hjw0dg}M5+6dy0RN-y&WOPtK
z=s&A(B7Njiyku;^tbdw0Xt6$^&SkL1htW3d&!zTDP~OV9|2->J0#~O2^>YJP%OXmB
zoejji1*S4M&=ylekavRI!Q7>U<0YD82Vv=4nqq96&m5awRr@$T43N}h+n_<cVO%xf
z!tNKF?x1!qX?)Yzw0llLQ7W(2M^V96-xP>7)%kBU`P|y&eM|6{jXB5P^`>gAo_p()
zBM;voZKPZGOUzB!Fog&cJgmjU%@O$o?QQ!GCL4oYOO|2&#^5kaIsuaI-yK3gwxEt0
z3Idv4UXOr!!t+ud8IwF5>3ImrU=hIBsC$AtE?4Vc%pDhtg`{8Dq(i}9Ip?>vq^A!`
zi%AJXBj0uN^B=(kB36Oy55_Wn;b>Wf{S_z10{&F$WnGO@sts6MC(F|i5i#{7aR%_|
z{Q)LFE)Y0XH^p@c`Z2f)B{(N(XLkc1$Y8EZC@3U1byuk_emyC*Cr<4KbzUW77g4&w
z7V~sv-YKVz+D-j?xrW$^4oG;A(QInI91gpH4)OUKzO?Pd3otu4RoAU?|E^m%-KQ$Q
z)8AbCUReu{b1jJa7D)59`pW`OwCwjhVFvamdN;X@Chqk;gvje*8-y_JhQnH)@CNhV
zNEce1u13ijShg<>Uc#8)GM3e;lx3&c2NQT(@Kh0~G#3<y%yY!$(&HXl_mlyXr|~zv
zc{9hO3TRz_Gf}!lEk0@Wo7d>a(e*<DJWs}~Nf)1AXC>UfeSg57C|PMJEq!RDx1rz>
zFOe)57q&WY_-gy(nZ_(W>^tk~`Pnfd?O~hbX5G9!9*fEt3);;dO=0la{a~ZY{;JCR
z_O9F$Wkjh_HQx%=K4(Mq$6GcdfU)i3RNet+sdZH5k6Yd#-CNa-JyaY}8sO&$MBc*;
zsq>D5`x@@X%cSpl?M|9)o*6yosQ%sx4tcY){`n>QA-H58o3}NFke&-ZHC!5>0Q0?l
zHd`o`oPUgY*iDK@A)Gd@Dgh?vZK+zlJ|4=;6eN3nW3>F6r;yG1^Q<11=P7KjKJer@
zxQ^W(3KCD2XUfC`%iv>gvhb^2pZ<1uJ%B}gXCl4yBO`d2@}_(4+7u{SHZNQ=O%PXL
z7QPWO&!mVH!swSmrjD-09FpC{&%u9gJdsbzpr%}ntCV79p#u+UrC#we>`8<$F<n<1
zQB1Y_AuBr-3#9Mr3kMn^8&7G#dj?euRv+%GF{;HWCb93eFTV|Ze}lo)p_b|jF|OU^
zgN%Rb<c!}Q_a`SZ?|M%I<`N>6rh~{A4O`fb_bqo_L@otJ!U-!mQT2AfPp$|k`(T62
z=cc^-mzJxx0m6as&U~+4p&AuGTB#2SGO$W_P80s^{dgY?AxF0Tny^T^%6Q81I3s2f
zZ1rLI;lkn<Tj5lNmf*okJ7_Z7tBXhb3$f}YCkK-v?HHiOtj2oM7z{{+K@`HSwTXei
zPHwxN>d#3!5wq?a<TNQIRC^5gAI>q84FZ@k$NDqlOhAT5Tgf~X9&g@@py4>o1MstM
z$pjn2KE~Hmu9wHR-ferG$Bk>))hg_NK%T)kT0-H*n_+ipjjTIDCE|@;hJ;&k{Cz%<
zUS#jT3;lFS9j=NHOGD-M8e%b>JRo1bsdW@4c>A-A@Q$S$J{jX%Y--#xs7R4^$xRm#
zp&5Q%=9|e;!+si*naPAgbL=f)j9gUHPO91+?-nDfa$LR*ZY~%pRaG+Ji{$clGGIJ)
zKC2$GR)Etcm_ex7SFB3K2I$<_c|DTgMA$fdB;n6kWR3GT*QjTZjDg#w4Da4%2&kRy
zKQct<i*Cok#*Ytn)r5R1=r3P{^<!U#N!=N$@I4~t{n%=|(h>bHfl)okAQ{%P<n9SE
zSQl@u&V!~<yL(Y|V4K5yyX5D0<R&KzkEM8Deqf@l^jts~m%=4ZI+9D%2YKDtE4xA9
z)?G>4{=6=cuihph3C%8x%Epz}YHx@y7wsBlXcz}Feu{;}4_dHab+0834$MEE&hMX&
zZp#Trf;R>FbPSSDx8D}XjBwsx>Ua3o7Rn7ZA|JtH;?nYyU7Bq=7F>UWOam9cT12kY
z&(pjd<f>p0o7rr@mzb=-eJw}m|M_x2D>1O;t&gxtT^Z?NW11W~(QhGuao3Up?S=8`
zTg^pYjdB`NNBggmH$;2!=QE285rF~CuVhL%DPz<Hw<Wh#smq2P>lHHU{T^gU!<Ydw
zNql<F^2R3_8uzC}ifN2*8~j^~?1dI+mx+sE-BA6?1>|Rf`nEG|t|Y&PeFDYa==0s~
z*ocNkVaph$m~e{SS?@eYeG_N-itnn>STrzDy%Ic>=hr1*o6oa4hSHTz<BgJGJsDg;
zNoX5PUbN#7@$2MDBoqF)GG#n8dV@udU4i?pPpMfm7P!hX&QO{UXB!UOtBX6F$ZA!I
zy3eYMj`O`<5j~awRg5{G@H4{sKAVkR?%;fx;)J&8wFgUDQw_X}9GFn=uDd>R<lf=9
zY4~^BjLc@<pc*@Z>h|l~%vqdV6MzRNzj>xFolC!5ZaTpEn~TFdx6xaV7lZm)uo`wa
zb5ep_<<lmckB$>$3tqU=dv{GaCMzW5M;yo22q$n(qnXlVgYj&a2r*4ux&{Lgo4jg-
ztF)mj#0+6mWYw|*tweq)a5`;V-bTd?hVJC4XpU$GI+ykFF0u1T;|;@q6x-k6B|FRu
zna@GTG^P?9h8s8rzqJ_q4eW{;App{s!Q>VNxK9&HfZwdxuMawJbO^w-ENZQ<>_Mll
zI*3>56NQ_<nV2Tx$oPhbd_ScZWQCXe9@K_ip9^E5{$jYqG^J+_P`mNy*D8v>TPkDI
z6P%cc@ZC6p4@VhA6Sf7;#pFEo6EwMZBs@lef$E#$se!xk8->ndUnUxs$jartvH3!P
zv=n>!#y_=u&H^v32S2bCN_NOk)?l=S8Iv7qjChfGhwoMbN`6D;NlYIhj!~Bwgwxyw
zfv=x6eR5~O(WkGf7{1xV+jAMkK2e*|BD=9eAtr5A$#89N1vpf+I2Q#-jlg|m_5~y5
z2n)FJeUPt@Lbi)@qNj`|I<TmQ7kv=Pu1whcJsggLk;LJ_$3^p?*6caDn@^>hmyVVd
zXRpMN8gsvfvSH|OH!pnL>vsY!(11`KqyRfU&gi*cUr8b_`x|pgua3wgY<_&G3bjKb
z?0LQHKuvE4au$T%mbV|f>7fQergXN|c`<x{hHHg@L21s`<&9~yqc|nM`<oyh%~}Q^
zmB9kmluH3WHlt!0>opf(%PV3~h={DD&84`S*!AlkA#>CX4O{ng8BXA?tIGv?yb(u(
z`f*F5=Ja+FiYv@kSEzuABU)Tt-N-94PXraO)YVvn`j_n23)fGTsNI}oY8b8+7+l+G
z^otPW$T5yq=EW3#&5N+2D!mYYYgLf0q1n7Co%@d{;HAn$TeQ1Cp>w#@t!l-@pD6IK
ztTI+Y%RlNQGQF=COhhGfVzjH$?rk1Ly%Si`PI_d7AaQdGUxj{73fX){x*awg$L5?-
zvj3X1Q&%*A;k@$T2ZPvr4Hk+$4XvY60UOQtV6NN1tqRSl=Y^?WIbVGusRQGb5&E4f
zjz7i~Zb_Uoc#}275XhHc<v{PQ!j7?MDqVDDM@M|+bt?+SJ|rI5#mT78`qI0yb4BR_
z<71|*IkfpA%yT;i@dvPQ@?y>Q$;ZBtIv<=pik~B!hR7lx${}bz-&3Ed2RH)wXk4y%
zhh#6(FuPvX{(AeUXDN5L>~bMlJ=HZukt8iE*(rT$eIMLU-~@BOekOdA{0Xr|&D3zQ
zMVq8ewhas@@(gcPI>R)8WH@KH>w8qP;#E)O@a;cQdk5&$dK;^lXEWgEoYDII;_XcG
z-}+?LXVaK=8X2q}tRlbugo>+w`%1cDuTZos$vrSzq2+V;MYTk$Ytk2$f{N~o$*Q`g
z1>FETN^hG~UYmwTjoifm^73{a&$3j~pg5YrU33pMCV|nllV<*(n-hyBr6zA{W_q5G
zX%+1hT9@<aK#Ls?+imuP3S_{%=rEyc?|56k^14)MW50gg#8c5%`yA}Nft;I{I^^Xx
z(=|e0x{*^hxpb#F>*y{fH*#V0<i4^=0<^|l*e_qGoY}_$7Wuj2YCD7s17@JD=!#2=
zVp)UFONt_a?7=?hI&)U-iffjE#r@VJPV1=xzQS$L30nODZ2yOkNe0N2D+JjaYLo*u
z7q3zsdRVp5)vzyHdB8<JZvpF(s%TNAzM8c{87zbiey1vu^xL-(l5Kz3o@*NbY0DOk
zACd3yDz<|=>vb@IP_}E(Hc9z+#nClvm;tIijKR4KG$j0#%CLj7IDIFZ2{+E<a=hIR
zD;NxGoQISCu#(+UXJg7_DTVNZ_cN<l5Hm?5=CH!bT1A-pmQ;pSmn;(Zv+TjyKBF~#
z9drdJZ|#ysqa;=;I?*J>nVZxnC%u-_B#L^2JP5L$n0GYhH;NxOV@95^XR+rCRO_-<
z4=+Ta@9E_q%3EaGzbn^?&5zzT8j+lz6w?N-_(bqq?Pp()WuLnrG83_QC?1AvtwAiO
zX>Qzk$osX0WjCMPo9ZIgFVEuC*+OtgjBD5x9&Q!hy6M~&!;Oy7+KM{Cs%W;ySv|%E
zo|Z3KlK@BPxa(7DXrA-R8QQ6B{RYgHHSKPLO*`@@tvl%Zh3!5p;phJC)bMk?ru)~b
zyig_PTwq3#`K6xD&08=nQBzp*@%D}Cxis0!VP=;}xKR+hTk0@)I`1T-oZNf;<R>Jj
z{79;0fIDO~Ow>93`II^7*?O2Q6?b_v*B@EC^GJrpfUszoKJax1+SOg>32N=$n7z~J
z81@%mmcy2$yhGO~>mR;|vFQHV;{09d)`CDI;0!sty;Z{E!cgMt+<OBS8^q_PG>oS=
zSe-GDV~fN^y0tS?y|dos6yAGxcvEHGlr>}MwS73<@T1q7YpC`#VL-WLk2Z0x(fOLO
z+tO4iuRkjgNGropW~1GYJ!T4{ag)4cte}%^c+?5}Wxp9vLg$@x^GY8QUt%Slu4c-N
z<X<<kk4S54wFBBU7VV)OUW$nBUjX6P;nkhe9ul*z|ICcb_&EZQVnOXY&e$~TOFcdK
zFAa{5CVU`%Qj1&bi6o=`%A2>#5&o<vEb*K@wZ&o+HS9g2(nVHNos8l_vSC^8Du1Ko
zxTiDRd%6=>=VLvmulutuE~O|V+P~u3n9@Y94`+PRWIY-!GLPUDt{ugPM;G7!Blewu
zCTENH1g@>~H%^DMKaS8E#2eY%663)E<(%q|4~2gF7d+Lnje)f?>VJjjGURtXwGBT^
z<gMHWN)s;K<M+ujGCS|)$eAXA1LioBbsgXA81-N39-8~PUavicWgTT=@)EA{GwohB
zyi8>kyV_cOl<NnzaZnxmpb;s*Yx?iDC2wLbElx!01@EeEobgmip)8_(>;1i7f;C$*
z2>kRdXvC$uy2KklPVBTB>e~HYcGcFgaD9Sd8}4D;l_Q#RMF!lA5n8=51z+cjO&f;)
zaH_9)-d3Hxe<thGB$-j$pea`tue^zJV!9nG&|rFQN}^cNm@O6OD?nxzhLUAazr6<S
zqT3QG3!W6TY8ktpc|Yq)zLXR@Nhrl;@RCpn?N!Q-_>lw#LRW*0_9>P;faU0^90Gtv
zIQ}|ZI*6nsh_Qer2j91qWg+u!PBd>;sBB6X%26nQ>9J#mnntN(Z&Vhnn$<1?iUYUa
zm|x9e&>R5=3`^3@>6%6HJe#`sdSZ2+k_Gn`R={XKlp!emouQ;=?qQdCf-FV*lN2d=
zl;nI>ACwMvihcIyy)UkSz`<H4yKI;v-?*RNB`ylkd8==kD<JSt!u#tVA`YdYM<Zo`
zmDH$!e%W1bxx4kS$t3VmtM2Bi@h@Vn&D83c1S_npOBdoZkz6IuzNaXme)&92X`B#L
z*RUVFA^3&#=SpO;PrK^;Xcp~H5hi7=L4IzuQo_)m-Gc*U6cC}3m4LUGr^nL1Q0sWu
zM=lW`z~<D@!TB|tzRR||?qvDUxZtSmtA6lakR-UetlSfijb8r&<-X%3T9+B~utpXE
zq+Ncaji-h)?45T#z?(Q0F%pNY@FS2fMG!`kep$Rp4UkSUf?otJ!@7{vC7wEU_XEM>
z%Sk%l+-lv>1cU68;m}7OGnGn2yB?T6J7$83Q)pqdjsc%zaE+uiC|FZyvn~=JNC_~C
z6k8W7cd%mW5}nbfk0_D}IGN{BiV)CtW8k?OB0#Uc8K&g93>3&TG*`<#V8j}r5(czG
z2}6N_kuP>R(Q|>hlbX;5w!R@pG%*-sa7lhN*XzKFpm9GExJgRrn{_k~_ifZ^dzGqk
z6NG|(v)o<k1%^N;c^RWnU+G*Ny_8+b1x?wpqW=dLnt?LHXDyeb=5@^TeAIZ1Wk&#n
z6K*V0*adyexfxr|bSn44dVQ1*X`_#pa|)3meZ*>PF94OUrxvD*5U7@yOl{`*cFDj8
z3mLcK6_|kOsvTiA2GlfXDK)`oO8~_9R^7H%GK?A3Ly)!CQYO5<7UV4S)}0AgF6gCp
zlX_stUB|4A2^S05h(dWjK(F4#0wnmG5OFsYkn18~7?Inx-><`Fg$vb+Lh8zf)zbMF
zkCi{r$lpyD%1&+vhWR24)7q3il{0C*;3q>K=-lf4&n!SYFcqXZf|`SFhgk1YujBR3
zp_(Dss2vuGPP<n>l&4t^M&0u&8Zn3fd1f8ii7&QM=$R8a?u86OF~@Y4$?sJGj5hR|
zPJep-ride>RE<o!qb85;M1_mz`snLixVP=#MDRzsKOFe2&?YZDK92?)Om&OYSZj*N
zrEfPB`{1RQT_6_^oP@~>0a6*vvSSn8MN=&27t4EKlanaDz08u$8zZHT>kV((X;&3e
zU)AjWb~NaH=m)nTNd{0vL>|Qn8Nj2I`VZS1Us5x<9I2O^sS;BAxQhEg;E)u<9>0|5
zhVv$WVysfDt0JAeCo=Dsp)?1;&;MY5lu%mBk>D`69gKHi!sQ2Vze*35WX)gI%$9N5
zeG*3a6p}DNMUA|f@{@ebH8@3V6*&?v1%r>Jl=9n@CulUUy{J5LqPfFz;$s+0v@n|%
zeBI=<Q;)R=z}6c6KHxRiW;%bXrI(JWrBd_47ZuyJ%@WE4sf)l}5P|RqB;*Q5$*>_J
zm)5qv%ohGSmHcjq49!?=8c5%s<S`^->5@2?#?Wp4APhM>2k~=jhgjYwsJ3n9?i9m)
zD8J3Q<>@&vLU^afE3uc$;rvbm<lbHKr3TYMozkyoMc;guA(6e<{g|gQE4%>5O+omP
zUD6&&V_&*|N_d*|wG+RBs|Hk!FR+ZiCXPZL4k?_Onfm~<b)h{XDbTv4zCEeBB#~cT
zL5EOExX3$f+9H#tEPvXn#Zw>ujocwJlw-hr0qruBbx53gk6|-quH2ijieWcu@Nscw
zWi)07cEVUA!k(Qx8KU}$n>|o&OfA!~JjATyagYW}S7M;j#}JHmJ0F?%IxNqZz78li
zZau2jgSPvwY4<*r#R~JDdwZ4LsRfvn=91sm?z(KaU>_u-PPjs!qVzd-A;=>6R@Dmx
zjF<tNj0Pt-MjiR!@2-^fSo?a-P4q6jR4@4OzCGgS#F>>*w0WRti4?tPGFyp3d8TUo
zH`5ds^ua5xU8w@#gxR(`x8GqVNNRn)KwkgW=-iLSfJ9}4owG_TfzK;@e3fpMGJ4LJ
z!A|yZ7?K^c3r8L|w=<!?wYgBKe;bY9J;*BP?B@<B@oSz7=g;(GZ6^fs<f!unq=+Sk
zi3Do-2I<sBf0MGP%5b#$Bd)-2Y~C8Hx4c&^v}l))kUWyS;o*&Ytd!NwcZP$%^D}%|
zI&}J4fvqvf`h#(OwV+J<eh8=f?ZD`rogJNWX5X@_;amS26EGSmJS_V_5sL(Lf34v0
zODUjB1eV~}FR21%Oc%gi*=K(J^2|x_&!fCmr@YVK2}Ib3BNqi(*f8BuEr8O)J-r5^
z`0tFk@;31x^|6&Yew<*)PhfIoecY~e0~)OpFNrVbrry(;W+P~K_!JeK7VStC)8PV_
zoaKoWZ`JD|`~@aGXLx%5L|Egw<Lvf&*&jm;@(_-x*v67$7ChyU1)+Erw&L_;VjV8{
zx|;MUzl=T?o!!~`g-eCckesC`%p^_JnN><%{6$Jzd<IPc6Q>q=(5)sXEHt|--Q0!Q
zHCrOYU2%h@Rzbw-ELT7xTSgzHed&`*edq%>$sF1r8Kk~jw|G|*c=%{>P}G+*5+51T
zmad_6obO&TABJ%3G~F~cTxupAQ~za&J|jOL1kQfK+<*z0)96#{9N`PtbiP5NFeyE^
zM<uF`x^yW4)Sjj{;My_7?Le~??g|x4bvCVg&qNc&9B6aCbsoF9-SzRnMe*rVbYZTZ
zDA({ZIz7FmKq!&dw`?!=0qx{F3AsLghd>#1#<J|LI<nOxoq7H>5|GG~lop}UI|nqr
zR|lIIL}<(n)o5`?&7Io!w+*Ovn*<MGx*%|Tog$%!rg8uVIm#mlzDxo@wN*wMM%vuy
zW!!TL+xc_p+C>vi2*9QrSq_pOJQ7tytbsF^c?ib>3!E_;ZE9VAcJ$OyO>+bw`Cj_j
zRhflMBk2wvPKa!evSIoUF3`lG`m0KT$HU#dHOzZ8l55<+#fp&HmifREEk;Y>xcFTA
z(jn=_kb|f0JqD}HmJq>5NyAYsaRO`d#HrP91s*tsYmt0Yn2Qt4w)KS#<fabFqWJ}~
zSv2A3xY#K`_lF!gj56r({j7I(Xn05?TUZcUHQeUeV*8eqBKwPS1pZ7On50n`k$ND`
zJ8g^%oht_>bf2P}a<H8h!>bah9NC8nS$TGMy8gfR&io(h_U-?g%dQ&Rkjo`S2pJ&~
z1}Wl_gc#c(8k8-2_NGFXEFn8XwyZJsG3hE(*>|#yl4Z&akugQXeY&de<I_Lzz3<2U
zxPS8)Gv}P=aUAdCy_~PtbM~*j5UzHS!BhtZ^=P;M88`{YvLwrxQUVQO{t4cR794*2
z*4VEn7Eg(fT}dPgW&tH+TU>qf&EAJ|E@_5G{w_0oYAn+^?C|9qaq8*6o3mRtA{GqZ
z)4y1^jEedULk;q3`Azer5;gmP455aC7BX$RSmZX86n2Oe`4Bhi$rzir^B6FU7`i@G
zc+AiHNNS2^Vb=3v{U?(5J`^s03eWods*&GZVqla_VG?VQ1Np0%wDLZazj%F&8&_v@
z!<79bb8rZAtMx<EQ0G$n4u1XeFU=nIPgc?&n#8USAM4)e5ocY_{2OchP<J9EP+2>t
zx#1EJG7%9geRmdk?vi{5&7awpH@Mie#}ye^4NLR@gI9#R&ES<|&$FanJo~gzn`f9|
zh>bp0^BF2VoGmpP?&O8Y%CWbIpw=ApH!!@a94|L8Dq8!v1xoj><S00w2l+Zrg~fXi
zYD}>%lo3{|5j2%EW+5pkZN4^!_8sOeZ;a^cxR|2M7{ir6u2eM6b+6TNa#cMp3JKda
zNxl=~a#reEj>W=Ub!(A;o`S%lqVp&9p#J%w475}Y|C+P#s22N1l8GDQ?!EB_gzOob
z>VCJg1RvPi`7$I|?05-Kvwc{EyXW~<5G?$1k5=Ob5e8>J%(J|k!6R-ZUMv%BIZ~yI
zh&qWRAl8`b0)jWAm;bD}Z!sT03)Igl3fXpGlqY;T{G!Al=u~aGE2ohW4_TTn4Xu;m
zO*t|(hLt-YeA?452-9JGDB~Wu@{}yd<mf*uo2~KpypC#2SgNk1uJj#`NX@vr-=eCf
zq&toaY%oGfIw7d4Cyw9$E^}EmNR$m0NF?HEvWqW(+>zpOMe@E31i=WAs1#gIuJDJT
zEWr-3U3b;%*y`4topd0=-0%<BpPV;iGPg!1?x@9A)Fvq5-JE8v?Or)Ps|oiyX~TPX
z-p9V7jRDkqA;ev)d?j`F>Y7Be`LPS%N6d_?A}emiZWf=w-6uYkhM+#ofgO~9nAy5Y
zT-AXIi|C%oOb?HwuRV>*sS~dymy-m{Eo$sg&0Q%1<Ve{(vX1f1=6_@=tsBY()g6a8
zFnP0oNDIhi!3U&{CTz?Ms-O5$*xA;!;n4}nmwszCLA6(CO8|1V{<LT3)@_exNh+|$
zjWxtB$9lrb3O&uIAE!ZP`21&<V^^eYM9S7-UwiPUZN(T-=ERwqXtsVpC~)pqR-a>5
zYT4{pn?A8x>20+pN*dKFMZss)JbQdew@6HI$1})GyYT>^958#<MA48$?^Z+cnnh?W
zGJ1KiXgRL6NEt{F<R^FD%R5QyV2#79<1G8lbx@5HK#@E9jD2G;eiE+JeM_Vv*J#H~
zDJ-FGu8nK<y8MT?ap9zGW3>obmR9V}n(T|24p2mik#_!TEB^hgx)X?xNACiOg=5&s
zJwu|s0`cojR9g>jxj`R`>qK<Az2CH4LOcpG&h4JBavEq>q#A^In55TBa`-7wUN9R9
zJTZi~biir_FgsuQ8eM_Npve<eZ~-1kmNn&J4%Teo4}*m$q81Ks#9MO0zY$Er62t}<
z@ShpSC;4hm5IpSl>7O4EM|}HR7d)!8>6T}~>}t5`)d*z`mh{O`3^0q1dyx4n5>I@K
zy(t<zGLag27jkx^ZTls{n>9B00>AhYo-_IIt8V;i=+hCPbQ=xqd?=CD+8RyM`EGmI
zkO#u>-a(%qJJx6#|K}O@jqgj+k=$}h0eYjicp)g7B|gAI-V!~PGQP}?(F{J{!T*qb
zL-RwS-2;yhXB!~v{%5XD0M!rCR&i~+P;=m5qZqwn${|ht>qq3bzBB!kM*_DKRTB$_
z8GX(0&=AbH(?7C16)Vv5TN@qSAZ13_I6c&KQpGKSQ~MBKdXzkXVQH9j5JP%xwi$F;
z$Umb)eBetyoO$b(9x863kg>X+)v3-}eXu`Z=S{WBg^S+RS>glR45VMSc>^JT$FmAX
z)>(?_pf(f=BAVKIf+o=YLdIG-uV&94W>zaj32cZNcv5}5=*McdTLO8mflG+FS+muQ
zAc-OWgDN3IrKS&#>SY}t&@C_=+&QOljGH_bccSVsrmIBl1QWPA3*o_NA|Atogd?qG
zdB|q3<L{&#6#ba7J8zW#2gZ|tt*LE!L`^q9&ECMQ93U??6ozyorQD8cixFH$yyW(~
z&kkq>wsCxX8WffuKz%8n<+<a~-kUey;MpAv=L#dQ1(4Pe{)xHi&%M`(Eq}Y0Yo|x*
zr^v~X>d!=mlfgdwIVti{r{aU2D<Drm?XsjaS%)_C^2YZX@XW*RO2i%3LfP<;6Nfph
zzRD?jf_>qI-hxQS2Zj%=Dp*?8-trApy~&&NSnh4Cq#cl`S*Dc@NgPM79Y9n}Az!Yy
zpry4w+Ty;wfBko2+mrF~DxKKRv|Bdh!$wc@ppdkT)P*a2IC$xyM_=OY%~*mPn+FzM
zcew=N^%d2vGOlGnq2MM~#9<Q9W5)e7=OWTBs2Jr<gek<zTw#SEM1>)U{A8bzf5eYf
zmJoeDiaZHK>+4|YbUj)2CrkUcxf`wW;9MIST*SkR{(Vz>eh?qQnC2BL<~OY8cz7n5
z3*cAbftxI{ad?~y^@LcTe^sC`<}}$prE=+|lOCUo5f=aG*!lVzX+K5Y7p0vLLa*2i
zo^uwBsqS2?LwD;$h1|?DSbeXHT?&MOshB)^Ts3>K2G$H5F@?6i_qo09ejZd-c0sbJ
z&4q5<zfRCb;s}t|`_H$R7j~4oyIb+lfa}gC?OxgKhjr~G?~Z>Ar9BqNB7|B6U9I*Z
z2v95GBoEHkKHti$_0zq<n$zuBInp|w$jxHIr=6zU+b<{mFeJU?71)yFfH@{lC&y(q
ziyv}wLp@TvUn;FIVv?5_#!qOp8j91{M#0F|)9FjOjTehAq4>$xUoR)=Oi$89QMue?
zJFX{{Q(lBlsZ*kpgZAv~GcPsX!qA2I#NexMsHlCT^N7nung#Mla|?Fr&<w@`dNTt`
zGNuuBDB&}Ny=Tg$C@Q|ot=OxM7Mc`aX<N6yo}nrfc7^6`w6^YbS9?5Tz;e$g7|e4m
zJBQx{X>ruc{<Q$vntkYPnHz6D9WVg~<(6Jjb5j?1FnL<*dn)EA-}<BPZe+T&$3alD
z{JBD+A>I+wWFV7)KH&QhVLj!o3w%bcpB5XOr^cfLt&@Gb5iYLK@$)==XvTwF@N5-0
zng;;7K`fPr!aFZ-IRL#WrBJz>klFS+=Ku-+rZ6_}y&m1Il@{9EKKZ9MoMxxs?H;Gy
z|Ei;i**rc_Kl3f5mZf$2h{Wwr&F>19bjD&cLi$RV+YRESnS*!uK!jaR$y`WjPe<o7
z;6Uu=Pct%T7;jsK5MsYs)k?M5A^r8RvRf^-nFz=za5q)ctgo>}-#uSb`H?sKiK%hX
zs&dG+^of_y%^;0dmtDNbsbTlU_s(p+is~DOaOW1kYH+sE#^<60Of(oNGmpvP;Wcg{
zJoF&>XoH3Hi~&F;!V|i6u)Z}7pK{e2jIb_o2aPxn3w}74uY>JguM40~S_hR~sg3p-
zQ9?H8U%)Q41Io&`_)m?Wuavw}(CBPEG9V?<gL~$IXF+7=U)Kmm%yh-?me@@RVX6<V
z>)FaXrpd_;`S%y66Mcnhwx6dSX{Y(@D%=3B;=rTDWXm%3-S=C{=4Hln#92K|-UBvj
z%gAQ5n*7_248L;16Xw>gYU`YYklEC=llRvSRw%BpKoIS(_olG!LuS-0;S=%{t!A7~
zI5?vl%&~4OSQc}Zq^e0r%^N$#!vT|zBo(yW{&;zb!_WV%GT5!4s_r!DUkRKkbxp##
zU+yH@d9KFBlE#kFw|kcap*N&dVqx!Ss&5G|DOnPl<%V#j;hitl$BfiA%;-)NlHcaM
zaN6Ul$Zc?M<z@eUPS8za#L4E2Dd$G~hoUxKNwl&kJ+IPnA2%j%H*K#4a`qm5w60k(
z+@wehtTlq5LXW+X=xLZKg^x`)S`E+`ovP!J-eg}?bZ9OgndCG+VxUWpPKSNu&1(&u
zx_8y8qGjv+o>H{uVD?f-d@RM+%g{kp^hvm9O#K7l(~~X$O)+`@6oE`L$?Z5?do~Ja
zQLSu7``mfe4C1zc$2NFM&=Jg!Ya`wXHs?1BS-rP;IqI2{DV#|RoG|ub^7`Loi^r)e
z7{mp0Ud{7%t<&yb+z)KZt6^G>5$D>R(5);zk~RZMeE`ap%*OUa71PJzSFp|+Kdh{~
z`FL5aMwV7qovNLBnQk2$2aWSXk^)))(ucU2<<cP2x@0@v?E~iuc9pqN43%p)_{LAP
zNJ#<@HHv-Q&Nj~FR=pX!s;t6wh#v>_|89hmg5GdZ;h1?A79zo=VqW<iY@M5N`6Kw|
z(j0lj7Y~V^FqC2^6AIUgYSS2jgWK12n*Eo@q|;a5*1jZdHa5ZJeCGXM(hv0=Q{Yp2
z*uHgYQNdU`BNWgs691%HOBs@F_++n}AouZ_mAaqB$w2GRy4~#ZicWB@N=biyQ_DGd
zjAnCj!$#{9y@Z{(#{dLwgPh>|kKp#Uz?C!j;Y8z>mt6R%NZ;#BFc`2OSKs#Ly-`;Z
zC7Ej#Zm}=$Mw;lkqKX_zE`Jsb9~<<GNthWhCV!0H0~_S;eC-~%YMCASlF!ZPZ%1n+
zt#F*nUKDAhRvxQ{5`KQ5Ouy0Lu|YQTqgUrAQfVP+TngqOXUSi$^X7Y>cu}3eh18u?
z2z@}30Jp_6r%R%Uo+?7fZCs3_hyc~GK38$_oc3x5g<-jPM)cSazuV1_;N(Zz4vNfb
z(gTsL_bWm&wXc$x{D%*)anQcGY$mE#_&$`yPol;g^BPQ6D5du1Jt^4xs`5V7!MWSw
z{xhe9I`6fHyWPo+*?i9tpz_2bw+?2>XqT)CC|#VWU&rYR(;C7zW+yTZxp!l+U6mrA
z=#*LWrVvv|SaWXH8yCvP^!?ntf)nV%Q%*#jDNX-<IQMa508i?tzdP;o)oL77x0Ji;
z6ttMigpuSWS85|&VBY@uh<-}%IE;%_BHrg|X-<mGIdE!pAx*Y7!Rkt-<`Y);fXywW
z$|GO2Xyt4zxXWlOu~d)CF}8nYnjb5>pu+@N;}`+5b1xwY@b|wIfr0aTF8WVT6Xj0h
zcLVIl(>VULIHqr<PrXJ(+!&niXg%MeO7eHw4W(r~iO+7?%-#jPN$2c%VdHCxeAgZA
z6yTW49<dpL{TwbFk}F8&?Gd$_6TFE&-x_7klfFFgzU=Y)1l59wM_ZH@Vr5?t{ULjV
z1AZc`DSHM-?3b?zWuPqFTk1o^8(h$WuqEK%hIU=J!^Avk%!DEBXqH6&1RhB8klXeG
zF_Bok+!gD`Vbd~^ejw-3>38#-TFalG4UD!!GX{58?>b+SCn*_z4O~&#&W|`fh3mfP
zxdo*HAOYKy$>v{Q3+HU(`S&LQz=7XgKynpkT$KIEbTB%4TYERi7}nsr*I|}gn>G!M
z6-6{|+Fo@R^f$&|Q??Jbn%%NL1%oY7)aC!qSw1pg3tVN3n<C25KL8Z<zz`RPMS7{V
zXIzHM4w<vWbS;asCb`Wyq`7^&leS>H9vAD|r|va(zn~8!9N=K`j!OIju5u@W58q$d
z>%xR#X!Aus`agK0n*8+aolaB%V;$I%141WHs>{{!8<e6vO0+|OQ66gptVlDTysWu?
zz+cML0k-2ZlyNh0LfJ5u%9XevCf7u-A-!!{EG6Y@cJ7n8TJ7MDo}R#M3C<K^mpM33
zcbYsW96f0T7Sg%CwDKd?``9l^04J(~Sl3^XV{sbc4HH*=yP*w@oV1J=-)j-hO?IA*
zVdoS`_rEw!H*-QfK;XO#jfW2YK!|9ogR*$N;Utty!gW0(cx66F%Uap0Mv&y~$FwRC
zQAAtY(F>DJHDOD-ymM{r%GDTcJov=aXnsBG&qDG8)43k}v{E1WxtS>Vh$~pd79qkB
zIIf4o^W`EAhiG?OhmX8?rkbvwnAY)r3l}F!k5Ww><xBko<IJax1=U=J_yH)vdVmO>
zL0DT9y`D_hEYDKAz#z#v(>*H2?fheWtUB#CgK__Sm~zz6K%*wj**b#a^RWBb>-S1S
zgrP6{lQyxpLMXYjPK+SMDUNT0B5ec~!nB&$PRTe5#C#S!aZ-Jq8DM~3HKal<%&1eL
z;Vne=j-ZZ&(&Oy)d%R$lTz$TuL&=GOp(s6v#1O^GVa-BNDp^#Jo}23orBdQ1qeb_-
zQ!?CrOwtb_nKXd0@d8|5>J9Ej$I~pjfI%NsJ~e4?m}_87@&Atx{OLRsW`+E~?s6S2
zE;mK<mN3;Q{c@^9jhaD`R$ceS*PV#QaeyM-YL7Ns%nAoCeL&(7QIsNZixS16Mr{}1
z;rSEa)69n#!?|y3&j451GU?n1^k*jIAnblF5r%KOdaXkMdMCy=#xKTyXos0Uld;$k
z*2GXWu!s-|5tQ`_&1}6X$9&|n7D(L|jYAn6!FyBn%LL1JSe*@sobX5&Y`gVN@!r;z
zsC83r8xdKgig`;h)*S`h_uLu#LFHa+_C@}}(CC3YrXLdGO|DzNk?1!u;$Hxdni|XY
znBtBu73<6oAH6oc-krcw4Gz6>aX?H?P!9b0mD)%0!GoIo!TK|f%!$HnXUFda2ULgb
zj|+O0>~K?pO*#D>_8bVH$S3zK&L4-3D-*EZ7F~KM5w^XhA8Dm^J4dX@SSG`^?GeTP
zVda7v9@W;hq9AHFZ5b@I-D|cff<E_eU3n~<O#bn62o*3y$4&~&a=R35o|wu1j)Lyq
zx)a8mJnPh;>uCEA?OB~dg7CHjAtCJU#GG<-{p(M%T^Ma7sC@j;a6RA8c<%?{d&A&Y
zdNWa0E&|YlWxBmQ%bCa9T@1hZzP+$^IW@qlml(N+XL$~AQY(7^iJuL`6xj2n3bVVi
zG%Zn9V=n#6T$`<biI@0}iN5s=n58=NE+|0A50(pATsE7S^ovsVvcnJRT3+XY+t1qM
z*4Jgi<i%EIe(c$x3eWxZTEMjT;!{zzCi!V-wy=-+ng;KO*_J6v#Lf%_<G~19kY)+8
z7=F2!DfjHDW*je(!$*So$jR$q2?lf~`p>4Q?T`dY;LcjoBsHv6Qol^}<m2{0#=HF8
z9<{azB1UODLOA{K)Rg_TTi`nVabgB_a24S^J8^we3u+<$ifvia@SDL>R<FI)*-~#V
z(DL}0FP&LPEu-4voFB;9HlMta$<O*D2e-8YK{RD4Zq(PL(D3g^d|)b9#hQSdSAF|V
zS~JeL=#OUU->^lOIY81RGD=BBv5b#=t`-|MVm*(=i?x4+^xv>xUx8xbqx{7|13C`;
z?k0ytVN{b|#`sgSGTz@tOUeOdcV_Op@EaPV`hhbq1$AjsDuSYDJAv0=byfe*_qJXH
zz5g)@dGd^|PCGT$lWMl7K=;1xe@3QS@7$g8U+07$AXgF4GHR~7xLEje1aQzBC1Te9
zy?YhH%ubFq)wBJmoF8xHKCTBwj<+dBHt1H0cgc#VOz;2g|E`C?l3%J(fR~w<LMJ(v
zL})dZK05#ivKQnP)!p+Z2Dxfe$D2yag#L5BM=rC0q^fP`#Hi_4x)d^!QO@|l)Z>*M
z#j$OBJD!vl_I2JvE&RWI75OCis%|xV0a|oG*4`;;C0DDdZk4*~6wy3y7Z3xeg;^qM
z<CLT`&PveJDQVxIwlHRYaFG539(nu{SWpg1ye9Q_gkNH2CDrO-Muhw7hutvH$^LNJ
z+zlCf&zNQZtQT-P?RO5;pP(<2uziK)zfNkuqB>v)7PGl2C-<K?)4g_uVdViQk1|}D
z%x{Yap{-8`HB1OI?EZD6{F)*5EU1nHpH7tjY$d;5XM#|{Db&krc1ZEJY64eYuG}Cl
zch%y`e_e+COx<nm5YT+5rkd)1PZ=2UziBC3L@RwJCdl~SZv^~`$p3<6{ohi+ar!^5
g*#3WKWmgmq(}Qb%mu7%p!(cMdy>z}>$KlR@0j%;XYybcN

literal 18102
zcmeEt_dna;7k`y1t<|bhvr1{HU3*hi)ZUvKu~+O}lxnS3?Zj3)s6A7Zv_*oF*s)^=
zL2Tc6f4<-U;`@WRkL12y=iGbFy=Ok-b+l9|?=arEapMN1+M8E;H*OHG5yq!vq=Zjl
zL-%#UKXR`(robCF?$TcW5#7iJF%Z5a0_v$M-GIQEuV;_|6g3rZ+^9>wdv1H{#*Nn+
zYOfRx0*SU~wW~nmS=h5QBMmO%T9No_R}Y)}KUo#f4O5P<6cSrgn{<_JXtHfc;|1^k
zS)+LX(gi*AQjLsh)zt83{QdZgVy1}mTV72#$yu4-P1PUW9P^P$V6|z%8?Pk~tM@+*
z<ZO4Mf?cbd=j74J4GyF3Sj$7B?PjE>EW&KlH6okrE2r95PUrH=CvUDt#U&#78rOp{
zB2r(%4{Befen#?N5A<#`3_radY}_DjyM9KE#YQ3HdQeVGv35O^(=U?9@_JBACHVhe
z^8fbar!{Hf-;BK2-IL|b0ff@=;=jqO?Q`yoOK_sOz$Ko}_(Sc&Um7t>mjCvY?$Pk=
z*c;&<_gfIT+2<#FTbs$Hj<%pHt6#BdCRhFA^*_5DLdcBW!53;D(H(sM$?V~7qr4``
zAu<l8ECKX+_*-78E0E0d({7i;^ov?+Ada&ZEm;1`-0xag#rC8Jx$*DM59<Epl{(Vl
zAWy3FO`(@bYEEdOyL<l$Z~W14NTiO|<E!2;9=G=oE$T&Ln6Oj}Q(z<1wfvo*5(leN
zKK*8$Wc#16`5hFnG*QQU%BeNIy%Z}Mz<;y#wOUU95vvo<54bVz6D3K96Ds-|K9F2p
zajjg8+{CB%?+r#=9c*b^zLMZBrbSY2T+{CFiR>aF#rZZsnZ~YZCB7%>R3XK!lC8)1
zMPDmjae;7t^ZYrCojJ@`vODx2=WsaCoXprl^vttM%tG=E&D0FJ@I*uipW!}hD@%w6
zQRQh};v=*!&-d#mO}E@V?defvwEw-^*@<F}uFrBZ;_9ta-|BXK<MjE-5R=j|sqJ}%
z4kZFRSU&lS`2EFJ`K0|Tcap=9ygV6%yW%yMq`i?=<OOLJK|?=-hlc8l{v1#JsP<sc
z?!#T!?9ZU(jpS?F<*xT;G=*XKZc)Ws!3b#27iW@PzSNClzvj4I<A4B;l$yBsJ4wkQ
zSQBuq$8`IDYZJ_;f5mVyq`ZQ1)PIiEe+FfSAlg+O>5fKiXI=hVewRo-y~5*&&2<_r
zW~x4#UjcFqJ%{bFt;kcbnoa~hP0cAn`oYrFe<wH0QNfxfZ9by4#zG}CrPa>!s&b{g
z#lx*o)kffh`YE40`SoJPcu5a#(eR)pbiS8>#r&h5GIb^DuIFoEx>?TLgRZ4G#zuM&
zOO^_aka{tw3FkKyE(5wp?EYZCUZI{m!?1`hU#Ux%Q)GhcAUWo=LQnDH(;vSs%*mcV
z*R^_XP!k7}DlK^)eegQ6H+e&`I$&ZK?y=7dCkUwrgW*o4$_qQhy0_{0&9(SF^bvQr
zgfBhKryZlWT+(hoOS&CRV1d&wok&(UerVpQaSmr7Jw8)k${z{_eek(Fn~7r2h`da!
z1QdkfjwW#vKdQm=R`1l_=fDV$sci@o2S?ps=<pv0iS$eUNRUT>cp@I2nPmpH_}@oD
z8-YiFYEkr4tJ})|Z7)PPzs7Dv$E4Cl>jwA7e>X2L>->pNf&(F_?K=yxmVP~_h2y^e
zo61j^YCGZ;{tvIzvb6WZ5XiG!%KiSFTD6VtuK-}+i00DK^W?8o>U}pP>Y!1m_*(Bb
z7>Q3EUNFM3$3bM$N3l=H7D)(agQ&AcLOsdGf~QR2l=0E%wch*16NS-zCV_tC+a|#+
z#c^53o$2yv_cgLJWVoD_ehpvLar%{VeJhvqQmpY3-WajFzeu-OK#EIKCp}a_mV}Vx
zTx%4+V1CVR&yFJo1l(d+2pu_GB?0!VrbM5pmzbd^1+j;!${hI1D|U;3kd*8BB|`a#
zpe1^CkOcFOx<@lR+`URB=315hsej%VJT3+@&pt_I25_OUeP{dJ5|GUE!Ltb2ET_qb
z*CKL$PO(P+(*29oOAsw5I=F4}m%?+B#Tsj}zPyOGnGIP(-zX`=!|&O#2YMP0IMpup
z_F!yr2)VNtp{MJmw$n}S3l)`jSUaDn>2VPUXR=HlPRA)Vsq{)3;);^R&>v*RzpKrk
zBjJ;Ip<%dK)4oidE~CcNnPu-VMVl+Phr#j}_WnmwSC-PhXopRi<{Gl+UKZE0=Bl@G
zI%kqCdUl;S2Ck$T_xwraE?o$nLspyj!okwLmI1S(7y4g|#S-Q6#b+OQjCjp8faUr`
z#Su|l*5*xVvT?FcZsdVu4wJc3DcG26ME=w|skA%4DIE$&UFK65<ZFQ>gjkNl_y2Hb
zoVQNR(;Ji@dv-y(PB+%`3~D&bA4Z+LC%*k1U;1Ds!zy^aWL;LBKrBHnU;b>?zgJRz
zQ~c~NM3g(yb5gES$~qzhtkA{UvwIc#GzOoPM!or&`2+H0vAznlzvoZIyTkRda{Yzl
zy`=~bYQ7ICaQd`L7HNM8!Sn{f9rLT><ST|GX36{6$9u^U%&bAnUmmOJxffBbX(GV_
zKg^~1wxK~~79kR8KK`s%mF*d{e+A-n&e7C}p5Ss!+Z)67KRY`wh>jOFA*#t_iw3_>
zQftKeYQxXAo2NZ^o--|+vt2H*eNX-LS+Ll(>iEL>tX_UR&DiRZn8%F7g{PF&YCCQ*
zCW7rK&F0ULcEstwVU;?a`!${&5PvzO$W)}(tsIVVpBJM(Lwn9Js+cEas|IQ9Rtrx-
zw4+&M&vgw;HRf}pw=J*gYE%I_1Gh=X_aUKzINMrR6lC`&F|~<l#<%4R=(&SL=&{!`
z+pEiqe`m=rxh$#Ho}Uh9p;jWh(pvzx?L_W8=dt@n1fD<EZpSGtJz3~jr`qF9lM)Dh
zRQ4`|Y~ynW6|-HyqRt%Dwy2GTI#zc<Ia5cA*Xa-PaP*x{a|Qppj|)X`GOMi1_?x0p
zd}Np249O1-+s+!IFxFX$grTFI*-cp=#UhF|O);l-%n1zyP4^>>3}Y~e4GcwEp#z>q
zojzd)AFi==fiLl}u?=_}wjLoj-A--m=;<uG(rff}bvAgj`VUi2$vZDZzxSvK1z$db
zLq|B=#Q!jWj6epjmQG^&%6V0w=Qj-nuve1ln<zP^D#VHtM0>}%uIp;WHU>C+OQFST
z83eRaPUi->*MhtdsX?|mcB@H_5Nj-xSkRJkK@Hnaims(4OzWHb47=%67?XZ0=C2zP
z{#zy!&7Npq;pC9mzw@krS`zlM?CL{{?yfPHr$^gpnFsM#9AIBgzwrO0ITC(wDj0B|
z`!P1$MeJm_L0Ho(eDCap9h}#UDes8rT@5cqQ$)Y&3q|D0Mjz`e_Rcp=d*FH%Mh%9}
z7zZLbDOEvR;$<0qjrMLwmDU$L&t}VRN-=#{fk;D(y5S2AUgEM#GDbs_bsZYg^{AiA
z#M?`ns6g7$Njmvv%iO%2c8+uCW#LTHf#pw+ei9rYH|YDn&B2`c-bhn(d7F~Gcc}x}
zh8|jtDKBM&%0yoK{D6XpZ&Ss@v4u~#X0HkKzIo~5?pez|{lpb+>KcyiZtFguS(ZBA
z>ojdyfg&td@%I0ggFIV=V?58%_43VbmADp@r6JDWtM2uhGd{|KHN2MhT3tuib@F*~
z+gQPQz;my9k53R=aXWue)*r`212@2;Iy=Ywi35eI71fyKKdFkNUf$FN`<r{8R%$_!
zA>1K>68z@VPB(*h`t7T}z%98oKXf@u0Kg&ZyQ&kiB9Dqda$?ds1n;i+&Y%#0sX5kb
z7D5A=HT7S52@yr7)`HFNVX#;JXt(NcIOomHk;x(X!|Yn(bIpLmfL;k6^+Bs7=WR|y
z-`pBDQa!wP@+X*-+{tok!De7bn1r>d9?+N3ry_Cbqwur*;gz*Ri|J=(Gvr>9o89C#
zt+rXf@f;9PE8kw$zuZe}A{3uCdAbGYl0Hj~Cr0WDOciSWj<}j|!?gpr(Pw!9h+JU2
z2>f0E06jBrPs%#Yl!cPV`+eVxjL(%TEcxhPE#%?$E3O+HFt>inT^`*cJ~G}Lxm|O0
zT2l%Z;Me{#_B4s(?<e+IfBXQG$}L%OgO#QxTAhe`0ecS6RsJy5_P@kIu7wk7)hTmc
z+3LS}Ak7=KkhA`F`e`tW1)l8^HOZ;4!xeHK#&hn=QT*544+GehIco@Fb2r1!2B7DW
z*_iy%zEkeY2189Vey%&^-*ASmk^Mm(ji$00{k`6=OofWsWP`^-)_F*+6%p0B;?C|E
z2FMJLu;Mq%DSAo_39w?hPOqyfaWIg%?#HFpd$lNMW5w6QNtp>Jg7t4e)19?ztw-*K
z_f=Lx2NyzFt6jD%1j=P8oR%Iz(I@bL@6o~)tD=?Mjpp3Vo@~D-OTYdQOtNWxj2GpL
zO_4j0{gTZXdaG0IPzHa7A?=y?+xgqfk&7gg;kee0Wk3e9euAZ+g3*_k!B9`cX@HiQ
z=Wi^NwM0SSWmCJ{C|9^Ks&dU+p-N+K;SS(uNiZ3@PQ5FkMAX3Ee`m&Y8-Sgt<9Nf!
zEhfFh3X^iFgnqWI`Wu5Grk0`c9m(fiYaItpfq~}Jz+O8yS5HQG`CFVwxj_IQ2=Q4F
z3=8sqKjfPebH4sHI_}5ip99RYn;`0tr)?qB-DtFApRCtFdEz8f;4&oZ%xZ>U5(?5g
zL-3?(OMUx_ZSSj!1(JlaP<wjORBMvsz?-<FPWHRfNf<N2jrDuduS1(D?OwX|ar>mB
zkLL88a{z{{joG4d?30AVjys?~Xe;yzci1LE-xC+R5E>TxxMksUgT#)cdm{wXvI%Lg
zs8Ye4dCaznzF`zs4t``I|CuX;(M`eg$B_)T1cr4wRyuqxbm)W$nprJ^b$Mn%y}6WS
z;dFY8m7LgmdkRt`;&Sq-SmUOOgWK8+#(Jsu#tPWYOj_~8cN@~<Tk{Wn38tjrC0r$)
zlV;RXwNrvC0KC@XR7-Smyl?=`37IiUMf!`$3{($CN2&>8-!Tk|R)5FBE8SF$Ele0w
zX!9i>@?q8*XLy3+hj!JUfJ9O^W4(eVw)%?Xifs$0hK!$YS=t_2jLM|bnGsomEEDM|
z#qxnx0TnZe3zwp}4)=;xO}HfYu$~{J5eceoU*DKqD*7N3z6TXIn8H1YUFC9Z$$Ha@
zKdFikrW}z(36y?<1A|JHVTZ!e14)^GdlUHBEk^}q38!8KSwsu3&ebrZ99W%rpg&})
z0(DAhZ=wCGXI@SohH~8MAHvIl?_HJ_?A!eIKU&^$JJjRbj)C)~&WFuz(<_}cP3E~4
zNS)`gA;f~tobLO`KWo_uo(h+Hez5a*OKx!1JM6TcE0qX;CAE$1=Ly~*={fU!xtZ`R
zG0R+@A-hN=(RB^JGb+d;6X?}eT;lv;mEhq8*h#3lzPrjpS%p-e3MDJ6NC0H|$(ilM
ze@9+=yf#er%56WAxt)faY5W%h6w5bn2+P(9zkp1hp7KraS1t;T;phVNxbvX_Zb%=f
zlL^35%Fs#d{eU-&b1dT6|8!^9U^vS}ZI&|kCph?fD1DMek*qW;OOspG$?`F~9LR}y
zC8AzUUnZ6lVpuZD=J_ks&$j=Y>)NXJs8<~;cque9LBkT+qwvM6%YC}(v6`#x3*3ZV
zvSzZizQ_xD!F+1Ld|F0w&vG9#1ds>!m_|(UiQ9-**MbsVAZBjAdgHCwTPQxd-sW`Y
zOUKZ$(Wfgh@nRhQDk$I-4S4tTurdD=9IaVB`cJoRFcRK|b69+>e)h~-6Od@9gX!-`
z(3}pO(U#eZS+HPiY2gBgPy!z1Nq8^v(#FhCFUk(RFEr4OR~LaoEnPl4QYe>d8N&}9
zP2q)hOrLmUYQ*{h{?Tv>{``oPy}5!mi^|~K_C4l@lo-+I6Bn|Q&^?(}G0kYHD~~1Q
z<m=$_vJD(w-+W{Va#_v}#=mw^z?`i%ZDc4tb$YZ3FaL;JP#yqOB?&}rJg}w!E!|es
zN~<hCma#DANgOy|^~Mp0?vOv-asG<Lx1%rkzuj-&KTeGo@4t7Bb;#$ydR`rQN{?)4
z8QaCjzfQj)EVtAu)NOjVhD{!@bt|>|)&c0~h)JYkx^)~kgY;LDT<CC~mgolMwc3&|
z-O1&q+xFac^FL=k4lsZO*l!hT@dm>;zv`>xY01rikco;pm%>vs<TY_rq7(Nch(ag4
zQu$kX_tj^<VJa;eqoh*r%Fy3l8f|&>x8!muK!S9{vfW?vDuMRWO%Ca3!sb#by(A(w
zyNt+gN8-#9W%H?B=<%Q<f2EbfcBLOP8$lkfXCtaPx6H#IGUe-b&(<#Q-9ls)4g2JU
z{My+8i0tH%oXS3`-us7FeI3{5_bBIb)s5NYjyOy#*??p4$BEjISJ!>{&yW1&Xw-*Z
z;fUaIT$}1F-!#6yeCyqFwKh`p4{;#|6L4>g5g@jV6E#1U{o-UpBwN}k3&0#viv$Q<
zMMSXvpt(eQ((IP)-v3y$K77B`MALI>{~{-D-|XD3MA{I(=OOOkZ_*I|$!F+cQ4oop
z+)N9*0PpaALEBm!FCrS-ow2F@RE?2Z4xhCZvD1n%bCqS=pIl2IpCoCCof=>|v(e$(
zk=)*vtRBNU#t7WB(^^TJM=!tK&FK;+@6nEh7p~;G?oAICEgrY~($1uM&q-Ea2yppp
zEpN4Hedwxm-*KNknB<J7Zs0e_&S^4Avl2l2dCT(6(uD>#3DGR{9(3a8axNbQtYI)U
zGcDXZ8wk612qG(u02oyU$cd%~?LtQa$v|msW<?0>Nw-d67tcf1W0^e4!t>_6Yxx7K
zVZSck!|2e{p_D7RED@y`YkG?vyo}Z70?Ro5{0p!EQV+$d(Z_R2^Z8~6H(Tg#H%}>9
z(8JTno2M@yJ2W(VYB8gJ4fW}*xcQPI!I(wBvWG%5h1>G$5Ap%-va=+m=(l-mzfA>_
zK$fWT!ida!m(yfx?uH2RhKBb6l=V#%T#}x?vIaA8bARD+^2BI~XL)%+t7hX8)8M>r
zt;+M6W^{PSLdio4t3~G<cI4m=4-4h4Y@@rGTP-K6Ii)B*R?o6MTR|qYhGqoYN#{~P
z%AY|!X;*>M%vCme{Jz{Q(_%3b0{Y9*A?7eJ?;xNvVS?^>!1_{n>=)-xo%W+Zfr_MV
zk^@lVz_pazaU@ru$bQr5)=l`?eK-e1Ej8t`2AXLxz_Px%0jyYi+fwXNmhq2Lh;pB=
zL(JU(LGEWcH!0K9X3bxWv;7S$J1KaxQgm=Hpkc0rLW$hBh7gnbrv7X?Vz>GEXGG{N
zRhozVXG2oQ#YZ3YW}neLKmw=Tn?HQbq<JD(!)4X0SHpykF6uc}>hO3vhZmhgOJ;ZR
z^9Foa4eN8%d|KU%dx@%EX8b!r)S&kVuMe8H6XDybnUT*qzf9{AACl+aNwkO)L*2tH
zUOY_R<qh&&z4(PsOvQPR77m~asRQM``LONSE|<I#Wk}(4pCQ8LW-y#_2Q)ja&8&6R
zF4>(`n~m#KG!KdQf-_=82$@PWkPX2)aB|l!K9OEbv-tF4$h}@(T;TXj^$ip=&#3@l
zW{25Hg-cwV2n+oP+BZSuT3lhny%$b0Jb;+s{c{%&B<l4c$ZI6d4#tovr~uBpa8zhD
zQsAlf)v>Ip{+tKHx#K{?fy<WWIkSQ^4uV}yI&a4NT}`Ait+gmN-}>xIcdX5aWo_Qj
zxZG;WgN5D&l2>*DjxTJKD*Ml#lXiosipA!ZNc^}Qg65vip-UPL&tyHr^-^7J4Lc>3
zRtE&2%tu?3v@Q!GA!Y766?VRK2hQ{A=Pc};-$~?_cV==!m?wR4jki!Bk@)V(IoECb
z@SPsJ>?`T>E!TimHWu;Z{0P`3ja9_U3-qoD5bg^wz8ph+CfMP#NS<JqoDi2?&rj)#
z*~gIpaDcVo84ItNvNb^KjYr6h2}eQep-`PNmND0b(|bHqGim90bkP(!Po%uqsCEK;
zNgDP-{$N?<0iAf7?jUqmSam0C=`07<G<en{9Cf0;XZ9`o8xyaRRH|;{)a;4nz6r|L
zs`UNVVs8DV47W^YhHe_dIR{~hVx9h#Q<!#nbrzv?ysaDrOHkR{lRKL={pk^GBu&d8
znS<MFTV?iywXT)h%3(}ia?$IJ%XuVLsPWCvJo<RJ-Tym3CjFnoj<7W?v5%RftKopr
zSrA3jNtUJLlh|%bK1A5`y$qOn+pOem`Q~JlX60vmlvsp75cO;j2ApB3JOsF(26O0u
zwF-F2PRXF93ZMMNd2Dud<T8wTb{-l9{Pg%percu}CzPekZtJ9#`?l(ngK$1IXFjG%
z9wGU4)Au#Kv)1k)IoQdpl`8>SD;;BZdq0WaT!Ugm(?3^r2e^PKk@Iik(ChV`@squk
zDplTn^%fj5{(!Ab^!Ag%9pSrZ@tNOo!6Kd^3aq!M-J6CnHi9!5M9&Q2jP286Bc+?)
z$NNIMg`)%)-ZefR{H483*@UFUt2=E@=t6GMX1e=REPLsB2zk>^F)muMEBCDPiMBL_
zJtmUneoz7_-T4dIH2?$6FdP-D0^#zVE7J6-6eNc;E>i5l2>tblJk<NfwQ?WG`1&1|
z`v6|8WfX;G=;P`p@-OM7Lqw9%S~s6YZ}S^OBk%JDyHozonKxsV{OVogsYoGgx&BF~
z%^?=|EcQU+4=w6v08w({hWB>JO{lbG?oSZRwb#y1kDogAg5a*aDtnJd3@1@n9!jHN
z?{uA`$=28$62$e)XMkqio)c-80rJ0@nM5ZaB#HS}tTx_*Pl9E|zwI+F`!d%)l%?D3
z)32PBtm$jMj~0LX3{-g#WCGOD*(6pQ`qk~lM7%F?j5W)*&|ppps8j#-v!xRCW80<Z
z@BHD0Qlo#@*RHL<CoV1AV;jPDO7A99Ia^yE;!=6kX_M<K3#e12%NJScN`%zj-6Cw-
zg~j59FyJCcO99Hyk&kI;)~AHKyK{q}{}v}wQsk<~0O4HUS%I-2{{xU%K-*otX^gl-
z2Nj#q1T68rZuMW|A!-g$<y|)skboT}fIMaa^5H;^VxC;s&Ex6H;)Nx!c~fO1*h(tJ
zzaR}-F7X_N3OWB#yId1n&_Q*cG`^AVEh8n8Ai5UsJ|meEHa<a@3(NX<Ki6C~Vu^k`
z7WeUn<{Lziheb13a`Q+8*?|FBUMluW9T;GE-x{IZH2hE~kA>)(RmVu2e5og0!MLwb
zF}|CSPOK_+^=ipWlfmbPXc(KF<Pd^t_24$M51-kqlmRkkz^!4VP^%^CEHbygU#gVR
z($4reK)7Ft5CS4WnA}RM`|KCiI|I`RNtP&mA=|H~o%Q}>+8t2Af)q2Anvq4T%|KCP
zF4J&($@2WZPb|UsC<lv(nn#s5uq%99qj{txk!-ZXo_{z__9=Ar$eSg2YcYYlV~kN+
zf~&I(kI9etQkSQ@@!CVOKptRx3~D+7ZAj8hiy=WZaWKWZSC9+mh={2$FW{6t8SRa7
zK!WF-oNJyo`N8?tSp3a7`R5Dm{ja?nNk^ppx5zfA<PKas7eyPqScOs??H<X`;C8J7
zw;NG1yq3{D7xUPwODt2~?nsH|u6lghL6K^5aDw4YISrQ=?)mehj%k@Xo!(3RwI7?f
z{Fzg&#?I~p!>M-6?-QT4CV0ETcecyB^CGkKWv@`TdQM{RvY+&~1FWk`(V)5qHcxp@
z8o8dhfGoqgIPl-QxJ1onT*0Ewnd|9d6+bB?Vjsdea&zbkWxQ`kIKK(B7OyS@Qm=1j
z;|{&BY#Tt{M70D)g3lCj=_7m>yROnT)gu&;y&k53aMsnD{q~~tW;Wv~Fv<q5Yg;8L
zKWn3Qx8VHJI&Rnc61I%U0I25e`C)PF=evhAtU<2#b0bE#&JRI{i*LP(yZEsVG6+z7
z>KO7QF}_qzXZz}h&28F+j{XtjM!C>2i*0Hpt{brppSRR0{<4K1FXYSky0^SiP*$ds
zX$DI)MNvNr(QZ+*&t`0oNAa{AurT3*I;K=EsMq3`CPDdG`s-Sh8Ei%pcc`$AJA1uL
zotP8TXV#k-ug)@$tNE34;6xBavK>;&s-Hpo9VkX}F0&UBE?cC$hKnNVM7EB_;=IdS
zH(fGnnSa%0CF5f^M?^VewvDaBL^shCt-VgHP^>>wb<OL6!w_sIz3=mMlkM4d@wLLa
z+XrK!YsMCjzK$Q~mKz2|dBe?4#)h^Q@s}gTQsH6!bzbK=v)V%y>ym_k9zB?u|91P;
zM5yu`|E3<ijP)+A6PZ5=<bpY@85RUl$>r1?8Y-vOyNZE1?Ib6Emhd?@d0N{HVH`m6
z5prvK*HPEmld9n>%|*TGR1uyv01xwY`njx#us$5JZ2&Ki^>k~YrWyBz9L6g<N95g+
z8sGUw6;1FR5nLV*GH^1f-1fpdEL(P7n$L=iZgp-CNe~h_uD;e6XV9w<&*_?$BfGLY
zLw36-qZ$IYN;vX&9Rd<#qwpg5u5Td$dGdMLyZErM4o^tO{@=o-+PbiZHE~FPbhPz2
z;n+^?cRc5)4KtNEz*MQhC-eCLmS+JGqcR}Xm;=z^$yJE(b8q(^e!L83Y<5?i)1U7b
z%VO&J^X(e<-^aO3S2a3st8-fG)0RezT&1zkxx&k&1Z%U{9_ZICbiP$3Yo9<uyrhra
zVpDqTzn#X3>4qcPLKFT8vvQny$PjYvu9sjjZlQInrLG6qFSxuunb^Q=)oohYpFWSV
zUtH~OAB=wZI;K`sP*gpexL`!MW~@#J*Fh{T`lviiD1x!QhjtJMXNEE>3AmzM%l5)F
zA`sb#2)@Rtf^#p1I5E;(id`Q-H~EI!qCtI019i0a%>hdB8Nmx+c)nJo?fvM(Pwq{d
zFt7PK*I8Id&QkWV*-7x;86Dov1nEOPhuGWOGcoRh28EtqqH=4OE;_6fYGmX805L)|
zZ}hlet$99)nHFUyj8b1(<@(k6vv`vAV^|~3_wvUN<?>6XOl+&_p8FYN6XKi4)Fa@!
zUd<Ac<F1WnGu8RY%0uq~sWz;-D#3($PH8<xrk_!<voadd7&Y$l&dGz<Z4VgeKBn0L
zFsW}vo@Jv2x)?}s?c@GBn7TdIwwYI6GAc0wbLcoKmWrnO<KVQlsnpCF4*lp8nM=7E
z>uuC9&*{{081L-R&*g#4&zyUg`2$Rv3fg{<9$6_sZ|&H_ohlr90P9_lTOTY(rJBTC
zF1Bm`)AZd@4*nhUZ`4E>_>O@wY2Nd07g38GG185D(Yh5m=N%8cZx$4f<;jY5#7&s9
zV_4yh%d)W7U)*DSZ#rvD^B^3qt0f9%ORO{*D2tu!kw5yxHRC{ptg*c^@|baFy1C}K
zzitmdn`k%vZ^hx6`@Agdd+`kKi-*+^6w$@`S7|Ls*>0|Sb$&!vUEQQ3wsrK`_#I3u
z@$Q3AdD4awahE5<{<btfV_wI@?nR#x3{*v~!{7~`#vWa}&Z_aW8W(~Ixui^0Zqx?O
zcTOJ6{e*I~j0IZLDj2tWfYdCPiodB?RSqnK?8ydEYREfs2~)1Q(9+t0e*6rXdTK~G
z5q8LQXXSn&*OC$~EjO=s>WnXXtlaDpbE<dyB2X2KsF|@3n7TQ$5ex6?P2qb9YhuqN
z)fD=^{Ini0Ny^!vwZf|0WW&c~gUKm=(07XOVkbr08wNCuklD*TEO3q;_S@8~FIb&i
z{O-MRUBLgShUa$G^;t3&m~BoOTF<yWgP=Q@ZyK#va7FiI7toMsi@oMOnP`1D-~!~%
zM2F#L3rKq?uyKr-43WBpRo&TctW~@hONQE5?Hb9=KzaG&`RG18yv0JpI}tppVG-9j
zwidgKzu1>)?%-ECCFIK`!DBx@FFoO1yL;>t|A^b~U%LRTbI^gt#elZI>#{tVL1K^S
zi0x{R?VnB4oI*wa<Bn>hz0uSg4W9hUGt3oST@mY$XeU#YrnTWlK-AgmYDKSdFPLs(
zNwrZyQD^6YaQ=NulIO^reD|o?gF{2nJB&@f)}Mr)pYfWgLCa7?o&E4FV|)7o0UmqM
znB9`z^kz^3CKQTi1b88bky$ln6z?;6UM$D}6#D1ui}mO_9+ce98*l=8y5=XfghLw~
z1Sp49o<^{_7^p3^&Xsf-C&sU*xsZ%bvAh<IFk-?$@VbdnubHseuK4YphbfL#`86r^
zd*&_*=*+#WG_y4HH=&2p?)QpOF0;SE<y&1XBRU>*>F<r!O|Qgm=RUKf^JteETRnVt
z9|Mp-Gkunb{}NrYGTjvxGLpW|NOwR;_I&uNt}RjRFJE=<jigi^UkSG`IK|vDadQ#A
zQpr(_GWjvCT4Wk+We3BL;fBN=LLs@5X-%7_qL1*)OL0mQb>a{9;9qDi%{%SVZWB5c
zwB%&iC{)JG2KuF}Q8^{9XHIP@{zMR={CV#N$?{$tz^7vDmGXL0RAa-WJI%e6<JJ;u
z4$`+&Tp5)Iou&Q7Si*fqqyA2>a5>DaG5-6grG04bK5z;&_D)ivJU;d?sfekFF^|a#
z-B>3hz{)G(*h#%+@KvM93)3)E3c<oB)npGt^wzb*cHU}%9v2HRFa)_lBOFg)(}&%3
zx7tTtX9uS#Ui@wT&Hqz0U0_Tuw_J>?cxu$YS@Bo?_k&fl+~khX5_w{mw=D<m_e=YK
z&`0gNH_tviO~4EvEx*ecKYrnNadIl*aY)Eb73DIsYJLvU`l#*BrcGH9vp$w`xiHN>
z>Gz1m;c3*Nxi%AA-%__$LGft;>GfWO^oNXCM!FlGSBO<!4P>9x2-O~SHiU*T6d$=i
z-aZTV+tCD;GinqOgq)1Gt>&OPE5y?sB*q&qgJuWydwsRm&_87PLL)N2WI@oPl?kOb
zo#)UQ`W|X4kET{uqN`sE`LB#0Fjsns2uyA;c@sB)`80Dx=AMet15*wS>kb?IQ=WBN
zNf6>c-<)>WVbNDzHO82^ml+khzsn4LTbcW5OXN})g~=cq+iH;BIBJ1vrs}(==ta38
zL|Eo79&X=}+~jt5!A%M83CG)82B|9~6Py`?J=3`Gp`mrdfC_ZXfjO7mRnGn76^D0A
zYenoG5ig$g(tQxQ5xqkwK_E{PPDnE=hBkEKQN4=Djm+hW3p4(Zp3jXxpIS$7hrGOI
z$av{ZUDu+Vnca+GFf{hCR$DO<!VjTjY8&6^w-=7Y$|4)rIz7F$tNl@B`Lu!hP;7bv
zd3}ILl0etf`r&@N@UZ(W*5{W~<EQJvxKic7)%I8%mlz#u5vombay9q+(C|us*kOZ^
zciq<ER7txmXlsP#(JNyBpj*YW%M?RaT(Gs?jkBo%Wvk=@^1{Rt46n>*XkkMJ-^szx
z7o@r0>Z@o^>Rz-Zq>6v1+xs;SHdndAAtdDPu3Jz`k*-e9$xH1_u7CbL34Cl=%_Kz>
zHYoC~e5y_EDg1tuaD&AFB)@&Bt@t}ArTyc>%~2;$wkg@{hrwL_E>(|8)zST<4g4En
zw`4IgyTJG*<j|v8ZUAk&HJLlf`hyq@sBtYM*Nv4dt6?6{8M3^Xu@EYh!)w2u4X8{C
ziWN4ju3t0uX?ElLEBmzg(DXUd%;eJ8?nnFf(NWVD8VdbuB`d0BzV>;}c(~a<0icgy
zKzD>a4Ah@Om0OhN3TZV;H;#s#nw*+q44}?_^8Sy@h#_TyBd{&@@o`4Jb$7#&N!zxg
zJwBOaM%epCLq+^Co!Zy*$^-AT5s2j@n84v6p=Ks%nQFzTLD@gCBKgsBM*reVBKkg#
zDDC81;-U}FEXuwEIwvW7ceXrj4@@2Fb(wl#Ijgbcy<@QpI^?&Df3?v#YvCLXHzQ&(
z7>V;Dwnw%%h}8_}08rzs84_<cfzN+$5V|H9`XNS#%x)861`lj9C(67RhZzVSik<$V
zt}B~;?E1IJ%f1f}_rC3d>#ac8kaMp&F0X*CJi`ley!#UnFm!s=UpvW(uEyI(p=PYa
z0@ts#3%H;W$q(zBYGowq^mNBGNeG~R+@>ut4`8X)SQ<h4Kp^j!bqTJ;ay)ykc_M8b
zApt^WM?7{Wy7s2}Z63nJ9IU?$FE7NnJyd-xlh?``iq}tljC%JpgkphNvS#nQy5(O3
zAJ<qFxo@W(if<E<JT33%yc&cg%#N@+>)j(0x3?I%xP==T+sIute;{Wx%6MHsdvZux
zdeVEt(5y_;GWAlAjr8fH+)p@|{xlNW;zb(7&$Hfq&S{|f>}q7Qmm2eia;-`pZBaLG
z2E4eR)=a47w`AmCbR=v1IMvzAM=b`r+Qdj#2ISgm4@llu{~QfW2wIHtvMRbu9m<88
ziw2bSw-qcsCL|N<oFo-_{ZM`tGm>a=K()V?>!FO<5?c_jXJR-KfAy!zr^}mDa{leR
zxS`d*v%?Mfs46Gju#l}fm{Q)v*CRuS!xfJeOw3B7LSTSVXZVgu+v(Dun~YDeUn-e4
zm-)LpkP<hQtwQ9m=O;Kt2Z|^8n3zI5`ps3lDIf#zKy%_y7Gu(YmQO`*Q1ytggPZVr
zirH#+y8v)f|1g*pi8WVJbF;w?*JGQpxmyPlb#@D-@)yo)#8EWuP19ffpQ9)v409B5
z<HpvM173^2;&rl#ES2puMmGA7&$J|JWLFoec7+&RrGqI6*%!(2o%%-uIFH}I6%AVT
zl`TBzz(#oVj>gp`?}!mwbnu2>wkl2OJ2<{2;*}xp835S5X(pjqlqmLGuC{G%$4VMh
zR6{hoF>Ks8+4hGMCCn@rD)DWV(U($pW>`Ig(fqTt`*vfmq+OGcg>k?AfYqWXXY@oL
zLAmt<RJCXQz>l-*H>DJJPJczdrn?AyT~^w6nn+xqGW$RgED+8G=mD}EcUgyvg`A(G
z$(NY8@n1aO=AIdv0}z0JzvX+i059C<Mthb;a?;0TeE6Q&4-20Wf+jzY$F#jc>y&k?
zdj(Ud!)DQ`^%mA-nf*6wm15Q!;o>JQ!02u>RpM5)!q`uE|8-fWBv#p}O||N}w`PK|
zk26tI?T&LM$lPXcKA62A5%_Udo6*@p-GS!(amaHbmh<zmNG&?B*q+^OC1J554jF+q
zB4+l`;N_eph@U2&{!QpbYp#3a1?{(~%yRpm5g2fQ)^vTF4+#@-#Z(&b;6ynMZ19LP
z-A#O{7k^#o{8#&m;*z1N*}Id$4QV3pEP+ZHUwgSIH;IWK6w3cs)*m!mX{)x#uv641
z7p-FlIF@60^;x<szcjW8FWg35?ps>;1YG|REQX)a>@!KeJS$c9)Jel6oX*r(NTQ_y
z<qBFMA*n0Gn%=#hyb3-`vR?BF6ZHVZXmZM8bQ(-o8U*A2Ajf-A(0?{>3_Uf}VVo;!
z&~7R~jwlRRB)4thmPJEFkAMpz6$n@w)g#OIZ+C#4uI8>KS!1lx*|2|!ep%x-JZlEU
zlli<QQ;z~HwaXvF4!6q<4SH}FP^v#prdOC*_PYdCKCnDfF0E^wg3e?>{d2|6K}O+L
z1N6|mOT4K@o>Uf4%1*d+JBg%XDs;Z#P*GX>gy4LLff**+13W9E1KMtty=fA4VpGy!
z+{9=Em<lSx@qSCSFDa1DkE$obdVZGmPXS*5sY#WR0EU%ZbJpkK=@esc?S1aw(dcam
z4JG{%v|6AXXNLai_QDmBwOe-hD5fq+oK#;?6%N^y?A}aI7@`MOuD#`XNj_C&%{`Ha
zU3<lsT%#rdn2mrR_A@~VhcsqwOmnBD3CrE3{r!xtfqA%fPyX1Z-y`}%Ei#QqOl0Id
zpxT9q7;GZsSyUp+_&2DeF}JrX?98l^W;`^a@xxB4S(<N`XLXZOIJI8;8y5qKsSUZ|
zmhsG(Cl8tqFIRv+kdw1v-J<irs3$j3QQpKkGvOF%bXF3}lnWpab^%<IZpO(0LXAW~
zc2yeH1m<t2p~ggKt1w!W{r?W9=qlG5+Gsbl0UqZet4bfT6$XIVn>SC)Q!x>XUQWw%
z(LwFMa8j#VgAsQ9et(@kq5f)kYGj+PN=>Wj)XJ~KS{n17Nl?NhoNLqLBxNONw@<3e
zVVm^y7p-;-`@CpbUDjjTrxd|1o~#mVa*x&yf-Jca(fELJf7LGIp}&$oVvJSZRcu)E
zbOT=p>|IJ)LX3db?Fbm}@Vuu^UKr}QkIN0+|1<X?%QNC^+V*4MY`dN;V6<@g9;gF5
zwKd1=dMiwaP~l{e1&9aC1R4huip@W4XOa>YS94310t*yXK+vnxBKA<<(1&9gK_rZb
zv$dk3%F}>znV6xh0mq0uU75wH<+t|BZ-r-Cev#jo`*vL}D#?|XWJBIl4t8Vzbsk4Y
z6b&25BatIzuq^wg=f^tg#<%dj|CK7mx+rN(<u+ts9{2kT$G`kJ)DEfAjp%HZJmP3R
zi0XFISI~xbwP=(to7dM@(;6->Sw*!~7VzH?qCIBoe+JVAx)x<6bDI|K`3&^B4woRx
z<pNA<GvFo}Qj|RT(PDVUG><42TS3v#{eTnka#wJc#c}sQ{indxNR6i_(Mtp6-)RS1
z)7kiUPx|2pl1%&Tn8VD=gai)`Bm$TA?$P!(j-f?3(~(8T_(f46^cmNKSmf7!t~VtC
zz+`o5vL!Cx40GP!2s?$p`OTkIuSzl_sA9CD6xDYVET6hp7cG0QP<%R`EkBgA^ZG-d
z%uR({M0^HI2^D|a(i;x+qOKgT>kG-iTDtfMH6g5b_WIEof%~6IJsMmos<+)4fw#x!
zZw71SNWT}(y#Pl=sy0g9q6@D0gO_3ZdhcLIlA-4_CDMb#=pQ*m2I(4rtS-wSTg~s2
z515rihyujVvd3|*(mGxawQMNs@+|(w;<R6Kw;GYpIjG18=CR>*X)LCt>iw?T2brhy
z_xcwdKZI33*kfLdVN?_$MVw%iRuVV&j;1{Y9Dqd@4Xvn9kVXAB2oZ6c&t5q90diw8
zJ1_Gy{tpOV)B=|QtfOe!l1|rJF}XnCKvaL|7>2<64rCKXT^iRr!z0C%01g-N#7T)I
zwqLERva<-4MZ-cbSb~ym`Iz;OcD|es?eb&3V@0PqleMCmC24P1$V=VV+jfe`YcdoP
zbO3)*?O0w?j#Nuxe@EJu_;>SMIF3PVNiWW1J3)hKF;-3}M(p10st%7Zt2{W)8X11K
z(z#9T2|zsrt848|&Hf7YN++2%dmb2f_HL^%zB>sD>-<`yBKI$D*l&!xBhKQJKHS8=
z8w%>a<xA6V^_F6-iVoc>-uVe`wg=+qOuEdV2|o_DdMX`zfZj0=&G0Am9LqrYrX5r1
z|A7a&u_v0FGT+x9t>z7i*%r;_kjDXU!K@A7IWdGDildp06X{1PPF?ubq?zQCe2j8t
zlilsV>kJLG>A_`z*4M7`={(S?${$IUOJP&}AZ^_DM|7<a)H8C7dq(=5bg=m%;+^NG
z>{tS?eip{&(*ulWhs^d0+>!U+=<^IW6k-fUfC0YA)jN{yZP6)Rzq1e0y3TzME5zBu
zmOhbJ;ON`P5Qc++Rg%z>ZY*mmiDdP5i@04DkxvDt*>~%*+l_<IXkhLoM7w|ZYg0lu
zAm?4lN6c^jb1H=#TSV0NRA!^3v4EJWf;kQ$s9Yz)Qq{6o_Mev(a-tH$FebT%^LK1u
z(N{q@bE8vD-8deULO$oI`lYA9V-)yeb{S@t=&zd176@tZS#8m~O>Yf4z67z9gD?gy
z<d|~WCof9ZdQNd^49}qF`vc?tMy&VKZzVi68Gb5wfG3G|<xz?8Yk#xP_ai(9&4HEN
z5NT8BuYFEusoFdjTp5CVppe4sb9+v>Wn^y9hBz3WG+*~A_tX5_TN1E6<@)$Yi}M##
zv%m-|`abeFuJ%EZ_g|+q1;fC?s9X&5wEqQUh1IqBq4Mcz*<%X7*%Oz{MAm^gR$}e+
z;9A~}(H8_2(s(|xrL`fl^TlrMXS}6q$kCR!a6Nl(5R39`tvJetSZaKz%CcRAV8m`L
z$BI-yHN}lMoJP6O;|eXTQjFmT!IzR4-BcVQiT44p9aL>Moz5QXI~@jfsoLzzOko(1
zawp9vv|#5&Ayrpue_?yKo9U+#4a~l_WDJ;aHJjeSH`avlJ)_PmiN)>rzkF?qAdvJJ
zBwK4W^yRuSA;^#NWXpDMLy~NwC%qur<Gwc~Y45s`X0+Wyt9zcmYa(=PrHV+>z9e81
z9<fk`V&&eZ(~Qn`9_rbl2OVCxUxW@Pmn#>Rf6d_(uKnSv8m4;aL9xKA_1dI*uoqAU
zlT3%T{OY{N&*-a=%+yv=_$gOkaperNqXd8T)05SuYmbSR*uaA=sM54D(41^3;5<~U
zz-#E{W`naNLNb-PwR(FGv+S$m+ZimEOl>U}0}IWyU)HLuSDfsQzI|KPS@|i7XzVml
z;M)x67c720nnf}|&+G;h7u5x84`<AR3`<^CLJ8=YFWGgVB?dt=L?M;W!TqL>1qB>#
z1GmRB5r7i2ljH?%O~}T4Ey2>BtEJ}sBO(*Z0?7cF|L|mr{w!P;-ytxL3lb_$_VX8x
z)e~pUipRzgWZ<~qF8dZ&pLbCeD1Ix)ZL+S2{qaZ2Ly^=!w=;^1i+<V`6eUUQE!dBg
zkg^REoRmryl_pkyKLn*)Ob@-kxMx7J5$U1BEmernEC~saOGWI31o_cv(NK?MDsd}a
z2$#l>@Yl&AtVDipd{~kcLb*$d!PNF6yDa~@*Te8IX4L@v><S>D^(fTSne2Fw+3YD8
ztzG%kxg~G!l<D!NP0vyz$_h6HdACc%(h4ioB^<8+t_NtKp!_$zx`+jA_V!NKl-)-~
zuzbyQW8fPr#P_@HllA=|@aj63$pOG;mf(z8sYt&wKIE$2mIPCTRpr(^@cd{ep%tt6
zk*z8N=C{x(7^7(#NM%Wev5ctcyJ5)zQk}80&?qaI2gaz)b!j^`?X}BuhJN2|lg`pr
zWzLf<=xT#fa46(Y?b!s=B*ENQsssobq8RBT1P`=zVisI$<$`NIp&VT!l?ZzgBa*6q
zk(C-mOU--lHvzum50S79)L%eUsF+;^B?85_bGqPJhNp7I2#mM#3^d}(xAPfHmvTFk
zj#Q80m7A(L8}>uX!@WsB9%a~AN$0IVW)N(yPbBuF(`G1$DTkyVRaAq0?rgOMU*E(y
z<Ur5D=(!nK<i{3{Ukp9FDMe;v;Gj>;BOR>&0q&f^^&s!6*wj^w_GUvn?Y@SQQz_pi
z5Mj$WDxV_@C$y_TV1)Lx)EaHB21xdUsID%d7|%JH9&rf7{0pYhwy^LIiIR-9EPl_S
zx#_C-Y8;nF<^VmL#eTPqdhd&lma<KS-#}$!ma=tC*Z<vUsAM~M8(5nYlmWNJFgA8R
z|Hf{qd|zIG(4&-P)mge^7R;OGDw2t!g8@3&!UQ0mFNtJ#!o#-|PN4<k;QDr8lyz0D
z82LKZy3ALfyzBeug{?*}@V6^_3iQzTQfph60@K<p+1v=mLV=zPFU(qxH(Yh_{noFU
zBU$A_0~z_-&*j!m^HZtlSk384_XoBF=P|<bVT5>@6jfg2{V6qz<dEP0?<*r76iha|
zjyhy45WL|PiX3#hDP_WOvGe61agxl7IliB9lR4&K2HcH_;Bu`Es(LDBz9~F~%_4Db
z%aKKWSCfzr`C;mMmSX+cNv;g1Qt6Qqv{Vzl`A0~ZP*%A^XfwP*mfMV%^1yzy-%r;S
zh0ZK7=#?ftK&WGIU-7;fgaF9Gf(aF)0C!a#fTGH%cFy*sLK1O_1lO&;mHcNp*_W8s
znZt#4fvPx<`=g4VFG66B-L(T4uy~!*IJ0K&iSMX`HB!FSUh~T2SN`Q6?z|Wd)ub3g
z{jyOc+Qs7_v~22=5k2v6i#Aa-!r?YA7X^dw(YB~{zwlRx*1H4un>uFHP`l8+Fz$3y
zUGdBs{-j6#TN8U2l*UTm=G$kOfh93|k+w*!%|LjpcRowD-7lYMc!0G;a=Ti)G+C;f
z6n<pSREMqS6aeYl>;3F*glec}iNS$BUBp(#)(ctC#@wxl3<XR1xk!-(5t7;FY$mWr
zSP72lv~C!KTx1~iFB`hs1>9O7Cq5@C2hy%1;rkG3R3&vP!=BkS((2#Y9W3{p`Cf#0
z;}n)7gs9;n#H`KkJx1sB%<p2(@Kgxd74d6c9I9r|lt-{I^Vv{*$HTW10fy(`C2a_K
z#J-Bn)&RzrEnJ}6Z>7!$9Mo~{$t6#*>=3rCO>1la9hS2Vzw1(Ed$q7Rq{L1Hb$*kl
zO4nSY2-b?4@k3Xda)QKzSk_tyId`WIX~eE_EgGa$9tShKCteQ23-J0`T5h-F#ju|t
z?{xS4Jb_PEr_q=G&z?7RLgF@IkL9djTy0X`jn8(T=7k+i=Ard@qZOGGaG^^IBcBS|
zV)33G^8EWDatN2&msA9hPQIUP={QSF&y42M6Eh@GtjzqbBA7pNhz0S&;7`!pXQRQQ
zp=sq7yiLdgsAzd7NbcFa=r0UoC#gjK6x+PFC%knMC5Q*Es<rE9+YCSr3yrIvuUOEu
zxK(1qD*xR!*p<ihkDI6J-i5rERNmXiET7&zOGN!e)0IcGs6x&YEKb^!j?Z>5MK86A
zDA(pqYCuK-wRcZ11jO&IGkYIRhkZ8wf?*p#Z&4D!g|kyXU*vXP#&I}ANqU^l6=z=r
z)??HT<?x}WDm;f@Pm6f3!PVqN=FxJ=BCyGki{9pcSCs5IHf-YW2Nd(U4!ke@<5S>v
z?15UJPXD<GV44JYt7!kWCSkjTrL8-QRi*|JR)L*9lnz7CYpmZTgP#H{ms|O`bT%k$
zu0<pyHAQPJWVzaBbCd(FgZW$wcl7txZ|9rdAn{i^R^?VYp5LBC@EJbOgEbbGgaLc&
zr^zvIW0&Q`{I}SqA=?Ce<JWZ01EMg-n|TQ9<gPbwZqu0|3C2>^x@A$O+7Yp>qO=me
zgftyqfr?*4b*-T;+q<m13lX6LF<0l-?4hTBs$3#g4ZsR=o+H5doLC@aBs0MM8iLv&
zQ$)R%Kl*n3WpiPt)0<L0@E!5-gYTG<(#+V6)OvONid(iThJUs&H<~<ER(plbS-i$s
zBAG7a8Utz;3nj}gbeSryZv8`Qk0bj)a{gVlB~9sBvh_d@Cwcp_00&6FJV0OKdh&q1
zLY}gYhmY?g_KUm#0pyXUtElV0APT#?@l_d^R)Nt`Dyukp#xAOm*MYPDK{Y=Ug?%`S
zn0-+;q6BvYeui+De$sxL;j%wW%+U?n42yXGS9E|R+h5qL+SKFm|0Q9)@adKrm>kn!
z4D_9Tay)5}8pk+>i9UI<y_Iiv*)Ncd{fId9`yWS^HVW3a-Uoh3x$HrwNh<M?9<n!d
z{cD56w@RWU7h`_M_ZHmiZM~K#zc)R8;3GTu#ze6@IDHeB7vlxH6vm|dJB8E3N;^Y1
z382Dp`Hd=)iWSr>YmfD;<&lg(q?h1wg;P>suI1O(AK4YFqAZn@9_H1twKH#!doso1
z2>T9}6FYl^$!Iw{RgdvuT}lkF3=TCXC)sb#^{^6+Td*KJt4IH67**%=27{I%H2gj0
zWJv$j@8}H_;rSbJ87$rKiRQ9e?J?B<{$LHX)2$?w6|ucm^jY&Y1N^^DOe2!*E@WqE
z=z!wXJL{Y59Aw`a2Z>dlEiG>naM(*$2MNF^aa&N)+VLW^X4X_7MfGXLsvzh;P%Q!C
zTgWhKil6b#I5N=$@JTAxt$z13Z$fk#Jt?jiyZo%ScClYiBPTC^1}vG#?ppZ!XK9A<
z+77e*GS)A2Ns<m{?MdqV585p?5Sja4VTga3<TWx}oDTJ!c9eY6{Yju_g@lmK-^o<=
z4sV->!}Q(7J)MY*Nl~loO>^1LY<z2MK^A%~`zPn8MfIPg-dl?7wRO~n?0Las{w7<A
z|But##KFo2P|2LoktWaRQ;N+&N$JZx|BP0uElzC-EN=2Y{C$7lL;S7bTsW8WXE#>*
zzI?c&&wx{yb$TEq<j)^+@#s_n$uU&|Xj``+Y`J<ec%3$`r)M=<Y5{sgn5obR5<Rx4
zQJJ2=c!Q5*95ntzmWK=H)BC_KC;!hbCbPVDUH$Sw^`6fk*KG26u;-)G)_vy_=Ng=Q
zR#k6d#QjqB?w7qMvp@VODw_q~z%Rt}g7xUM?=NRQXN}&*xozX6n!mseYPYY>-yrAw
z^Y?Kv`}SH>mZDY9V#Gly`;{?sRboq;S^w{A=Zda=seB*Lv3tR$bCJ(kzOCC*DtyS^
zO7IYS{&~6dmtH^j%;i5fjUmDmWQ_?6+ZV4D%Yv`k7FKMtuH@agsqW9ATi+#a?BALD
zO<ni9so=5yKjJw4|J<*(@XwE&*vhjGa=G{HqJx051R#5zlN{D}7++p)z3G1FzPGie
zpVxj6bgVr*b@Qoz%edpEx5w4g{Q0l<9CZ3Z67XP!dLiJxbLSsZ?A8DO?q3cx@^RUr
zA2wP~@Bj4#Zp&u#)Oz%4_33q@yIyHUR{;}tas2PgE&<vc=iTLX?|-u^VJW_Kv;5uG
zx!@iC1(Fx0H+WU`etqejYykF)8rv6Dr4{>spIWl%U)cqjW^mRt1RaF1_WgUgmHXZH
zzK&2o|E!7)?DXUm;OPlb_il35=DdA>{f^CTpYyRdL0O`ACga}4NuhHO*d5v6r?LMX
za6|d4ywy)Z366!iil0Yyjr1vyx|g4LUf32td!n;peqHgm`?>R7=QSH&Uw>cp(6hIb
z?eoon+Yo>U5kvtMxkvOb{QE~;agMFzw5AVxQ-zLf&oQ0=k`5AP`_i~@QQfl;j(DGc
zHr213rvBX=nf+%c>+Q#!cjAFNu8V~Z?KfQ67r(VMO!fYr+;7W)&QMut$^7etzytgL
zC1JV!9G_c$WE`HV7*n|L*WSX{Z~U2efYe+~g`E*X^8-fKflJpYJ7&;U{*iP#$q)Mk
YiMSP<@2pM(kC|ccboFyt=akR{01B1@?*IS*


From 47a3277d121a175abf67a2575e49dc5327a6b58f Mon Sep 17 00:00:00 2001
From: JohnsonLee <0xjohnsonlee@gmail.com>
Date: Fri, 4 Oct 2024 17:26:57 +0800
Subject: [PATCH 053/128] feat: customize channel information for sqlness tests
 (#4729)

* feat: add pg and mysql server_address options

* feat: start pg and mysql server(standalone)

* feat: start pg and mysql in distribute

* feat: finally get there, specify postgres sqlness

* feat: support mysql sqlness

* fix: license

* fix: remove unused import

* fix: toml

* fix: clippy

* refactor: BeginProtocolInterceptorFactory to ProtocolInterceptorFactory

* fix: sqlness pg connect

* fix: clippy

* Apply suggestions from code review

Co-authored-by: Yingwen <realevenyag@gmail.com>

* fix: rustfmt

* fix: reconnect pg and mysql when restart

* test: add mysql related sqlness

* fix: wait for start while restarting

* fix: clippy

* fix: cargo lock conflict

fix: Cargo.lock conflict

* fix: usage of '@@tx_isolation' in sqlness

* fix: typos

* feat: retry with backoff when create client

* fix: use millisecond rather than microseconds in backoff

---------

Co-authored-by: Yingwen <realevenyag@gmail.com>
---
 Cargo.lock                                    | 797 ++++++++++++------
 src/log-store/src/kafka/client_manager.rs     |   2 +-
 .../src/python/rspython/builtins/test.rs      |   2 +-
 tests/cases/standalone/common/mysql.result    |  29 +
 tests/cases/standalone/common/mysql.sql       |   8 +
 .../common/system/pg_catalog.result           | 140 ++-
 .../standalone/common/system/pg_catalog.sql   |  28 +-
 tests/conf/standalone-test.toml.template      |  13 +
 tests/runner/Cargo.toml                       |   3 +
 tests/runner/src/env.rs                       | 381 +++++++--
 tests/runner/src/main.rs                      |  40 +-
 tests/runner/src/protocol_interceptor.rs      |  48 ++
 12 files changed, 1125 insertions(+), 366 deletions(-)
 create mode 100644 tests/cases/standalone/common/mysql.result
 create mode 100644 tests/cases/standalone/common/mysql.sql
 create mode 100644 tests/runner/src/protocol_interceptor.rs

diff --git a/Cargo.lock b/Cargo.lock
index ab5043939a..418e68a88a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -226,7 +226,7 @@ dependencies = [
  "paste",
  "prost 0.12.6",
  "serde_json",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "tonic-build",
 ]
 
@@ -459,7 +459,7 @@ dependencies = [
  "arrow-schema",
  "chrono",
  "half 2.4.1",
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "lexical-core",
  "num",
  "serde",
@@ -534,7 +534,7 @@ dependencies = [
  "memchr",
  "num",
  "regex",
- "regex-syntax 0.8.4",
+ "regex-syntax 0.8.5",
 ]
 
 [[package]]
@@ -607,11 +607,11 @@ dependencies = [
 
 [[package]]
 name = "async-compression"
-version = "0.4.12"
+version = "0.4.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa"
+checksum = "7e614738943d3f68c628ae3dbce7c3daffb196665f82f8c8ea6b65de73c79429"
 dependencies = [
- "brotli 6.0.0",
+ "brotli 7.0.0",
  "bzip2",
  "flate2",
  "futures-core",
@@ -654,14 +654,14 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "async-stream"
-version = "0.3.5"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
+checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476"
 dependencies = [
  "async-stream-impl",
  "futures-core",
@@ -670,13 +670,13 @@ dependencies = [
 
 [[package]]
 name = "async-stream-impl"
-version = "0.3.5"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
+checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -687,13 +687,13 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de"
 
 [[package]]
 name = "async-trait"
-version = "0.1.82"
+version = "0.1.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1"
+checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -778,7 +778,7 @@ dependencies = [
  "digest",
  "notify",
  "sha1",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sql",
  "tokio",
 ]
@@ -791,7 +791,7 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -802,9 +802,9 @@ checksum = "7460f7dd8e100147b82a63afca1a20eb6c231ee36b90ba7272e14951cb58af59"
 
 [[package]]
 name = "autocfg"
-version = "1.3.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
+checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
 
 [[package]]
 name = "autotools"
@@ -890,7 +890,7 @@ dependencies = [
  "heck 0.4.1",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -976,6 +976,17 @@ dependencies = [
  "smallvec",
 ]
 
+[[package]]
+name = "bigdecimal"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa"
+dependencies = [
+ "num-bigint",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "bigdecimal"
 version = "0.4.5"
@@ -998,6 +1009,25 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "bindgen"
+version = "0.59.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8"
+dependencies = [
+ "bitflags 1.3.2",
+ "cexpr",
+ "clang-sys",
+ "lazy_static",
+ "lazycell",
+ "peeking_take_while",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash 1.1.0",
+ "shlex",
+]
+
 [[package]]
 name = "bindgen"
 version = "0.69.4"
@@ -1017,7 +1047,7 @@ dependencies = [
  "regex",
  "rustc-hash 1.1.0",
  "shlex",
- "syn 2.0.77",
+ "syn 2.0.79",
  "which",
 ]
 
@@ -1036,7 +1066,7 @@ dependencies = [
  "regex",
  "rustc-hash 1.1.0",
  "shlex",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -1160,7 +1190,7 @@ dependencies = [
  "proc-macro-crate 3.2.0",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
  "syn_derive",
 ]
 
@@ -1177,9 +1207,9 @@ dependencies = [
 
 [[package]]
 name = "brotli"
-version = "6.0.0"
+version = "7.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b"
+checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -1224,7 +1254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c"
 dependencies = [
  "memchr",
- "regex-automata 0.4.7",
+ "regex-automata 0.4.8",
  "serde",
 ]
 
@@ -1237,6 +1267,12 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "bufstream"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8"
+
 [[package]]
 name = "build-data"
 version = "0.2.1"
@@ -1346,7 +1382,7 @@ dependencies = [
  "common-macro",
  "common-meta",
  "moka",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "substrait 0.9.3",
 ]
 
@@ -1416,7 +1452,7 @@ dependencies = [
  "rustc-hash 2.0.0",
  "serde_json",
  "session",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sql",
  "store-api",
  "table",
@@ -1435,9 +1471,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.1.21"
+version = "1.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0"
+checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938"
 dependencies = [
  "jobserver",
  "libc",
@@ -1504,7 +1540,7 @@ version = "0.13.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6026d8cd82ada8bbcfe337805dd1eb6afdc9e80fa4d57e977b3a36315e0c5525"
 dependencies = [
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "lazy_static",
  "num-traits",
  "regex",
@@ -1654,9 +1690,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.18"
+version = "4.5.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3"
+checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -1664,9 +1700,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.18"
+version = "4.5.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b"
+checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1683,7 +1719,7 @@ dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -1731,7 +1767,7 @@ dependencies = [
  "query",
  "rand",
  "serde_json",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "substrait 0.37.3",
  "substrait 0.9.3",
  "tokio",
@@ -1771,7 +1807,7 @@ dependencies = [
  "cache",
  "catalog",
  "chrono",
- "clap 4.5.18",
+ "clap 4.5.19",
  "client",
  "common-base",
  "common-catalog",
@@ -1818,7 +1854,7 @@ dependencies = [
  "servers",
  "session",
  "similar-asserts",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "store-api",
  "substrait 0.9.3",
  "table",
@@ -1877,7 +1913,7 @@ dependencies = [
  "futures",
  "paste",
  "serde",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "toml 0.8.19",
  "zeroize",
 ]
@@ -1889,7 +1925,7 @@ dependencies = [
  "chrono",
  "common-error",
  "common-macro",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "tokio",
 ]
 
@@ -1909,7 +1945,7 @@ dependencies = [
  "num_cpus",
  "serde",
  "serde_json",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sysinfo",
  "temp-env",
  "tempfile",
@@ -1945,7 +1981,7 @@ dependencies = [
  "rand",
  "regex",
  "serde",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "strum 0.25.0",
  "tokio",
  "tokio-util",
@@ -1957,20 +1993,20 @@ dependencies = [
 name = "common-decimal"
 version = "0.9.3"
 dependencies = [
- "bigdecimal",
+ "bigdecimal 0.4.5",
  "common-error",
  "common-macro",
  "rust_decimal",
  "serde",
  "serde_json",
- "snafu 0.8.4",
+ "snafu 0.8.5",
 ]
 
 [[package]]
 name = "common-error"
 version = "0.9.3"
 dependencies = [
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "strum 0.25.0",
  "tonic 0.11.0",
 ]
@@ -1986,7 +2022,7 @@ dependencies = [
  "common-macro",
  "common-query",
  "session",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sql",
 ]
 
@@ -2021,7 +2057,7 @@ dependencies = [
  "serde",
  "serde_json",
  "session",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sql",
  "statrs",
  "store-api",
@@ -2066,7 +2102,7 @@ dependencies = [
  "lazy_static",
  "prost 0.12.6",
  "rand",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "tokio",
  "tonic 0.11.0",
  "tower",
@@ -2086,7 +2122,7 @@ dependencies = [
  "datatypes",
  "paste",
  "prost 0.12.6",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "table",
 ]
 
@@ -2099,9 +2135,9 @@ dependencies = [
  "datatypes",
  "proc-macro2",
  "quote",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "static_assertions",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -2110,7 +2146,7 @@ version = "0.9.3"
 dependencies = [
  "common-error",
  "common-macro",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "tempfile",
  "tikv-jemalloc-ctl",
  "tikv-jemalloc-sys",
@@ -2163,7 +2199,7 @@ dependencies = [
  "serde_json",
  "serde_with",
  "session",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "store-api",
  "strum 0.25.0",
  "table",
@@ -2200,7 +2236,7 @@ dependencies = [
  "serde",
  "serde_json",
  "smallvec",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "tokio",
  "uuid",
 ]
@@ -2231,7 +2267,7 @@ dependencies = [
  "datatypes",
  "futures-util",
  "serde",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sqlparser 0.45.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=54a267ac89c09b11c0c88934690530807185d3e7)",
  "sqlparser_derive 0.1.1",
  "statrs",
@@ -2254,7 +2290,7 @@ dependencies = [
  "pin-project",
  "serde",
  "serde_json",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "tokio",
 ]
 
@@ -2272,7 +2308,7 @@ dependencies = [
  "paste",
  "prometheus",
  "serde",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "tokio",
  "tokio-metrics",
  "tokio-metrics-collector",
@@ -2333,7 +2369,7 @@ dependencies = [
  "rand",
  "serde",
  "serde_json",
- "snafu 0.8.4",
+ "snafu 0.8.5",
 ]
 
 [[package]]
@@ -2361,11 +2397,11 @@ dependencies = [
  "rskafka",
  "rustls 0.23.13",
  "rustls-native-certs 0.7.3",
- "rustls-pemfile 2.1.3",
+ "rustls-pemfile 2.2.0",
  "serde",
  "serde_json",
  "serde_with",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "tokio",
  "toml 0.8.19",
 ]
@@ -2624,7 +2660,7 @@ dependencies = [
  "anes",
  "cast",
  "ciborium",
- "clap 4.5.18",
+ "clap 4.5.19",
  "criterion-plot",
  "is-terminal",
  "itertools 0.10.5",
@@ -2809,7 +2845,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim 0.11.1",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -2831,7 +2867,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
 dependencies = [
  "darling_core 0.20.10",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -2863,7 +2899,7 @@ dependencies = [
  "arrow-array",
  "arrow-ipc",
  "arrow-schema",
- "async-compression 0.4.12",
+ "async-compression 0.4.13",
  "async-trait",
  "bytes",
  "bzip2",
@@ -2886,7 +2922,7 @@ dependencies = [
  "glob",
  "half 2.4.1",
  "hashbrown 0.14.5",
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "itertools 0.12.1",
  "log",
  "num_cpus",
@@ -3045,10 +3081,10 @@ dependencies = [
  "datafusion-expr",
  "datafusion-physical-expr",
  "hashbrown 0.14.5",
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "itertools 0.12.1",
  "log",
- "regex-syntax 0.8.4",
+ "regex-syntax 0.8.5",
 ]
 
 [[package]]
@@ -3073,7 +3109,7 @@ dependencies = [
  "half 2.4.1",
  "hashbrown 0.14.5",
  "hex",
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "itertools 0.12.1",
  "log",
  "paste",
@@ -3115,7 +3151,7 @@ dependencies = [
  "futures",
  "half 2.4.1",
  "hashbrown 0.14.5",
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "itertools 0.12.1",
  "log",
  "once_cell",
@@ -3206,7 +3242,7 @@ dependencies = [
  "serde_json",
  "servers",
  "session",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "store-api",
  "substrait 0.9.3",
  "table",
@@ -3239,7 +3275,7 @@ dependencies = [
  "paste",
  "serde",
  "serde_json",
- "snafu 0.8.4",
+ "snafu 0.8.5",
 ]
 
 [[package]]
@@ -3322,7 +3358,7 @@ checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -3333,7 +3369,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -3396,7 +3432,7 @@ dependencies = [
  "darling 0.20.10",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -3426,7 +3462,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4abae7035bf79b9877b779505d8cf3749285b80c43941eda66604841889451dc"
 dependencies = [
  "derive_builder_core 0.20.1",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -3446,10 +3482,21 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
  "unicode-xid",
 ]
 
+[[package]]
+name = "derive_utils"
+version = "0.14.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65f152f4b8559c4da5d574bafc7af85454d706b4c5fe8b530d508cacbb6807ea"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.79",
+]
+
 [[package]]
 name = "diff"
 version = "0.1.13"
@@ -3604,7 +3651,7 @@ dependencies = [
  "serde",
  "thiserror",
  "time",
- "winnow 0.6.18",
+ "winnow 0.6.20",
 ]
 
 [[package]]
@@ -3658,7 +3705,7 @@ dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -3670,7 +3717,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -3834,7 +3881,7 @@ dependencies = [
  "object-store",
  "serde",
  "serde_json",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "store-api",
  "table",
  "tokio",
@@ -3888,9 +3935,9 @@ dependencies = [
 
 [[package]]
 name = "flate2"
-version = "1.0.33"
+version = "1.0.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253"
+checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0"
 dependencies = [
  "crc32fast",
  "libz-sys",
@@ -3968,7 +4015,7 @@ dependencies = [
  "servers",
  "session",
  "smallvec",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "store-api",
  "strum 0.25.0",
  "substrait 0.9.3",
@@ -3992,6 +4039,21 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.1"
@@ -4067,7 +4129,7 @@ dependencies = [
  "serde_json",
  "servers",
  "session",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sql",
  "sqlparser 0.45.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=54a267ac89c09b11c0c88934690530807185d3e7)",
  "store-api",
@@ -4109,7 +4171,7 @@ checksum = "e99b8b3c28ae0e84b604c75f721c21dc77afb3706076af5e8216d15fd1deaae3"
 dependencies = [
  "frunk_proc_macro_helpers",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -4121,7 +4183,7 @@ dependencies = [
  "frunk_core",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -4133,7 +4195,7 @@ dependencies = [
  "frunk_core",
  "frunk_proc_macro_helpers",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -4263,7 +4325,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -4437,7 +4499,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http 0.2.12",
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "slab",
  "tokio",
  "tokio-util",
@@ -4499,6 +4561,12 @@ dependencies = [
  "allocator-api2",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb"
+
 [[package]]
 name = "hashlink"
 version = "0.8.4"
@@ -4734,9 +4802,9 @@ checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f"
 
 [[package]]
 name = "httparse"
-version = "1.9.4"
+version = "1.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9"
+checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946"
 
 [[package]]
 name = "httpdate"
@@ -4817,7 +4885,7 @@ dependencies = [
  "proc-macro-crate 1.3.1",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -4832,7 +4900,7 @@ dependencies = [
  "rust-sitter",
  "rust-sitter-tool",
  "slotmap",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -4841,7 +4909,7 @@ version = "0.6.2"
 source = "git+https://github.com/GreptimeTeam/hydroflow.git?branch=main#b072ee026f97f8537165e1fb247101e0ab2fb320"
 dependencies = [
  "auto_impl",
- "clap 4.5.18",
+ "clap 4.5.19",
  "data-encoding",
  "itertools 0.10.5",
  "prettyplease",
@@ -4851,7 +4919,7 @@ dependencies = [
  "serde",
  "serde_json",
  "slotmap",
- "syn 2.0.77",
+ "syn 2.0.79",
  "webbrowser",
 ]
 
@@ -4865,7 +4933,7 @@ dependencies = [
  "proc-macro-crate 1.3.1",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -4977,9 +5045,9 @@ dependencies = [
 
 [[package]]
 name = "hyper-util"
-version = "0.1.8"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba"
+checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b"
 dependencies = [
  "bytes",
  "futures-channel",
@@ -4990,7 +5058,6 @@ dependencies = [
  "pin-project-lite",
  "socket2 0.5.7",
  "tokio",
- "tower",
  "tower-service",
  "tracing",
 ]
@@ -5081,9 +5148,9 @@ dependencies = [
  "prost 0.12.6",
  "rand",
  "regex",
- "regex-automata 0.4.7",
+ "regex-automata 0.4.8",
  "serde",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "tantivy",
  "tantivy-jieba",
  "tempfile",
@@ -5105,12 +5172,12 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.5.0"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5"
+checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da"
 dependencies = [
  "equivalent",
- "hashbrown 0.14.5",
+ "hashbrown 0.15.0",
  "serde",
 ]
 
@@ -5127,7 +5194,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88"
 dependencies = [
  "ahash 0.8.11",
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "is-terminal",
  "itoa",
  "log",
@@ -5209,6 +5276,15 @@ version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f958d3d68f4167080a18141e10381e7634563984a537f2a49a30fd8e53ac5767"
 
+[[package]]
+name = "io-enum"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53b53d712d99a73eec59ee5e4fe6057f8052142d38eeafbbffcb06b36d738a6e"
+dependencies = [
+ "derive_utils",
+]
+
 [[package]]
 name = "ipnet"
 version = "2.10.0"
@@ -5217,9 +5293,9 @@ checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4"
 
 [[package]]
 name = "iri-string"
-version = "0.7.5"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c25163201be6ded9e686703e85532f8f852ea1f92ba625cb3c51f7fe6d07a4a"
+checksum = "44bd7eced44cfe2cebc674adb2a7124a754a4b5269288d22e9f39f8fada3562d"
 dependencies = [
  "memchr",
  "serde",
@@ -5399,7 +5475,7 @@ dependencies = [
  "fast-float",
  "itoa",
  "nom",
- "ordered-float 4.2.2",
+ "ordered-float 4.3.0",
  "rand",
  "ryu",
  "serde_json",
@@ -5439,7 +5515,7 @@ checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f"
 dependencies = [
  "base64 0.21.7",
  "js-sys",
- "pem",
+ "pem 3.0.4",
  "ring 0.17.8",
  "serde",
  "serde_json",
@@ -5474,7 +5550,7 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d"
 dependencies = [
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
 ]
 
 [[package]]
@@ -5533,9 +5609,9 @@ dependencies = [
  "jsonpath-rust",
  "k8s-openapi",
  "kube-core",
- "pem",
+ "pem 3.0.4",
  "rustls 0.23.13",
- "rustls-pemfile 2.1.3",
+ "rustls-pemfile 2.2.0",
  "secrecy",
  "serde",
  "serde_json",
@@ -5575,7 +5651,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_json",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -5667,7 +5743,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -5691,6 +5767,15 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
 
+[[package]]
+name = "lexical"
+version = "6.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7aefb36fd43fef7003334742cbf77b243fcd36418a1d1bdd480d613a67968f6"
+dependencies = [
+ "lexical-core",
+]
+
 [[package]]
 name = "lexical-core"
 version = "0.8.5"
@@ -5757,9 +5842,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.158"
+version = "0.2.159"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
+checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
 
 [[package]]
 name = "libfuzzer-sys"
@@ -5818,7 +5903,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
 dependencies = [
  "bitflags 2.6.0",
  "libc",
- "redox_syscall 0.5.4",
+ "redox_syscall 0.5.7",
 ]
 
 [[package]]
@@ -5906,7 +5991,7 @@ dependencies = [
  "rskafka",
  "serde",
  "serde_json",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "store-api",
  "tokio",
  "uuid",
@@ -5925,7 +6010,7 @@ dependencies = [
  "num-traits",
  "quote",
  "regex",
- "regex-syntax 0.8.4",
+ "regex-syntax 0.8.5",
  "serde",
  "vergen",
 ]
@@ -5940,7 +6025,7 @@ dependencies = [
  "cactus",
  "cfgrammar",
  "filetime",
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "lazy_static",
  "lrtable",
  "num-traits",
@@ -5966,6 +6051,15 @@ dependencies = [
  "vob",
 ]
 
+[[package]]
+name = "lru"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6e8aaa3f231bb4bd57b84b2d5dc3ae7f350265df8aa96492e0bc394a1571909"
+dependencies = [
+ "hashbrown 0.12.3",
+]
+
 [[package]]
 name = "lru"
 version = "0.12.4"
@@ -5977,18 +6071,18 @@ dependencies = [
 
 [[package]]
 name = "lz4"
-version = "1.27.0"
+version = "1.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a231296ca742e418c43660cb68e082486ff2538e8db432bc818580f3965025ed"
+checksum = "4d1febb2b4a79ddd1980eede06a8f7902197960aa0383ffcfdd62fe723036725"
 dependencies = [
  "lz4-sys",
 ]
 
 [[package]]
 name = "lz4-sys"
-version = "1.11.0"
+version = "1.11.1+lz4-1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcb44a01837a858d47e5a630d2ccf304c8efcc4b83b8f9f75b7a9ee4fcc6e57d"
+checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6"
 dependencies = [
  "cc",
  "libc",
@@ -6200,7 +6294,7 @@ dependencies = [
  "meta-srv",
  "rand",
  "serde",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "tokio",
  "tokio-stream",
  "tonic 0.11.0",
@@ -6216,7 +6310,7 @@ dependencies = [
  "api",
  "async-trait",
  "chrono",
- "clap 4.5.18",
+ "clap 4.5.19",
  "client",
  "common-base",
  "common-catalog",
@@ -6254,7 +6348,7 @@ dependencies = [
  "serde_json",
  "servers",
  "session",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "store-api",
  "table",
  "tokio",
@@ -6311,7 +6405,7 @@ dependencies = [
  "object-store",
  "prometheus",
  "serde_json",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "store-api",
  "tokio",
 ]
@@ -6449,7 +6543,7 @@ dependencies = [
  "serde_with",
  "session",
  "smallvec",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "store-api",
  "strum 0.25.0",
  "table",
@@ -6556,6 +6650,32 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
 
+[[package]]
+name = "mysql"
+version = "23.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05f11339ca5c251941805d51362a07823605a80586ced92914ab7de84fba813f"
+dependencies = [
+ "bufstream",
+ "bytes",
+ "crossbeam",
+ "flate2",
+ "io-enum",
+ "libc",
+ "lru 0.8.1",
+ "mysql_common 0.29.2",
+ "named_pipe",
+ "native-tls",
+ "once_cell",
+ "pem 1.1.1",
+ "percent-encoding",
+ "serde",
+ "serde_json",
+ "socket2 0.4.10",
+ "twox-hash",
+ "url",
+]
+
 [[package]]
 name = "mysql-common-derive"
 version = "0.30.2"
@@ -6569,7 +6689,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
  "termcolor",
  "thiserror",
 ]
@@ -6587,7 +6707,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
  "termcolor",
  "thiserror",
 ]
@@ -6606,11 +6726,11 @@ dependencies = [
  "futures-util",
  "keyed_priority_queue",
  "lazy_static",
- "lru",
+ "lru 0.12.4",
  "mio 0.8.11",
  "mysql_common 0.31.0",
  "once_cell",
- "pem",
+ "pem 3.0.4",
  "percent-encoding",
  "pin-project",
  "rand",
@@ -6629,6 +6749,43 @@ dependencies = [
  "webpki-roots 0.25.4",
 ]
 
+[[package]]
+name = "mysql_common"
+version = "0.29.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9006c95034ccf7b903d955f210469119f6c3477fc9c9e7a7845ce38a3e665c2a"
+dependencies = [
+ "base64 0.13.1",
+ "bigdecimal 0.3.1",
+ "bindgen 0.59.2",
+ "bitflags 1.3.2",
+ "bitvec",
+ "byteorder",
+ "bytes",
+ "cc",
+ "cmake",
+ "crc32fast",
+ "flate2",
+ "frunk",
+ "lazy_static",
+ "lexical",
+ "num-bigint",
+ "num-traits",
+ "rand",
+ "regex",
+ "rust_decimal",
+ "saturating",
+ "serde",
+ "serde_json",
+ "sha1",
+ "sha2",
+ "smallvec",
+ "subprocess",
+ "thiserror",
+ "time",
+ "uuid",
+]
+
 [[package]]
 name = "mysql_common"
 version = "0.31.0"
@@ -6636,7 +6793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06f19e4cfa0ab5a76b627cec2d81331c49b034988eaf302c3bafeada684eadef"
 dependencies = [
  "base64 0.21.7",
- "bigdecimal",
+ "bigdecimal 0.4.5",
  "bindgen 0.70.1",
  "bitflags 2.6.0",
  "bitvec",
@@ -6675,7 +6832,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "478b0ff3f7d67b79da2b96f56f334431aef65e15ba4b29dd74a4236e29582bdc"
 dependencies = [
  "base64 0.21.7",
- "bigdecimal",
+ "bigdecimal 0.4.5",
  "bindgen 0.70.1",
  "bitflags 2.6.0",
  "bitvec",
@@ -6737,6 +6894,32 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "named_pipe"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad9c443cce91fc3e12f017290db75dde490d685cdaaf508d7159d7cf41f0eb2b"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "native-tls"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466"
+dependencies = [
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
 [[package]]
 name = "ndk-context"
 version = "0.1.1"
@@ -6916,7 +7099,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -7078,9 +7261,12 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.19.0"
+version = "1.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
+checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1"
+dependencies = [
+ "portable-atomic",
+]
 
 [[package]]
 name = "oneshot"
@@ -7153,12 +7339,50 @@ dependencies = [
  "tokio-rustls 0.26.0",
 ]
 
+[[package]]
+name = "openssl"
+version = "0.10.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1"
+dependencies = [
+ "bitflags 2.6.0",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.79",
+]
+
 [[package]]
 name = "openssl-probe"
 version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
+[[package]]
+name = "openssl-sys"
+version = "0.9.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "opentelemetry"
 version = "0.21.0"
@@ -7167,7 +7391,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a"
 dependencies = [
  "futures-core",
  "futures-sink",
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "js-sys",
  "once_cell",
  "pin-project-lite",
@@ -7258,7 +7482,7 @@ dependencies = [
  "glob",
  "once_cell",
  "opentelemetry 0.21.0",
- "ordered-float 4.2.2",
+ "ordered-float 4.3.0",
  "percent-encoding",
  "rand",
  "thiserror",
@@ -7280,7 +7504,7 @@ dependencies = [
  "glob",
  "once_cell",
  "opentelemetry 0.22.0",
- "ordered-float 4.2.2",
+ "ordered-float 4.3.0",
  "percent-encoding",
  "rand",
  "thiserror",
@@ -7329,7 +7553,7 @@ dependencies = [
  "regex",
  "serde_json",
  "session",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sql",
  "sqlparser 0.45.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=54a267ac89c09b11c0c88934690530807185d3e7)",
  "store-api",
@@ -7392,9 +7616,9 @@ dependencies = [
 
 [[package]]
 name = "ordered-float"
-version = "4.2.2"
+version = "4.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6"
+checksum = "44d501f1a72f71d3c063a6bbc8f7271fa73aa09fe5d6283b6571e2ed176a2537"
 dependencies = [
  "num-traits",
 ]
@@ -7531,7 +7755,7 @@ dependencies = [
  "cfg-if",
  "libc",
  "petgraph",
- "redox_syscall 0.5.4",
+ "redox_syscall 0.5.7",
  "smallvec",
  "thread-id",
  "windows-targets 0.52.6",
@@ -7597,7 +7821,7 @@ dependencies = [
  "itertools 0.10.5",
  "serde",
  "serde_json",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sql",
  "sqlparser 0.45.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=54a267ac89c09b11c0c88934690530807185d3e7)",
  "store-api",
@@ -7669,6 +7893,21 @@ dependencies = [
  "hmac",
 ]
 
+[[package]]
+name = "peeking_take_while"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
+
+[[package]]
+name = "pem"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8"
+dependencies = [
+ "base64 0.13.1",
+]
+
 [[package]]
 name = "pem"
 version = "3.0.4"
@@ -7740,7 +7979,7 @@ dependencies = [
  "pest_meta",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -7761,7 +8000,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
 dependencies = [
  "fixedbitset",
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
 ]
 
 [[package]]
@@ -7852,7 +8091,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -7915,7 +8154,7 @@ dependencies = [
  "serde",
  "serde_json",
  "session",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sql",
  "table",
  "tokio",
@@ -7996,9 +8235,9 @@ dependencies = [
 
 [[package]]
 name = "pkg-config"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
+checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
 
 [[package]]
 name = "plotters"
@@ -8037,7 +8276,7 @@ dependencies = [
  "datanode",
  "frontend",
  "meta-srv",
- "snafu 0.8.4",
+ "snafu 0.8.5",
 ]
 
 [[package]]
@@ -8053,9 +8292,9 @@ dependencies = [
 
 [[package]]
 name = "portable-atomic"
-version = "1.8.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d30538d42559de6b034bc76fd6dd4c38961b1ee5c6c56e3808c50128fdbc22ce"
+checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2"
 
 [[package]]
 name = "postgres-protocol"
@@ -8195,7 +8434,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba"
 dependencies = [
  "proc-macro2",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -8214,7 +8453,7 @@ version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b"
 dependencies = [
- "toml_edit 0.22.21",
+ "toml_edit 0.22.22",
 ]
 
 [[package]]
@@ -8324,7 +8563,7 @@ dependencies = [
  "prost 0.12.6",
  "query",
  "session",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "tokio",
 ]
 
@@ -8388,7 +8627,7 @@ dependencies = [
  "prost 0.12.6",
  "prost-types 0.12.6",
  "regex",
- "syn 2.0.77",
+ "syn 2.0.79",
  "tempfile",
 ]
 
@@ -8434,7 +8673,7 @@ dependencies = [
  "itertools 0.12.1",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -8447,7 +8686,7 @@ dependencies = [
  "itertools 0.13.0",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -8541,7 +8780,7 @@ dependencies = [
 name = "puffin"
 version = "0.9.3"
 dependencies = [
- "async-compression 0.4.12",
+ "async-compression 0.4.13",
  "async-trait",
  "async-walkdir",
  "auto_impl",
@@ -8560,7 +8799,7 @@ dependencies = [
  "serde",
  "serde_json",
  "sha2",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "tokio",
  "tokio-util",
  "uuid",
@@ -8628,7 +8867,7 @@ dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -8641,7 +8880,7 @@ dependencies = [
  "proc-macro2",
  "pyo3-build-config",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -8717,7 +8956,7 @@ dependencies = [
  "rand",
  "regex",
  "session",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sql",
  "sqlparser 0.45.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=54a267ac89c09b11c0c88934690530807185d3e7)",
  "statrs",
@@ -8909,9 +9148,9 @@ dependencies = [
 
 [[package]]
 name = "raw-cpuid"
-version = "11.1.0"
+version = "11.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d"
+checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0"
 dependencies = [
  "bitflags 2.6.0",
 ]
@@ -8959,9 +9198,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.5.4"
+version = "0.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853"
+checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f"
 dependencies = [
  "bitflags 2.6.0",
 ]
@@ -8994,19 +9233,19 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "regex"
-version = "1.10.6"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
+checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-automata 0.4.7",
- "regex-syntax 0.8.4",
+ "regex-automata 0.4.8",
+ "regex-syntax 0.8.5",
 ]
 
 [[package]]
@@ -9020,13 +9259,13 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
+checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax 0.8.4",
+ "regex-syntax 0.8.5",
 ]
 
 [[package]]
@@ -9043,9 +9282,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
 
 [[package]]
 name = "regex-syntax"
-version = "0.8.4"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
+checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
 [[package]]
 name = "regress"
@@ -9105,9 +9344,9 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.12.7"
+version = "0.12.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63"
+checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b"
 dependencies = [
  "base64 0.22.1",
  "bytes",
@@ -9129,8 +9368,8 @@ dependencies = [
  "pin-project-lite",
  "quinn",
  "rustls 0.23.13",
- "rustls-native-certs 0.7.3",
- "rustls-pemfile 2.1.3",
+ "rustls-native-certs 0.8.0",
+ "rustls-pemfile 2.2.0",
  "rustls-pki-types",
  "serde",
  "serde_json",
@@ -9372,7 +9611,7 @@ dependencies = [
  "regex",
  "relative-path",
  "rustc_version",
- "syn 2.0.77",
+ "syn 2.0.79",
  "unicode-ident",
 ]
 
@@ -9384,7 +9623,7 @@ checksum = "b3a8fb4672e840a587a66fc577a5491375df51ddb88f2a2c2a792598c326fe14"
 dependencies = [
  "quote",
  "rand",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -9407,7 +9646,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rust-embed-utils",
- "syn 2.0.77",
+ "syn 2.0.79",
  "walkdir",
 ]
 
@@ -9617,7 +9856,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5"
 dependencies = [
  "openssl-probe",
- "rustls-pemfile 2.1.3",
+ "rustls-pemfile 2.2.0",
  "rustls-pki-types",
  "schannel",
  "security-framework",
@@ -9630,7 +9869,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a"
 dependencies = [
  "openssl-probe",
- "rustls-pemfile 2.1.3",
+ "rustls-pemfile 2.2.0",
  "rustls-pki-types",
  "schannel",
  "security-framework",
@@ -9647,19 +9886,18 @@ dependencies = [
 
 [[package]]
 name = "rustls-pemfile"
-version = "2.1.3"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425"
+checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
 dependencies = [
- "base64 0.22.1",
  "rustls-pki-types",
 ]
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.8.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0"
+checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55"
 
 [[package]]
 name = "rustls-webpki"
@@ -10134,7 +10372,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde_derive_internals",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -10192,7 +10430,7 @@ dependencies = [
  "serde",
  "servers",
  "session",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sql",
  "table",
  "tokio",
@@ -10235,7 +10473,7 @@ dependencies = [
  "heck 0.4.1",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -10313,7 +10551,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -10324,7 +10562,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -10333,7 +10571,7 @@ version = "1.0.128"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
 dependencies = [
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "itoa",
  "memchr",
  "ryu",
@@ -10358,14 +10596,14 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "serde_spanned"
-version = "0.6.7"
+version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d"
+checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
 dependencies = [
  "serde",
 ]
@@ -10379,7 +10617,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "serde",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -10396,15 +10634,15 @@ dependencies = [
 
 [[package]]
 name = "serde_with"
-version = "3.9.0"
+version = "3.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857"
+checksum = "9720086b3357bcb44fce40117d769a4d068c70ecfa190850a980a71755f66fcc"
 dependencies = [
  "base64 0.22.1",
  "chrono",
  "hex",
  "indexmap 1.9.3",
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "serde",
  "serde_derive",
  "serde_json",
@@ -10414,14 +10652,14 @@ dependencies = [
 
 [[package]]
 name = "serde_with_macros"
-version = "3.9.0"
+version = "3.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350"
+checksum = "5f1abbfe725f27678f4663bcacb75a83e829fd464c25d78dd038a3a29e307cec"
 dependencies = [
  "darling 0.20.10",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -10430,7 +10668,7 @@ version = "0.9.34+deprecated"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
 dependencies = [
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "itoa",
  "ryu",
  "serde",
@@ -10516,14 +10754,14 @@ dependencies = [
  "reqwest",
  "rust-embed",
  "rustls 0.23.13",
- "rustls-pemfile 2.1.3",
+ "rustls-pemfile 2.2.0",
  "rustls-pki-types",
  "schemars",
  "script",
  "serde",
  "serde_json",
  "session",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "snap",
  "sql",
  "strum 0.25.0",
@@ -10560,7 +10798,7 @@ dependencies = [
  "common-time",
  "derive_builder 0.12.0",
  "meter-core",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sql",
 ]
 
@@ -10766,11 +11004,11 @@ dependencies = [
 
 [[package]]
 name = "snafu"
-version = "0.8.4"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b835cb902660db3415a672d862905e791e54d306c6e8189168c7f3d9ae1c79d"
+checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019"
 dependencies = [
- "snafu-derive 0.8.4",
+ "snafu-derive 0.8.5",
 ]
 
 [[package]]
@@ -10787,14 +11025,14 @@ dependencies = [
 
 [[package]]
 name = "snafu-derive"
-version = "0.8.4"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38d1e02fca405f6280643174a50c942219f0bbf4dbf7d480f1dd864d6f211ae5"
+checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917"
 dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -10894,7 +11132,7 @@ dependencies = [
  "lazy_static",
  "regex",
  "serde_json",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sqlparser 0.45.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=54a267ac89c09b11c0c88934690530807185d3e7)",
  "sqlparser_derive 0.1.1",
  "table",
@@ -10933,18 +11171,21 @@ name = "sqlness-runner"
 version = "0.9.3"
 dependencies = [
  "async-trait",
- "clap 4.5.18",
+ "clap 4.5.19",
  "client",
  "common-error",
  "common-query",
  "common-recordbatch",
  "common-time",
+ "datatypes",
+ "mysql",
  "serde",
  "serde_json",
  "sqlness",
  "tempfile",
  "tinytemplate",
  "tokio",
+ "tokio-postgres",
 ]
 
 [[package]]
@@ -10988,7 +11229,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -10998,7 +11239,7 @@ source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=54a267ac89c09
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -11167,7 +11408,7 @@ dependencies = [
  "futures",
  "serde",
  "serde_json",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "strum 0.25.0",
  "tokio",
 ]
@@ -11288,7 +11529,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -11301,7 +11542,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -11330,7 +11571,7 @@ dependencies = [
  "datatypes",
  "promql",
  "prost 0.12.6",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "substrait 0.37.3",
  "tokio",
 ]
@@ -11354,7 +11595,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_yaml",
- "syn 2.0.77",
+ "syn 2.0.79",
  "typify",
  "walkdir",
 ]
@@ -11375,7 +11616,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_yaml",
- "syn 2.0.77",
+ "syn 2.0.79",
  "typify",
  "walkdir",
 ]
@@ -11388,9 +11629,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "symbolic-common"
-version = "12.11.1"
+version = "12.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fdf97c441f18a4f92425b896a4ec7a27e03631a0b1047ec4e34e9916a9a167e"
+checksum = "366f1b4c6baf6cfefc234bbd4899535fca0b06c74443039a73f6dfb2fad88d77"
 dependencies = [
  "debugid",
  "memmap2 0.9.5",
@@ -11400,9 +11641,9 @@ dependencies = [
 
 [[package]]
 name = "symbolic-demangle"
-version = "12.11.1"
+version = "12.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc8ece6b129e97e53d1fbb3f61d33a6a9e5369b11d01228c068094d6d134eaea"
+checksum = "aba05ba5b9962ea5617baf556293720a8b2d0a282aa14ee4bf10e22efc7da8c8"
 dependencies = [
  "cpp_demangle",
  "rustc-demangle",
@@ -11422,9 +11663,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.77"
+version = "2.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed"
+checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -11459,7 +11700,7 @@ dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -11545,7 +11786,7 @@ dependencies = [
  "paste",
  "serde",
  "serde_json",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "store-api",
  "tokio",
  "tokio-util",
@@ -11579,7 +11820,7 @@ dependencies = [
  "itertools 0.12.1",
  "levenshtein_automata",
  "log",
- "lru",
+ "lru 0.12.4",
  "lz4_flex 0.11.3",
  "measure_time",
  "memmap2 0.9.5",
@@ -11654,7 +11895,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18"
 dependencies = [
  "byteorder",
- "regex-syntax 0.8.4",
+ "regex-syntax 0.8.5",
  "utf8-ranges",
 ]
 
@@ -11733,9 +11974,9 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.12.0"
+version = "3.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"
+checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b"
 dependencies = [
  "cfg-if",
  "fastrand",
@@ -11811,7 +12052,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_yaml",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sql",
  "sqlparser 0.45.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=54a267ac89c09b11c0c88934690530807185d3e7)",
  "sqlx",
@@ -11834,7 +12075,7 @@ dependencies = [
  "cache",
  "catalog",
  "chrono",
- "clap 4.5.18",
+ "clap 4.5.19",
  "client",
  "cmd",
  "common-base",
@@ -11881,7 +12122,7 @@ dependencies = [
  "servers",
  "session",
  "similar-asserts",
- "snafu 0.8.4",
+ "snafu 0.8.5",
  "sql",
  "sqlx",
  "store-api",
@@ -11937,7 +12178,7 @@ checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -12124,7 +12365,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -12291,7 +12532,7 @@ dependencies = [
  "serde",
  "serde_spanned",
  "toml_datetime",
- "toml_edit 0.22.21",
+ "toml_edit 0.22.22",
 ]
 
 [[package]]
@@ -12309,22 +12550,22 @@ version = "0.19.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
 dependencies = [
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "toml_datetime",
  "winnow 0.5.40",
 ]
 
 [[package]]
 name = "toml_edit"
-version = "0.22.21"
+version = "0.22.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf"
+checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
 dependencies = [
- "indexmap 2.5.0",
+ "indexmap 2.6.0",
  "serde",
  "serde_spanned",
  "toml_datetime",
- "winnow 0.6.18",
+ "winnow 0.6.20",
 ]
 
 [[package]]
@@ -12375,7 +12616,7 @@ dependencies = [
  "percent-encoding",
  "pin-project",
  "prost 0.12.6",
- "rustls-pemfile 2.1.3",
+ "rustls-pemfile 2.2.0",
  "rustls-pki-types",
  "tokio",
  "tokio-rustls 0.25.0",
@@ -12397,7 +12638,7 @@ dependencies = [
  "proc-macro2",
  "prost-build 0.12.6",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -12440,7 +12681,7 @@ version = "0.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140"
 dependencies = [
- "async-compression 0.4.12",
+ "async-compression 0.4.13",
  "base64 0.21.7",
  "bitflags 2.6.0",
  "bytes",
@@ -12527,7 +12768,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -12784,7 +13025,7 @@ checksum = "70b20a22c42c8f1cd23ce5e34f165d4d37038f5b663ad20fb6adbdf029172483"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -12812,7 +13053,7 @@ dependencies = [
  "semver",
  "serde",
  "serde_json",
- "syn 2.0.77",
+ "syn 2.0.79",
  "thiserror",
  "unicode-ident",
 ]
@@ -12830,7 +13071,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_tokenstream",
- "syn 2.0.77",
+ "syn 2.0.79",
  "typify-impl",
 ]
 
@@ -12871,9 +13112,9 @@ checksum = "fe4fa6e588762366f1eb4991ce59ad1b93651d0b769dfb4e4d1c5c4b943d1159"
 
 [[package]]
 name = "ucd-trie"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9"
+checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
 
 [[package]]
 name = "uname"
@@ -13011,9 +13252,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-bidi"
-version = "0.3.15"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
+checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893"
 
 [[package]]
 name = "unicode-casing"
@@ -13038,9 +13279,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-properties"
-version = "0.1.2"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524"
+checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0"
 
 [[package]]
 name = "unicode-segmentation"
@@ -13154,7 +13395,7 @@ checksum = "ee1cd046f83ea2c4e920d6ee9f7c3537ef928d75dce5d84a87c2c5d6b3999a3a"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -13270,7 +13511,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
  "wasm-bindgen-shared",
 ]
 
@@ -13304,7 +13545,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -13317,9 +13558,9 @@ checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
 
 [[package]]
 name = "wasm-streams"
-version = "0.4.0"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129"
+checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd"
 dependencies = [
  "futures-util",
  "js-sys",
@@ -13427,7 +13668,7 @@ version = "1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d"
 dependencies = [
- "redox_syscall 0.5.4",
+ "redox_syscall 0.5.7",
  "wasite",
  "web-sys",
 ]
@@ -13796,9 +14037,9 @@ dependencies = [
 
 [[package]]
 name = "winnow"
-version = "0.6.18"
+version = "0.6.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
+checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b"
 dependencies = [
  "memchr",
 ]
@@ -13832,7 +14073,7 @@ dependencies = [
  "chrono",
  "der 0.7.9",
  "hex",
- "pem",
+ "pem 3.0.4",
  "ring 0.17.8",
  "signature",
  "spki 0.7.3",
@@ -13888,7 +14129,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -13908,7 +14149,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.77",
+ "syn 2.0.79",
 ]
 
 [[package]]
diff --git a/src/log-store/src/kafka/client_manager.rs b/src/log-store/src/kafka/client_manager.rs
index 6337683c93..589d5a900e 100644
--- a/src/log-store/src/kafka/client_manager.rs
+++ b/src/log-store/src/kafka/client_manager.rs
@@ -132,7 +132,7 @@ impl ClientManager {
     }
 
     async fn try_create_client(&self, provider: &Arc<KafkaProvider>) -> Result<Client> {
-        // Sets to Retry to retry connecting if the kafka cluter replies with an UnknownTopic error.
+        // Sets to Retry to retry connecting if the kafka cluster replies with an UnknownTopic error.
         // That's because the topic is believed to exist as the metasrv is expected to create required topics upon start.
         // The reconnecting won't stop until succeed or a different error returns.
         let client = self
diff --git a/src/script/src/python/rspython/builtins/test.rs b/src/script/src/python/rspython/builtins/test.rs
index 3908ab9eef..541f28c66a 100644
--- a/src/script/src/python/rspython/builtins/test.rs
+++ b/src/script/src/python/rspython/builtins/test.rs
@@ -145,7 +145,7 @@ enum PyValue {
 }
 
 impl PyValue {
-    /// compare if results is just as expect, not using PartialEq because it is not transtive .e.g. [1,2,3] == len(3) == [4,5,6]
+    /// compare if results is just as expect, not using PartialEq because it is not transitive .e.g. [1,2,3] == len(3) == [4,5,6]
     fn just_as_expect(&self, other: &Self) -> bool {
         match (self, other) {
             (PyValue::FloatVec(a), PyValue::FloatVec(b)) => a
diff --git a/tests/cases/standalone/common/mysql.result b/tests/cases/standalone/common/mysql.result
new file mode 100644
index 0000000000..980990fa7a
--- /dev/null
+++ b/tests/cases/standalone/common/mysql.result
@@ -0,0 +1,29 @@
+-- SQLNESS PROTOCOL MYSQL
+SELECt @@tx_isolation;
+
++-----------------+
+| @@tx_isolation; |
++-----------------+
+| 0               |
++-----------------+
+
+-- SQLNESS PROTOCOL MYSQL
+SELECT @@version_comment;
+
++--------------------+
+| @@version_comment; |
++--------------------+
+| 0                  |
++--------------------+
+
+-- SQLNESS PROTOCOL MYSQL
+SHOW DATABASES;
+
++--------------------+
+| Database           |
++--------------------+
+| greptime_private   |
+| information_schema |
+| public             |
++--------------------+
+
diff --git a/tests/cases/standalone/common/mysql.sql b/tests/cases/standalone/common/mysql.sql
new file mode 100644
index 0000000000..31b2db23f9
--- /dev/null
+++ b/tests/cases/standalone/common/mysql.sql
@@ -0,0 +1,8 @@
+-- SQLNESS PROTOCOL MYSQL
+SELECt @@tx_isolation;
+
+-- SQLNESS PROTOCOL MYSQL
+SELECT @@version_comment;
+
+-- SQLNESS PROTOCOL MYSQL
+SHOW DATABASES;
diff --git a/tests/cases/standalone/common/system/pg_catalog.result b/tests/cases/standalone/common/system/pg_catalog.result
index d30355352f..4e3e217b2e 100644
--- a/tests/cases/standalone/common/system/pg_catalog.result
+++ b/tests/cases/standalone/common/system/pg_catalog.result
@@ -3,11 +3,49 @@ create database pg_catalog;
 
 Error: 1004(InvalidArguments), Schema pg_catalog already exists
 
-select * from pg_catalog.pg_type order by oid;
+-- make sure all the pg_catalog tables are only visible to postgres
+select * from pg_catalog.pg_class;
+
+Error: 4001(TableNotFound), Failed to plan SQL: Table not found: greptime.pg_catalog.pg_class
+
+select * from pg_catalog.pg_namespace;
+
+Error: 4001(TableNotFound), Failed to plan SQL: Table not found: greptime.pg_catalog.pg_namespace
+
+select * from pg_catalog.pg_type;
 
 Error: 4001(TableNotFound), Failed to plan SQL: Table not found: greptime.pg_catalog.pg_type
 
+-- SQLNESS PROTOCOL POSTGRES
+select * from pg_catalog.pg_type order by oid;
+
++-----+-----------+--------+
+| oid | typname   | typlen |
++-----+-----------+--------+
+| 1   | String    | -1     |
+| 2   | Binary    | -1     |
+| 3   | Int8      | 1      |
+| 4   | Int16     | 2      |
+| 5   | Int32     | 4      |
+| 6   | Int64     | 8      |
+| 7   | UInt8     | 1      |
+| 8   | UInt16    | 2      |
+| 9   | UInt32    | 4      |
+| 10  | UInt64    | 8      |
+| 11  | Float32   | 4      |
+| 12  | Float64   | 8      |
+| 13  | Decimal   | 16     |
+| 14  | Date      | 4      |
+| 15  | DateTime  | 8      |
+| 16  | Timestamp | 8      |
+| 17  | Time      | 8      |
+| 18  | Duration  | 8      |
+| 19  | Interval  | 16     |
+| 20  | List      | -1     |
++-----+-----------+--------+
+
 -- \d
+-- SQLNESS PROTOCOL POSTGRES
 SELECT n.nspname as "Schema",
   c.relname as "Name",
   CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 't' THEN 'TOAST table' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END as "Type",
@@ -21,9 +59,14 @@ WHERE c.relkind IN ('r','p','v','m','S','f','')
   AND pg_catalog.pg_table_is_visible(c.oid)
 ORDER BY 1,2;
 
-Error: 4001(TableNotFound), Failed to plan SQL: Table not found: greptime.pg_catalog.pg_class
++--------+---------+-------+-------+
+| Schema | Name    | Type  | Owner |
++--------+---------+-------+-------+
+| public | numbers | table |       |
++--------+---------+-------+-------+
 
 -- \dt
+-- SQLNESS PROTOCOL POSTGRES
 SELECT n.nspname as "Schema",
   c.relname as "Name",
   CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 't' THEN 'TOAST table' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END as "Type",
@@ -37,22 +80,36 @@ WHERE c.relkind IN ('r','p','')
   AND pg_catalog.pg_table_is_visible(c.oid)
 ORDER BY 1,2;
 
-Error: 4001(TableNotFound), Failed to plan SQL: Table not found: greptime.pg_catalog.pg_class
++--------+---------+-------+-------+
+| Schema | Name    | Type  | Owner |
++--------+---------+-------+-------+
+| public | numbers | table |       |
++--------+---------+-------+-------+
 
 -- make sure oid of namespace keep stable
+-- SQLNESS PROTOCOL POSTGRES
 SELECT * FROM pg_namespace ORDER BY oid;
 
-Error: 4001(TableNotFound), Failed to plan SQL: Table not found: greptime.public.pg_namespace
++------------+--------------------+
+| oid        | nspname            |
++------------+--------------------+
+| 667359454  | pg_catalog         |
+| 3174397350 | information_schema |
+| 3338153620 | greptime_private   |
+| 3927743705 | public             |
++------------+--------------------+
 
-create
-database my_db;
+-- SQLNESS PROTOCOL POSTGRES
+create database my_db;
 
 Affected Rows: 1
 
+-- SQLNESS PROTOCOL POSTGRES
 use my_db;
 
-Affected Rows: 0
 
+
+-- SQLNESS PROTOCOL POSTGRES
 create table foo
 (
     ts TIMESTAMP TIME INDEX
@@ -61,6 +118,7 @@ create table foo
 Affected Rows: 0
 
 -- show tables in `my_db`
+-- SQLNESS PROTOCOL POSTGRES
 select relname
 from pg_catalog.pg_class
 where relnamespace = (
@@ -69,9 +127,14 @@ where relnamespace = (
     where nspname = 'my_db'
 );
 
-Error: 4001(TableNotFound), Failed to plan SQL: Table not found: greptime.pg_catalog.pg_class
++---------+
+| relname |
++---------+
+| foo     |
++---------+
 
 -- \dt
+-- SQLNESS PROTOCOL POSTGRES
 SELECT n.nspname as "Schema",
   c.relname as "Name",
   CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 't' THEN 'TOAST table' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END as "Type",
@@ -85,9 +148,15 @@ WHERE c.relkind IN ('r','p','')
   AND pg_catalog.pg_table_is_visible(c.oid)
 ORDER BY 1,2;
 
-Error: 4001(TableNotFound), Failed to plan SQL: Table not found: greptime.pg_catalog.pg_class
++--------+---------+-------+-------+
+| Schema | Name    | Type  | Owner |
++--------+---------+-------+-------+
+| my_db  | foo     | table |       |
+| public | numbers | table |       |
++--------+---------+-------+-------+
 
 -- show tables in `my_db`, `public`
+-- SQLNESS PROTOCOL POSTGRES
 select relname
 from pg_catalog.pg_class
 where relnamespace in (
@@ -97,8 +166,14 @@ where relnamespace in (
 )
 order by relname;
 
-Error: 4001(TableNotFound), Failed to plan SQL: Table not found: greptime.pg_catalog.pg_class
++---------+
+| relname |
++---------+
+| foo     |
+| numbers |
++---------+
 
+-- SQLNESS PROTOCOL POSTGRES
 select relname
 from pg_catalog.pg_class
 where relnamespace in (
@@ -107,8 +182,13 @@ where relnamespace in (
     where nspname like 'my%'
 );
 
-Error: 4001(TableNotFound), Failed to plan SQL: Table not found: greptime.pg_catalog.pg_class
++---------+
+| relname |
++---------+
+| foo     |
++---------+
 
+-- SQLNESS PROTOCOL POSTGRES
 select relnamespace, relname, relkind
 from pg_catalog.pg_class
 where relnamespace in (
@@ -120,34 +200,58 @@ where relnamespace in (
 )
 order by relnamespace, relname;
 
-Error: 4001(TableNotFound), Failed to plan SQL: Table not found: greptime.pg_catalog.pg_class
++--------------+---------+---------+
+| relnamespace | relname | relkind |
++--------------+---------+---------+
+| 434869349    | foo     | r       |
++--------------+---------+---------+
 
+-- SQLNESS PROTOCOL POSTGRES
 use public;
 
-Affected Rows: 0
 
+
+-- SQLNESS PROTOCOL POSTGRES
 drop schema my_db;
 
 Affected Rows: 0
 
+-- SQLNESS PROTOCOL POSTGRES
 use pg_catalog;
 
-Affected Rows: 0
+
 
 -- pg_class
+-- SQLNESS PROTOCOL POSTGRES
 desc table pg_class;
 
-Error: 4001(TableNotFound), Table not found: pg_class
++--------------+--------+-----+------+---------+---------------+
+| Column       | Type   | Key | Null | Default | Semantic Type |
++--------------+--------+-----+------+---------+---------------+
+| oid          | UInt32 |     | NO   |         | FIELD         |
+| relname      | String |     | NO   |         | FIELD         |
+| relnamespace | UInt32 |     | NO   |         | FIELD         |
+| relkind      | String |     | NO   |         | FIELD         |
+| relowner     | UInt32 |     | NO   |         | FIELD         |
++--------------+--------+-----+------+---------+---------------+
 
+-- SQLNESS PROTOCOL POSTGRES
 desc table pg_namespace;
 
-Error: 4001(TableNotFound), Table not found: pg_namespace
++---------+--------+-----+------+---------+---------------+
+| Column  | Type   | Key | Null | Default | Semantic Type |
++---------+--------+-----+------+---------+---------------+
+| oid     | UInt32 |     | NO   |         | FIELD         |
+| nspname | String |     | NO   |         | FIELD         |
++---------+--------+-----+------+---------+---------------+
 
+-- SQLNESS PROTOCOL POSTGRES
 drop table my_db.foo;
 
-Error: 4001(TableNotFound), Table not found: greptime.my_db.foo
+Failed to execute query, encountered: Error { kind: Db, cause: Some(DbError { severity: "ERROR", parsed_severity: None, code: SqlState(E42P01), message: "Table not found: greptime.my_db.foo", detail: None, hint: None, position: None, where_: None, schema: None, table: None, column: None, datatype: None, constraint: None, file: None, line: None, routine: None }) }
 
+-- SQLNESS PROTOCOL POSTGRES
 use public;
 
-Affected Rows: 0
+
 
diff --git a/tests/cases/standalone/common/system/pg_catalog.sql b/tests/cases/standalone/common/system/pg_catalog.sql
index 50fb7b9745..712302254d 100644
--- a/tests/cases/standalone/common/system/pg_catalog.sql
+++ b/tests/cases/standalone/common/system/pg_catalog.sql
@@ -1,9 +1,16 @@
 -- should not able to create pg_catalog
 create database pg_catalog;
 
+-- make sure all the pg_catalog tables are only visible to postgres
+select * from pg_catalog.pg_class;
+select * from pg_catalog.pg_namespace;
+select * from pg_catalog.pg_type;
+
+-- SQLNESS PROTOCOL POSTGRES
 select * from pg_catalog.pg_type order by oid;
 
 -- \d
+-- SQLNESS PROTOCOL POSTGRES
 SELECT n.nspname as "Schema",
   c.relname as "Name",
   CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 't' THEN 'TOAST table' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END as "Type",
@@ -18,7 +25,7 @@ WHERE c.relkind IN ('r','p','v','m','S','f','')
 ORDER BY 1,2;
 
 -- \dt
-
+-- SQLNESS PROTOCOL POSTGRES
 SELECT n.nspname as "Schema",
   c.relname as "Name",
   CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 't' THEN 'TOAST table' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END as "Type",
@@ -33,19 +40,23 @@ WHERE c.relkind IN ('r','p','')
 ORDER BY 1,2;
 
 -- make sure oid of namespace keep stable
+-- SQLNESS PROTOCOL POSTGRES
 SELECT * FROM pg_namespace ORDER BY oid;
 
-create
-database my_db;
+-- SQLNESS PROTOCOL POSTGRES
+create database my_db;
 
+-- SQLNESS PROTOCOL POSTGRES
 use my_db;
 
+-- SQLNESS PROTOCOL POSTGRES
 create table foo
 (
     ts TIMESTAMP TIME INDEX
 );
 
 -- show tables in `my_db`
+-- SQLNESS PROTOCOL POSTGRES
 select relname
 from pg_catalog.pg_class
 where relnamespace = (
@@ -55,6 +66,7 @@ where relnamespace = (
 );
 
 -- \dt
+-- SQLNESS PROTOCOL POSTGRES
 SELECT n.nspname as "Schema",
   c.relname as "Name",
   CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 't' THEN 'TOAST table' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END as "Type",
@@ -69,6 +81,7 @@ WHERE c.relkind IN ('r','p','')
 ORDER BY 1,2;
 
 -- show tables in `my_db`, `public`
+-- SQLNESS PROTOCOL POSTGRES
 select relname
 from pg_catalog.pg_class
 where relnamespace in (
@@ -78,6 +91,7 @@ where relnamespace in (
 )
 order by relname;
 
+-- SQLNESS PROTOCOL POSTGRES
 select relname
 from pg_catalog.pg_class
 where relnamespace in (
@@ -86,6 +100,7 @@ where relnamespace in (
     where nspname like 'my%'
 );
 
+-- SQLNESS PROTOCOL POSTGRES
 select relnamespace, relname, relkind
 from pg_catalog.pg_class
 where relnamespace in (
@@ -97,17 +112,24 @@ where relnamespace in (
 )
 order by relnamespace, relname;
 
+-- SQLNESS PROTOCOL POSTGRES
 use public;
 
+-- SQLNESS PROTOCOL POSTGRES
 drop schema my_db;
 
+-- SQLNESS PROTOCOL POSTGRES
 use pg_catalog;
 
 -- pg_class
+-- SQLNESS PROTOCOL POSTGRES
 desc table pg_class;
 
+-- SQLNESS PROTOCOL POSTGRES
 desc table pg_namespace;
 
+-- SQLNESS PROTOCOL POSTGRES
 drop table my_db.foo;
 
+-- SQLNESS PROTOCOL POSTGRES
 use public;
diff --git a/tests/conf/standalone-test.toml.template b/tests/conf/standalone-test.toml.template
index 2e30ac35c2..f96a815762 100644
--- a/tests/conf/standalone-test.toml.template
+++ b/tests/conf/standalone-test.toml.template
@@ -24,6 +24,19 @@ data_home = '{data_home}'
 addr = '127.0.0.1:4001'
 runtime_size = 8
 
+[mysql]
+enable = true
+addr = "127.0.0.1:4002"
+runtime_size = 2
+
+[mysql.tls]
+mode = "disable"
+
+[postgres]
+enable = true
+addr = "127.0.0.1:4003"
+runtime_size = 2
+
 [procedure]
 max_retry_times = 3
 retry_delay = "500ms"
diff --git a/tests/runner/Cargo.toml b/tests/runner/Cargo.toml
index 1bac0c5933..641f07c2c6 100644
--- a/tests/runner/Cargo.toml
+++ b/tests/runner/Cargo.toml
@@ -15,8 +15,11 @@ common-error.workspace = true
 common-query.workspace = true
 common-recordbatch.workspace = true
 common-time.workspace = true
+datatypes = { workspace = true }
+mysql = { version = "23.0.1" }
 serde.workspace = true
 serde_json.workspace = true
+tokio-postgres = { workspace = true }
 # sqlness 0.6.0 have a bug causing `cargo sqlness` to fail(see https://github.com/CeresDB/sqlness/issues/68) which is fixed in 0.6.1
 sqlness = "0.6.1"
 tempfile.workspace = true
diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs
index 9d71489f8c..0dc0e3a9e8 100644
--- a/tests/runner/src/env.rs
+++ b/tests/runner/src/env.rs
@@ -12,10 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::borrow::Cow;
 use std::fmt::Display;
 use std::fs::OpenOptions;
 use std::io;
 use std::io::Write;
+use std::net::SocketAddr;
 use std::path::{Path, PathBuf};
 use std::process::{Child, Command};
 use std::sync::atomic::{AtomicU32, Ordering};
@@ -30,15 +32,25 @@ use client::{
 use common_error::ext::ErrorExt;
 use common_query::{Output, OutputData};
 use common_recordbatch::RecordBatches;
+use datatypes::data_type::ConcreteDataType;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::schema::{ColumnSchema, Schema};
+use datatypes::vectors::{StringVectorBuilder, VectorRef};
+use mysql::prelude::Queryable;
+use mysql::{Conn as MySqlClient, Row as MySqlRow};
 use serde::Serialize;
 use sqlness::{Database, EnvController, QueryContext};
 use tinytemplate::TinyTemplate;
 use tokio::sync::Mutex as TokioMutex;
+use tokio_postgres::{Client as PgClient, SimpleQueryMessage as PgRow};
 
-use crate::util;
+use crate::protocol_interceptor::{MYSQL, PROTOCOL_KEY};
+use crate::{util, ServerAddr};
 
 const METASRV_ADDR: &str = "127.0.0.1:3002";
-const SERVER_ADDR: &str = "127.0.0.1:4001";
+const GRPC_SERVER_ADDR: &str = "127.0.0.1:4001";
+const MYSQL_SERVER_ADDR: &str = "127.0.0.1:4002";
+const POSTGRES_SERVER_ADDR: &str = "127.0.0.1:4003";
 const DEFAULT_LOG_LEVEL: &str = "--log-level=debug,hyper=warn,tower=warn,datafusion=warn,reqwest=warn,sqlparser=warn,h2=info,opendal=info";
 
 #[derive(Clone)]
@@ -55,7 +67,7 @@ pub enum WalConfig {
 #[derive(Clone)]
 pub struct Env {
     sqlness_home: PathBuf,
-    server_addr: Option<String>,
+    server_addrs: ServerAddr,
     wal: WalConfig,
 
     /// The path to the directory that contains the pre-built GreptimeDB binary.
@@ -86,21 +98,21 @@ impl EnvController for Env {
 impl Env {
     pub fn new(
         data_home: PathBuf,
-        server_addr: Option<String>,
+        server_addrs: ServerAddr,
         wal: WalConfig,
         bins_dir: Option<PathBuf>,
     ) -> Self {
         Self {
             sqlness_home: data_home,
-            server_addr,
+            server_addrs,
             wal,
             bins_dir: Arc::new(Mutex::new(bins_dir)),
         }
     }
 
     async fn start_standalone(&self) -> GreptimeDB {
-        if let Some(server_addr) = self.server_addr.clone() {
-            self.connect_db(&server_addr)
+        if self.server_addrs.server_addr.is_some() {
+            self.connect_db(&self.server_addrs).await
         } else {
             self.build_db();
             self.setup_wal();
@@ -109,25 +121,18 @@ impl Env {
 
             let server_process = self.start_server("standalone", &db_ctx, true).await;
 
-            let client = Client::with_urls(vec![SERVER_ADDR]);
-            let db = DB::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client);
+            let mut greptimedb = self.connect_db(&Default::default()).await;
+            greptimedb.server_processes = Some(Arc::new(Mutex::new(vec![server_process])));
+            greptimedb.is_standalone = true;
+            greptimedb.ctx = db_ctx;
 
-            GreptimeDB {
-                server_processes: Some(Arc::new(Mutex::new(vec![server_process]))),
-                metasrv_process: None,
-                frontend_process: None,
-                flownode_process: None,
-                client: TokioMutex::new(db),
-                ctx: db_ctx,
-                is_standalone: true,
-                env: self.clone(),
-            }
+            greptimedb
         }
     }
 
     async fn start_distributed(&self) -> GreptimeDB {
-        if let Some(server_addr) = self.server_addr.clone() {
-            self.connect_db(&server_addr)
+        if self.server_addrs.server_addr.is_some() {
+            self.connect_db(&self.server_addrs).await
         } else {
             self.build_db();
             self.setup_wal();
@@ -145,29 +150,92 @@ impl Env {
 
             let flownode = self.start_server("flownode", &db_ctx, true).await;
 
-            let client = Client::with_urls(vec![SERVER_ADDR]);
-            let db = DB::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client);
+            let mut greptimedb = self.connect_db(&Default::default()).await;
 
-            GreptimeDB {
-                server_processes: Some(Arc::new(Mutex::new(vec![
-                    datanode_1, datanode_2, datanode_3,
-                ]))),
-                metasrv_process: Some(meta_server),
-                frontend_process: Some(frontend),
-                flownode_process: Some(flownode),
-                client: TokioMutex::new(db),
-                ctx: db_ctx,
-                is_standalone: false,
-                env: self.clone(),
-            }
+            greptimedb.metasrv_process = Some(meta_server);
+            greptimedb.server_processes = Some(Arc::new(Mutex::new(vec![
+                datanode_1, datanode_2, datanode_3,
+            ])));
+            greptimedb.frontend_process = Some(frontend);
+            greptimedb.flownode_process = Some(flownode);
+            greptimedb.is_standalone = false;
+            greptimedb.ctx = db_ctx;
+
+            greptimedb
         }
     }
 
-    fn connect_db(&self, server_addr: &str) -> GreptimeDB {
-        let client = Client::with_urls(vec![server_addr.to_owned()]);
-        let db = DB::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, client);
+    async fn create_pg_client(&self, pg_server_addr: &str) -> PgClient {
+        let sockaddr: SocketAddr = pg_server_addr.parse().expect(
+            "Failed to parse the Postgres server address. Please check if the address is in the format of `ip:port`.",
+        );
+        let mut config = tokio_postgres::config::Config::new();
+        config.host(sockaddr.ip().to_string());
+        config.port(sockaddr.port());
+        config.dbname(DEFAULT_SCHEMA_NAME);
+
+        // retry to connect to Postgres server until success
+        const MAX_RETRY: usize = 3;
+        let mut backoff = Duration::from_millis(500);
+        for _ in 0..MAX_RETRY {
+            if let Ok((pg_client, conn)) = config.connect(tokio_postgres::NoTls).await {
+                tokio::spawn(conn);
+                return pg_client;
+            }
+            tokio::time::sleep(backoff).await;
+            backoff *= 2;
+        }
+        panic!("Failed to connect to Postgres server. Please check if the server is running.");
+    }
+
+    async fn create_mysql_client(&self, mysql_server_addr: &str) -> MySqlClient {
+        let sockaddr: SocketAddr = mysql_server_addr.parse().expect(
+            "Failed to parse the MySQL server address. Please check if the address is in the format of `ip:port`.",
+        );
+        let ops = mysql::OptsBuilder::new()
+            .ip_or_hostname(Some(sockaddr.ip().to_string()))
+            .tcp_port(sockaddr.port())
+            .db_name(Some(DEFAULT_SCHEMA_NAME));
+        // retry to connect to MySQL server until success
+        const MAX_RETRY: usize = 3;
+        let mut backoff = Duration::from_millis(500);
+
+        for _ in 0..MAX_RETRY {
+            // exponential backoff
+            if let Ok(client) = mysql::Conn::new(ops.clone()) {
+                return client;
+            }
+            tokio::time::sleep(backoff).await;
+            backoff *= 2;
+        }
+
+        panic!("Failed to connect to MySQL server. Please check if the server is running.")
+    }
+
+    async fn connect_db(&self, server_addr: &ServerAddr) -> GreptimeDB {
+        let grpc_server_addr = server_addr
+            .server_addr
+            .clone()
+            .unwrap_or(GRPC_SERVER_ADDR.to_owned());
+        let pg_server_addr = server_addr
+            .pg_server_addr
+            .clone()
+            .unwrap_or(POSTGRES_SERVER_ADDR.to_owned());
+        let mysql_server_addr = server_addr
+            .mysql_server_addr
+            .clone()
+            .unwrap_or(MYSQL_SERVER_ADDR.to_owned());
+
+        let grpc_client = Client::with_urls(vec![grpc_server_addr.clone()]);
+        let db = DB::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, grpc_client);
+        let pg_client = self.create_pg_client(&pg_server_addr).await;
+
+        let mysql_client = self.create_mysql_client(&mysql_server_addr).await;
+
         GreptimeDB {
-            client: TokioMutex::new(db),
+            grpc_client: TokioMutex::new(db),
+            pg_client: TokioMutex::new(pg_client),
+            mysql_client: TokioMutex::new(mysql_client),
             server_processes: None,
             metasrv_process: None,
             frontend_process: None,
@@ -217,9 +285,15 @@ impl Env {
             .open(stdout_file_name)
             .unwrap();
 
-        let (args, check_ip_addr) = match subcommand {
-            "datanode" => self.datanode_start_args(db_ctx),
-            "flownode" => self.flownode_start_args(db_ctx, &self.sqlness_home),
+        let (args, check_ip_addrs) = match subcommand {
+            "datanode" => {
+                let (args, addr) = self.datanode_start_args(db_ctx);
+                (args, vec![addr])
+            }
+            "flownode" => {
+                let (args, addr) = self.flownode_start_args(db_ctx, &self.sqlness_home);
+                (args, vec![addr])
+            }
             "standalone" => {
                 let args = vec![
                     DEFAULT_LOG_LEVEL.to_string(),
@@ -233,7 +307,7 @@ impl Env {
                     self.generate_config_file(subcommand, db_ctx),
                     "--http-addr=127.0.0.1:5002".to_string(),
                 ];
-                (args, SERVER_ADDR.to_string())
+                (args, vec![GRPC_SERVER_ADDR.to_string()])
             }
             "frontend" => {
                 let args = vec![
@@ -247,7 +321,7 @@ impl Env {
                         self.sqlness_home.display()
                     ),
                 ];
-                (args, SERVER_ADDR.to_string())
+                (args, vec![GRPC_SERVER_ADDR.to_string()])
             }
             "metasrv" => {
                 let args = vec![
@@ -266,16 +340,18 @@ impl Env {
                     "-c".to_string(),
                     self.generate_config_file(subcommand, db_ctx),
                 ];
-                (args, METASRV_ADDR.to_string())
+                (args, vec![METASRV_ADDR.to_string()])
             }
             _ => panic!("Unexpected subcommand: {subcommand}"),
         };
 
-        if util::check_port(check_ip_addr.parse().unwrap(), Duration::from_secs(1)).await {
-            panic!(
-                "Port {check_ip_addr} is already in use, please check and retry.",
-                check_ip_addr = check_ip_addr
-            );
+        for check_ip_addr in &check_ip_addrs {
+            if util::check_port(check_ip_addr.parse().unwrap(), Duration::from_secs(1)).await {
+                panic!(
+                    "Port {check_ip_addr} is already in use, please check and retry.",
+                    check_ip_addr = check_ip_addr
+                );
+            }
         }
 
         #[cfg(not(windows))]
@@ -297,9 +373,11 @@ impl Env {
                 panic!("Failed to start the DB with subcommand {subcommand},Error: {error}")
             });
 
-        if !util::check_port(check_ip_addr.parse().unwrap(), Duration::from_secs(10)).await {
-            Env::stop_server(&mut process);
-            panic!("{subcommand} doesn't up in 10 seconds, quit.")
+        for check_ip_addr in &check_ip_addrs {
+            if !util::check_port(check_ip_addr.parse().unwrap(), Duration::from_secs(10)).await {
+                Env::stop_server(&mut process);
+                panic!("{subcommand} doesn't up in 10 seconds, quit.")
+            }
         }
 
         process
@@ -379,6 +457,9 @@ impl Env {
         };
 
         if let Some(server_process) = db.server_processes.clone() {
+            *db.pg_client.lock().await = self.create_pg_client(&self.pg_server_addr()).await;
+            *db.mysql_client.lock().await =
+                self.create_mysql_client(&self.mysql_server_addr()).await;
             let mut server_processes = server_process.lock().unwrap();
             *server_processes = new_server_processes;
         }
@@ -433,6 +514,20 @@ impl Env {
         conf_file
     }
 
+    fn pg_server_addr(&self) -> String {
+        self.server_addrs
+            .pg_server_addr
+            .clone()
+            .unwrap_or(POSTGRES_SERVER_ADDR.to_owned())
+    }
+
+    fn mysql_server_addr(&self) -> String {
+        self.server_addrs
+            .mysql_server_addr
+            .clone()
+            .unwrap_or(MYSQL_SERVER_ADDR.to_owned())
+    }
+
     /// Build the DB with `cargo build --bin greptime`
     fn build_db(&self) {
         if self.bins_dir.lock().unwrap().is_some() {
@@ -467,20 +562,51 @@ pub struct GreptimeDB {
     metasrv_process: Option<Child>,
     frontend_process: Option<Child>,
     flownode_process: Option<Child>,
-    client: TokioMutex<DB>,
+    grpc_client: TokioMutex<DB>,
+    pg_client: TokioMutex<PgClient>,
+    mysql_client: TokioMutex<MySqlClient>,
     ctx: GreptimeDBContext,
     is_standalone: bool,
     env: Env,
 }
 
-#[async_trait]
-impl Database for GreptimeDB {
-    async fn query(&self, ctx: QueryContext, query: String) -> Box<dyn Display> {
-        if ctx.context.contains_key("restart") && self.env.server_addr.is_none() {
-            self.env.restart_server(self).await;
+impl GreptimeDB {
+    async fn postgres_query(&self, _ctx: QueryContext, query: String) -> Box<dyn Display> {
+        let client = self.pg_client.lock().await;
+        match client.simple_query(&query).await {
+            Ok(rows) => Box::new(PostgresqlFormatter { rows }),
+            Err(e) => Box::new(format!("Failed to execute query, encountered: {:?}", e)),
         }
+    }
 
-        let mut client = self.client.lock().await;
+    async fn mysql_query(&self, _ctx: QueryContext, query: String) -> Box<dyn Display> {
+        let mut conn = self.mysql_client.lock().await;
+        let result = conn.query_iter(query);
+        Box::new(match result {
+            Ok(result) => {
+                let mut rows = vec![];
+                let affected_rows = result.affected_rows();
+                for row in result {
+                    match row {
+                        Ok(r) => rows.push(r),
+                        Err(e) => {
+                            return Box::new(format!("Failed to parse query result, err: {:?}", e))
+                        }
+                    }
+                }
+
+                if rows.is_empty() {
+                    format!("affected_rows: {}", affected_rows)
+                } else {
+                    format!("{}", MysqlFormatter { rows })
+                }
+            }
+            Err(e) => format!("Failed to execute query, err: {:?}", e),
+        })
+    }
+
+    async fn grpc_query(&self, _ctx: QueryContext, query: String) -> Box<dyn Display> {
+        let mut client = self.grpc_client.lock().await;
 
         let query_str = query.trim().to_lowercase();
 
@@ -542,6 +668,25 @@ impl Database for GreptimeDB {
     }
 }
 
+#[async_trait]
+impl Database for GreptimeDB {
+    async fn query(&self, ctx: QueryContext, query: String) -> Box<dyn Display> {
+        if ctx.context.contains_key("restart") && self.env.server_addrs.server_addr.is_none() {
+            self.env.restart_server(self).await;
+        }
+        if let Some(protocol) = ctx.context.get(PROTOCOL_KEY) {
+            // protocol is bound to be either "mysql" or "postgres"
+            if protocol == MYSQL {
+                self.mysql_query(ctx, query).await
+            } else {
+                self.postgres_query(ctx, query).await
+            }
+        } else {
+            self.grpc_query(ctx, query).await
+        }
+    }
+}
+
 impl GreptimeDB {
     fn stop(&mut self) {
         if let Some(server_processes) = self.server_processes.clone() {
@@ -575,7 +720,7 @@ impl GreptimeDB {
 
 impl Drop for GreptimeDB {
     fn drop(&mut self) {
-        if self.env.server_addr.is_none() {
+        if self.env.server_addrs.server_addr.is_none() {
             self.stop();
         }
     }
@@ -657,3 +802,117 @@ impl Display for ResultDisplayer {
         }
     }
 }
+
+struct PostgresqlFormatter {
+    pub rows: Vec<PgRow>,
+}
+
+impl Display for PostgresqlFormatter {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        if self.rows.is_empty() {
+            return f.write_fmt(format_args!("(Empty response)"));
+        }
+
+        // create schema
+        let schema = match &self.rows[0] {
+            PgRow::CommandComplete(affected_rows) => {
+                write!(
+                    f,
+                    "{}",
+                    ResultDisplayer {
+                        result: Ok(Output::new_with_affected_rows(*affected_rows as usize)),
+                    }
+                )?;
+                return Ok(());
+            }
+            PgRow::RowDescription(desc) => Arc::new(Schema::new(
+                desc.iter()
+                    .map(|column| {
+                        ColumnSchema::new(column.name(), ConcreteDataType::string_datatype(), false)
+                    })
+                    .collect(),
+            )),
+            _ => unreachable!(),
+        };
+        if schema.num_columns() == 0 {
+            return Ok(());
+        }
+
+        // convert to string vectors
+        let mut columns: Vec<StringVectorBuilder> = (0..schema.num_columns())
+            .map(|_| StringVectorBuilder::with_capacity(schema.num_columns()))
+            .collect();
+        for row in self.rows.iter().skip(1) {
+            if let PgRow::Row(row) = row {
+                for (i, column) in columns.iter_mut().enumerate().take(schema.num_columns()) {
+                    column.push(row.get(i));
+                }
+            }
+        }
+        let columns: Vec<VectorRef> = columns
+            .into_iter()
+            .map(|mut col| Arc::new(col.finish()) as VectorRef)
+            .collect();
+
+        // construct recordbatch
+        let recordbatches = RecordBatches::try_from_columns(schema, columns)
+            .expect("Failed to construct recordbatches from columns. Please check the schema.");
+        let result_displayer = ResultDisplayer {
+            result: Ok(Output::new_with_record_batches(recordbatches)),
+        };
+        write!(f, "{}", result_displayer)?;
+
+        Ok(())
+    }
+}
+
+struct MysqlFormatter {
+    pub rows: Vec<MySqlRow>,
+}
+
+impl Display for MysqlFormatter {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        if self.rows.is_empty() {
+            return f.write_fmt(format_args!("(Empty response)"));
+        }
+        // create schema
+        let head_column = &self.rows[0];
+        let head_binding = head_column.columns();
+        let names = head_binding
+            .iter()
+            .map(|column| column.name_str())
+            .collect::<Vec<Cow<str>>>();
+        let schema = Arc::new(Schema::new(
+            names
+                .iter()
+                .map(|name| {
+                    ColumnSchema::new(name.to_string(), ConcreteDataType::string_datatype(), false)
+                })
+                .collect(),
+        ));
+
+        // convert to string vectors
+        let mut columns: Vec<StringVectorBuilder> = (0..schema.num_columns())
+            .map(|_| StringVectorBuilder::with_capacity(schema.num_columns()))
+            .collect();
+        for row in self.rows.iter() {
+            for (i, name) in names.iter().enumerate() {
+                columns[i].push(row.get::<String, &str>(name).as_deref());
+            }
+        }
+        let columns: Vec<VectorRef> = columns
+            .into_iter()
+            .map(|mut col| Arc::new(col.finish()) as VectorRef)
+            .collect();
+
+        // construct recordbatch
+        let recordbatches = RecordBatches::try_from_columns(schema, columns)
+            .expect("Failed to construct recordbatches from columns. Please check the schema.");
+        let result_displayer = ResultDisplayer {
+            result: Ok(Output::new_with_record_batches(recordbatches)),
+        };
+        write!(f, "{}", result_displayer)?;
+
+        Ok(())
+    }
+}
diff --git a/tests/runner/src/main.rs b/tests/runner/src/main.rs
index 4474676802..eca72f280e 100644
--- a/tests/runner/src/main.rs
+++ b/tests/runner/src/main.rs
@@ -15,12 +15,15 @@
 #![allow(clippy::print_stdout)]
 
 use std::path::PathBuf;
+use std::sync::Arc;
 
 use clap::{Parser, ValueEnum};
 use env::{Env, WalConfig};
+use sqlness::interceptor::Registry;
 use sqlness::{ConfigBuilder, Runner};
 
 mod env;
+mod protocol_interceptor;
 mod util;
 
 #[derive(ValueEnum, Debug, Clone)]
@@ -30,6 +33,23 @@ enum Wal {
     Kafka,
 }
 
+// add a group to ensure that all server addresses are set together
+#[derive(clap::Args, Debug, Clone, Default)]
+#[group(multiple = true, requires_all=["server_addr", "pg_server_addr", "mysql_server_addr"])]
+struct ServerAddr {
+    /// Address of the grpc server.
+    #[clap(short, long)]
+    server_addr: Option<String>,
+
+    /// Address of the postgres server. Must be set if server_addr is set.
+    #[clap(short, long, requires = "server_addr")]
+    pg_server_addr: Option<String>,
+
+    /// Address of the mysql server. Must be set if server_addr is set.
+    #[clap(short, long, requires = "server_addr")]
+    mysql_server_addr: Option<String>,
+}
+
 #[derive(Parser, Debug)]
 #[clap(author, version, about, long_about = None)]
 /// SQL Harness for GrepTimeDB
@@ -50,9 +70,9 @@ struct Args {
     #[clap(short, long, default_value = ".*")]
     test_filter: String,
 
-    /// Address of the server.
-    #[clap(short, long)]
-    server_addr: Option<String>,
+    /// Addresses of the server.
+    #[command(flatten)]
+    server_addr: ServerAddr,
 
     /// The type of Wal.
     #[clap(short, long, default_value = "raft_engine")]
@@ -84,12 +104,19 @@ async fn main() {
         .unwrap();
     let sqlness_home = temp_dir.into_path();
 
+    let mut interceptor_registry: Registry = Default::default();
+    interceptor_registry.register(
+        protocol_interceptor::PREFIX,
+        Arc::new(protocol_interceptor::ProtocolInterceptorFactory),
+    );
+
     let config = ConfigBuilder::default()
         .case_dir(util::get_case_dir(args.case_dir))
         .fail_fast(args.fail_fast)
         .test_filter(args.test_filter)
         .follow_links(true)
         .env_config_file(args.env_config_file)
+        .interceptor_registry(interceptor_registry)
         .build()
         .unwrap();
 
@@ -107,7 +134,12 @@ async fn main() {
 
     let runner = Runner::new(
         config,
-        Env::new(sqlness_home.clone(), args.server_addr, wal, args.bins_dir),
+        Env::new(
+            sqlness_home.clone(),
+            args.server_addr.clone(),
+            wal,
+            args.bins_dir,
+        ),
     );
     runner.run().await.unwrap();
 
diff --git a/tests/runner/src/protocol_interceptor.rs b/tests/runner/src/protocol_interceptor.rs
new file mode 100644
index 0000000000..0f8a01e235
--- /dev/null
+++ b/tests/runner/src/protocol_interceptor.rs
@@ -0,0 +1,48 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sqlness::interceptor::{Interceptor, InterceptorFactory, InterceptorRef};
+use sqlness::SqlnessError;
+
+pub const PROTOCOL_KEY: &str = "protocol";
+pub const POSTGRES: &str = "postgres";
+pub const MYSQL: &str = "mysql";
+pub const PREFIX: &str = "PROTOCOL";
+
+pub struct ProtocolInterceptor {
+    protocol: String,
+}
+
+impl Interceptor for ProtocolInterceptor {
+    fn before_execute(&self, _: &mut Vec<String>, context: &mut sqlness::QueryContext) {
+        context
+            .context
+            .insert(PROTOCOL_KEY.to_string(), self.protocol.clone());
+    }
+}
+
+pub struct ProtocolInterceptorFactory;
+
+impl InterceptorFactory for ProtocolInterceptorFactory {
+    fn try_new(&self, ctx: &str) -> Result<InterceptorRef, SqlnessError> {
+        let protocol = ctx.to_lowercase();
+        match protocol.as_str() {
+            POSTGRES | MYSQL => Ok(Box::new(ProtocolInterceptor { protocol })),
+            _ => Err(SqlnessError::InvalidContext {
+                prefix: PREFIX.to_string(),
+                msg: format!("Unsupported protocol: {}", ctx),
+            }),
+        }
+    }
+}

From a283e13da744b6bbe3b08e7b0940ac2d3e03e4c6 Mon Sep 17 00:00:00 2001
From: Kaifeng Zheng <100595273+Kev1n8@users.noreply.github.com>
Date: Sat, 5 Oct 2024 02:05:40 +0800
Subject: [PATCH 054/128] feat: set max log files to 720 by default, info log
 only (#4787)

* feat: set max log files to 720 by default, info log only

* expose max_log_files in tomls

* include dir info when panicing, limit max_log_files of err_log to 30, and that of slow_queries to opt.max_log_files

* fix clippy

* update config.md

* update expected config str

* limit err_log max files size to `max_log_files` too, include err info when panicing, put `max_l_f` in right position

* fix typos

* chore: config

Co-authored-by: Lei, HUANG <6406592+v0y4g3r@users.noreply.github.com>

---------

Co-authored-by: dennis zhuang <killme2008@gmail.com>
Co-authored-by: Lei, HUANG <6406592+v0y4g3r@users.noreply.github.com>
---
 config/config.md                          |  5 +++
 config/datanode.example.toml              |  3 ++
 config/flownode.example.toml              |  3 ++
 config/frontend.example.toml              |  3 ++
 config/metasrv.example.toml               |  3 ++
 config/standalone.example.toml            |  3 ++
 src/common/telemetry/src/logging.rs       | 44 +++++++++++++++++++----
 src/log-store/src/kafka/client_manager.rs |  2 +-
 tests-integration/tests/http.rs           |  1 +
 9 files changed, 60 insertions(+), 7 deletions(-)

diff --git a/config/config.md b/config/config.md
index d413936ff3..6a48ed717e 100644
--- a/config/config.md
+++ b/config/config.md
@@ -161,6 +161,7 @@
 | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
+| `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
 | `logging.slow_query` | -- | -- | The slow query log options. |
@@ -251,6 +252,7 @@
 | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
+| `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
 | `logging.slow_query` | -- | -- | The slow query log options. |
@@ -320,6 +322,7 @@
 | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
+| `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
 | `logging.slow_query` | -- | -- | The slow query log options. |
@@ -476,6 +479,7 @@
 | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
+| `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
 | `logging.slow_query` | -- | -- | The slow query log options. |
@@ -526,6 +530,7 @@
 | `logging.otlp_endpoint` | String | `http://localhost:4317` | The OTLP tracing endpoint. |
 | `logging.append_stdout` | Bool | `true` | Whether to append logs to stdout. |
 | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
+| `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
 | `logging.tracing_sample_ratio` | -- | -- | The percentage of tracing will be sampled and exported.<br/>Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.<br/>ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
 | `logging.tracing_sample_ratio.default_ratio` | Float | `1.0` | -- |
 | `logging.slow_query` | -- | -- | The slow query log options. |
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index 06a59ebd6f..22670f857c 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -580,6 +580,9 @@ append_stdout = true
 ## The log format. Can be `text`/`json`.
 log_format = "text"
 
+## The maximum amount of log files.
+max_log_files = 720
+
 ## The percentage of tracing will be sampled and exported.
 ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
 ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0
diff --git a/config/flownode.example.toml b/config/flownode.example.toml
index 9d6d2fe5aa..34825542fa 100644
--- a/config/flownode.example.toml
+++ b/config/flownode.example.toml
@@ -78,6 +78,9 @@ append_stdout = true
 ## The log format. Can be `text`/`json`.
 log_format = "text"
 
+## The maximum amount of log files.
+max_log_files = 720
+
 ## The percentage of tracing will be sampled and exported.
 ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
 ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0
diff --git a/config/frontend.example.toml b/config/frontend.example.toml
index cc9698f61e..83e7808d46 100644
--- a/config/frontend.example.toml
+++ b/config/frontend.example.toml
@@ -185,6 +185,9 @@ append_stdout = true
 ## The log format. Can be `text`/`json`.
 log_format = "text"
 
+## The maximum amount of log files.
+max_log_files = 720
+
 ## The percentage of tracing will be sampled and exported.
 ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
 ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0
diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml
index 8431940b45..416f5ee6ef 100644
--- a/config/metasrv.example.toml
+++ b/config/metasrv.example.toml
@@ -172,6 +172,9 @@ append_stdout = true
 ## The log format. Can be `text`/`json`.
 log_format = "text"
 
+## The maximum amount of log files.
+max_log_files = 720
+
 ## The percentage of tracing will be sampled and exported.
 ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
 ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index 2d14b6550f..4c40d2dde6 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -624,6 +624,9 @@ append_stdout = true
 ## The log format. Can be `text`/`json`.
 log_format = "text"
 
+## The maximum amount of log files.
+max_log_files = 720
+
 ## The percentage of tracing will be sampled and exported.
 ## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
 ## ratio > 1 are treated as 1. Fractions < 0 are treated as 0
diff --git a/src/common/telemetry/src/logging.rs b/src/common/telemetry/src/logging.rs
index c21766f04d..4c5a9ee18f 100644
--- a/src/common/telemetry/src/logging.rs
+++ b/src/common/telemetry/src/logging.rs
@@ -54,6 +54,9 @@ pub struct LoggingOptions {
     /// The log format that can be one of "json" or "text". Default is "text".
     pub log_format: LogFormat,
 
+    /// The maximum number of log files set by default.
+    pub max_log_files: usize,
+
     /// Whether to append logs to stdout. Default is true.
     pub append_stdout: bool,
 
@@ -116,6 +119,8 @@ impl Default for LoggingOptions {
             tracing_sample_ratio: None,
             append_stdout: true,
             slow_query: SlowQueryOptions::default(),
+            // Rotation hourly, 24 files per day, keeps info log files of 30 days
+            max_log_files: 720,
         }
     }
 }
@@ -206,8 +211,17 @@ pub fn init_global_logging(
 
         // Configure the file logging layer with rolling policy.
         let file_logging_layer = if !opts.dir.is_empty() {
-            let rolling_appender =
-                RollingFileAppender::new(Rotation::HOURLY, &opts.dir, "greptimedb");
+            let rolling_appender = RollingFileAppender::builder()
+                .rotation(Rotation::HOURLY)
+                .filename_prefix("greptimedb")
+                .max_log_files(opts.max_log_files)
+                .build(&opts.dir)
+                .unwrap_or_else(|e| {
+                    panic!(
+                        "initializing rolling file appender at {} failed: {}",
+                        &opts.dir, e
+                    )
+                });
             let (writer, guard) = tracing_appender::non_blocking(rolling_appender);
             guards.push(guard);
 
@@ -228,8 +242,17 @@ pub fn init_global_logging(
 
         // Configure the error file logging layer with rolling policy.
         let err_file_logging_layer = if !opts.dir.is_empty() {
-            let rolling_appender =
-                RollingFileAppender::new(Rotation::HOURLY, &opts.dir, "greptimedb-err");
+            let rolling_appender = RollingFileAppender::builder()
+                .rotation(Rotation::HOURLY)
+                .filename_prefix("greptimedb-err")
+                .max_log_files(opts.max_log_files)
+                .build(&opts.dir)
+                .unwrap_or_else(|e| {
+                    panic!(
+                        "initializing rolling file appender at {} failed: {}",
+                        &opts.dir, e
+                    )
+                });
             let (writer, guard) = tracing_appender::non_blocking(rolling_appender);
             guards.push(guard);
 
@@ -256,8 +279,17 @@ pub fn init_global_logging(
         };
 
         let slow_query_logging_layer = if !opts.dir.is_empty() && opts.slow_query.enable {
-            let rolling_appender =
-                RollingFileAppender::new(Rotation::HOURLY, &opts.dir, "greptimedb-slow-queries");
+            let rolling_appender = RollingFileAppender::builder()
+                .rotation(Rotation::HOURLY)
+                .filename_prefix("greptimedb-slow-queries")
+                .max_log_files(opts.max_log_files)
+                .build(&opts.dir)
+                .unwrap_or_else(|e| {
+                    panic!(
+                        "initializing rolling file appender at {} failed: {}",
+                        &opts.dir, e
+                    )
+                });
             let (writer, guard) = tracing_appender::non_blocking(rolling_appender);
             guards.push(guard);
 
diff --git a/src/log-store/src/kafka/client_manager.rs b/src/log-store/src/kafka/client_manager.rs
index 589d5a900e..6dfb97b027 100644
--- a/src/log-store/src/kafka/client_manager.rs
+++ b/src/log-store/src/kafka/client_manager.rs
@@ -182,7 +182,7 @@ mod tests {
 
     use super::*;
 
-    /// Creates `num_topiocs` number of topics each will be decorated by the given decorator.
+    /// Creates `num_topics` number of topics each will be decorated by the given decorator.
     pub async fn create_topics<F>(
         num_topics: usize,
         decorator: F,
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index 5f3eb854bb..20a0f4edd2 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -833,6 +833,7 @@ max_retry_times = 3
 retry_delay = "500ms"
 
 [logging]
+max_log_files = 720
 append_stdout = true
 enable_otlp_tracing = false
 

From 4dea4cac47ebdd83cae2277392c61ac6fb9653b2 Mon Sep 17 00:00:00 2001
From: Ning Sun <sunng@protonmail.com>
Date: Tue, 8 Oct 2024 00:33:24 -0700
Subject: [PATCH 055/128] refactor: change sqlness ports to avoid conflict with
 local instance (#4794)

---
 .../information_schema/cluster_info.result    | 20 +++----
 .../information_schema/cluster_info.sql       | 10 ++--
 tests/conf/datanode-test.toml.template        |  4 +-
 tests/conf/standalone-test.toml.template      |  8 +--
 tests/runner/src/env.rs                       | 55 +++++++++++++------
 5 files changed, 59 insertions(+), 38 deletions(-)

diff --git a/tests/cases/distributed/information_schema/cluster_info.result b/tests/cases/distributed/information_schema/cluster_info.result
index a59d59ee60..9f19362591 100644
--- a/tests/cases/distributed/information_schema/cluster_info.result
+++ b/tests/cases/distributed/information_schema/cluster_info.result
@@ -20,52 +20,52 @@ DESC TABLE CLUSTER_INFO;
 -- SQLNESS REPLACE version node_version
 -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version
 -- SQLNESS REPLACE (\s[a-z0-9]{7,8}\s) Hash
--- SQLNESS REPLACE (\s[\-0-9T:\.]{15,}) Start_time
+-- SQLNESS REPLACE (\s[\-0-9T:\.]{19,}) Start_time
 -- SQLNESS REPLACE ((\d+(s|ms|m)\s)+) Duration
 -- SQLNESS REPLACE [\s\-]+
 SELECT * FROM CLUSTER_INFO ORDER BY peer_type;
 
-+++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|1|DATANODE|127.0.0.1:4101|Version|Hash|Start_time|Duration|Duration||2|DATANODE|127.0.0.1:4102|Version|Hash|Start_time|Duration|Duration||3|DATANODE|127.0.0.1:4103|Version|Hash|Start_time|Duration|Duration||0|FLOWNODE|127.0.0.1:6800|Version|Hash|Start_time|Duration|Duration||1|FRONTEND|127.0.0.1:4001|Version|Hash|Start_time|Duration|Duration||1|METASRV|127.0.0.1:3002|Version|Hash|Start_time|Duration||+++++++++
++++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|1|DATANODE|127.0.0.1:29411|Version|Hash|Start_time|Duration|Duration||2|DATANODE|127.0.0.1:29412|Version|Hash|Start_time|Duration|Duration||3|DATANODE|127.0.0.1:29413|Version|Hash|Start_time|Duration|Duration||0|FLOWNODE|127.0.0.1:29680|Version|Hash|Start_time|Duration|Duration||1|FRONTEND|127.0.0.1:29401|Version|Hash|Start_time|Duration|Duration||1|METASRV|127.0.0.1:29302|Version|Hash|Start_time|Duration||+++++++++
 
 -- SQLNESS REPLACE version node_version
 -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version
 -- SQLNESS REPLACE (\s[a-z0-9]{7,8}\s) Hash
--- SQLNESS REPLACE (\s[\-0-9T:\.]{15,}) Start_time
+-- SQLNESS REPLACE (\s[\-0-9T:\.]{19,}) Start_time
 -- SQLNESS REPLACE ((\d+(s|ms|m)\s)+) Duration
 -- SQLNESS REPLACE [\s\-]+
 SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE = 'METASRV' ORDER BY peer_type;
 
-+++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|1|METASRV|127.0.0.1:3002|Version|Hash|Start_time|Duration||+++++++++
++++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|1|METASRV|127.0.0.1:29302|Version|Hash|Start_time|Duration||+++++++++
 
 -- SQLNESS REPLACE version node_version
 -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version
 -- SQLNESS REPLACE (\s[a-z0-9]{7,8}\s) Hash
--- SQLNESS REPLACE (\s[\-0-9T:\.]{15,}) Start_time
+-- SQLNESS REPLACE (\s[\-0-9T:\.]{19,}) Start_time
 -- SQLNESS REPLACE ((\d+(s|ms|m)\s)+) Duration
 -- SQLNESS REPLACE [\s\-]+
 SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE = 'FRONTEND' ORDER BY peer_type;
 
-+++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|1|FRONTEND|127.0.0.1:4001|Version|Hash|Start_time|Duration|Duration|+++++++++
++++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|1|FRONTEND|127.0.0.1:29401|Version|Hash|Start_time|Duration|Duration|+++++++++
 
 -- SQLNESS REPLACE version node_version
 -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version
 -- SQLNESS REPLACE (\s[a-z0-9]{7,8}\s) Hash
--- SQLNESS REPLACE (\s[\-0-9T:\.]{15,}) Start_time
+-- SQLNESS REPLACE (\s[\-0-9T:\.]{19,}) Start_time
 -- SQLNESS REPLACE ((\d+(s|ms|m)\s)+) Duration
 -- SQLNESS REPLACE [\s\-]+
 SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE != 'FRONTEND' ORDER BY peer_type;
 
-+++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|1|DATANODE|127.0.0.1:4101|Version|Hash|Start_time|Duration|Duration||2|DATANODE|127.0.0.1:4102|Version|Hash|Start_time|Duration|Duration||3|DATANODE|127.0.0.1:4103|Version|Hash|Start_time|Duration|Duration||0|FLOWNODE|127.0.0.1:6800|Version|Hash|Start_time|Duration|Duration||1|METASRV|127.0.0.1:3002|Version|Hash|Start_time|Duration||+++++++++
++++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|1|DATANODE|127.0.0.1:29411|Version|Hash|Start_time|Duration|Duration||2|DATANODE|127.0.0.1:29412|Version|Hash|Start_time|Duration|Duration||3|DATANODE|127.0.0.1:29413|Version|Hash|Start_time|Duration|Duration||0|FLOWNODE|127.0.0.1:29680|Version|Hash|Start_time|Duration|Duration||1|METASRV|127.0.0.1:29302|Version|Hash|Start_time|Duration||+++++++++
 
 -- SQLNESS REPLACE version node_version
 -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version
 -- SQLNESS REPLACE (\s[a-z0-9]{7,8}\s) Hash
--- SQLNESS REPLACE (\s[\-0-9T:\.]{15,}) Start_time
+-- SQLNESS REPLACE (\s[\-0-9T:\.]{19,}) Start_time
 -- SQLNESS REPLACE ((\d+(s|ms|m)\s)+) Duration
 -- SQLNESS REPLACE [\s\-]+
 SELECT * FROM CLUSTER_INFO WHERE PEER_ID > 1 ORDER BY peer_type;
 
-+++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|2|DATANODE|127.0.0.1:4102|Version|Hash|Start_time|Duration|Duration||3|DATANODE|127.0.0.1:4103|Version|Hash|Start_time|Duration|Duration|+++++++++
++++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|2|DATANODE|127.0.0.1:29412|Version|Hash|Start_time|Duration|Duration||3|DATANODE|127.0.0.1:29413|Version|Hash|Start_time|Duration|Duration|+++++++++
 
 USE PUBLIC;
 
diff --git a/tests/cases/distributed/information_schema/cluster_info.sql b/tests/cases/distributed/information_schema/cluster_info.sql
index f017e2f7fd..ec0f3bb273 100644
--- a/tests/cases/distributed/information_schema/cluster_info.sql
+++ b/tests/cases/distributed/information_schema/cluster_info.sql
@@ -5,7 +5,7 @@ DESC TABLE CLUSTER_INFO;
 -- SQLNESS REPLACE version node_version
 -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version
 -- SQLNESS REPLACE (\s[a-z0-9]{7,8}\s) Hash
--- SQLNESS REPLACE (\s[\-0-9T:\.]{15,}) Start_time
+-- SQLNESS REPLACE (\s[\-0-9T:\.]{19,}) Start_time
 -- SQLNESS REPLACE ((\d+(s|ms|m)\s)+) Duration
 -- SQLNESS REPLACE [\s\-]+
 SELECT * FROM CLUSTER_INFO ORDER BY peer_type;
@@ -13,7 +13,7 @@ SELECT * FROM CLUSTER_INFO ORDER BY peer_type;
 -- SQLNESS REPLACE version node_version
 -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version
 -- SQLNESS REPLACE (\s[a-z0-9]{7,8}\s) Hash
--- SQLNESS REPLACE (\s[\-0-9T:\.]{15,}) Start_time
+-- SQLNESS REPLACE (\s[\-0-9T:\.]{19,}) Start_time
 -- SQLNESS REPLACE ((\d+(s|ms|m)\s)+) Duration
 -- SQLNESS REPLACE [\s\-]+
 SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE = 'METASRV' ORDER BY peer_type;
@@ -21,7 +21,7 @@ SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE = 'METASRV' ORDER BY peer_type;
 -- SQLNESS REPLACE version node_version
 -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version
 -- SQLNESS REPLACE (\s[a-z0-9]{7,8}\s) Hash
--- SQLNESS REPLACE (\s[\-0-9T:\.]{15,}) Start_time
+-- SQLNESS REPLACE (\s[\-0-9T:\.]{19,}) Start_time
 -- SQLNESS REPLACE ((\d+(s|ms|m)\s)+) Duration
 -- SQLNESS REPLACE [\s\-]+
 SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE = 'FRONTEND' ORDER BY peer_type;
@@ -29,7 +29,7 @@ SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE = 'FRONTEND' ORDER BY peer_type;
 -- SQLNESS REPLACE version node_version
 -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version
 -- SQLNESS REPLACE (\s[a-z0-9]{7,8}\s) Hash
--- SQLNESS REPLACE (\s[\-0-9T:\.]{15,}) Start_time
+-- SQLNESS REPLACE (\s[\-0-9T:\.]{19,}) Start_time
 -- SQLNESS REPLACE ((\d+(s|ms|m)\s)+) Duration
 -- SQLNESS REPLACE [\s\-]+
 SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE != 'FRONTEND' ORDER BY peer_type;
@@ -37,7 +37,7 @@ SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE != 'FRONTEND' ORDER BY peer_type;
 -- SQLNESS REPLACE version node_version
 -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version
 -- SQLNESS REPLACE (\s[a-z0-9]{7,8}\s) Hash
--- SQLNESS REPLACE (\s[\-0-9T:\.]{15,}) Start_time
+-- SQLNESS REPLACE (\s[\-0-9T:\.]{19,}) Start_time
 -- SQLNESS REPLACE ((\d+(s|ms|m)\s)+) Duration
 -- SQLNESS REPLACE [\s\-]+
 SELECT * FROM CLUSTER_INFO WHERE PEER_ID > 1 ORDER BY peer_type;
diff --git a/tests/conf/datanode-test.toml.template b/tests/conf/datanode-test.toml.template
index 04968bc956..5ed5352124 100644
--- a/tests/conf/datanode-test.toml.template
+++ b/tests/conf/datanode-test.toml.template
@@ -1,7 +1,7 @@
 node_id = 1
 mode = 'distributed'
 require_lease_before_startup = true
-rpc_addr = '127.0.0.1:4100'
+rpc_addr = '127.0.0.1:29410'
 rpc_hostname = '127.0.0.1'
 rpc_runtime_size = 8
 
@@ -24,7 +24,7 @@ type = 'File'
 data_home = '{data_home}'
 
 [meta_client_options]
-metasrv_addrs = ['127.0.0.1:3002']
+metasrv_addrs = ['127.0.0.1:29302']
 timeout_millis = 3000
 connect_timeout_millis = 5000
 tcp_nodelay = false
diff --git a/tests/conf/standalone-test.toml.template b/tests/conf/standalone-test.toml.template
index f96a815762..da51f3b3f4 100644
--- a/tests/conf/standalone-test.toml.template
+++ b/tests/conf/standalone-test.toml.template
@@ -20,13 +20,13 @@ linger = "5ms"
 type = 'File'
 data_home = '{data_home}'
 
-[grpc_options]
-addr = '127.0.0.1:4001'
+[grpc]
+addr = '127.0.0.1:29401'
 runtime_size = 8
 
 [mysql]
 enable = true
-addr = "127.0.0.1:4002"
+addr = "127.0.0.1:29402"
 runtime_size = 2
 
 [mysql.tls]
@@ -34,7 +34,7 @@ mode = "disable"
 
 [postgres]
 enable = true
-addr = "127.0.0.1:4003"
+addr = "127.0.0.1:29403"
 runtime_size = 2
 
 [procedure]
diff --git a/tests/runner/src/env.rs b/tests/runner/src/env.rs
index 0dc0e3a9e8..bb5d74a267 100644
--- a/tests/runner/src/env.rs
+++ b/tests/runner/src/env.rs
@@ -47,10 +47,10 @@ use tokio_postgres::{Client as PgClient, SimpleQueryMessage as PgRow};
 use crate::protocol_interceptor::{MYSQL, PROTOCOL_KEY};
 use crate::{util, ServerAddr};
 
-const METASRV_ADDR: &str = "127.0.0.1:3002";
-const GRPC_SERVER_ADDR: &str = "127.0.0.1:4001";
-const MYSQL_SERVER_ADDR: &str = "127.0.0.1:4002";
-const POSTGRES_SERVER_ADDR: &str = "127.0.0.1:4003";
+const METASRV_ADDR: &str = "127.0.0.1:29302";
+const GRPC_SERVER_ADDR: &str = "127.0.0.1:29401";
+const MYSQL_SERVER_ADDR: &str = "127.0.0.1:29402";
+const POSTGRES_SERVER_ADDR: &str = "127.0.0.1:29403";
 const DEFAULT_LOG_LEVEL: &str = "--log-level=debug,hyper=warn,tower=warn,datafusion=warn,reqwest=warn,sqlparser=warn,h2=info,opendal=info";
 
 #[derive(Clone)]
@@ -305,34 +305,55 @@ impl Env {
                     ),
                     "-c".to_string(),
                     self.generate_config_file(subcommand, db_ctx),
-                    "--http-addr=127.0.0.1:5002".to_string(),
+                    "--http-addr=127.0.0.1:29502".to_string(),
                 ];
-                (args, vec![GRPC_SERVER_ADDR.to_string()])
+                (
+                    args,
+                    vec![
+                        GRPC_SERVER_ADDR.to_string(),
+                        MYSQL_SERVER_ADDR.to_string(),
+                        POSTGRES_SERVER_ADDR.to_string(),
+                    ],
+                )
             }
             "frontend" => {
                 let args = vec![
                     DEFAULT_LOG_LEVEL.to_string(),
                     subcommand.to_string(),
                     "start".to_string(),
-                    "--metasrv-addrs=127.0.0.1:3002".to_string(),
-                    "--http-addr=127.0.0.1:5003".to_string(),
+                    "--metasrv-addrs=127.0.0.1:29302".to_string(),
+                    "--http-addr=127.0.0.1:29503".to_string(),
+                    format!("--rpc-addr={}", GRPC_SERVER_ADDR),
+                    format!("--mysql-addr={}", MYSQL_SERVER_ADDR),
+                    format!("--postgres-addr={}", POSTGRES_SERVER_ADDR),
                     format!(
                         "--log-dir={}/greptimedb-frontend/logs",
                         self.sqlness_home.display()
                     ),
                 ];
-                (args, vec![GRPC_SERVER_ADDR.to_string()])
+                (
+                    args,
+                    vec![
+                        GRPC_SERVER_ADDR.to_string(),
+                        MYSQL_SERVER_ADDR.to_string(),
+                        POSTGRES_SERVER_ADDR.to_string(),
+                    ],
+                )
             }
             "metasrv" => {
                 let args = vec![
                     DEFAULT_LOG_LEVEL.to_string(),
                     subcommand.to_string(),
                     "start".to_string(),
+                    "--bind-addr".to_string(),
+                    "127.0.0.1:29302".to_string(),
+                    "--server-addr".to_string(),
+                    "127.0.0.1:29302".to_string(),
                     "--backend".to_string(),
                     "memory-store".to_string(),
                     "--enable-region-failover".to_string(),
                     "false".to_string(),
-                    "--http-addr=127.0.0.1:5002".to_string(),
+                    "--http-addr=127.0.0.1:29502".to_string(),
                     format!(
                         "--log-dir={}/greptimedb-metasrv/logs",
                         self.sqlness_home.display()
@@ -396,15 +417,15 @@ impl Env {
             subcommand.to_string(),
             "start".to_string(),
         ];
-        args.push(format!("--rpc-addr=127.0.0.1:410{id}"));
-        args.push(format!("--http-addr=127.0.0.1:430{id}"));
+        args.push(format!("--rpc-addr=127.0.0.1:2941{id}"));
+        args.push(format!("--http-addr=127.0.0.1:2943{id}"));
         args.push(format!("--data-home={}", data_home.display()));
         args.push(format!("--log-dir={}/logs", data_home.display()));
         args.push(format!("--node-id={id}"));
         args.push("-c".to_string());
         args.push(self.generate_config_file(subcommand, db_ctx));
-        args.push("--metasrv-addrs=127.0.0.1:3002".to_string());
-        (args, format!("127.0.0.1:410{id}"))
+        args.push("--metasrv-addrs=127.0.0.1:29302".to_string());
+        (args, format!("127.0.0.1:2941{id}"))
     }
 
     fn flownode_start_args(
@@ -420,14 +441,14 @@ impl Env {
             subcommand.to_string(),
             "start".to_string(),
         ];
-        args.push(format!("--rpc-addr=127.0.0.1:680{id}"));
+        args.push(format!("--rpc-addr=127.0.0.1:2968{id}"));
         args.push(format!("--node-id={id}"));
         args.push(format!(
             "--log-dir={}/greptimedb-flownode/logs",
             sqlness_home.display()
         ));
-        args.push("--metasrv-addrs=127.0.0.1:3002".to_string());
-        (args, format!("127.0.0.1:680{id}"))
+        args.push("--metasrv-addrs=127.0.0.1:29302".to_string());
+        (args, format!("127.0.0.1:2968{id}"))
     }
 
     /// stop and restart the server process

From 2cdd103874a649282612bd05f94023fc0556f6b2 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Tue, 8 Oct 2024 17:02:06 +0800
Subject: [PATCH 056/128] feat: introduce
 `HeartbeatHandlerGroupBuilderCustomizer` (#4803)

* feat: introduce `HeartbeatHandlerGroupBuilderFinalizer`

* chore: rename to `HeartbeatHandlerGroupBuilderCustomizer`
---
 src/meta-srv/src/handler.rs         | 42 +++++++++++++++++++++--------
 src/meta-srv/src/metasrv/builder.rs |  2 +-
 2 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/src/meta-srv/src/handler.rs b/src/meta-srv/src/handler.rs
index 9d82920c08..b45a1ac7e3 100644
--- a/src/meta-srv/src/handler.rs
+++ b/src/meta-srv/src/handler.rs
@@ -527,11 +527,22 @@ impl HeartbeatHandlerGroupBuilder {
     }
 
     /// Builds the group of heartbeat handlers.
-    pub fn build(self) -> HeartbeatHandlerGroup {
-        HeartbeatHandlerGroup {
+    ///
+    /// Applies the customizer if it exists.
+    pub fn build(mut self) -> Result<HeartbeatHandlerGroup> {
+        if let Some(customizer) = self
+            .plugins
+            .as_ref()
+            .and_then(|plugins| plugins.get::<HeartbeatHandlerGroupBuilderCustomizerRef>())
+        {
+            debug!("Customizing the heartbeat handler group builder");
+            customizer.customize(&mut self)?;
+        }
+
+        Ok(HeartbeatHandlerGroup {
             handlers: self.handlers.into_iter().collect(),
             pushers: self.pushers,
-        }
+        })
     }
 
     /// Adds the handler after the specified handler.
@@ -582,6 +593,14 @@ impl HeartbeatHandlerGroupBuilder {
     }
 }
 
+pub type HeartbeatHandlerGroupBuilderCustomizerRef =
+    Arc<dyn HeartbeatHandlerGroupBuilderCustomizer>;
+
+/// The customizer of the [`HeartbeatHandlerGroupBuilder`].
+pub trait HeartbeatHandlerGroupBuilderCustomizer: Send + Sync {
+    fn customize(&self, builder: &mut HeartbeatHandlerGroupBuilder) -> Result<()>;
+}
+
 #[cfg(test)]
 mod tests {
 
@@ -670,7 +689,8 @@ mod tests {
     fn test_handler_group_builder() {
         let group = HeartbeatHandlerGroupBuilder::new(Pushers::default())
             .add_default_handlers()
-            .build();
+            .build()
+            .unwrap();
 
         let handlers = group.handlers;
         assert_eq!(12, handlers.len());
@@ -706,7 +726,7 @@ mod tests {
             )
             .unwrap();
 
-        let group = builder.build();
+        let group = builder.build().unwrap();
         let handlers = group.handlers;
         assert_eq!(13, handlers.len());
 
@@ -739,7 +759,7 @@ mod tests {
             .add_handler_before("ResponseHeaderHandler", CollectStatsHandler::default())
             .unwrap();
 
-        let group = builder.build();
+        let group = builder.build().unwrap();
         let handlers = group.handlers;
         assert_eq!(13, handlers.len());
 
@@ -772,7 +792,7 @@ mod tests {
             .add_handler_after("MailboxHandler", CollectStatsHandler::default())
             .unwrap();
 
-        let group = builder.build();
+        let group = builder.build().unwrap();
         let handlers = group.handlers;
         assert_eq!(13, handlers.len());
 
@@ -805,7 +825,7 @@ mod tests {
             .add_handler_after("CollectStatsHandler", ResponseHeaderHandler)
             .unwrap();
 
-        let group = builder.build();
+        let group = builder.build().unwrap();
         let handlers = group.handlers;
         assert_eq!(13, handlers.len());
 
@@ -838,7 +858,7 @@ mod tests {
             .replace_handler("MailboxHandler", CollectStatsHandler::default())
             .unwrap();
 
-        let group = builder.build();
+        let group = builder.build().unwrap();
         let handlers = group.handlers;
         assert_eq!(12, handlers.len());
 
@@ -870,7 +890,7 @@ mod tests {
             .replace_handler("CollectStatsHandler", ResponseHeaderHandler)
             .unwrap();
 
-        let group = builder.build();
+        let group = builder.build().unwrap();
         let handlers = group.handlers;
         assert_eq!(12, handlers.len());
 
@@ -902,7 +922,7 @@ mod tests {
             .replace_handler("ResponseHeaderHandler", CollectStatsHandler::default())
             .unwrap();
 
-        let group = builder.build();
+        let group = builder.build().unwrap();
         let handlers = group.handlers;
         assert_eq!(12, handlers.len());
 
diff --git a/src/meta-srv/src/metasrv/builder.rs b/src/meta-srv/src/metasrv/builder.rs
index 88d7e5bd6d..050f825174 100644
--- a/src/meta-srv/src/metasrv/builder.rs
+++ b/src/meta-srv/src/metasrv/builder.rs
@@ -363,7 +363,7 @@ impl MetasrvBuilder {
                     .with_region_failure_handler(region_failover_handler)
                     .with_region_lease_handler(Some(region_lease_handler))
                     .add_default_handlers()
-                    .build()
+                    .build()?
             }
         };
 

From 71a66d15f7e8353000f966dd1dc05bc58733cf15 Mon Sep 17 00:00:00 2001
From: localhost <xpaomian@gmail.com>
Date: Tue, 8 Oct 2024 20:11:09 +0800
Subject: [PATCH 057/128] chore: add json write (#4744)

* chore: add json write

* chore: add test for write json log api

* chore: enhancement of Error Handling

* chore: fix by pr comment

* chore: fix by pr comment

* chore: enhancement of error content and add some doc
---
 Cargo.lock                                    |   1 +
 src/pipeline/Cargo.toml                       |   1 +
 src/pipeline/src/etl/error.rs                 | 118 ++++---
 src/pipeline/src/etl/transform/transformer.rs |   1 +
 .../src/etl/transform/transformer/greptime.rs | 313 +++++++++++++++++-
 src/pipeline/src/lib.rs                       |   1 +
 src/servers/src/http/event.rs                 |  90 ++---
 tests-integration/tests/http.rs               |  71 ++++
 8 files changed, 505 insertions(+), 91 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 418e68a88a..c2356a4c2e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8141,6 +8141,7 @@ dependencies = [
  "futures",
  "greptime-proto",
  "itertools 0.10.5",
+ "jsonb",
  "lazy_static",
  "moka",
  "once_cell",
diff --git a/src/pipeline/Cargo.toml b/src/pipeline/Cargo.toml
index 5b85b999fe..2402605f7f 100644
--- a/src/pipeline/Cargo.toml
+++ b/src/pipeline/Cargo.toml
@@ -40,6 +40,7 @@ enum_dispatch = "0.3"
 futures.workspace = true
 greptime-proto.workspace = true
 itertools.workspace = true
+jsonb.workspace = true
 lazy_static.workspace = true
 moka = { workspace = true, features = ["sync"] }
 once_cell.workspace = true
diff --git a/src/pipeline/src/etl/error.rs b/src/pipeline/src/etl/error.rs
index 3680053ba0..0a28cb7423 100644
--- a/src/pipeline/src/etl/error.rs
+++ b/src/pipeline/src/etl/error.rs
@@ -40,7 +40,7 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("{processor} processor: missing field: {field}"))]
+    #[snafu(display("Processor {processor}: missing field: {field}"))]
     ProcessorMissingField {
         processor: String,
         field: String,
@@ -48,7 +48,7 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("{processor} processor: expect string value, but got {v:?}"))]
+    #[snafu(display("Processor {processor}: expect string value, but got {v:?}"))]
     ProcessorExpectString {
         processor: String,
         v: crate::etl::Value,
@@ -56,7 +56,7 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("{processor} processor: unsupported value {val}"))]
+    #[snafu(display("Processor {processor}: unsupported value {val}"))]
     ProcessorUnsupportedValue {
         processor: &'static str,
         val: String,
@@ -64,13 +64,13 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("processor key must be a string"))]
+    #[snafu(display("Processor key must be a string"))]
     ProcessorKeyMustBeString {
         #[snafu(implicit)]
         location: Location,
     },
 
-    #[snafu(display("{kind} processor: failed to parse {value}"))]
+    #[snafu(display("Processor {kind}: failed to parse {value}"))]
     ProcessorFailedToParseString {
         kind: String,
         value: String,
@@ -78,13 +78,13 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("processor must have a string key"))]
+    #[snafu(display("Processor must have a string key"))]
     ProcessorMustHaveStringKey {
         #[snafu(implicit)]
         location: Location,
     },
 
-    #[snafu(display("unsupported {processor} processor"))]
+    #[snafu(display("Unsupported {processor} processor"))]
     UnsupportedProcessor {
         processor: String,
         #[snafu(implicit)]
@@ -108,7 +108,7 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("failed to parse {key} as int: {value}"))]
+    #[snafu(display("Failed to parse {key} as int: {value}"))]
     FailedToParseIntKey {
         key: String,
         value: String,
@@ -126,7 +126,7 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("failed to parse {key} as float: {value}"))]
+    #[snafu(display("Failed to parse {key} as float: {value}"))]
     FailedToParseFloatKey {
         key: String,
         value: String,
@@ -136,7 +136,7 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("{kind} processor.{key} not found in intermediate keys"))]
+    #[snafu(display("Processor {kind}: {key} not found in intermediate keys"))]
     IntermediateKeyIndex {
         kind: String,
         key: String,
@@ -144,41 +144,41 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("{k} missing value in {s}"))]
+    #[snafu(display("Cmcd {k} missing value in {s}"))]
     CmcdMissingValue {
         k: String,
         s: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("{part} missing key in {s}"))]
+    #[snafu(display("Part: {part} missing key in {s}"))]
     CmcdMissingKey {
         part: String,
         s: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("key must be a string, but got {k:?}"))]
+    #[snafu(display("Key must be a string, but got {k:?}"))]
     KeyMustBeString {
         k: yaml_rust::Yaml,
         #[snafu(implicit)]
         location: Location,
     },
 
-    #[snafu(display("csv read error"))]
+    #[snafu(display("Csv read error"))]
     CsvRead {
         #[snafu(implicit)]
         location: Location,
         #[snafu(source)]
         error: csv::Error,
     },
-    #[snafu(display("expected at least one record from csv format, but got none"))]
+    #[snafu(display("Expected at least one record from csv format, but got none"))]
     CsvNoRecord {
         #[snafu(implicit)]
         location: Location,
     },
 
-    #[snafu(display("'{separator}' must be a single character, but got '{value}'"))]
+    #[snafu(display("Separator '{separator}' must be a single character, but got '{value}'"))]
     CsvSeparatorName {
         separator: &'static str,
         value: String,
@@ -186,7 +186,7 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("'{quote}' must be a single character, but got '{value}'"))]
+    #[snafu(display("Quote '{quote}' must be a single character, but got '{value}'"))]
     CsvQuoteName {
         quote: &'static str,
         value: String,
@@ -212,19 +212,19 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("failed to get local timezone"))]
+    #[snafu(display("Failed to get local timezone"))]
     DateFailedToGetLocalTimezone {
         #[snafu(implicit)]
         location: Location,
     },
 
-    #[snafu(display("failed to get timestamp"))]
+    #[snafu(display("Failed to get timestamp"))]
     DateFailedToGetTimestamp {
         #[snafu(implicit)]
         location: Location,
     },
 
-    #[snafu(display("{processor} processor: invalid format {s}"))]
+    #[snafu(display("Processor {processor}: invalid format {s}"))]
     DateInvalidFormat {
         s: String,
         processor: String,
@@ -245,20 +245,20 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("'{split}' exceeds the input"))]
+    #[snafu(display("Split: '{split}' exceeds the input"))]
     DissectSplitExceedsInput {
         split: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("'{split}' does not match the input '{input}'"))]
+    #[snafu(display("Split: '{split}' does not match the input '{input}'"))]
     DissectSplitNotMatchInput {
         split: String,
         input: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("consecutive names are not allowed: '{name1}' '{name2}'"))]
+    #[snafu(display("Consecutive names are not allowed: '{name1}' '{name2}'"))]
     DissectConsecutiveNames {
         name1: String,
         name2: String,
@@ -270,7 +270,7 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("'{m}' modifier already set, but found {modifier}"))]
+    #[snafu(display("Modifier '{m}' already set, but found {modifier}"))]
     DissectModifierAlreadySet {
         m: String,
         modifier: String,
@@ -304,23 +304,23 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("invalid resolution: {resolution}"))]
+    #[snafu(display("Invalid resolution: {resolution}"))]
     EpochInvalidResolution {
         resolution: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("pattern is required"))]
+    #[snafu(display("Pattern is required"))]
     GsubPatternRequired {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("replacement is required"))]
+    #[snafu(display("Replacement is required"))]
     GsubReplacementRequired {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("invalid regex pattern: {pattern}"))]
+    #[snafu(display("Invalid regex pattern: {pattern}"))]
     Regex {
         #[snafu(source)]
         error: regex::Error,
@@ -328,72 +328,72 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("separator is required"))]
+    #[snafu(display("Separator is required"))]
     JoinSeparatorRequired {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("invalid method: {method}"))]
+    #[snafu(display("Invalid method: {method}"))]
     LetterInvalidMethod {
         method: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("no named group found in regex {origin}"))]
+    #[snafu(display("No named group found in regex {origin}"))]
     RegexNamedGroupNotFound {
         origin: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("no valid field found in {processor} processor"))]
+    #[snafu(display("No valid field found in {processor} processor"))]
     RegexNoValidField {
         processor: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("no valid pattern found in {processor} processor"))]
+    #[snafu(display("No valid pattern found in {processor} processor"))]
     RegexNoValidPattern {
         processor: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("invalid method: {s}"))]
+    #[snafu(display("Invalid method: {s}"))]
     UrlEncodingInvalidMethod {
         s: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("url decoding error"))]
+    #[snafu(display("Url decoding error"))]
     UrlEncodingDecode {
         #[snafu(source)]
         error: std::string::FromUtf8Error,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("invalid transform on_failure value: {value}"))]
+    #[snafu(display("Invalid transform on_failure value: {value}"))]
     TransformOnFailureInvalidValue {
         value: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("transform element must be a map"))]
+    #[snafu(display("Transform element must be a map"))]
     TransformElementMustBeMap {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("transform {fields:?} type MUST BE set before default {default}"))]
+    #[snafu(display("Transform {fields:?} type MUST BE set before default {default}"))]
     TransformTypeMustBeSet {
         fields: String,
         default: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("transform cannot be empty"))]
+    #[snafu(display("Transform cannot be empty"))]
     TransformEmpty {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("column name must be unique, but got duplicated: {duplicates}"))]
+    #[snafu(display("Column name must be unique, but got duplicated: {duplicates}"))]
     TransformColumnNameMustBeUnique {
         duplicates: String,
         #[snafu(implicit)]
@@ -407,7 +407,7 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("transform must have exactly one field specified as timestamp Index, but got {count}: {columns}"))]
+    #[snafu(display("Transform must have exactly one field specified as timestamp Index, but got {count}: {columns}"))]
     TransformTimestampIndexCount {
         count: usize,
         columns: String,
@@ -425,13 +425,13 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("{ty} value not supported for Epoch"))]
+    #[snafu(display("Type: {ty} value not supported for Epoch"))]
     CoerceUnsupportedEpochType {
         ty: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("failed to coerce string value '{s}' to type '{ty}'"))]
+    #[snafu(display("Failed to coerce string value '{s}' to type '{ty}'"))]
     CoerceStringToType {
         s: String,
         ty: String,
@@ -440,7 +440,7 @@ pub enum Error {
     },
 
     #[snafu(display(
-        "invalid resolution: '{resolution}'. Available resolutions: {valid_resolution}"
+        "Invalid resolution: '{resolution}'. Available resolutions: {valid_resolution}"
     ))]
     ValueInvalidResolution {
         resolution: String,
@@ -449,14 +449,14 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("failed to parse type: '{t}'"))]
+    #[snafu(display("Failed to parse type: '{t}'"))]
     ValueParseType {
         t: String,
         #[snafu(implicit)]
         location: Location,
     },
 
-    #[snafu(display("failed to parse {ty}: {v}"))]
+    #[snafu(display("Failed to parse {ty}: {v}"))]
     ValueParseInt {
         ty: String,
         v: String,
@@ -466,7 +466,7 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("failed to parse {ty}: {v}"))]
+    #[snafu(display("Failed to parse {ty}: {v}"))]
     ValueParseFloat {
         ty: String,
         v: String,
@@ -476,7 +476,7 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("failed to parse {ty}: {v}"))]
+    #[snafu(display("Failed to parse {ty}: {v}"))]
     ValueParseBoolean {
         ty: String,
         v: String,
@@ -485,19 +485,19 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("default value not unsupported for type {value}"))]
+    #[snafu(display("Default value not unsupported for type {value}"))]
     ValueDefaultValueUnsupported {
         value: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("unsupported number type: {value}"))]
+    #[snafu(display("Unsupported number type: {value}"))]
     ValueUnsupportedNumberType {
         value: serde_json::Number,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("unsupported yaml type: {value:?}"))]
+    #[snafu(display("Unsupported yaml type: {value:?}"))]
     ValueUnsupportedYamlType {
         value: yaml_rust::Yaml,
         #[snafu(implicit)]
@@ -531,12 +531,26 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("unsupported index type: {value}"))]
+    #[snafu(display("Unsupported index type: {value}"))]
     UnsupportedIndexType {
         value: String,
         #[snafu(implicit)]
         location: Location,
     },
+    #[snafu(display("Unsupported number type: {value:?}"))]
+    UnsupportedNumberType {
+        value: serde_json::Number,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("Column datatype mismatch. For column: {column}, expected datatype: {expected}, actual datatype: {actual}"))]
+    IdentifyPipelineColumnTypeMismatch {
+        column: String,
+        expected: String,
+        actual: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
diff --git a/src/pipeline/src/etl/transform/transformer.rs b/src/pipeline/src/etl/transform/transformer.rs
index 28aacdcbfb..7a18524662 100644
--- a/src/pipeline/src/etl/transform/transformer.rs
+++ b/src/pipeline/src/etl/transform/transformer.rs
@@ -13,3 +13,4 @@
 // limitations under the License.
 
 pub mod greptime;
+pub use greptime::identity_pipeline;
diff --git a/src/pipeline/src/etl/transform/transformer/greptime.rs b/src/pipeline/src/etl/transform/transformer/greptime.rs
index 000a2ddc26..11d63c4503 100644
--- a/src/pipeline/src/etl/transform/transformer/greptime.rs
+++ b/src/pipeline/src/etl/transform/transformer/greptime.rs
@@ -16,13 +16,20 @@ pub mod coerce;
 
 use std::collections::HashSet;
 
+use ahash::HashMap;
+use api::helper::proto_value_type;
+use api::v1::column_data_type_extension::TypeExt;
+use api::v1::value::ValueData;
+use api::v1::{ColumnDataType, ColumnDataTypeExtension, JsonTypeExtension, SemanticType};
 use coerce::{coerce_columns, coerce_value};
 use greptime_proto::v1::{ColumnSchema, Row, Rows, Value as GreptimeValue};
 use itertools::Itertools;
+use serde_json::{Map, Number};
 
 use crate::etl::error::{
-    Result, TransformColumnNameMustBeUniqueSnafu, TransformEmptySnafu,
-    TransformMultipleTimestampIndexSnafu, TransformTimestampIndexCountSnafu,
+    IdentifyPipelineColumnTypeMismatchSnafu, Result, TransformColumnNameMustBeUniqueSnafu,
+    TransformEmptySnafu, TransformMultipleTimestampIndexSnafu, TransformTimestampIndexCountSnafu,
+    UnsupportedNumberTypeSnafu,
 };
 use crate::etl::field::{InputFieldInfo, OneInputOneOutputField};
 use crate::etl::transform::index::Index;
@@ -120,6 +127,7 @@ impl Transformer for GreptimeTransformer {
             if let Some(idx) = transform.index {
                 if idx == Index::Time {
                     match transform.real_fields.len() {
+                        //Safety unwrap is fine here because we have checked the length of real_fields
                         1 => timestamp_columns
                             .push(transform.real_fields.first().unwrap().input_name()),
                         _ => {
@@ -194,3 +202,304 @@ impl Transformer for GreptimeTransformer {
         &mut self.transforms
     }
 }
+
+/// This is used to record the current state schema information and a sequential cache of field names.
+/// As you traverse the user input JSON, this will change.
+/// It will record a superset of all user input schemas.
+#[derive(Debug, Default)]
+struct SchemaInfo {
+    /// schema info
+    schema: Vec<ColumnSchema>,
+    /// index of the column name
+    index: HashMap<String, usize>,
+}
+
+fn resolve_schema(
+    index: Option<usize>,
+    value_data: ValueData,
+    column_schema: ColumnSchema,
+    row: &mut Vec<GreptimeValue>,
+    schema_info: &mut SchemaInfo,
+) -> Result<()> {
+    if let Some(index) = index {
+        let api_value = GreptimeValue {
+            value_data: Some(value_data),
+        };
+        // Safety unwrap is fine here because api_value is always valid
+        let value_column_data_type = proto_value_type(&api_value).unwrap();
+        // Safety unwrap is fine here because index is always valid
+        let schema_column_data_type = schema_info.schema.get(index).unwrap().datatype();
+        if value_column_data_type != schema_column_data_type {
+            IdentifyPipelineColumnTypeMismatchSnafu {
+                column: column_schema.column_name,
+                expected: schema_column_data_type.as_str_name(),
+                actual: value_column_data_type.as_str_name(),
+            }
+            .fail()
+        } else {
+            row[index] = api_value;
+            Ok(())
+        }
+    } else {
+        let key = column_schema.column_name.clone();
+        schema_info.schema.push(column_schema);
+        schema_info.index.insert(key, schema_info.schema.len() - 1);
+        let api_value = GreptimeValue {
+            value_data: Some(value_data),
+        };
+        row.push(api_value);
+        Ok(())
+    }
+}
+
+fn resolve_number_schema(
+    n: Number,
+    column_name: String,
+    index: Option<usize>,
+    row: &mut Vec<GreptimeValue>,
+    schema_info: &mut SchemaInfo,
+) -> Result<()> {
+    let (value, datatype, semantic_type) = if n.is_i64() {
+        (
+            ValueData::I64Value(n.as_i64().unwrap()),
+            ColumnDataType::Int64 as i32,
+            SemanticType::Field as i32,
+        )
+    } else if n.is_u64() {
+        (
+            ValueData::U64Value(n.as_u64().unwrap()),
+            ColumnDataType::Uint64 as i32,
+            SemanticType::Field as i32,
+        )
+    } else if n.is_f64() {
+        (
+            ValueData::F64Value(n.as_f64().unwrap()),
+            ColumnDataType::Float64 as i32,
+            SemanticType::Field as i32,
+        )
+    } else {
+        return UnsupportedNumberTypeSnafu { value: n }.fail();
+    };
+    resolve_schema(
+        index,
+        value,
+        ColumnSchema {
+            column_name,
+            datatype,
+            semantic_type,
+            datatype_extension: None,
+            options: None,
+        },
+        row,
+        schema_info,
+    )
+}
+
+fn json_value_to_row(
+    schema_info: &mut SchemaInfo,
+    map: Map<String, serde_json::Value>,
+) -> Result<Row> {
+    let mut row: Vec<GreptimeValue> = Vec::with_capacity(schema_info.schema.len());
+    for _ in 0..schema_info.schema.len() {
+        row.push(GreptimeValue { value_data: None });
+    }
+    for (column_name, value) in map {
+        if column_name == DEFAULT_GREPTIME_TIMESTAMP_COLUMN {
+            continue;
+        }
+        let index = schema_info.index.get(&column_name).copied();
+        match value {
+            serde_json::Value::Null => {
+                // do nothing
+            }
+            serde_json::Value::String(s) => {
+                resolve_schema(
+                    index,
+                    ValueData::StringValue(s),
+                    ColumnSchema {
+                        column_name,
+                        datatype: ColumnDataType::String as i32,
+                        semantic_type: SemanticType::Field as i32,
+                        datatype_extension: None,
+                        options: None,
+                    },
+                    &mut row,
+                    schema_info,
+                )?;
+            }
+            serde_json::Value::Bool(b) => {
+                resolve_schema(
+                    index,
+                    ValueData::BoolValue(b),
+                    ColumnSchema {
+                        column_name,
+                        datatype: ColumnDataType::Boolean as i32,
+                        semantic_type: SemanticType::Field as i32,
+                        datatype_extension: None,
+                        options: None,
+                    },
+                    &mut row,
+                    schema_info,
+                )?;
+            }
+            serde_json::Value::Number(n) => {
+                resolve_number_schema(n, column_name, index, &mut row, schema_info)?;
+            }
+            serde_json::Value::Array(_) | serde_json::Value::Object(_) => {
+                resolve_schema(
+                    index,
+                    ValueData::BinaryValue(jsonb::Value::from(value).to_vec()),
+                    ColumnSchema {
+                        column_name,
+                        datatype: ColumnDataType::Binary as i32,
+                        semantic_type: SemanticType::Field as i32,
+                        datatype_extension: Some(ColumnDataTypeExtension {
+                            type_ext: Some(TypeExt::JsonType(JsonTypeExtension::JsonBinary.into())),
+                        }),
+                        options: None,
+                    },
+                    &mut row,
+                    schema_info,
+                )?;
+            }
+        }
+    }
+    Ok(Row { values: row })
+}
+
+/// Identity pipeline for Greptime
+/// This pipeline will convert the input JSON array to Greptime Rows
+/// 1. The pipeline will add a default timestamp column to the schema
+/// 2. The pipeline not resolve NULL value
+/// 3. The pipeline assumes that the json format is fixed
+/// 4. The pipeline will return an error if the same column datatype is mismatched
+/// 5. The pipeline will analyze the schema of each json record and merge them to get the final schema.
+pub fn identity_pipeline(array: Vec<serde_json::Value>) -> Result<Rows> {
+    let mut rows = Vec::with_capacity(array.len());
+
+    let mut schema = SchemaInfo::default();
+    for value in array {
+        if let serde_json::Value::Object(map) = value {
+            let row = json_value_to_row(&mut schema, map)?;
+            rows.push(row);
+        }
+    }
+    let greptime_timestamp_schema = ColumnSchema {
+        column_name: DEFAULT_GREPTIME_TIMESTAMP_COLUMN.to_string(),
+        datatype: ColumnDataType::TimestampNanosecond as i32,
+        semantic_type: SemanticType::Timestamp as i32,
+        datatype_extension: None,
+        options: None,
+    };
+    let ns = chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0);
+    let ts = GreptimeValue {
+        value_data: Some(ValueData::TimestampNanosecondValue(ns)),
+    };
+    let column_count = schema.schema.len();
+    for row in rows.iter_mut() {
+        let diff = column_count - row.values.len();
+        for _ in 0..diff {
+            row.values.push(GreptimeValue { value_data: None });
+        }
+        row.values.push(ts.clone());
+    }
+    schema.schema.push(greptime_timestamp_schema);
+    Ok(Rows {
+        schema: schema.schema,
+        rows,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::identity_pipeline;
+
+    #[test]
+    fn test_identify_pipeline() {
+        {
+            let array = vec![
+                serde_json::json!({
+                    "woshinull": null,
+                    "name": "Alice",
+                    "age": 20,
+                    "is_student": true,
+                    "score": 99.5,
+                    "hobbies": "reading",
+                    "address": "Beijing",
+                }),
+                serde_json::json!({
+                    "name": "Bob",
+                    "age": 21,
+                    "is_student": false,
+                    "score": "88.5",
+                    "hobbies": "swimming",
+                    "address": "Shanghai",
+                    "gaga": "gaga"
+                }),
+            ];
+            let rows = identity_pipeline(array);
+            assert!(rows.is_err());
+            assert_eq!(
+                rows.err().unwrap().to_string(),
+                "Column datatype mismatch. For column: score, expected datatype: FLOAT64, actual datatype: STRING".to_string(),
+            );
+        }
+        {
+            let array = vec![
+                serde_json::json!({
+                    "woshinull": null,
+                    "name": "Alice",
+                    "age": 20,
+                    "is_student": true,
+                    "score": 99.5,
+                    "hobbies": "reading",
+                    "address": "Beijing",
+                }),
+                serde_json::json!({
+                    "name": "Bob",
+                    "age": 21,
+                    "is_student": false,
+                    "score": 88,
+                    "hobbies": "swimming",
+                    "address": "Shanghai",
+                    "gaga": "gaga"
+                }),
+            ];
+            let rows = identity_pipeline(array);
+            assert!(rows.is_err());
+            assert_eq!(
+                rows.err().unwrap().to_string(),
+                "Column datatype mismatch. For column: score, expected datatype: FLOAT64, actual datatype: INT64".to_string(),
+            );
+        }
+        {
+            let array = vec![
+                serde_json::json!({
+                    "woshinull": null,
+                    "name": "Alice",
+                    "age": 20,
+                    "is_student": true,
+                    "score": 99.5,
+                    "hobbies": "reading",
+                    "address": "Beijing",
+                }),
+                serde_json::json!({
+                    "name": "Bob",
+                    "age": 21,
+                    "is_student": false,
+                    "score": 88.5,
+                    "hobbies": "swimming",
+                    "address": "Shanghai",
+                    "gaga": "gaga"
+                }),
+            ];
+            let rows = identity_pipeline(array);
+            assert!(rows.is_ok());
+            let rows = rows.unwrap();
+            assert_eq!(rows.schema.len(), 8);
+            assert_eq!(rows.rows.len(), 2);
+            assert_eq!(8, rows.rows[0].values.len());
+            assert_eq!(8, rows.rows[1].values.len());
+        }
+    }
+}
diff --git a/src/pipeline/src/lib.rs b/src/pipeline/src/lib.rs
index 8fc72c5844..36ef3023f6 100644
--- a/src/pipeline/src/lib.rs
+++ b/src/pipeline/src/lib.rs
@@ -18,6 +18,7 @@ mod metrics;
 
 pub use etl::error::Result;
 pub use etl::processor::Processor;
+pub use etl::transform::transformer::identity_pipeline;
 pub use etl::transform::{GreptimeTransformer, Transformer};
 pub use etl::value::{Array, Map, Value};
 pub use etl::{parse, Content, Pipeline};
diff --git a/src/servers/src/http/event.rs b/src/servers/src/http/event.rs
index 7877d2b84a..80f15ff211 100644
--- a/src/servers/src/http/event.rs
+++ b/src/servers/src/http/event.rs
@@ -50,6 +50,9 @@ use crate::metrics::{
 };
 use crate::query_handler::LogHandlerRef;
 
+const GREPTIME_INTERNAL_PIPELINE_NAME_PREFIX: &str = "greptime_";
+const GREPTIME_INTERNAL_IDENTITY_PIPELINE_NAME: &str = "greptime_identity";
+
 #[derive(Debug, Default, Serialize, Deserialize, JsonSchema)]
 pub struct LogIngesterQueryParams {
     pub table: Option<String>,
@@ -121,6 +124,12 @@ pub async fn add_pipeline(
             reason: "pipeline_name is required in path",
         }
     );
+    ensure!(
+        !pipeline_name.starts_with(GREPTIME_INTERNAL_PIPELINE_NAME_PREFIX),
+        InvalidParameterSnafu {
+            reason: "pipeline_name cannot start with greptime_",
+        }
+    );
     ensure!(
         !payload.is_empty(),
         InvalidParameterSnafu {
@@ -425,47 +434,54 @@ async fn ingest_logs_inner(
     let db = query_ctx.get_db_string();
     let exec_timer = std::time::Instant::now();
 
-    let pipeline = state
-        .get_pipeline(&pipeline_name, version, query_ctx.clone())
-        .await?;
-
-    let transform_timer = std::time::Instant::now();
-    let mut intermediate_state = pipeline.init_intermediate_state();
-
     let mut results = Vec::with_capacity(pipeline_data.len());
+    let transformed_data: Rows;
+    if pipeline_name == GREPTIME_INTERNAL_IDENTITY_PIPELINE_NAME {
+        let rows = pipeline::identity_pipeline(pipeline_data)
+            .context(PipelineTransformSnafu)
+            .context(PipelineSnafu)?;
+        transformed_data = rows;
+    } else {
+        let pipeline = state
+            .get_pipeline(&pipeline_name, version, query_ctx.clone())
+            .await?;
 
-    for v in pipeline_data {
-        pipeline
-            .prepare(v, &mut intermediate_state)
-            .inspect_err(|_| {
-                METRIC_HTTP_LOGS_TRANSFORM_ELAPSED
-                    .with_label_values(&[db.as_str(), METRIC_FAILURE_VALUE])
-                    .observe(transform_timer.elapsed().as_secs_f64());
-            })
-            .context(PipelineTransformSnafu)
-            .context(PipelineSnafu)?;
-        let r = pipeline
-            .exec_mut(&mut intermediate_state)
-            .inspect_err(|_| {
-                METRIC_HTTP_LOGS_TRANSFORM_ELAPSED
-                    .with_label_values(&[db.as_str(), METRIC_FAILURE_VALUE])
-                    .observe(transform_timer.elapsed().as_secs_f64());
-            })
-            .context(PipelineTransformSnafu)
-            .context(PipelineSnafu)?;
-        results.push(r);
-        pipeline.reset_intermediate_state(&mut intermediate_state);
+        let transform_timer = std::time::Instant::now();
+        let mut intermediate_state = pipeline.init_intermediate_state();
+
+        for v in pipeline_data {
+            pipeline
+                .prepare(v, &mut intermediate_state)
+                .inspect_err(|_| {
+                    METRIC_HTTP_LOGS_TRANSFORM_ELAPSED
+                        .with_label_values(&[db.as_str(), METRIC_FAILURE_VALUE])
+                        .observe(transform_timer.elapsed().as_secs_f64());
+                })
+                .context(PipelineTransformSnafu)
+                .context(PipelineSnafu)?;
+            let r = pipeline
+                .exec_mut(&mut intermediate_state)
+                .inspect_err(|_| {
+                    METRIC_HTTP_LOGS_TRANSFORM_ELAPSED
+                        .with_label_values(&[db.as_str(), METRIC_FAILURE_VALUE])
+                        .observe(transform_timer.elapsed().as_secs_f64());
+                })
+                .context(PipelineTransformSnafu)
+                .context(PipelineSnafu)?;
+            results.push(r);
+            pipeline.reset_intermediate_state(&mut intermediate_state);
+        }
+
+        METRIC_HTTP_LOGS_TRANSFORM_ELAPSED
+            .with_label_values(&[db.as_str(), METRIC_SUCCESS_VALUE])
+            .observe(transform_timer.elapsed().as_secs_f64());
+
+        transformed_data = Rows {
+            rows: results,
+            schema: pipeline.schemas().clone(),
+        };
     }
 
-    METRIC_HTTP_LOGS_TRANSFORM_ELAPSED
-        .with_label_values(&[db.as_str(), METRIC_SUCCESS_VALUE])
-        .observe(transform_timer.elapsed().as_secs_f64());
-
-    let transformed_data: Rows = Rows {
-        rows: results,
-        schema: pipeline.schemas().clone(),
-    };
-
     let insert_request = RowInsertRequest {
         rows: Some(transformed_data),
         table_name: table_name.clone(),
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index 20a0f4edd2..f70df8176a 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -86,6 +86,7 @@ macro_rules! http_tests {
                 test_pipeline_api,
                 test_test_pipeline_api,
                 test_plain_text_ingestion,
+                test_identify_pipeline,
 
                 test_otlp_metrics,
                 test_otlp_traces,
@@ -1076,6 +1077,21 @@ transform:
 "#;
 
     // 1. create pipeline
+    let res = client
+        .post("/v1/events/pipelines/greptime_guagua")
+        .header("Content-Type", "application/x-yaml")
+        .body(body)
+        .send()
+        .await;
+
+    assert_eq!(res.status(), StatusCode::BAD_REQUEST);
+    assert_eq!(
+        res.json::<serde_json::Value>().await["error"]
+            .as_str()
+            .unwrap(),
+        "Invalid request parameter: pipeline_name cannot start with greptime_"
+    );
+
     let res = client
         .post("/v1/events/pipelines/test")
         .header("Content-Type", "application/x-yaml")
@@ -1161,6 +1177,61 @@ transform:
     guard.remove_all().await;
 }
 
+pub async fn test_identify_pipeline(store_type: StorageType) {
+    common_telemetry::init_default_ut_logging();
+    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_pipeline_api").await;
+
+    // handshake
+    let client = TestClient::new(app);
+    let body = r#"{"__time__":1453809242,"__topic__":"","__source__":"10.170.***.***","ip":"10.200.**.***","time":"26/Jan/2016:19:54:02 +0800","url":"POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=<yourAccessKeyId>&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=<yourSignature>HTTP/1.1","status":"200","user-agent":"aliyun-sdk-java"}
+{"__time__":1453809242,"__topic__":"","__source__":"10.170.***.***","ip":"10.200.**.***","time":"26/Jan/2016:19:54:02 +0800","url":"POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=<yourAccessKeyId>&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=<yourSignature>HTTP/1.1","status":"200","user-agent":"aliyun-sdk-java","hasagei":"hasagei","dongdongdong":"guaguagua"}"#;
+    let res = client
+        .post("/v1/events/logs?db=public&table=logs&pipeline_name=greptime_identity")
+        .header("Content-Type", "application/json")
+        .body(body)
+        .send()
+        .await;
+
+    assert_eq!(res.status(), StatusCode::OK);
+
+    let body: serde_json::Value = res.json().await;
+
+    assert!(body.get("execution_time_ms").unwrap().is_number());
+    assert_eq!(body["output"][0]["affectedrows"], 2);
+
+    let res = client.get("/v1/sql?sql=select * from logs").send().await;
+
+    assert_eq!(res.status(), StatusCode::OK);
+
+    let line1_expected = r#"["10.170.***.***",1453809242,"","10.200.**.***","200","26/Jan/2016:19:54:02 +0800","POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=<yourAccessKeyId>&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=<yourSignature>HTTP/1.1","aliyun-sdk-java","guaguagua","hasagei",null]"#;
+    let line2_expected = r#"["10.170.***.***",1453809242,"","10.200.**.***","200","26/Jan/2016:19:54:02 +0800","POST/PutData?Category=YunOsAccountOpLog&AccessKeyId=<yourAccessKeyId>&Date=Fri%2C%2028%20Jun%202013%2006%3A53%3A30%20GMT&Topic=raw&Signature=<yourSignature>HTTP/1.1","aliyun-sdk-java",null,null,null]"#;
+    let res = client.get("/v1/sql?sql=select * from logs").send().await;
+    assert_eq!(res.status(), StatusCode::OK);
+    let resp: serde_json::Value = res.json().await;
+    let result = resp["output"][0]["records"]["rows"].as_array().unwrap();
+    assert_eq!(result.len(), 2);
+    let mut line1 = result[0].as_array().unwrap().clone();
+    let mut line2 = result[1].as_array().unwrap().clone();
+    assert!(line1.last().unwrap().is_i64());
+    assert!(line2.last().unwrap().is_i64());
+    *line1.last_mut().unwrap() = serde_json::Value::Null;
+    *line2.last_mut().unwrap() = serde_json::Value::Null;
+
+    assert_eq!(
+        line1,
+        serde_json::from_str::<Vec<Value>>(line1_expected).unwrap()
+    );
+    assert_eq!(
+        line2,
+        serde_json::from_str::<Vec<Value>>(line2_expected).unwrap()
+    );
+
+    let expected = r#"[["__source__","String","","YES","","FIELD"],["__time__","Int64","","YES","","FIELD"],["__topic__","String","","YES","","FIELD"],["ip","String","","YES","","FIELD"],["status","String","","YES","","FIELD"],["time","String","","YES","","FIELD"],["url","String","","YES","","FIELD"],["user-agent","String","","YES","","FIELD"],["dongdongdong","String","","YES","","FIELD"],["hasagei","String","","YES","","FIELD"],["greptime_timestamp","TimestampNanosecond","PRI","NO","","TIMESTAMP"]]"#;
+    validate_data(&client, "desc logs", expected).await;
+
+    guard.remove_all().await;
+}
+
 pub async fn test_test_pipeline_api(store_type: StorageType) {
     common_telemetry::init_default_ut_logging();
     let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_pipeline_api").await;

From 5f0a83b2b1876e1e4b0ce93ecd4c8c5a65f91c03 Mon Sep 17 00:00:00 2001
From: shuiyisong <113876041+shuiyisong@users.noreply.github.com>
Date: Wed, 9 Oct 2024 01:54:44 +0800
Subject: [PATCH 058/128] fix: ts conversion during transform phase (#4790)

* fix: allow ts conversion during transform phase

* chore: replace `unimplemented` with snafu
---
 src/pipeline/src/etl/error.rs                 | 11 ++++
 .../transform/transformer/greptime/coerce.rs  | 57 ++++++++++---------
 src/pipeline/tests/epoch.rs                   | 24 ++++----
 src/pipeline/tests/timestamp.rs               | 23 ++++----
 4 files changed, 66 insertions(+), 49 deletions(-)

diff --git a/src/pipeline/src/etl/error.rs b/src/pipeline/src/etl/error.rs
index 0a28cb7423..08654c2c82 100644
--- a/src/pipeline/src/etl/error.rs
+++ b/src/pipeline/src/etl/error.rs
@@ -438,6 +438,17 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+    #[snafu(display("failed to coerce complex value, not supported"))]
+    CoerceComplexType {
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("failed to coerce value: {msg}"))]
+    CoerceIncompatibleTypes {
+        msg: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
 
     #[snafu(display(
         "Invalid resolution: '{resolution}'. Available resolutions: {valid_resolution}"
diff --git a/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs b/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs
index 827613b02b..8d07b34d46 100644
--- a/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs
+++ b/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs
@@ -20,7 +20,8 @@ use greptime_proto::v1::{ColumnDataType, ColumnSchema, SemanticType};
 use snafu::ResultExt;
 
 use crate::etl::error::{
-    CoerceStringToTypeSnafu, CoerceUnsupportedEpochTypeSnafu, CoerceUnsupportedNullTypeSnafu,
+    CoerceComplexTypeSnafu, CoerceIncompatibleTypesSnafu, CoerceStringToTypeSnafu,
+    CoerceUnsupportedEpochTypeSnafu, CoerceUnsupportedNullTypeSnafu,
     CoerceUnsupportedNullTypeToSnafu, ColumnOptionsSnafu, Error, Result,
 };
 use crate::etl::transform::index::Index;
@@ -61,8 +62,7 @@ impl TryFrom<Value> for ValueData {
             }
             Value::Timestamp(Timestamp::Second(s)) => Ok(ValueData::TimestampSecondValue(s)),
 
-            Value::Array(_) => unimplemented!("Array type not supported"),
-            Value::Map(_) => unimplemented!("Object type not supported"),
+            Value::Array(_) | Value::Map(_) => CoerceComplexTypeSnafu.fail(),
         }
     }
 }
@@ -134,8 +134,7 @@ fn coerce_type(transform: &Transform) -> Result<ColumnDataType> {
         Value::Timestamp(Timestamp::Millisecond(_)) => Ok(ColumnDataType::TimestampMillisecond),
         Value::Timestamp(Timestamp::Second(_)) => Ok(ColumnDataType::TimestampSecond),
 
-        Value::Array(_) => unimplemented!("Array"),
-        Value::Map(_) => unimplemented!("Object"),
+        Value::Array(_) | Value::Map(_) => CoerceComplexTypeSnafu.fail(),
 
         Value::Null => CoerceUnsupportedNullTypeToSnafu {
             ty: transform.type_.to_str_type(),
@@ -176,19 +175,28 @@ pub(crate) fn coerce_value(val: &Value, transform: &Transform) -> Result<Option<
         Value::Boolean(b) => coerce_bool_value(*b, transform),
         Value::String(s) => coerce_string_value(s, transform),
 
-        Value::Timestamp(Timestamp::Nanosecond(ns)) => {
-            Ok(Some(ValueData::TimestampNanosecondValue(*ns)))
-        }
-        Value::Timestamp(Timestamp::Microsecond(us)) => {
-            Ok(Some(ValueData::TimestampMicrosecondValue(*us)))
-        }
-        Value::Timestamp(Timestamp::Millisecond(ms)) => {
-            Ok(Some(ValueData::TimestampMillisecondValue(*ms)))
-        }
-        Value::Timestamp(Timestamp::Second(s)) => Ok(Some(ValueData::TimestampSecondValue(*s))),
+        Value::Timestamp(input_timestamp) => match &transform.type_ {
+            Value::Timestamp(target_timestamp) => match target_timestamp {
+                Timestamp::Nanosecond(_) => Ok(Some(ValueData::TimestampNanosecondValue(
+                    input_timestamp.timestamp_nanos(),
+                ))),
+                Timestamp::Microsecond(_) => Ok(Some(ValueData::TimestampMicrosecondValue(
+                    input_timestamp.timestamp_micros(),
+                ))),
+                Timestamp::Millisecond(_) => Ok(Some(ValueData::TimestampMillisecondValue(
+                    input_timestamp.timestamp_millis(),
+                ))),
+                Timestamp::Second(_) => Ok(Some(ValueData::TimestampSecondValue(
+                    input_timestamp.timestamp(),
+                ))),
+            },
+            _ => CoerceIncompatibleTypesSnafu {
+                msg: "Timestamp can only be coerced to another timestamp",
+            }
+            .fail(),
+        },
 
-        Value::Array(_) => unimplemented!("Array type not supported"),
-        Value::Map(_) => unimplemented!("Object type not supported"),
+        Value::Array(_) | Value::Map(_) => CoerceComplexTypeSnafu.fail(),
     }
 }
 
@@ -220,8 +228,7 @@ fn coerce_bool_value(b: bool, transform: &Transform) -> Result<Option<ValueData>
             }
         },
 
-        Value::Array(_) => unimplemented!("Array type not supported"),
-        Value::Map(_) => unimplemented!("Object type not supported"),
+        Value::Array(_) | Value::Map(_) => return CoerceComplexTypeSnafu.fail(),
 
         Value::Null => return Ok(None),
     };
@@ -257,8 +264,7 @@ fn coerce_i64_value(n: i64, transform: &Transform) -> Result<Option<ValueData>>
             }
         },
 
-        Value::Array(_) => unimplemented!("Array type not supported"),
-        Value::Map(_) => unimplemented!("Object type not supported"),
+        Value::Array(_) | Value::Map(_) => return CoerceComplexTypeSnafu.fail(),
 
         Value::Null => return Ok(None),
     };
@@ -294,8 +300,7 @@ fn coerce_u64_value(n: u64, transform: &Transform) -> Result<Option<ValueData>>
             }
         },
 
-        Value::Array(_) => unimplemented!("Array type not supported"),
-        Value::Map(_) => unimplemented!("Object type not supported"),
+        Value::Array(_) | Value::Map(_) => return CoerceComplexTypeSnafu.fail(),
 
         Value::Null => return Ok(None),
     };
@@ -331,8 +336,7 @@ fn coerce_f64_value(n: f64, transform: &Transform) -> Result<Option<ValueData>>
             }
         },
 
-        Value::Array(_) => unimplemented!("Array type not supported"),
-        Value::Map(_) => unimplemented!("Object type not supported"),
+        Value::Array(_) | Value::Map(_) => return CoerceComplexTypeSnafu.fail(),
 
         Value::Null => return Ok(None),
     };
@@ -407,8 +411,7 @@ fn coerce_string_value(s: &String, transform: &Transform) -> Result<Option<Value
             None => CoerceUnsupportedEpochTypeSnafu { ty: "String" }.fail(),
         },
 
-        Value::Array(_) => unimplemented!("Array type not supported"),
-        Value::Map(_) => unimplemented!("Object type not supported"),
+        Value::Array(_) | Value::Map(_) => CoerceComplexTypeSnafu.fail(),
 
         Value::Null => Ok(None),
     }
diff --git a/src/pipeline/tests/epoch.rs b/src/pipeline/tests/epoch.rs
index 35a2ab635c..84662793b9 100644
--- a/src/pipeline/tests/epoch.rs
+++ b/src/pipeline/tests/epoch.rs
@@ -200,6 +200,8 @@ transform:
 
 #[test]
 fn test_default_wrong_resolution() {
+    // given a number, we have no ways to guess its resolution
+    // but we can convert resolution during transform phase
     let test_input = r#"
     {
         "input_s": "1722580862",
@@ -209,28 +211,30 @@ fn test_default_wrong_resolution() {
     let pipeline_yaml = r#"
 processors:
   - epoch:
-      fields: 
-        - input_s
-        - input_nano
+      field: input_s
+      resolution: s
+  - epoch:
+      field: input_nano
+      resolution: ns
 
 transform:
   - fields:
       - input_s
-    type: epoch, s
+    type: epoch, ms
   - fields:
       - input_nano
-    type: epoch, nano
+    type: epoch, ms
 "#;
 
     let expected_schema = vec![
         common::make_column_schema(
             "input_s".to_string(),
-            ColumnDataType::TimestampSecond,
+            ColumnDataType::TimestampMillisecond,
             SemanticType::Field,
         ),
         common::make_column_schema(
             "input_nano".to_string(),
-            ColumnDataType::TimestampNanosecond,
+            ColumnDataType::TimestampMillisecond,
             SemanticType::Field,
         ),
         common::make_column_schema(
@@ -242,14 +246,12 @@ transform:
 
     let output = common::parse_and_exec(test_input, pipeline_yaml);
     assert_eq!(output.schema, expected_schema);
-    // this is actually wrong
-    // TODO(shuiyisong): add check for type when converting epoch
     assert_eq!(
         output.rows[0].values[0].value_data,
-        Some(ValueData::TimestampMillisecondValue(1722580862))
+        Some(ValueData::TimestampMillisecondValue(1722580862000))
     );
     assert_eq!(
         output.rows[0].values[1].value_data,
-        Some(ValueData::TimestampMillisecondValue(1722583122284583936))
+        Some(ValueData::TimestampMillisecondValue(1722583122284))
     );
 }
diff --git a/src/pipeline/tests/timestamp.rs b/src/pipeline/tests/timestamp.rs
index cba68d6cb1..85cf0643fc 100644
--- a/src/pipeline/tests/timestamp.rs
+++ b/src/pipeline/tests/timestamp.rs
@@ -318,6 +318,7 @@ transform:
 
 #[test]
 fn test_timestamp_default_wrong_resolution() {
+    // same as test_default_wrong_resolution from epoch tests
     let test_input = r#"
     {
         "input_s": "1722580862",
@@ -327,28 +328,30 @@ fn test_timestamp_default_wrong_resolution() {
     let pipeline_yaml = r#"
 processors:
   - timestamp:
-      fields: 
-        - input_s
-        - input_nano
+      field: input_s
+      resolution: s
+  - timestamp:
+      field: input_nano
+      resolution: ns
 
 transform:
   - fields:
       - input_s
-    type: timestamp, s
+    type: timestamp, ms
   - fields:
       - input_nano
-    type: timestamp, nano
+    type: timestamp, ms
 "#;
 
     let expected_schema = vec![
         common::make_column_schema(
             "input_s".to_string(),
-            ColumnDataType::TimestampSecond,
+            ColumnDataType::TimestampMillisecond,
             SemanticType::Field,
         ),
         common::make_column_schema(
             "input_nano".to_string(),
-            ColumnDataType::TimestampNanosecond,
+            ColumnDataType::TimestampMillisecond,
             SemanticType::Field,
         ),
         common::make_column_schema(
@@ -360,14 +363,12 @@ transform:
 
     let output = common::parse_and_exec(test_input, pipeline_yaml);
     assert_eq!(output.schema, expected_schema);
-    // this is actually wrong
-    // TODO(shuiyisong): add check for type when converting epoch
     assert_eq!(
         output.rows[0].values[0].value_data,
-        Some(ValueData::TimestampMillisecondValue(1722580862))
+        Some(ValueData::TimestampMillisecondValue(1722580862000))
     );
     assert_eq!(
         output.rows[0].values[1].value_data,
-        Some(ValueData::TimestampMillisecondValue(1722583122284583936))
+        Some(ValueData::TimestampMillisecondValue(1722583122284))
     );
 }

From 19918928c59e43bd9d8b5f0bc58ffbf6702ed232 Mon Sep 17 00:00:00 2001
From: Ning Sun <sunng@protonmail.com>
Date: Tue, 8 Oct 2024 19:38:44 -0700
Subject: [PATCH 059/128] feat: add function to aggregate path into a geojson
 path (#4798)

* feat: add geojson function to aggregate paths

* test: add sqlness results

* test: add sqlness

* refactor: corrected to aggregation function

* chore: update comments

* fix: make linter happy again

* refactor: rename to remove `geo` from `geojson` function name

The return type is not geojson at all. It's just compatible with geojson's
coordinates part and superset's deckgl path plugin.
---
 src/common/function/src/scalars/aggregate.rs  |   3 +
 .../function/src/scalars/aggregate/argmax.rs  |   5 +-
 .../function/src/scalars/aggregate/argmin.rs  |   5 +-
 .../function/src/scalars/aggregate/diff.rs    |   4 +-
 .../function/src/scalars/aggregate/mean.rs    |   4 +-
 .../function/src/scalars/aggregate/polyval.rs |   3 +-
 .../scalars/aggregate/scipy_stats_norm_cdf.rs |   4 +-
 .../scalars/aggregate/scipy_stats_norm_pdf.rs |   4 +-
 src/common/function/src/scalars/geo.rs        |   2 +
 .../function/src/scalars/geo/encoding.rs      | 223 ++++++++++++++++++
 .../function/src/scalars/geo/helpers.rs       |  61 +++++
 src/common/macro/src/aggr_func.rs             |  18 +-
 src/common/macro/tests/test_derive.rs         |   2 +-
 src/datatypes/src/data_type.rs                |   9 +
 src/query/src/tests/my_sum_udaf_example.rs    |   1 +
 .../standalone/common/function/geo.result     |  25 ++
 .../cases/standalone/common/function/geo.sql  |  13 +
 17 files changed, 367 insertions(+), 19 deletions(-)
 create mode 100644 src/common/function/src/scalars/geo/encoding.rs
 create mode 100644 src/common/function/src/scalars/geo/helpers.rs

diff --git a/src/common/function/src/scalars/aggregate.rs b/src/common/function/src/scalars/aggregate.rs
index c6875a680c..f7ce316b6c 100644
--- a/src/common/function/src/scalars/aggregate.rs
+++ b/src/common/function/src/scalars/aggregate.rs
@@ -31,6 +31,7 @@ pub use polyval::PolyvalAccumulatorCreator;
 pub use scipy_stats_norm_cdf::ScipyStatsNormCdfAccumulatorCreator;
 pub use scipy_stats_norm_pdf::ScipyStatsNormPdfAccumulatorCreator;
 
+use super::geo::encoding::JsonPathEncodeFunctionCreator;
 use crate::function_registry::FunctionRegistry;
 
 /// A function creates `AggregateFunctionCreator`.
@@ -91,5 +92,7 @@ impl AggregateFunctions {
         register_aggr_func!("argmin", 1, ArgminAccumulatorCreator);
         register_aggr_func!("scipystatsnormcdf", 2, ScipyStatsNormCdfAccumulatorCreator);
         register_aggr_func!("scipystatsnormpdf", 2, ScipyStatsNormPdfAccumulatorCreator);
+
+        register_aggr_func!("json_encode_path", 3, JsonPathEncodeFunctionCreator);
     }
 }
diff --git a/src/common/function/src/scalars/aggregate/argmax.rs b/src/common/function/src/scalars/aggregate/argmax.rs
index c5c5264f19..4749ff9a3a 100644
--- a/src/common/function/src/scalars/aggregate/argmax.rs
+++ b/src/common/function/src/scalars/aggregate/argmax.rs
@@ -16,7 +16,10 @@ use std::cmp::Ordering;
 use std::sync::Arc;
 
 use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
-use common_query::error::{BadAccumulatorImplSnafu, CreateAccumulatorSnafu, Result};
+use common_query::error::{
+    BadAccumulatorImplSnafu, CreateAccumulatorSnafu, InvalidInputStateSnafu, Result,
+};
+use common_query::logical_plan::accumulator::AggrFuncTypeStore;
 use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
 use common_query::prelude::*;
 use datatypes::prelude::*;
diff --git a/src/common/function/src/scalars/aggregate/argmin.rs b/src/common/function/src/scalars/aggregate/argmin.rs
index 7233f43b77..fe89184460 100644
--- a/src/common/function/src/scalars/aggregate/argmin.rs
+++ b/src/common/function/src/scalars/aggregate/argmin.rs
@@ -16,7 +16,10 @@ use std::cmp::Ordering;
 use std::sync::Arc;
 
 use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
-use common_query::error::{BadAccumulatorImplSnafu, CreateAccumulatorSnafu, Result};
+use common_query::error::{
+    BadAccumulatorImplSnafu, CreateAccumulatorSnafu, InvalidInputStateSnafu, Result,
+};
+use common_query::logical_plan::accumulator::AggrFuncTypeStore;
 use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
 use common_query::prelude::*;
 use datatypes::prelude::*;
diff --git a/src/common/function/src/scalars/aggregate/diff.rs b/src/common/function/src/scalars/aggregate/diff.rs
index b83ed6d004..25d1614e4b 100644
--- a/src/common/function/src/scalars/aggregate/diff.rs
+++ b/src/common/function/src/scalars/aggregate/diff.rs
@@ -17,8 +17,10 @@ use std::sync::Arc;
 
 use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use common_query::error::{
-    CreateAccumulatorSnafu, DowncastVectorSnafu, FromScalarValueSnafu, Result,
+    CreateAccumulatorSnafu, DowncastVectorSnafu, FromScalarValueSnafu, InvalidInputStateSnafu,
+    Result,
 };
+use common_query::logical_plan::accumulator::AggrFuncTypeStore;
 use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
 use common_query::prelude::*;
 use datatypes::prelude::*;
diff --git a/src/common/function/src/scalars/aggregate/mean.rs b/src/common/function/src/scalars/aggregate/mean.rs
index 3dc3e18535..ed66c90bdb 100644
--- a/src/common/function/src/scalars/aggregate/mean.rs
+++ b/src/common/function/src/scalars/aggregate/mean.rs
@@ -17,8 +17,10 @@ use std::sync::Arc;
 
 use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use common_query::error::{
-    BadAccumulatorImplSnafu, CreateAccumulatorSnafu, DowncastVectorSnafu, Result,
+    BadAccumulatorImplSnafu, CreateAccumulatorSnafu, DowncastVectorSnafu, InvalidInputStateSnafu,
+    Result,
 };
+use common_query::logical_plan::accumulator::AggrFuncTypeStore;
 use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
 use common_query::prelude::*;
 use datatypes::prelude::*;
diff --git a/src/common/function/src/scalars/aggregate/polyval.rs b/src/common/function/src/scalars/aggregate/polyval.rs
index ae6ca101c4..bc3986fd0e 100644
--- a/src/common/function/src/scalars/aggregate/polyval.rs
+++ b/src/common/function/src/scalars/aggregate/polyval.rs
@@ -18,8 +18,9 @@ use std::sync::Arc;
 use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use common_query::error::{
     self, BadAccumulatorImplSnafu, CreateAccumulatorSnafu, DowncastVectorSnafu,
-    FromScalarValueSnafu, InvalidInputColSnafu, Result,
+    FromScalarValueSnafu, InvalidInputColSnafu, InvalidInputStateSnafu, Result,
 };
+use common_query::logical_plan::accumulator::AggrFuncTypeStore;
 use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
 use common_query::prelude::*;
 use datatypes::prelude::*;
diff --git a/src/common/function/src/scalars/aggregate/scipy_stats_norm_cdf.rs b/src/common/function/src/scalars/aggregate/scipy_stats_norm_cdf.rs
index e6c92225a6..09a9c820d8 100644
--- a/src/common/function/src/scalars/aggregate/scipy_stats_norm_cdf.rs
+++ b/src/common/function/src/scalars/aggregate/scipy_stats_norm_cdf.rs
@@ -17,8 +17,10 @@ use std::sync::Arc;
 use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use common_query::error::{
     self, BadAccumulatorImplSnafu, CreateAccumulatorSnafu, DowncastVectorSnafu,
-    FromScalarValueSnafu, GenerateFunctionSnafu, InvalidInputColSnafu, Result,
+    FromScalarValueSnafu, GenerateFunctionSnafu, InvalidInputColSnafu, InvalidInputStateSnafu,
+    Result,
 };
+use common_query::logical_plan::accumulator::AggrFuncTypeStore;
 use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
 use common_query::prelude::*;
 use datatypes::prelude::*;
diff --git a/src/common/function/src/scalars/aggregate/scipy_stats_norm_pdf.rs b/src/common/function/src/scalars/aggregate/scipy_stats_norm_pdf.rs
index 3045ae8665..2d5025ea3a 100644
--- a/src/common/function/src/scalars/aggregate/scipy_stats_norm_pdf.rs
+++ b/src/common/function/src/scalars/aggregate/scipy_stats_norm_pdf.rs
@@ -17,8 +17,10 @@ use std::sync::Arc;
 use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use common_query::error::{
     self, BadAccumulatorImplSnafu, CreateAccumulatorSnafu, DowncastVectorSnafu,
-    FromScalarValueSnafu, GenerateFunctionSnafu, InvalidInputColSnafu, Result,
+    FromScalarValueSnafu, GenerateFunctionSnafu, InvalidInputColSnafu, InvalidInputStateSnafu,
+    Result,
 };
+use common_query::logical_plan::accumulator::AggrFuncTypeStore;
 use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
 use common_query::prelude::*;
 use datatypes::prelude::*;
diff --git a/src/common/function/src/scalars/geo.rs b/src/common/function/src/scalars/geo.rs
index 8ad6a7aef2..37b6c0704b 100644
--- a/src/common/function/src/scalars/geo.rs
+++ b/src/common/function/src/scalars/geo.rs
@@ -13,8 +13,10 @@
 // limitations under the License.
 
 use std::sync::Arc;
+pub(crate) mod encoding;
 mod geohash;
 mod h3;
+mod helpers;
 
 use geohash::{GeohashFunction, GeohashNeighboursFunction};
 
diff --git a/src/common/function/src/scalars/geo/encoding.rs b/src/common/function/src/scalars/geo/encoding.rs
new file mode 100644
index 0000000000..0600120598
--- /dev/null
+++ b/src/common/function/src/scalars/geo/encoding.rs
@@ -0,0 +1,223 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use common_error::ext::{BoxedError, PlainError};
+use common_error::status_code::StatusCode;
+use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
+use common_query::error::{self, InvalidFuncArgsSnafu, InvalidInputStateSnafu, Result};
+use common_query::logical_plan::accumulator::AggrFuncTypeStore;
+use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
+use common_query::prelude::AccumulatorCreatorFunction;
+use common_time::Timestamp;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::value::{ListValue, Value};
+use datatypes::vectors::VectorRef;
+use snafu::{ensure, ResultExt};
+
+use super::helpers::{ensure_columns_len, ensure_columns_n};
+
+/// Accumulator of lat, lng, timestamp tuples
+#[derive(Debug)]
+pub struct JsonPathAccumulator {
+    timestamp_type: ConcreteDataType,
+    lat: Vec<Option<f64>>,
+    lng: Vec<Option<f64>>,
+    timestamp: Vec<Option<Timestamp>>,
+}
+
+impl JsonPathAccumulator {
+    fn new(timestamp_type: ConcreteDataType) -> Self {
+        Self {
+            lat: Vec::default(),
+            lng: Vec::default(),
+            timestamp: Vec::default(),
+            timestamp_type,
+        }
+    }
+}
+
+impl Accumulator for JsonPathAccumulator {
+    fn state(&self) -> Result<Vec<Value>> {
+        Ok(vec![
+            Value::List(ListValue::new(
+                self.lat.iter().map(|i| Value::from(*i)).collect(),
+                ConcreteDataType::float64_datatype(),
+            )),
+            Value::List(ListValue::new(
+                self.lng.iter().map(|i| Value::from(*i)).collect(),
+                ConcreteDataType::float64_datatype(),
+            )),
+            Value::List(ListValue::new(
+                self.timestamp.iter().map(|i| Value::from(*i)).collect(),
+                self.timestamp_type.clone(),
+            )),
+        ])
+    }
+
+    fn update_batch(&mut self, columns: &[VectorRef]) -> Result<()> {
+        // update batch as in datafusion just provides the accumulator original
+        //  input.
+        //
+        // columns is vec of [`lat`, `lng`, `timestamp`]
+        // where
+        // - `lat` is a vector of `Value::Float64` or similar type. Each item in
+        //  the vector is a row in given dataset.
+        // - so on so forth for `lng` and `timestamp`
+        ensure_columns_n!(columns, 3);
+
+        let lat = &columns[0];
+        let lng = &columns[1];
+        let ts = &columns[2];
+
+        let size = lat.len();
+
+        for idx in 0..size {
+            self.lat.push(lat.get(idx).as_f64_lossy());
+            self.lng.push(lng.get(idx).as_f64_lossy());
+            self.timestamp.push(ts.get(idx).as_timestamp());
+        }
+
+        Ok(())
+    }
+
+    fn merge_batch(&mut self, states: &[VectorRef]) -> Result<()> {
+        // merge batch as in datafusion gives state accumulated from the data
+        //  returned from child accumulators' state() call
+        // In our particular implementation, the data structure is like
+        //
+        // states is vec of [`lat`, `lng`, `timestamp`]
+        // where
+        // - `lat` is a vector of `Value::List`. Each item in the list is all
+        //  coordinates from a child accumulator.
+        // - so on so forth for `lng` and `timestamp`
+
+        ensure_columns_n!(states, 3);
+
+        let lat_lists = &states[0];
+        let lng_lists = &states[1];
+        let ts_lists = &states[2];
+
+        let len = lat_lists.len();
+
+        for idx in 0..len {
+            if let Some(lat_list) = lat_lists
+                .get(idx)
+                .as_list()
+                .map_err(BoxedError::new)
+                .context(error::ExecuteSnafu)?
+            {
+                for v in lat_list.items() {
+                    self.lat.push(v.as_f64_lossy());
+                }
+            }
+
+            if let Some(lng_list) = lng_lists
+                .get(idx)
+                .as_list()
+                .map_err(BoxedError::new)
+                .context(error::ExecuteSnafu)?
+            {
+                for v in lng_list.items() {
+                    self.lng.push(v.as_f64_lossy());
+                }
+            }
+
+            if let Some(ts_list) = ts_lists
+                .get(idx)
+                .as_list()
+                .map_err(BoxedError::new)
+                .context(error::ExecuteSnafu)?
+            {
+                for v in ts_list.items() {
+                    self.timestamp.push(v.as_timestamp());
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    fn evaluate(&self) -> Result<Value> {
+        let mut work_vec: Vec<(&Option<f64>, &Option<f64>, &Option<Timestamp>)> = self
+            .lat
+            .iter()
+            .zip(self.lng.iter())
+            .zip(self.timestamp.iter())
+            .map(|((a, b), c)| (a, b, c))
+            .collect();
+
+        // sort by timestamp, we treat null timestamp as 0
+        work_vec.sort_unstable_by_key(|tuple| tuple.2.unwrap_or_else(|| Timestamp::new_second(0)));
+
+        let result = serde_json::to_string(
+            &work_vec
+                .into_iter()
+                // note that we transform to lng,lat for geojson compatibility
+                .map(|(lat, lng, _)| vec![lng, lat])
+                .collect::<Vec<Vec<&Option<f64>>>>(),
+        )
+        .map_err(|e| {
+            BoxedError::new(PlainError::new(
+                format!("Serialization failure: {}", e),
+                StatusCode::EngineExecuteQuery,
+            ))
+        })
+        .context(error::ExecuteSnafu)?;
+
+        Ok(Value::String(result.into()))
+    }
+}
+
+/// This function accept rows of lat, lng and timestamp, sort with timestamp and
+/// encoding them into a geojson-like path.
+///
+/// Example:
+///
+/// ```sql
+/// SELECT json_encode_path(lat, lon, timestamp) FROM table [group by ...];
+/// ```
+///
+#[as_aggr_func_creator]
+#[derive(Debug, Default, AggrFuncTypeStore)]
+pub struct JsonPathEncodeFunctionCreator {}
+
+impl AggregateFunctionCreator for JsonPathEncodeFunctionCreator {
+    fn creator(&self) -> AccumulatorCreatorFunction {
+        let creator: AccumulatorCreatorFunction = Arc::new(move |types: &[ConcreteDataType]| {
+            let ts_type = types[2].clone();
+            Ok(Box::new(JsonPathAccumulator::new(ts_type)))
+        });
+
+        creator
+    }
+
+    fn output_type(&self) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::string_datatype())
+    }
+
+    fn state_types(&self) -> Result<Vec<ConcreteDataType>> {
+        let input_types = self.input_types()?;
+        ensure!(input_types.len() == 3, InvalidInputStateSnafu);
+
+        let timestamp_type = input_types[2].clone();
+
+        Ok(vec![
+            ConcreteDataType::list_datatype(ConcreteDataType::float64_datatype()),
+            ConcreteDataType::list_datatype(ConcreteDataType::float64_datatype()),
+            ConcreteDataType::list_datatype(timestamp_type),
+        ])
+    }
+}
diff --git a/src/common/function/src/scalars/geo/helpers.rs b/src/common/function/src/scalars/geo/helpers.rs
new file mode 100644
index 0000000000..f07eaefb15
--- /dev/null
+++ b/src/common/function/src/scalars/geo/helpers.rs
@@ -0,0 +1,61 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+macro_rules! ensure_columns_len {
+    ($columns:ident) => {
+        ensure!(
+            $columns.windows(2).all(|c| c[0].len() == c[1].len()),
+            InvalidFuncArgsSnafu {
+                err_msg: "The length of input columns are in different size"
+            }
+        )
+    };
+    ($column_a:ident, $column_b:ident, $($column_n:ident),*) => {
+        ensure!(
+            {
+                let mut result = $column_a.len() == $column_b.len();
+                $(
+                result = result && ($column_a.len() == $column_n.len());
+                )*
+                result
+            }
+            InvalidFuncArgsSnafu {
+                err_msg: "The length of input columns are in different size"
+            }
+        )
+    };
+}
+
+pub(super) use ensure_columns_len;
+
+macro_rules! ensure_columns_n {
+    ($columns:ident, $n:literal) => {
+        ensure!(
+            $columns.len() == $n,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of arguments is not correct, expect {}, provided : {}",
+                    stringify!($n),
+                    $columns.len()
+                ),
+            }
+        );
+
+        if $n > 1 {
+            ensure_columns_len!($columns);
+        }
+    };
+}
+
+pub(super) use ensure_columns_n;
diff --git a/src/common/macro/src/aggr_func.rs b/src/common/macro/src/aggr_func.rs
index 4c3ccccdee..2e17e70b5a 100644
--- a/src/common/macro/src/aggr_func.rs
+++ b/src/common/macro/src/aggr_func.rs
@@ -21,23 +21,19 @@ use syn::{parse_macro_input, DeriveInput, ItemStruct};
 pub(crate) fn impl_aggr_func_type_store(ast: &DeriveInput) -> TokenStream {
     let name = &ast.ident;
     let gen = quote! {
-        use common_query::logical_plan::accumulator::AggrFuncTypeStore;
-        use common_query::error::{InvalidInputStateSnafu, Error as QueryError};
-        use datatypes::prelude::ConcreteDataType;
-
-        impl AggrFuncTypeStore for #name {
-            fn input_types(&self) -> std::result::Result<Vec<ConcreteDataType>, QueryError> {
+        impl common_query::logical_plan::accumulator::AggrFuncTypeStore for #name {
+            fn input_types(&self) -> std::result::Result<Vec<datatypes::prelude::ConcreteDataType>, common_query::error::Error> {
                 let input_types = self.input_types.load();
-                snafu::ensure!(input_types.is_some(), InvalidInputStateSnafu);
+                snafu::ensure!(input_types.is_some(), common_query::error::InvalidInputStateSnafu);
                 Ok(input_types.as_ref().unwrap().as_ref().clone())
             }
 
-            fn set_input_types(&self, input_types: Vec<ConcreteDataType>) -> std::result::Result<(), QueryError> {
+            fn set_input_types(&self, input_types: Vec<datatypes::prelude::ConcreteDataType>) -> std::result::Result<(), common_query::error::Error> {
                 let old = self.input_types.swap(Some(std::sync::Arc::new(input_types.clone())));
                 if let Some(old) = old {
-                    snafu::ensure!(old.len() == input_types.len(), InvalidInputStateSnafu);
+                    snafu::ensure!(old.len() == input_types.len(), common_query::error::InvalidInputStateSnafu);
                     for (x, y) in old.iter().zip(input_types.iter()) {
-                        snafu::ensure!(x == y, InvalidInputStateSnafu);
+                        snafu::ensure!(x == y, common_query::error::InvalidInputStateSnafu);
                     }
                 }
                 Ok(())
@@ -51,7 +47,7 @@ pub(crate) fn impl_as_aggr_func_creator(_args: TokenStream, input: TokenStream)
     let mut item_struct = parse_macro_input!(input as ItemStruct);
     if let syn::Fields::Named(ref mut fields) = item_struct.fields {
         let result = syn::Field::parse_named.parse2(quote! {
-            input_types: arc_swap::ArcSwapOption<Vec<ConcreteDataType>>
+            input_types: arc_swap::ArcSwapOption<Vec<datatypes::prelude::ConcreteDataType>>
         });
         match result {
             Ok(field) => fields.named.push(field),
diff --git a/src/common/macro/tests/test_derive.rs b/src/common/macro/tests/test_derive.rs
index 9c648c788d..c614800572 100644
--- a/src/common/macro/tests/test_derive.rs
+++ b/src/common/macro/tests/test_derive.rs
@@ -24,5 +24,5 @@ struct Foo {}
 fn test_derive() {
     let _ = Foo::default();
     assert_fields!(Foo: input_types);
-    assert_impl_all!(Foo: std::fmt::Debug, Default, AggrFuncTypeStore);
+    assert_impl_all!(Foo: std::fmt::Debug, Default, common_query::logical_plan::accumulator::AggrFuncTypeStore);
 }
diff --git a/src/datatypes/src/data_type.rs b/src/datatypes/src/data_type.rs
index 495c47dc5f..12f5c8d1f2 100644
--- a/src/datatypes/src/data_type.rs
+++ b/src/datatypes/src/data_type.rs
@@ -249,6 +249,15 @@ impl ConcreteDataType {
         ]
     }
 
+    pub fn timestamps() -> Vec<ConcreteDataType> {
+        vec![
+            ConcreteDataType::timestamp_second_datatype(),
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            ConcreteDataType::timestamp_microsecond_datatype(),
+            ConcreteDataType::timestamp_nanosecond_datatype(),
+        ]
+    }
+
     /// Convert arrow data type to [ConcreteDataType].
     ///
     /// # Panics
diff --git a/src/query/src/tests/my_sum_udaf_example.rs b/src/query/src/tests/my_sum_udaf_example.rs
index 8220dcf72d..63653ffbce 100644
--- a/src/query/src/tests/my_sum_udaf_example.rs
+++ b/src/query/src/tests/my_sum_udaf_example.rs
@@ -19,6 +19,7 @@ use std::sync::Arc;
 use common_function::scalars::aggregate::AggregateFunctionMeta;
 use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
 use common_query::error::{CreateAccumulatorSnafu, Result as QueryResult};
+use common_query::logical_plan::accumulator::AggrFuncTypeStore;
 use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
 use common_query::prelude::*;
 use common_recordbatch::{RecordBatch, RecordBatches};
diff --git a/tests/cases/standalone/common/function/geo.result b/tests/cases/standalone/common/function/geo.result
index 8954447650..4f9d168ac0 100644
--- a/tests/cases/standalone/common/function/geo.result
+++ b/tests/cases/standalone/common/function/geo.result
@@ -236,3 +236,28 @@ SELECT geohash_neighbours(37.76938, -122.3889, 11);
 | [9q8yygxnefv, 9q8yygxnefu, 9q8yygxnefs, 9q8yygxnefk, 9q8yygxnefm, 9q8yygxnefq, 9q8yygxnefw, 9q8yygxnefy] |
 +----------------------------------------------------------------------------------------------------------+
 
+SELECT json_encode_path(37.76938, -122.3889, 1728083375::TimestampSecond);
+
++----------------------------------------------------------------------------------------------------------------------+
+| json_encode_path(Float64(37.76938),Float64(-122.3889),arrow_cast(Int64(1728083375),Utf8("Timestamp(Second, None)"))) |
++----------------------------------------------------------------------------------------------------------------------+
+| [[-122.3889,37.76938]]                                                                                               |
++----------------------------------------------------------------------------------------------------------------------+
+
+SELECT json_encode_path(lat, lon, ts)
+FROM(
+        SELECT 37.76938 AS lat, -122.3889 AS lon, 1728083375::TimestampSecond AS ts
+        UNION ALL
+        SELECT 37.76928 AS lat, -122.3839 AS lon, 1728083373::TimestampSecond AS ts
+        UNION ALL
+        SELECT 37.76930 AS lat, -122.3820 AS lon, 1728083379::TimestampSecond AS ts
+        UNION ALL
+        SELECT 37.77001 AS lat, -122.3888 AS lon, 1728083372::TimestampSecond AS ts
+);
+
++-------------------------------------------------------------------------------------+
+| json_encode_path(lat,lon,ts)                                                        |
++-------------------------------------------------------------------------------------+
+| [[-122.3888,37.77001],[-122.3839,37.76928],[-122.3889,37.76938],[-122.382,37.7693]] |
++-------------------------------------------------------------------------------------+
+
diff --git a/tests/cases/standalone/common/function/geo.sql b/tests/cases/standalone/common/function/geo.sql
index be2b3947bb..cd9d403e6e 100644
--- a/tests/cases/standalone/common/function/geo.sql
+++ b/tests/cases/standalone/common/function/geo.sql
@@ -66,3 +66,16 @@ SELECT geohash(37.76938, -122.3889, 11::UInt32);
 SELECT geohash(37.76938, -122.3889, 11::UInt64);
 
 SELECT geohash_neighbours(37.76938, -122.3889, 11);
+
+SELECT json_encode_path(37.76938, -122.3889, 1728083375::TimestampSecond);
+
+SELECT json_encode_path(lat, lon, ts)
+FROM(
+        SELECT 37.76938 AS lat, -122.3889 AS lon, 1728083375::TimestampSecond AS ts
+        UNION ALL
+        SELECT 37.76928 AS lat, -122.3839 AS lon, 1728083373::TimestampSecond AS ts
+        UNION ALL
+        SELECT 37.76930 AS lat, -122.3820 AS lon, 1728083379::TimestampSecond AS ts
+        UNION ALL
+        SELECT 37.77001 AS lat, -122.3888 AS lon, 1728083372::TimestampSecond AS ts
+);

From 0dd11f53f55ff1b342fda40d20a5125f568b244a Mon Sep 17 00:00:00 2001
From: Kaifeng Zheng <100595273+Kev1n8@users.noreply.github.com>
Date: Wed, 9 Oct 2024 15:11:57 +0800
Subject: [PATCH 060/128] feat: add json format output for http interface
 (#4797)

* feat: json output format for http

* feat: add json result test case

* fix: typo and refactor a piece of code

* fix: cargo check

* move affected_rows to top level
---
 src/servers/src/http.rs             |  32 +++++++
 src/servers/src/http/handler.rs     |   2 +
 src/servers/src/http/json_result.rs | 137 ++++++++++++++++++++++++++++
 tests-integration/tests/http.rs     |  18 +++-
 4 files changed, 188 insertions(+), 1 deletion(-)
 create mode 100644 src/servers/src/http/json_result.rs

diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs
index a2b72b548b..953ff9e73a 100644
--- a/src/servers/src/http.rs
+++ b/src/servers/src/http.rs
@@ -59,6 +59,7 @@ use crate::http::error_result::ErrorResponse;
 use crate::http::greptime_result_v1::GreptimedbV1Response;
 use crate::http::influxdb::{influxdb_health, influxdb_ping, influxdb_write_v1, influxdb_write_v2};
 use crate::http::influxdb_result_v1::InfluxdbV1Response;
+use crate::http::json_result::JsonResponse;
 use crate::http::prometheus::{
     build_info_query, format_query, instant_query, label_values_query, labels_query, range_query,
     series_query,
@@ -97,6 +98,7 @@ pub mod error_result;
 pub mod greptime_manage_resp;
 pub mod greptime_result_v1;
 pub mod influxdb_result_v1;
+pub mod json_result;
 pub mod table_result;
 
 #[cfg(any(test, feature = "testing"))]
@@ -279,6 +281,7 @@ pub enum ResponseFormat {
     #[default]
     GreptimedbV1,
     InfluxdbV1,
+    Json,
 }
 
 impl ResponseFormat {
@@ -289,6 +292,7 @@ impl ResponseFormat {
             "table" => Some(ResponseFormat::Table),
             "greptimedb_v1" => Some(ResponseFormat::GreptimedbV1),
             "influxdb_v1" => Some(ResponseFormat::InfluxdbV1),
+            "json" => Some(ResponseFormat::Json),
             _ => None,
         }
     }
@@ -300,6 +304,7 @@ impl ResponseFormat {
             ResponseFormat::Table => "table",
             ResponseFormat::GreptimedbV1 => "greptimedb_v1",
             ResponseFormat::InfluxdbV1 => "influxdb_v1",
+            ResponseFormat::Json => "json",
         }
     }
 }
@@ -356,6 +361,7 @@ pub enum HttpResponse {
     Error(ErrorResponse),
     GreptimedbV1(GreptimedbV1Response),
     InfluxdbV1(InfluxdbV1Response),
+    Json(JsonResponse),
 }
 
 impl HttpResponse {
@@ -366,6 +372,7 @@ impl HttpResponse {
             HttpResponse::Table(resp) => resp.with_execution_time(execution_time).into(),
             HttpResponse::GreptimedbV1(resp) => resp.with_execution_time(execution_time).into(),
             HttpResponse::InfluxdbV1(resp) => resp.with_execution_time(execution_time).into(),
+            HttpResponse::Json(resp) => resp.with_execution_time(execution_time).into(),
             HttpResponse::Error(resp) => resp.with_execution_time(execution_time).into(),
         }
     }
@@ -375,6 +382,7 @@ impl HttpResponse {
             HttpResponse::Csv(resp) => resp.with_limit(limit).into(),
             HttpResponse::Table(resp) => resp.with_limit(limit).into(),
             HttpResponse::GreptimedbV1(resp) => resp.with_limit(limit).into(),
+            HttpResponse::Json(resp) => resp.with_limit(limit).into(),
             _ => self,
         }
     }
@@ -407,6 +415,7 @@ impl IntoResponse for HttpResponse {
             HttpResponse::Table(resp) => resp.into_response(),
             HttpResponse::GreptimedbV1(resp) => resp.into_response(),
             HttpResponse::InfluxdbV1(resp) => resp.into_response(),
+            HttpResponse::Json(resp) => resp.into_response(),
             HttpResponse::Error(resp) => resp.into_response(),
         }
     }
@@ -452,6 +461,12 @@ impl From<InfluxdbV1Response> for HttpResponse {
     }
 }
 
+impl From<JsonResponse> for HttpResponse {
+    fn from(value: JsonResponse) -> Self {
+        HttpResponse::Json(value)
+    }
+}
+
 async fn serve_api(Extension(api): Extension<OpenApi>) -> impl IntoApiResponse {
     Json(api)
 }
@@ -1131,6 +1146,7 @@ mod test {
             ResponseFormat::Csv,
             ResponseFormat::Table,
             ResponseFormat::Arrow,
+            ResponseFormat::Json,
         ] {
             let recordbatches =
                 RecordBatches::try_new(schema.clone(), vec![recordbatch.clone()]).unwrap();
@@ -1141,6 +1157,7 @@ mod test {
                 ResponseFormat::Table => TableResponse::from_output(outputs).await,
                 ResponseFormat::GreptimedbV1 => GreptimedbV1Response::from_output(outputs).await,
                 ResponseFormat::InfluxdbV1 => InfluxdbV1Response::from_output(outputs, None).await,
+                ResponseFormat::Json => JsonResponse::from_output(outputs).await,
             };
 
             match json_resp {
@@ -1210,6 +1227,21 @@ mod test {
                     assert_eq!(rb.num_columns(), 2);
                     assert_eq!(rb.num_rows(), 4);
                 }
+
+                HttpResponse::Json(resp) => {
+                    let output = &resp.output()[0];
+                    if let GreptimeQueryOutput::Records(r) = output {
+                        assert_eq!(r.num_rows(), 4);
+                        assert_eq!(r.num_cols(), 2);
+                        assert_eq!(r.schema.column_schemas[0].name, "numbers");
+                        assert_eq!(r.schema.column_schemas[0].data_type, "UInt32");
+                        assert_eq!(r.rows[0][0], serde_json::Value::from(1));
+                        assert_eq!(r.rows[0][1], serde_json::Value::Null);
+                    } else {
+                        panic!("invalid output type");
+                    }
+                }
+
                 HttpResponse::Error(err) => unreachable!("{err:?}"),
             }
         }
diff --git a/src/servers/src/http/handler.rs b/src/servers/src/http/handler.rs
index 1befc22240..4925c79639 100644
--- a/src/servers/src/http/handler.rs
+++ b/src/servers/src/http/handler.rs
@@ -39,6 +39,7 @@ use crate::http::csv_result::CsvResponse;
 use crate::http::error_result::ErrorResponse;
 use crate::http::greptime_result_v1::GreptimedbV1Response;
 use crate::http::influxdb_result_v1::InfluxdbV1Response;
+use crate::http::json_result::JsonResponse;
 use crate::http::table_result::TableResponse;
 use crate::http::{
     ApiState, Epoch, GreptimeOptionsConfigState, GreptimeQueryOutput, HttpRecordsOutput,
@@ -138,6 +139,7 @@ pub async fn sql(
         ResponseFormat::Table => TableResponse::from_output(outputs).await,
         ResponseFormat::GreptimedbV1 => GreptimedbV1Response::from_output(outputs).await,
         ResponseFormat::InfluxdbV1 => InfluxdbV1Response::from_output(outputs, epoch).await,
+        ResponseFormat::Json => JsonResponse::from_output(outputs).await,
     };
 
     if let Some(limit) = query_params.limit {
diff --git a/src/servers/src/http/json_result.rs b/src/servers/src/http/json_result.rs
new file mode 100644
index 0000000000..bf4e4d7770
--- /dev/null
+++ b/src/servers/src/http/json_result.rs
@@ -0,0 +1,137 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use axum::http::{header, HeaderValue};
+use axum::response::{IntoResponse, Response};
+use common_error::status_code::StatusCode;
+use common_query::Output;
+use mime_guess::mime;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use serde_json::{json, Map, Value};
+
+use super::process_with_limit;
+use crate::http::error_result::ErrorResponse;
+use crate::http::header::{GREPTIME_DB_HEADER_EXECUTION_TIME, GREPTIME_DB_HEADER_FORMAT};
+use crate::http::{handler, GreptimeQueryOutput, HttpResponse, ResponseFormat};
+
+/// The json format here is different from the default json output of `GreptimedbV1` result.
+/// `JsonResponse` is intended to make it easier for user to consume data.
+#[derive(Serialize, Deserialize, Debug, JsonSchema)]
+pub struct JsonResponse {
+    output: Vec<GreptimeQueryOutput>,
+    execution_time_ms: u64,
+}
+
+impl JsonResponse {
+    pub async fn from_output(outputs: Vec<crate::error::Result<Output>>) -> HttpResponse {
+        match handler::from_output(outputs).await {
+            Err(err) => HttpResponse::Error(err),
+            Ok((output, _)) => {
+                if output.len() > 1 {
+                    HttpResponse::Error(ErrorResponse::from_error_message(
+                        StatusCode::InvalidArguments,
+                        "cannot output multi-statements result in json format".to_string(),
+                    ))
+                } else {
+                    HttpResponse::Json(JsonResponse {
+                        output,
+                        execution_time_ms: 0,
+                    })
+                }
+            }
+        }
+    }
+
+    pub fn output(&self) -> &[GreptimeQueryOutput] {
+        &self.output
+    }
+
+    pub fn with_execution_time(mut self, execution_time: u64) -> Self {
+        self.execution_time_ms = execution_time;
+        self
+    }
+
+    pub fn execution_time_ms(&self) -> u64 {
+        self.execution_time_ms
+    }
+
+    pub fn with_limit(mut self, limit: usize) -> Self {
+        self.output = process_with_limit(self.output, limit);
+        self
+    }
+}
+
+impl IntoResponse for JsonResponse {
+    fn into_response(mut self) -> Response {
+        debug_assert!(
+            self.output.len() <= 1,
+            "self.output has extra elements: {}",
+            self.output.len()
+        );
+
+        let execution_time = self.execution_time_ms;
+        let payload = match self.output.pop() {
+            None => String::default(),
+            Some(GreptimeQueryOutput::AffectedRows(n)) => json!({
+                "data": [],
+                "affected_rows": n,
+                "execution_time_ms": execution_time,
+            })
+            .to_string(),
+
+            Some(GreptimeQueryOutput::Records(records)) => {
+                let schema = records.schema();
+
+                let data: Vec<Map<String, Value>> = records
+                    .rows
+                    .iter()
+                    .map(|row| {
+                        schema
+                            .column_schemas
+                            .iter()
+                            .enumerate()
+                            .map(|(i, col)| (col.name.clone(), row[i].clone()))
+                            .collect::<Map<String, Value>>()
+                    })
+                    .collect();
+
+                json!({
+                    "data": data,
+                    "execution_time_ms": execution_time,
+                })
+                .to_string()
+            }
+        };
+
+        (
+            [
+                (
+                    header::CONTENT_TYPE,
+                    HeaderValue::from_static(mime::APPLICATION_JSON.as_ref()),
+                ),
+                (
+                    GREPTIME_DB_HEADER_FORMAT.clone(),
+                    HeaderValue::from_static(ResponseFormat::Json.as_str()),
+                ),
+                (
+                    GREPTIME_DB_HEADER_EXECUTION_TIME.clone(),
+                    HeaderValue::from(execution_time),
+                ),
+            ],
+            payload,
+        )
+            .into_response()
+    }
+}
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index f70df8176a..7a030aad56 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -182,6 +182,22 @@ pub async fn test_sql_api(store_type: StorageType) {
         })).unwrap()
     );
 
+    // test json result format
+    let res = client
+        .get("/v1/sql?format=json&sql=select * from numbers limit 10")
+        .send()
+        .await;
+    assert_eq!(res.status(), StatusCode::OK);
+
+    let body = res.json::<Value>().await;
+    let data = body.get("data").expect("Missing 'data' field in response");
+
+    let expected = json!([
+        {"number": 0}, {"number": 1}, {"number": 2}, {"number": 3}, {"number": 4},
+        {"number": 5}, {"number": 6}, {"number": 7}, {"number": 8}, {"number": 9}
+    ]);
+    assert_eq!(data, &expected);
+
     // test insert and select
     let res = client
         .get("/v1/sql?sql=insert into demo values('host', 66.6, 1024, 0)")
@@ -1307,7 +1323,7 @@ transform:
         .send()
         .await;
     assert_eq!(res.status(), StatusCode::OK);
-    let body: serde_json::Value = res.json().await;
+    let body: Value = res.json().await;
     let schema = &body["schema"];
     let rows = &body["rows"];
     assert_eq!(

From a8ed3db0aa272396c5818d42303e263a8faf4be4 Mon Sep 17 00:00:00 2001
From: discord9 <55937128+discord9@users.noreply.github.com>
Date: Wed, 9 Oct 2024 17:37:27 +0800
Subject: [PATCH 061/128] feat: Merge sort Logical plan (#4768)

* feat(WIP): MergeSort

* wip

* feat: MergeSort LogicalPlan

* update sqlness result

* Apply suggestions from code review

Co-authored-by: Lei, HUANG <6406592+v0y4g3r@users.noreply.github.com>

* refactor: per review advice

* refactor: more per review

* chore: per review

---------

Co-authored-by: Ruihang Xia <waynestxia@gmail.com>
Co-authored-by: Lei, HUANG <6406592+v0y4g3r@users.noreply.github.com>
---
 src/query/src/dist_plan.rs                    |   3 +-
 src/query/src/dist_plan/analyzer.rs           |   4 +-
 src/query/src/dist_plan/commutativity.rs      |   9 +-
 src/query/src/dist_plan/merge_scan.rs         |   8 ++
 src/query/src/dist_plan/merge_sort.rs         | 124 ++++++++++++++++++
 src/query/src/dist_plan/planner.rs            |  62 ++++++++-
 src/query/src/query_engine/state.rs           |   3 +-
 .../explain/multi_partitions.result           |   2 +-
 .../standalone/common/order/order_by.result   |   2 +-
 9 files changed, 207 insertions(+), 10 deletions(-)
 create mode 100644 src/query/src/dist_plan/merge_sort.rs

diff --git a/src/query/src/dist_plan.rs b/src/query/src/dist_plan.rs
index 6ab93d4e1d..b2bad029c4 100644
--- a/src/query/src/dist_plan.rs
+++ b/src/query/src/dist_plan.rs
@@ -15,8 +15,9 @@
 mod analyzer;
 mod commutativity;
 mod merge_scan;
+mod merge_sort;
 mod planner;
 
 pub use analyzer::DistPlannerAnalyzer;
 pub use merge_scan::{MergeScanExec, MergeScanLogicalPlan};
-pub use planner::DistExtensionPlanner;
+pub use planner::{DistExtensionPlanner, MergeSortExtensionPlanner};
diff --git a/src/query/src/dist_plan/analyzer.rs b/src/query/src/dist_plan/analyzer.rs
index c182eee1cd..335c23c58a 100644
--- a/src/query/src/dist_plan/analyzer.rs
+++ b/src/query/src/dist_plan/analyzer.rs
@@ -160,7 +160,6 @@ impl PlanRewriter {
         {
             return true;
         }
-
         match Categorizer::check_plan(plan, self.partition_cols.clone()) {
             Commutativity::Commutative => {}
             Commutativity::PartialCommutative => {
@@ -265,9 +264,10 @@ impl PlanRewriter {
 
         // add merge scan as the new root
         let mut node = MergeScanLogicalPlan::new(on_node, false).into_logical_plan();
+
         // expand stages
         for new_stage in self.stage.drain(..) {
-            node = new_stage.with_new_exprs(new_stage.expressions(), vec![node.clone()])?
+            node = new_stage.with_new_exprs(new_stage.expressions(), vec![node.clone()])?;
         }
         self.set_expanded();
 
diff --git a/src/query/src/dist_plan/commutativity.rs b/src/query/src/dist_plan/commutativity.rs
index c6b3ca2c62..8166400b8f 100644
--- a/src/query/src/dist_plan/commutativity.rs
+++ b/src/query/src/dist_plan/commutativity.rs
@@ -21,6 +21,7 @@ use promql::extension_plan::{
     EmptyMetric, InstantManipulate, RangeManipulate, SeriesDivide, SeriesNormalize,
 };
 
+use crate::dist_plan::merge_sort::{merge_sort_transformer, MergeSortLogicalPlan};
 use crate::dist_plan::MergeScanLogicalPlan;
 
 #[allow(dead_code)]
@@ -68,8 +69,9 @@ impl Categorizer {
                 }
 
                 // sort plan needs to consider column priority
-                // We can implement a merge-sort on partial ordered data
-                Commutativity::PartialCommutative
+                // Change Sort to MergeSort which assumes the input streams are already sorted hence can be more efficient
+                // We should ensure the number of partition is not smaller than the number of region at present. Otherwise this would result in incorrect output.
+                Commutativity::ConditionalCommutative(Some(Arc::new(merge_sort_transformer)))
             }
             LogicalPlan::Join(_) => Commutativity::NonCommutative,
             LogicalPlan::CrossJoin(_) => Commutativity::NonCommutative,
@@ -118,7 +120,8 @@ impl Categorizer {
                 || name == SeriesNormalize::name()
                 || name == RangeManipulate::name()
                 || name == SeriesDivide::name()
-                || name == MergeScanLogicalPlan::name() =>
+                || name == MergeScanLogicalPlan::name()
+                || name == MergeSortLogicalPlan::name() =>
             {
                 Commutativity::Unimplemented
             }
diff --git a/src/query/src/dist_plan/merge_scan.rs b/src/query/src/dist_plan/merge_scan.rs
index a3fb8004cf..8ed77eabd4 100644
--- a/src/query/src/dist_plan/merge_scan.rs
+++ b/src/query/src/dist_plan/merge_scan.rs
@@ -298,6 +298,14 @@ impl MergeScanExec {
     pub fn sub_stage_metrics(&self) -> Vec<RecordBatchMetrics> {
         self.sub_stage_metrics.lock().unwrap().clone()
     }
+
+    pub fn partition_count(&self) -> usize {
+        self.target_partition
+    }
+
+    pub fn region_count(&self) -> usize {
+        self.regions.len()
+    }
 }
 
 impl ExecutionPlan for MergeScanExec {
diff --git a/src/query/src/dist_plan/merge_sort.rs b/src/query/src/dist_plan/merge_sort.rs
new file mode 100644
index 0000000000..df0a9fda10
--- /dev/null
+++ b/src/query/src/dist_plan/merge_sort.rs
@@ -0,0 +1,124 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Merge sort logical plan for distributed query execution, roughly corresponding to the
+//! `SortPreservingMergeExec` operator in datafusion
+//!
+
+use std::fmt;
+use std::sync::Arc;
+
+use datafusion_common::{DataFusionError, Result};
+use datafusion_expr::{Expr, Extension, LogicalPlan, UserDefinedLogicalNodeCore};
+
+/// MergeSort Logical Plan, have same field as `Sort`, but indicate it is a merge sort,
+/// which assume each input partition is a sorted stream, and will use `SortPreserveingMergeExec`
+/// to merge them into a single sorted stream.
+#[derive(Hash, PartialEq, Eq, Clone)]
+pub struct MergeSortLogicalPlan {
+    pub expr: Vec<Expr>,
+    pub input: Arc<LogicalPlan>,
+    pub fetch: Option<usize>,
+}
+
+impl fmt::Debug for MergeSortLogicalPlan {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        UserDefinedLogicalNodeCore::fmt_for_explain(self, f)
+    }
+}
+
+impl MergeSortLogicalPlan {
+    pub fn new(input: Arc<LogicalPlan>, expr: Vec<Expr>, fetch: Option<usize>) -> Self {
+        Self { input, expr, fetch }
+    }
+
+    pub fn name() -> &'static str {
+        "MergeSort"
+    }
+
+    /// Create a [`LogicalPlan::Extension`] node from this merge sort plan
+    pub fn into_logical_plan(self) -> LogicalPlan {
+        LogicalPlan::Extension(Extension {
+            node: Arc::new(self),
+        })
+    }
+
+    /// Convert self to a [`Sort`] logical plan with same input and expressions
+    pub fn into_sort(self) -> LogicalPlan {
+        LogicalPlan::Sort(datafusion::logical_expr::Sort {
+            input: self.input.clone(),
+            expr: self.expr,
+            fetch: self.fetch,
+        })
+    }
+}
+
+impl UserDefinedLogicalNodeCore for MergeSortLogicalPlan {
+    fn name(&self) -> &str {
+        Self::name()
+    }
+
+    // Allow optimization here
+    fn inputs(&self) -> Vec<&LogicalPlan> {
+        vec![self.input.as_ref()]
+    }
+
+    fn schema(&self) -> &datafusion_common::DFSchemaRef {
+        self.input.schema()
+    }
+
+    // Allow further optimization
+    fn expressions(&self) -> Vec<datafusion_expr::Expr> {
+        self.expr.clone()
+    }
+
+    fn fmt_for_explain(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "MergeSort: ")?;
+        for (i, expr_item) in self.expr.iter().enumerate() {
+            if i > 0 {
+                write!(f, ", ")?;
+            }
+            write!(f, "{expr_item}")?;
+        }
+        if let Some(a) = self.fetch {
+            write!(f, ", fetch={a}")?;
+        }
+        Ok(())
+    }
+
+    fn with_exprs_and_inputs(
+        &self,
+        exprs: Vec<datafusion::prelude::Expr>,
+        mut inputs: Vec<LogicalPlan>,
+    ) -> Result<Self> {
+        let mut zelf = self.clone();
+        zelf.expr = exprs;
+        zelf.input = Arc::new(inputs.pop().ok_or_else(|| {
+            DataFusionError::Internal("Expected exactly one input with MergeSort".to_string())
+        })?);
+        Ok(zelf)
+    }
+}
+
+/// Turn `Sort` into `MergeSort` if possible
+pub fn merge_sort_transformer(plan: &LogicalPlan) -> Option<LogicalPlan> {
+    if let LogicalPlan::Sort(sort) = plan {
+        Some(
+            MergeSortLogicalPlan::new(sort.input.clone(), sort.expr.clone(), sort.fetch)
+                .into_logical_plan(),
+        )
+    } else {
+        None
+    }
+}
diff --git a/src/query/src/dist_plan/planner.rs b/src/query/src/dist_plan/planner.rs
index a94a798461..833ff1c9bc 100644
--- a/src/query/src/dist_plan/planner.rs
+++ b/src/query/src/dist_plan/planner.rs
@@ -25,7 +25,7 @@ use datafusion::execution::context::SessionState;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion::physical_planner::{ExtensionPlanner, PhysicalPlanner};
 use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion, TreeNodeVisitor};
-use datafusion_common::TableReference;
+use datafusion_common::{DataFusionError, TableReference};
 use datafusion_expr::{LogicalPlan, UserDefinedLogicalNode};
 use session::context::QueryContext;
 use snafu::{OptionExt, ResultExt};
@@ -35,9 +35,69 @@ use table::table::adapter::DfTableProviderAdapter;
 use table::table_name::TableName;
 
 use crate::dist_plan::merge_scan::{MergeScanExec, MergeScanLogicalPlan};
+use crate::dist_plan::merge_sort::MergeSortLogicalPlan;
 use crate::error::{CatalogSnafu, TableNotFoundSnafu};
 use crate::region_query::RegionQueryHandlerRef;
 
+/// Planner for convert merge sort logical plan to physical plan
+/// it is currently a fallback to sort, and doesn't change the execution plan:
+/// `MergeSort(MergeScan) -> Sort(MergeScan) - to physical plan -> ...`
+/// It should be applied after `DistExtensionPlanner`
+///
+/// (Later when actually impl this merge sort)
+///
+/// We should ensure the number of partition is not smaller than the number of region at present. Otherwise this would result in incorrect output.
+pub struct MergeSortExtensionPlanner {}
+
+#[async_trait]
+impl ExtensionPlanner for MergeSortExtensionPlanner {
+    async fn plan_extension(
+        &self,
+        planner: &dyn PhysicalPlanner,
+        node: &dyn UserDefinedLogicalNode,
+        _logical_inputs: &[&LogicalPlan],
+        physical_inputs: &[Arc<dyn ExecutionPlan>],
+        session_state: &SessionState,
+    ) -> Result<Option<Arc<dyn ExecutionPlan>>> {
+        if let Some(merge_sort) = node.as_any().downcast_ref::<MergeSortLogicalPlan>() {
+            if let LogicalPlan::Extension(ext) = &merge_sort.input.as_ref()
+                && ext
+                    .node
+                    .as_any()
+                    .downcast_ref::<MergeScanLogicalPlan>()
+                    .is_some()
+            {
+                let merge_scan_exec = physical_inputs
+                    .first()
+                    .and_then(|p| p.as_any().downcast_ref::<MergeScanExec>())
+                    .ok_or(DataFusionError::Internal(format!(
+                        "Expect MergeSort's input is a MergeScanExec, found {:?}",
+                        physical_inputs
+                    )))?;
+
+                let partition_cnt = merge_scan_exec.partition_count();
+                let region_cnt = merge_scan_exec.region_count();
+                // if partition >= region, we know that every partition stream of merge scan is ordered
+                // and we only need to do a merge sort, otherwise fallback to quick sort
+                let can_merge_sort = partition_cnt >= region_cnt;
+                if can_merge_sort {
+                    // TODO(discord9): use `SortPreversingMergeExec here`
+                }
+                // for now merge sort only exist in logical plan, and have the same effect as `Sort`
+                // doesn't change the execution plan, this will change in the future
+                let ret = planner
+                    .create_physical_plan(&merge_sort.clone().into_sort(), session_state)
+                    .await?;
+                Ok(Some(ret))
+            } else {
+                Ok(None)
+            }
+        } else {
+            Ok(None)
+        }
+    }
+}
+
 pub struct DistExtensionPlanner {
     catalog_manager: CatalogManagerRef,
     region_query_handler: RegionQueryHandlerRef,
diff --git a/src/query/src/query_engine/state.rs b/src/query/src/query_engine/state.rs
index f0e2ef53a5..97386e9b2d 100644
--- a/src/query/src/query_engine/state.rs
+++ b/src/query/src/query_engine/state.rs
@@ -42,7 +42,7 @@ use promql::extension_plan::PromExtensionPlanner;
 use table::table::adapter::DfTableProviderAdapter;
 use table::TableRef;
 
-use crate::dist_plan::{DistExtensionPlanner, DistPlannerAnalyzer};
+use crate::dist_plan::{DistExtensionPlanner, DistPlannerAnalyzer, MergeSortExtensionPlanner};
 use crate::optimizer::count_wildcard::CountWildcardToTimeIndexRule;
 use crate::optimizer::parallelize_scan::ParallelizeScan;
 use crate::optimizer::remove_duplicate::RemoveDuplicate;
@@ -295,6 +295,7 @@ impl DfQueryPlanner {
                 catalog_manager,
                 region_query_handler,
             )));
+            planners.push(Arc::new(MergeSortExtensionPlanner {}));
         }
         Self {
             physical_planner: DefaultPhysicalPlanner::with_extension_planners(planners),
diff --git a/tests/cases/distributed/explain/multi_partitions.result b/tests/cases/distributed/explain/multi_partitions.result
index 6bbb6abb8e..c06869d459 100644
--- a/tests/cases/distributed/explain/multi_partitions.result
+++ b/tests/cases/distributed/explain/multi_partitions.result
@@ -25,7 +25,7 @@ explain SELECT * FROM demo WHERE ts > cast(1000000000 as timestamp) ORDER BY hos
 +-+-+
 | plan_type_| plan_|
 +-+-+
-| logical_plan_| Sort: demo.host ASC NULLS LAST_|
+| logical_plan_| MergeSort: demo.host ASC NULLS LAST_|
 |_|_MergeScan [is_placeholder=false]_|
 | physical_plan | SortPreservingMergeExec: [host@0 ASC NULLS LAST]_|
 |_|_SortExec: expr=[host@0 ASC NULLS LAST], preserve_partitioning=[true]_|
diff --git a/tests/cases/standalone/common/order/order_by.result b/tests/cases/standalone/common/order/order_by.result
index f6385537c3..8cf92d9fbf 100644
--- a/tests/cases/standalone/common/order/order_by.result
+++ b/tests/cases/standalone/common/order/order_by.result
@@ -294,7 +294,7 @@ explain analyze select tag from t where num > 6 order by ts desc limit 2;
 +-+-+-+
 | 0_| 0_|_GlobalLimitExec: skip=0, fetch=2 REDACTED
 |_|_|_SortPreservingMergeExec: [ts@1 DESC] REDACTED
-|_|_|_SortExec: TopK(fetch=2), expr=[ts@1 DESC], preserve_partitioning=[true] REDACTED
+|_|_|_SortExec: expr=[ts@1 DESC], preserve_partitioning=[true] REDACTED
 |_|_|_MergeScanExec: REDACTED
 |_|_|_|
 | 1_| 0_|_GlobalLimitExec: skip=0, fetch=2 REDACTED

From 288fdc31453f7db461a6218328850bfc45081d39 Mon Sep 17 00:00:00 2001
From: Yohan Wal <59358312+CookiePieWw@users.noreply.github.com>
Date: Thu, 10 Oct 2024 22:15:34 +0800
Subject: [PATCH 062/128] feat: json_path_exists udf (#4807)

* feat: json_path_exists udf

* chore: fix comments

* fix: caution when copy&paste QAQ
---
 src/common/function/src/scalars/json.rs       |   3 +
 .../src/scalars/json/json_path_exists.rs      | 172 ++++++++++++++++++
 .../common/function/json/json.result          |  33 ++++
 .../standalone/common/function/json/json.sql  |   8 +
 4 files changed, 216 insertions(+)
 create mode 100644 src/common/function/src/scalars/json/json_path_exists.rs
 create mode 100644 tests/cases/standalone/common/function/json/json.result
 create mode 100644 tests/cases/standalone/common/function/json/json.sql

diff --git a/src/common/function/src/scalars/json.rs b/src/common/function/src/scalars/json.rs
index 279577b495..133eb145a7 100644
--- a/src/common/function/src/scalars/json.rs
+++ b/src/common/function/src/scalars/json.rs
@@ -15,6 +15,7 @@
 use std::sync::Arc;
 mod json_get;
 mod json_is;
+mod json_path_exists;
 mod json_to_string;
 mod parse_json;
 
@@ -46,5 +47,7 @@ impl JsonFunction {
         registry.register(Arc::new(JsonIsBool));
         registry.register(Arc::new(JsonIsArray));
         registry.register(Arc::new(JsonIsObject));
+
+        registry.register(Arc::new(json_path_exists::JsonPathExistsFunction));
     }
 }
diff --git a/src/common/function/src/scalars/json/json_path_exists.rs b/src/common/function/src/scalars/json/json_path_exists.rs
new file mode 100644
index 0000000000..9254b5f396
--- /dev/null
+++ b/src/common/function/src/scalars/json/json_path_exists.rs
@@ -0,0 +1,172 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::{self, Display};
+
+use common_query::error::{InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu};
+use common_query::prelude::Signature;
+use datafusion::logical_expr::Volatility;
+use datatypes::data_type::ConcreteDataType;
+use datatypes::prelude::VectorRef;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::vectors::{BooleanVectorBuilder, MutableVector};
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+
+/// Check if the given JSON data contains the given JSON path.
+#[derive(Clone, Debug, Default)]
+pub struct JsonPathExistsFunction;
+
+const NAME: &str = "json_path_exists";
+
+impl Function for JsonPathExistsFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::boolean_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::exact(
+            vec![
+                ConcreteDataType::json_datatype(),
+                ConcreteDataType::string_datatype(),
+            ],
+            Volatility::Immutable,
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly two, have: {}",
+                    columns.len()
+                ),
+            }
+        );
+        let jsons = &columns[0];
+        let paths = &columns[1];
+
+        let size = jsons.len();
+        let datatype = jsons.data_type();
+        let mut results = BooleanVectorBuilder::with_capacity(size);
+
+        match datatype {
+            // JSON data type uses binary vector
+            ConcreteDataType::Binary(_) => {
+                for i in 0..size {
+                    let json = jsons.get_ref(i);
+                    let path = paths.get_ref(i);
+
+                    let json = json.as_binary();
+                    let path = path.as_string();
+                    let result = match (json, path) {
+                        (Ok(Some(json)), Ok(Some(path))) => {
+                            let json_path = jsonb::jsonpath::parse_json_path(path.as_bytes());
+                            match json_path {
+                                Ok(json_path) => jsonb::path_exists(json, json_path).ok(),
+                                Err(_) => None,
+                            }
+                        }
+                        _ => None,
+                    };
+
+                    results.push(result);
+                }
+            }
+            _ => {
+                return UnsupportedInputDataTypeSnafu {
+                    function: NAME,
+                    datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
+                }
+                .fail();
+            }
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+impl Display for JsonPathExistsFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "JSON_PATH_EXISTS")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_query::prelude::TypeSignature;
+    use datatypes::scalars::ScalarVector;
+    use datatypes::vectors::{BinaryVector, StringVector};
+
+    use super::*;
+
+    #[test]
+    fn test_json_path_exists_function() {
+        let json_path_exists = JsonPathExistsFunction;
+
+        assert_eq!("json_path_exists", json_path_exists.name());
+        assert_eq!(
+            ConcreteDataType::boolean_datatype(),
+            json_path_exists
+                .return_type(&[ConcreteDataType::json_datatype()])
+                .unwrap()
+        );
+
+        assert!(matches!(json_path_exists.signature(),
+                         Signature {
+                             type_signature: TypeSignature::Exact(valid_types),
+                             volatility: Volatility::Immutable
+                         } if  valid_types == vec![ConcreteDataType::json_datatype(), ConcreteDataType::string_datatype()]
+        ));
+
+        let json_strings = [
+            r#"{"a": {"b": 2}, "b": 2, "c": 3}"#,
+            r#"{"a": 4, "b": {"c": 6}, "c": 6}"#,
+            r#"{"a": 7, "b": 8, "c": {"a": 7}}"#,
+            r#"{"a": 7, "b": 8, "c": {"a": 7}}"#,
+        ];
+        let paths = vec!["$.a.b.c", "$.b", "$.c.a", ".d"];
+        let results = [false, true, true, false];
+
+        let jsonbs = json_strings
+            .iter()
+            .map(|s| {
+                let value = jsonb::parse_value(s.as_bytes()).unwrap();
+                value.to_vec()
+            })
+            .collect::<Vec<_>>();
+
+        let json_vector = BinaryVector::from_vec(jsonbs);
+        let path_vector = StringVector::from_vec(paths);
+        let args: Vec<VectorRef> = vec![Arc::new(json_vector), Arc::new(path_vector)];
+        let vector = json_path_exists
+            .eval(FunctionContext::default(), &args)
+            .unwrap();
+
+        assert_eq!(4, vector.len());
+        for (i, gt) in results.iter().enumerate() {
+            let result = vector.get_ref(i);
+            let result = result.as_boolean().unwrap().unwrap();
+            assert_eq!(*gt, result);
+        }
+    }
+}
diff --git a/tests/cases/standalone/common/function/json/json.result b/tests/cases/standalone/common/function/json/json.result
new file mode 100644
index 0000000000..bae3313548
--- /dev/null
+++ b/tests/cases/standalone/common/function/json/json.result
@@ -0,0 +1,33 @@
+--- json_path_exists ---
+SELECT json_path_exists(parse_json('{"a": 1, "b": 2}'), '$.a');
+
++--------------------------------------------------------------------+
+| json_path_exists(parse_json(Utf8("{"a": 1, "b": 2}")),Utf8("$.a")) |
++--------------------------------------------------------------------+
+| true                                                               |
++--------------------------------------------------------------------+
+
+SELECT json_path_exists(parse_json('{"a": 1, "b": 2}'), '$.c');
+
++--------------------------------------------------------------------+
+| json_path_exists(parse_json(Utf8("{"a": 1, "b": 2}")),Utf8("$.c")) |
++--------------------------------------------------------------------+
+| false                                                              |
++--------------------------------------------------------------------+
+
+SELECT json_path_exists(parse_json('[1, 2]'), '[0]');
+
++----------------------------------------------------------+
+| json_path_exists(parse_json(Utf8("[1, 2]")),Utf8("[0]")) |
++----------------------------------------------------------+
+| true                                                     |
++----------------------------------------------------------+
+
+SELECT json_path_exists(parse_json('[1, 2]'), '[2]');
+
++----------------------------------------------------------+
+| json_path_exists(parse_json(Utf8("[1, 2]")),Utf8("[2]")) |
++----------------------------------------------------------+
+| false                                                    |
++----------------------------------------------------------+
+
diff --git a/tests/cases/standalone/common/function/json/json.sql b/tests/cases/standalone/common/function/json/json.sql
new file mode 100644
index 0000000000..76914e2575
--- /dev/null
+++ b/tests/cases/standalone/common/function/json/json.sql
@@ -0,0 +1,8 @@
+--- json_path_exists ---
+SELECT json_path_exists(parse_json('{"a": 1, "b": 2}'), '$.a');
+
+SELECT json_path_exists(parse_json('{"a": 1, "b": 2}'), '$.c');
+
+SELECT json_path_exists(parse_json('[1, 2]'), '[0]');
+
+SELECT json_path_exists(parse_json('[1, 2]'), '[2]');

From 695ff1e03713c554b50eb9f6245dbebc8decb91c Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Fri, 11 Oct 2024 10:40:51 +0800
Subject: [PATCH 063/128] feat: expose `RegionMigrationManagerRef` (#4812)

* chore: expose `RegionMigrationProcedureTask`

* fix: fix typos

* chore: expose `tracker`
---
 src/meta-srv/src/procedure/region_migration.rs         | 6 ++++--
 src/meta-srv/src/procedure/region_migration/manager.rs | 4 ++--
 src/meta-srv/src/region/supervisor.rs                  | 8 ++++----
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/meta-srv/src/procedure/region_migration.rs b/src/meta-srv/src/procedure/region_migration.rs
index 9b5bdfc006..72b886418c 100644
--- a/src/meta-srv/src/procedure/region_migration.rs
+++ b/src/meta-srv/src/procedure/region_migration.rs
@@ -43,8 +43,10 @@ use common_procedure::error::{
     Error as ProcedureError, FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu,
 };
 use common_procedure::{Context as ProcedureContext, LockKey, Procedure, Status, StringKey};
-pub use manager::RegionMigrationProcedureTask;
-use manager::{RegionMigrationProcedureGuard, RegionMigrationProcedureTracker};
+use manager::RegionMigrationProcedureGuard;
+pub use manager::{
+    RegionMigrationManagerRef, RegionMigrationProcedureTask, RegionMigrationProcedureTracker,
+};
 use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt};
 use store_api::storage::RegionId;
diff --git a/src/meta-srv/src/procedure/region_migration/manager.rs b/src/meta-srv/src/procedure/region_migration/manager.rs
index 01ea887ca9..1059c1794a 100644
--- a/src/meta-srv/src/procedure/region_migration/manager.rs
+++ b/src/meta-srv/src/procedure/region_migration/manager.rs
@@ -44,7 +44,7 @@ pub struct RegionMigrationManager {
 }
 
 #[derive(Default, Clone)]
-pub(crate) struct RegionMigrationProcedureTracker {
+pub struct RegionMigrationProcedureTracker {
     running_procedures: Arc<RwLock<HashMap<RegionId, RegionMigrationProcedureTask>>>,
 }
 
@@ -149,7 +149,7 @@ impl RegionMigrationManager {
     }
 
     /// Returns the [`RegionMigrationProcedureTracker`].
-    pub(crate) fn tracker(&self) -> &RegionMigrationProcedureTracker {
+    pub fn tracker(&self) -> &RegionMigrationProcedureTracker {
         &self.tracker
     }
 
diff --git a/src/meta-srv/src/region/supervisor.rs b/src/meta-srv/src/region/supervisor.rs
index 8367acbb26..80f6c84865 100644
--- a/src/meta-srv/src/region/supervisor.rs
+++ b/src/meta-srv/src/region/supervisor.rs
@@ -223,7 +223,7 @@ impl RegionFailureDetectorController for RegionFailureDetectorControl {
             .send(Event::RegisterFailureDetectors(detecting_regions))
             .await
         {
-            error!(err; "RegionSupervisor is stop receiving heartbeat");
+            error!(err; "RegionSupervisor has stop receiving heartbeat.");
         }
     }
 
@@ -233,7 +233,7 @@ impl RegionFailureDetectorController for RegionFailureDetectorControl {
             .send(Event::DeregisterFailureDetectors(detecting_regions))
             .await
         {
-            error!(err; "RegionSupervisor is stop receiving heartbeat");
+            error!(err; "RegionSupervisor has stop receiving heartbeat.");
         }
     }
 }
@@ -251,13 +251,13 @@ impl HeartbeatAcceptor {
     /// Accepts heartbeats from datanodes.
     pub(crate) async fn accept(&self, heartbeat: DatanodeHeartbeat) {
         if let Err(err) = self.sender.send(Event::HeartbeatArrived(heartbeat)).await {
-            error!(err; "RegionSupervisor is stop receiving heartbeat");
+            error!(err; "RegionSupervisor has stop receiving heartbeat.");
         }
     }
 }
 
 impl RegionSupervisor {
-    /// Returns a a mpsc channel with a buffer capacity of 1024 for sending and receiving `Event` messages.
+    /// Returns a mpsc channel with a buffer capacity of 1024 for sending and receiving `Event` messages.
     pub(crate) fn channel() -> (Sender<Event>, Receiver<Event>) {
         tokio::sync::mpsc::channel(1024)
     }

From 4b34f610aa35d8847c7d713cd7bb588e571f5df3 Mon Sep 17 00:00:00 2001
From: jeremyhi <jiachun_feng@proton.me>
Date: Fri, 11 Oct 2024 11:13:49 +0800
Subject: [PATCH 064/128] feat: information extension (#4811)

* feat: information extension

* Update manager.rs

Co-authored-by: Weny Xu <wenymedia@gmail.com>

* chore: by comment

---------

Co-authored-by: Weny Xu <wenymedia@gmail.com>
---
 src/catalog/src/error.rs                      |   7 +-
 src/catalog/src/kvbackend/manager.rs          |  26 ++--
 .../src/system_schema/information_schema.rs   |  42 ++++++-
 .../information_schema/cluster_info.rs        |  69 ++---------
 .../information_schema/procedure_info.rs      |  83 ++-----------
 .../information_schema/region_statistics.rs   |  30 +----
 src/catalog/src/system_schema/utils.rs        |  55 ++-------
 src/catalog/src/table_source.rs               |   6 +-
 src/cmd/src/cli/repl.rs                       |   6 +-
 src/cmd/src/flownode.rs                       |   7 +-
 src/cmd/src/frontend.rs                       |   8 +-
 src/cmd/src/lib.rs                            |  76 ++++++++++++
 src/cmd/src/standalone.rs                     | 112 ++++++++++++++++--
 tests-integration/src/cluster.rs              |   6 +-
 tests-integration/src/standalone.rs           |   4 +-
 15 files changed, 284 insertions(+), 253 deletions(-)

diff --git a/src/catalog/src/error.rs b/src/catalog/src/error.rs
index dd7071b095..0d9e96ab6a 100644
--- a/src/catalog/src/error.rs
+++ b/src/catalog/src/error.rs
@@ -89,9 +89,8 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to get procedure client in {mode} mode"))]
-    GetProcedureClient {
-        mode: String,
+    #[snafu(display("Failed to get information extension client"))]
+    GetInformationExtension {
         #[snafu(implicit)]
         location: Location,
     },
@@ -301,7 +300,7 @@ impl ErrorExt for Error {
             | Error::CacheNotFound { .. }
             | Error::CastManager { .. }
             | Error::Json { .. }
-            | Error::GetProcedureClient { .. }
+            | Error::GetInformationExtension { .. }
             | Error::ProcedureIdNotFound { .. } => StatusCode::Unexpected,
 
             Error::ViewPlanColumnsChanged { .. } => StatusCode::InvalidArguments,
diff --git a/src/catalog/src/kvbackend/manager.rs b/src/catalog/src/kvbackend/manager.rs
index e3377afbe1..ca20805d37 100644
--- a/src/catalog/src/kvbackend/manager.rs
+++ b/src/catalog/src/kvbackend/manager.rs
@@ -21,7 +21,6 @@ use common_catalog::consts::{
     DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME, NUMBERS_TABLE_ID,
     PG_CATALOG_NAME,
 };
-use common_config::Mode;
 use common_error::ext::BoxedError;
 use common_meta::cache::{LayeredCacheRegistryRef, ViewInfoCacheRef};
 use common_meta::key::catalog_name::CatalogNameKey;
@@ -34,7 +33,6 @@ use common_meta::kv_backend::KvBackendRef;
 use common_procedure::ProcedureManagerRef;
 use futures_util::stream::BoxStream;
 use futures_util::{StreamExt, TryStreamExt};
-use meta_client::client::MetaClient;
 use moka::sync::Cache;
 use partition::manager::{PartitionRuleManager, PartitionRuleManagerRef};
 use session::context::{Channel, QueryContext};
@@ -50,7 +48,7 @@ use crate::error::{
     CacheNotFoundSnafu, GetTableCacheSnafu, InvalidTableInfoInCatalogSnafu, ListCatalogsSnafu,
     ListSchemasSnafu, ListTablesSnafu, Result, TableMetadataManagerSnafu,
 };
-use crate::information_schema::InformationSchemaProvider;
+use crate::information_schema::{InformationExtensionRef, InformationSchemaProvider};
 use crate::kvbackend::TableCacheRef;
 use crate::system_schema::pg_catalog::PGCatalogProvider;
 use crate::system_schema::SystemSchemaProvider;
@@ -63,9 +61,8 @@ use crate::CatalogManager;
 /// comes from `SystemCatalog`, which is static and read-only.
 #[derive(Clone)]
 pub struct KvBackendCatalogManager {
-    mode: Mode,
-    /// Only available in `Distributed` mode.
-    meta_client: Option<Arc<MetaClient>>,
+    /// Provides the extension methods for the `information_schema` tables
+    information_extension: InformationExtensionRef,
     /// Manages partition rules.
     partition_manager: PartitionRuleManagerRef,
     /// Manages table metadata.
@@ -82,15 +79,13 @@ const CATALOG_CACHE_MAX_CAPACITY: u64 = 128;
 
 impl KvBackendCatalogManager {
     pub fn new(
-        mode: Mode,
-        meta_client: Option<Arc<MetaClient>>,
+        information_extension: InformationExtensionRef,
         backend: KvBackendRef,
         cache_registry: LayeredCacheRegistryRef,
         procedure_manager: Option<ProcedureManagerRef>,
     ) -> Arc<Self> {
         Arc::new_cyclic(|me| Self {
-            mode,
-            meta_client,
+            information_extension,
             partition_manager: Arc::new(PartitionRuleManager::new(
                 backend.clone(),
                 cache_registry
@@ -118,20 +113,15 @@ impl KvBackendCatalogManager {
         })
     }
 
-    /// Returns the server running mode.
-    pub fn running_mode(&self) -> &Mode {
-        &self.mode
-    }
-
     pub fn view_info_cache(&self) -> Result<ViewInfoCacheRef> {
         self.cache_registry.get().context(CacheNotFoundSnafu {
             name: "view_info_cache",
         })
     }
 
-    /// Returns the `[MetaClient]`.
-    pub fn meta_client(&self) -> Option<Arc<MetaClient>> {
-        self.meta_client.clone()
+    /// Returns the [`InformationExtension`].
+    pub fn information_extension(&self) -> InformationExtensionRef {
+        self.information_extension.clone()
     }
 
     pub fn partition_manager(&self) -> PartitionRuleManagerRef {
diff --git a/src/catalog/src/system_schema/information_schema.rs b/src/catalog/src/system_schema/information_schema.rs
index 9fa31b85fd..4101887cb4 100644
--- a/src/catalog/src/system_schema/information_schema.rs
+++ b/src/catalog/src/system_schema/information_schema.rs
@@ -32,7 +32,11 @@ use std::collections::HashMap;
 use std::sync::{Arc, Weak};
 
 use common_catalog::consts::{self, DEFAULT_CATALOG_NAME, INFORMATION_SCHEMA_NAME};
+use common_error::ext::ErrorExt;
+use common_meta::cluster::NodeInfo;
+use common_meta::datanode::RegionStat;
 use common_meta::key::flow::FlowMetadataManager;
+use common_procedure::ProcedureInfo;
 use common_recordbatch::SendableRecordBatchStream;
 use datatypes::schema::SchemaRef;
 use lazy_static::lazy_static;
@@ -45,7 +49,7 @@ use views::InformationSchemaViews;
 
 use self::columns::InformationSchemaColumns;
 use super::{SystemSchemaProviderInner, SystemTable, SystemTableRef};
-use crate::error::Result;
+use crate::error::{Error, Result};
 use crate::system_schema::information_schema::cluster_info::InformationSchemaClusterInfo;
 use crate::system_schema::information_schema::flows::InformationSchemaFlows;
 use crate::system_schema::information_schema::information_memory_table::get_schema_columns;
@@ -318,3 +322,39 @@ where
         InformationTable::to_stream(self, request)
     }
 }
+
+pub type InformationExtensionRef = Arc<dyn InformationExtension<Error = Error> + Send + Sync>;
+
+/// The `InformationExtension` trait provides the extension methods for the `information_schema` tables.
+#[async_trait::async_trait]
+pub trait InformationExtension {
+    type Error: ErrorExt;
+
+    /// Gets the nodes information.
+    async fn nodes(&self) -> std::result::Result<Vec<NodeInfo>, Self::Error>;
+
+    /// Gets the procedures information.
+    async fn procedures(&self) -> std::result::Result<Vec<(String, ProcedureInfo)>, Self::Error>;
+
+    /// Gets the region statistics.
+    async fn region_stats(&self) -> std::result::Result<Vec<RegionStat>, Self::Error>;
+}
+
+pub struct NoopInformationExtension;
+
+#[async_trait::async_trait]
+impl InformationExtension for NoopInformationExtension {
+    type Error = Error;
+
+    async fn nodes(&self) -> std::result::Result<Vec<NodeInfo>, Self::Error> {
+        Ok(vec![])
+    }
+
+    async fn procedures(&self) -> std::result::Result<Vec<(String, ProcedureInfo)>, Self::Error> {
+        Ok(vec![])
+    }
+
+    async fn region_stats(&self) -> std::result::Result<Vec<RegionStat>, Self::Error> {
+        Ok(vec![])
+    }
+}
diff --git a/src/catalog/src/system_schema/information_schema/cluster_info.rs b/src/catalog/src/system_schema/information_schema/cluster_info.rs
index e85e210304..1ab700497c 100644
--- a/src/catalog/src/system_schema/information_schema/cluster_info.rs
+++ b/src/catalog/src/system_schema/information_schema/cluster_info.rs
@@ -17,13 +17,10 @@ use std::time::Duration;
 
 use arrow_schema::SchemaRef as ArrowSchemaRef;
 use common_catalog::consts::INFORMATION_SCHEMA_CLUSTER_INFO_TABLE_ID;
-use common_config::Mode;
 use common_error::ext::BoxedError;
-use common_meta::cluster::{ClusterInfo, NodeInfo, NodeStatus};
-use common_meta::peer::Peer;
+use common_meta::cluster::NodeInfo;
 use common_recordbatch::adapter::RecordBatchStreamAdapter;
 use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
-use common_telemetry::warn;
 use common_time::timestamp::Timestamp;
 use datafusion::execution::TaskContext;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
@@ -40,7 +37,7 @@ use snafu::ResultExt;
 use store_api::storage::{ScanRequest, TableId};
 
 use super::CLUSTER_INFO;
-use crate::error::{CreateRecordBatchSnafu, InternalSnafu, ListNodesSnafu, Result};
+use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result};
 use crate::system_schema::information_schema::{InformationTable, Predicates};
 use crate::system_schema::utils;
 use crate::CatalogManager;
@@ -70,7 +67,6 @@ const INIT_CAPACITY: usize = 42;
 pub(super) struct InformationSchemaClusterInfo {
     schema: SchemaRef,
     catalog_manager: Weak<dyn CatalogManager>,
-    start_time_ms: u64,
 }
 
 impl InformationSchemaClusterInfo {
@@ -78,7 +74,6 @@ impl InformationSchemaClusterInfo {
         Self {
             schema: Self::schema(),
             catalog_manager,
-            start_time_ms: common_time::util::current_time_millis() as u64,
         }
     }
 
@@ -100,11 +95,7 @@ impl InformationSchemaClusterInfo {
     }
 
     fn builder(&self) -> InformationSchemaClusterInfoBuilder {
-        InformationSchemaClusterInfoBuilder::new(
-            self.schema.clone(),
-            self.catalog_manager.clone(),
-            self.start_time_ms,
-        )
+        InformationSchemaClusterInfoBuilder::new(self.schema.clone(), self.catalog_manager.clone())
     }
 }
 
@@ -144,7 +135,6 @@ impl InformationTable for InformationSchemaClusterInfo {
 
 struct InformationSchemaClusterInfoBuilder {
     schema: SchemaRef,
-    start_time_ms: u64,
     catalog_manager: Weak<dyn CatalogManager>,
 
     peer_ids: Int64VectorBuilder,
@@ -158,11 +148,7 @@ struct InformationSchemaClusterInfoBuilder {
 }
 
 impl InformationSchemaClusterInfoBuilder {
-    fn new(
-        schema: SchemaRef,
-        catalog_manager: Weak<dyn CatalogManager>,
-        start_time_ms: u64,
-    ) -> Self {
+    fn new(schema: SchemaRef, catalog_manager: Weak<dyn CatalogManager>) -> Self {
         Self {
             schema,
             catalog_manager,
@@ -174,56 +160,17 @@ impl InformationSchemaClusterInfoBuilder {
             start_times: TimestampMillisecondVectorBuilder::with_capacity(INIT_CAPACITY),
             uptimes: StringVectorBuilder::with_capacity(INIT_CAPACITY),
             active_times: StringVectorBuilder::with_capacity(INIT_CAPACITY),
-            start_time_ms,
         }
     }
 
     /// Construct the `information_schema.cluster_info` virtual table
     async fn make_cluster_info(&mut self, request: Option<ScanRequest>) -> Result<RecordBatch> {
         let predicates = Predicates::from_scan_request(&request);
-        let mode = utils::running_mode(&self.catalog_manager)?.unwrap_or(Mode::Standalone);
-
-        match mode {
-            Mode::Standalone => {
-                let build_info = common_version::build_info();
-
-                self.add_node_info(
-                    &predicates,
-                    NodeInfo {
-                        // For the standalone:
-                        // - id always 0
-                        // - empty string for peer_addr
-                        peer: Peer {
-                            id: 0,
-                            addr: "".to_string(),
-                        },
-                        last_activity_ts: -1,
-                        status: NodeStatus::Standalone,
-                        version: build_info.version.to_string(),
-                        git_commit: build_info.commit_short.to_string(),
-                        // Use `self.start_time_ms` instead.
-                        // It's not precise but enough.
-                        start_time_ms: self.start_time_ms,
-                    },
-                );
-            }
-            Mode::Distributed => {
-                if let Some(meta_client) = utils::meta_client(&self.catalog_manager)? {
-                    let node_infos = meta_client
-                        .list_nodes(None)
-                        .await
-                        .map_err(BoxedError::new)
-                        .context(ListNodesSnafu)?;
-
-                    for node_info in node_infos {
-                        self.add_node_info(&predicates, node_info);
-                    }
-                } else {
-                    warn!("Could not find meta client in distributed mode.");
-                }
-            }
+        let information_extension = utils::information_extension(&self.catalog_manager)?;
+        let node_infos = information_extension.nodes().await?;
+        for node_info in node_infos {
+            self.add_node_info(&predicates, node_info);
         }
-
         self.finish()
     }
 
diff --git a/src/catalog/src/system_schema/information_schema/procedure_info.rs b/src/catalog/src/system_schema/information_schema/procedure_info.rs
index 56c36c2210..6e3c0b1f46 100644
--- a/src/catalog/src/system_schema/information_schema/procedure_info.rs
+++ b/src/catalog/src/system_schema/information_schema/procedure_info.rs
@@ -14,14 +14,10 @@
 
 use std::sync::{Arc, Weak};
 
-use api::v1::meta::{ProcedureMeta, ProcedureStatus};
 use arrow_schema::SchemaRef as ArrowSchemaRef;
 use common_catalog::consts::INFORMATION_SCHEMA_PROCEDURE_INFO_TABLE_ID;
-use common_config::Mode;
 use common_error::ext::BoxedError;
-use common_meta::ddl::{ExecutorContext, ProcedureExecutor};
-use common_meta::rpc::procedure;
-use common_procedure::{ProcedureInfo, ProcedureState};
+use common_procedure::ProcedureInfo;
 use common_recordbatch::adapter::RecordBatchStreamAdapter;
 use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
 use common_time::timestamp::Timestamp;
@@ -38,10 +34,7 @@ use snafu::ResultExt;
 use store_api::storage::{ScanRequest, TableId};
 
 use super::PROCEDURE_INFO;
-use crate::error::{
-    ConvertProtoDataSnafu, CreateRecordBatchSnafu, GetProcedureClientSnafu, InternalSnafu,
-    ListProceduresSnafu, ProcedureIdNotFoundSnafu, Result,
-};
+use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result};
 use crate::system_schema::information_schema::{InformationTable, Predicates};
 use crate::system_schema::utils;
 use crate::CatalogManager;
@@ -167,45 +160,11 @@ impl InformationSchemaProcedureInfoBuilder {
     /// Construct the `information_schema.procedure_info` virtual table
     async fn make_procedure_info(&mut self, request: Option<ScanRequest>) -> Result<RecordBatch> {
         let predicates = Predicates::from_scan_request(&request);
-        let mode = utils::running_mode(&self.catalog_manager)?.unwrap_or(Mode::Standalone);
-        match mode {
-            Mode::Standalone => {
-                if let Some(procedure_manager) = utils::procedure_manager(&self.catalog_manager)? {
-                    let procedures = procedure_manager
-                        .list_procedures()
-                        .await
-                        .map_err(BoxedError::new)
-                        .context(ListProceduresSnafu)?;
-                    for procedure in procedures {
-                        self.add_procedure(
-                            &predicates,
-                            procedure.state.as_str_name().to_string(),
-                            procedure,
-                        );
-                    }
-                } else {
-                    return GetProcedureClientSnafu { mode: "standalone" }.fail();
-                }
-            }
-            Mode::Distributed => {
-                if let Some(meta_client) = utils::meta_client(&self.catalog_manager)? {
-                    let procedures = meta_client
-                        .list_procedures(&ExecutorContext::default())
-                        .await
-                        .map_err(BoxedError::new)
-                        .context(ListProceduresSnafu)?;
-                    for procedure in procedures.procedures {
-                        self.add_procedure_info(&predicates, procedure)?;
-                    }
-                } else {
-                    return GetProcedureClientSnafu {
-                        mode: "distributed",
-                    }
-                    .fail();
-                }
-            }
-        };
-
+        let information_extension = utils::information_extension(&self.catalog_manager)?;
+        let procedures = information_extension.procedures().await?;
+        for (status, procedure_info) in procedures {
+            self.add_procedure(&predicates, status, procedure_info);
+        }
         self.finish()
     }
 
@@ -247,34 +206,6 @@ impl InformationSchemaProcedureInfoBuilder {
         self.lock_keys.push(Some(&lock_keys));
     }
 
-    fn add_procedure_info(
-        &mut self,
-        predicates: &Predicates,
-        procedure: ProcedureMeta,
-    ) -> Result<()> {
-        let pid = match procedure.id {
-            Some(pid) => pid,
-            None => return ProcedureIdNotFoundSnafu {}.fail(),
-        };
-        let pid = procedure::pb_pid_to_pid(&pid)
-            .map_err(BoxedError::new)
-            .context(ConvertProtoDataSnafu)?;
-        let status = ProcedureStatus::try_from(procedure.status)
-            .map(|v| v.as_str_name())
-            .unwrap_or("Unknown")
-            .to_string();
-        let procedure_info = ProcedureInfo {
-            id: pid,
-            type_name: procedure.type_name,
-            start_time_ms: procedure.start_time_ms,
-            end_time_ms: procedure.end_time_ms,
-            state: ProcedureState::Running,
-            lock_keys: procedure.lock_keys,
-        };
-        self.add_procedure(predicates, status, procedure_info);
-        Ok(())
-    }
-
     fn finish(&mut self) -> Result<RecordBatch> {
         let columns: Vec<VectorRef> = vec![
             Arc::new(self.procedure_ids.finish()),
diff --git a/src/catalog/src/system_schema/information_schema/region_statistics.rs b/src/catalog/src/system_schema/information_schema/region_statistics.rs
index 07b94ede54..e92558acd0 100644
--- a/src/catalog/src/system_schema/information_schema/region_statistics.rs
+++ b/src/catalog/src/system_schema/information_schema/region_statistics.rs
@@ -16,13 +16,10 @@ use std::sync::{Arc, Weak};
 
 use arrow_schema::SchemaRef as ArrowSchemaRef;
 use common_catalog::consts::INFORMATION_SCHEMA_REGION_STATISTICS_TABLE_ID;
-use common_config::Mode;
 use common_error::ext::BoxedError;
-use common_meta::cluster::ClusterInfo;
 use common_meta::datanode::RegionStat;
 use common_recordbatch::adapter::RecordBatchStreamAdapter;
 use common_recordbatch::{DfSendableRecordBatchStream, RecordBatch, SendableRecordBatchStream};
-use common_telemetry::tracing::warn;
 use datafusion::execution::TaskContext;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
 use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
@@ -34,7 +31,7 @@ use snafu::ResultExt;
 use store_api::storage::{ScanRequest, TableId};
 
 use super::{InformationTable, REGION_STATISTICS};
-use crate::error::{CreateRecordBatchSnafu, InternalSnafu, ListRegionStatsSnafu, Result};
+use crate::error::{CreateRecordBatchSnafu, InternalSnafu, Result};
 use crate::information_schema::Predicates;
 use crate::system_schema::utils;
 use crate::CatalogManager;
@@ -167,28 +164,11 @@ impl InformationSchemaRegionStatisticsBuilder {
         request: Option<ScanRequest>,
     ) -> Result<RecordBatch> {
         let predicates = Predicates::from_scan_request(&request);
-        let mode = utils::running_mode(&self.catalog_manager)?.unwrap_or(Mode::Standalone);
-
-        match mode {
-            Mode::Standalone => {
-                // TODO(weny): implement it
-            }
-            Mode::Distributed => {
-                if let Some(meta_client) = utils::meta_client(&self.catalog_manager)? {
-                    let region_stats = meta_client
-                        .list_region_stats()
-                        .await
-                        .map_err(BoxedError::new)
-                        .context(ListRegionStatsSnafu)?;
-                    for region_stat in region_stats {
-                        self.add_region_statistic(&predicates, region_stat);
-                    }
-                } else {
-                    warn!("Meta client is not available");
-                }
-            }
+        let information_extension = utils::information_extension(&self.catalog_manager)?;
+        let region_stats = information_extension.region_stats().await?;
+        for region_stat in region_stats {
+            self.add_region_statistic(&predicates, region_stat);
         }
-
         self.finish()
     }
 
diff --git a/src/catalog/src/system_schema/utils.rs b/src/catalog/src/system_schema/utils.rs
index b9786bc260..8d8af84bf0 100644
--- a/src/catalog/src/system_schema/utils.rs
+++ b/src/catalog/src/system_schema/utils.rs
@@ -12,48 +12,33 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub mod tables;
+use std::sync::Weak;
 
-use std::sync::{Arc, Weak};
-
-use common_config::Mode;
 use common_meta::key::TableMetadataManagerRef;
-use common_procedure::ProcedureManagerRef;
-use meta_client::client::MetaClient;
 use snafu::OptionExt;
 
-use crate::error::{Result, UpgradeWeakCatalogManagerRefSnafu};
+use crate::error::{GetInformationExtensionSnafu, Result, UpgradeWeakCatalogManagerRefSnafu};
+use crate::information_schema::InformationExtensionRef;
 use crate::kvbackend::KvBackendCatalogManager;
 use crate::CatalogManager;
 
-/// Try to get the server running mode from `[CatalogManager]` weak reference.
-pub fn running_mode(catalog_manager: &Weak<dyn CatalogManager>) -> Result<Option<Mode>> {
+pub mod tables;
+
+/// Try to get the `[InformationExtension]` from `[CatalogManager]` weak reference.
+pub fn information_extension(
+    catalog_manager: &Weak<dyn CatalogManager>,
+) -> Result<InformationExtensionRef> {
     let catalog_manager = catalog_manager
         .upgrade()
         .context(UpgradeWeakCatalogManagerRefSnafu)?;
 
-    Ok(catalog_manager
+    let information_extension = catalog_manager
         .as_any()
         .downcast_ref::<KvBackendCatalogManager>()
-        .map(|manager| manager.running_mode())
-        .copied())
-}
+        .map(|manager| manager.information_extension())
+        .context(GetInformationExtensionSnafu)?;
 
-/// Try to get the `[MetaClient]` from `[CatalogManager]` weak reference.
-pub fn meta_client(catalog_manager: &Weak<dyn CatalogManager>) -> Result<Option<Arc<MetaClient>>> {
-    let catalog_manager = catalog_manager
-        .upgrade()
-        .context(UpgradeWeakCatalogManagerRefSnafu)?;
-
-    let meta_client = match catalog_manager
-        .as_any()
-        .downcast_ref::<KvBackendCatalogManager>()
-    {
-        None => None,
-        Some(manager) => manager.meta_client(),
-    };
-
-    Ok(meta_client)
+    Ok(information_extension)
 }
 
 /// Try to get the `[TableMetadataManagerRef]` from `[CatalogManager]` weak reference.
@@ -69,17 +54,3 @@ pub fn table_meta_manager(
         .downcast_ref::<KvBackendCatalogManager>()
         .map(|manager| manager.table_metadata_manager_ref().clone()))
 }
-
-/// Try to get the `[ProcedureManagerRef]` from `[CatalogManager]` weak reference.
-pub fn procedure_manager(
-    catalog_manager: &Weak<dyn CatalogManager>,
-) -> Result<Option<ProcedureManagerRef>> {
-    let catalog_manager = catalog_manager
-        .upgrade()
-        .context(UpgradeWeakCatalogManagerRefSnafu)?;
-
-    Ok(catalog_manager
-        .as_any()
-        .downcast_ref::<KvBackendCatalogManager>()
-        .and_then(|manager| manager.procedure_manager()))
-}
diff --git a/src/catalog/src/table_source.rs b/src/catalog/src/table_source.rs
index 09d3d9d2fd..ca9c5b89d3 100644
--- a/src/catalog/src/table_source.rs
+++ b/src/catalog/src/table_source.rs
@@ -259,7 +259,6 @@ mod tests {
 
     use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
     use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry};
-    use common_config::Mode;
     use common_meta::cache::{CacheRegistryBuilder, LayeredCacheRegistryBuilder};
     use common_meta::key::TableMetadataManager;
     use common_meta::kv_backend::memory::MemoryKvBackend;
@@ -269,6 +268,8 @@ mod tests {
     use datafusion::logical_expr::builder::LogicalTableSource;
     use datafusion::logical_expr::{col, lit, LogicalPlan, LogicalPlanBuilder};
 
+    use crate::information_schema::NoopInformationExtension;
+
     struct MockDecoder;
     impl MockDecoder {
         pub fn arc() -> Arc<Self> {
@@ -323,8 +324,7 @@ mod tests {
         );
 
         let catalog_manager = KvBackendCatalogManager::new(
-            Mode::Standalone,
-            None,
+            Arc::new(NoopInformationExtension),
             backend.clone(),
             layered_cache_registry,
             None,
diff --git a/src/cmd/src/cli/repl.rs b/src/cmd/src/cli/repl.rs
index b55b1c44d6..0138471227 100644
--- a/src/cmd/src/cli/repl.rs
+++ b/src/cmd/src/cli/repl.rs
@@ -46,12 +46,12 @@ use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
 use crate::cli::cmd::ReplCommand;
 use crate::cli::helper::RustylineHelper;
 use crate::cli::AttachCommand;
-use crate::error;
 use crate::error::{
     CollectRecordBatchesSnafu, ParseSqlSnafu, PlanStatementSnafu, PrettyPrintRecordBatchesSnafu,
     ReadlineSnafu, ReplCreationSnafu, RequestDatabaseSnafu, Result, StartMetaClientSnafu,
     SubstraitEncodeLogicalPlanSnafu,
 };
+use crate::{error, DistributedInformationExtension};
 
 /// Captures the state of the repl, gathers commands and executes them one by one
 pub struct Repl {
@@ -275,9 +275,9 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
         .build(),
     );
 
+    let information_extension = Arc::new(DistributedInformationExtension::new(meta_client.clone()));
     let catalog_manager = KvBackendCatalogManager::new(
-        Mode::Distributed,
-        Some(meta_client.clone()),
+        information_extension,
         cached_meta_backend.clone(),
         layered_cache_registry,
         None,
diff --git a/src/cmd/src/flownode.rs b/src/cmd/src/flownode.rs
index 78dfc90607..d2a8454085 100644
--- a/src/cmd/src/flownode.rs
+++ b/src/cmd/src/flownode.rs
@@ -41,7 +41,7 @@ use crate::error::{
     MissingConfigSnafu, Result, ShutdownFlownodeSnafu, StartFlownodeSnafu,
 };
 use crate::options::{GlobalOptions, GreptimeOptions};
-use crate::{log_versions, App};
+use crate::{log_versions, App, DistributedInformationExtension};
 
 pub const APP_NAME: &str = "greptime-flownode";
 
@@ -269,9 +269,10 @@ impl StartCommand {
             .build(),
         );
 
+        let information_extension =
+            Arc::new(DistributedInformationExtension::new(meta_client.clone()));
         let catalog_manager = KvBackendCatalogManager::new(
-            opts.mode,
-            Some(meta_client.clone()),
+            information_extension,
             cached_meta_backend.clone(),
             layered_cache_registry.clone(),
             None,
diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs
index 7678e90c88..2f10b722c7 100644
--- a/src/cmd/src/frontend.rs
+++ b/src/cmd/src/frontend.rs
@@ -38,7 +38,6 @@ use frontend::server::Services;
 use meta_client::{MetaClientOptions, MetaClientType};
 use query::stats::StatementStatistics;
 use servers::tls::{TlsMode, TlsOption};
-use servers::Mode;
 use snafu::{OptionExt, ResultExt};
 use tracing_appender::non_blocking::WorkerGuard;
 
@@ -47,7 +46,7 @@ use crate::error::{
     Result, StartFrontendSnafu,
 };
 use crate::options::{GlobalOptions, GreptimeOptions};
-use crate::{log_versions, App};
+use crate::{log_versions, App, DistributedInformationExtension};
 
 type FrontendOptions = GreptimeOptions<frontend::frontend::FrontendOptions>;
 
@@ -316,9 +315,10 @@ impl StartCommand {
             .build(),
         );
 
+        let information_extension =
+            Arc::new(DistributedInformationExtension::new(meta_client.clone()));
         let catalog_manager = KvBackendCatalogManager::new(
-            Mode::Distributed,
-            Some(meta_client.clone()),
+            information_extension,
             cached_meta_backend.clone(),
             layered_cache_registry.clone(),
             None,
diff --git a/src/cmd/src/lib.rs b/src/cmd/src/lib.rs
index 5797ef5a85..80e1628bc0 100644
--- a/src/cmd/src/lib.rs
+++ b/src/cmd/src/lib.rs
@@ -15,7 +15,17 @@
 #![feature(assert_matches, let_chains)]
 
 use async_trait::async_trait;
+use catalog::information_schema::InformationExtension;
+use client::api::v1::meta::ProcedureStatus;
+use common_error::ext::BoxedError;
+use common_meta::cluster::{ClusterInfo, NodeInfo};
+use common_meta::datanode::RegionStat;
+use common_meta::ddl::{ExecutorContext, ProcedureExecutor};
+use common_meta::rpc::procedure;
+use common_procedure::{ProcedureInfo, ProcedureState};
 use common_telemetry::{error, info};
+use meta_client::MetaClientRef;
+use snafu::ResultExt;
 
 use crate::error::Result;
 
@@ -94,3 +104,69 @@ fn log_env_flags() {
         info!("argument: {}", argument);
     }
 }
+
+pub struct DistributedInformationExtension {
+    meta_client: MetaClientRef,
+}
+
+impl DistributedInformationExtension {
+    pub fn new(meta_client: MetaClientRef) -> Self {
+        Self { meta_client }
+    }
+}
+
+#[async_trait::async_trait]
+impl InformationExtension for DistributedInformationExtension {
+    type Error = catalog::error::Error;
+
+    async fn nodes(&self) -> std::result::Result<Vec<NodeInfo>, Self::Error> {
+        self.meta_client
+            .list_nodes(None)
+            .await
+            .map_err(BoxedError::new)
+            .context(catalog::error::ListNodesSnafu)
+    }
+
+    async fn procedures(&self) -> std::result::Result<Vec<(String, ProcedureInfo)>, Self::Error> {
+        let procedures = self
+            .meta_client
+            .list_procedures(&ExecutorContext::default())
+            .await
+            .map_err(BoxedError::new)
+            .context(catalog::error::ListProceduresSnafu)?
+            .procedures;
+        let mut result = Vec::with_capacity(procedures.len());
+        for procedure in procedures {
+            let pid = match procedure.id {
+                Some(pid) => pid,
+                None => return catalog::error::ProcedureIdNotFoundSnafu {}.fail(),
+            };
+            let pid = procedure::pb_pid_to_pid(&pid)
+                .map_err(BoxedError::new)
+                .context(catalog::error::ConvertProtoDataSnafu)?;
+            let status = ProcedureStatus::try_from(procedure.status)
+                .map(|v| v.as_str_name())
+                .unwrap_or("Unknown")
+                .to_string();
+            let procedure_info = ProcedureInfo {
+                id: pid,
+                type_name: procedure.type_name,
+                start_time_ms: procedure.start_time_ms,
+                end_time_ms: procedure.end_time_ms,
+                state: ProcedureState::Running,
+                lock_keys: procedure.lock_keys,
+            };
+            result.push((status, procedure_info));
+        }
+
+        Ok(result)
+    }
+
+    async fn region_stats(&self) -> std::result::Result<Vec<RegionStat>, Self::Error> {
+        self.meta_client
+            .list_region_stats()
+            .await
+            .map_err(BoxedError::new)
+            .context(catalog::error::ListRegionStatsSnafu)
+    }
+}
diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
index 4335bd5447..d4a0d823c9 100644
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -17,14 +17,18 @@ use std::{fs, path};
 
 use async_trait::async_trait;
 use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry};
+use catalog::information_schema::InformationExtension;
 use catalog::kvbackend::KvBackendCatalogManager;
 use clap::Parser;
+use client::api::v1::meta::RegionRole;
 use common_base::Plugins;
 use common_catalog::consts::{MIN_USER_FLOW_ID, MIN_USER_TABLE_ID};
 use common_config::{metadata_store_dir, Configurable, KvBackendConfig};
 use common_error::ext::BoxedError;
 use common_meta::cache::LayeredCacheRegistryBuilder;
 use common_meta::cache_invalidator::CacheInvalidatorRef;
+use common_meta::cluster::{NodeInfo, NodeStatus};
+use common_meta::datanode::RegionStat;
 use common_meta::ddl::flow_meta::{FlowMetadataAllocator, FlowMetadataAllocatorRef};
 use common_meta::ddl::table_meta::{TableMetadataAllocator, TableMetadataAllocatorRef};
 use common_meta::ddl::{DdlContext, NoopRegionFailureDetectorControl, ProcedureExecutorRef};
@@ -33,10 +37,11 @@ use common_meta::key::flow::{FlowMetadataManager, FlowMetadataManagerRef};
 use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
 use common_meta::kv_backend::KvBackendRef;
 use common_meta::node_manager::NodeManagerRef;
+use common_meta::peer::Peer;
 use common_meta::region_keeper::MemoryRegionKeeper;
 use common_meta::sequence::SequenceBuilder;
 use common_meta::wal_options_allocator::{WalOptionsAllocator, WalOptionsAllocatorRef};
-use common_procedure::ProcedureManagerRef;
+use common_procedure::{ProcedureInfo, ProcedureManagerRef};
 use common_telemetry::info;
 use common_telemetry::logging::{LoggingOptions, TracingOptions};
 use common_time::timezone::set_default_timezone;
@@ -44,6 +49,7 @@ use common_version::{short_version, version};
 use common_wal::config::DatanodeWalConfig;
 use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
 use datanode::datanode::{Datanode, DatanodeBuilder};
+use datanode::region_server::RegionServer;
 use file_engine::config::EngineConfig as FileEngineConfig;
 use flow::{FlowWorkerManager, FlownodeBuilder, FrontendInvoker};
 use frontend::frontend::FrontendOptions;
@@ -478,9 +484,18 @@ impl StartCommand {
             .build(),
         );
 
+        let datanode = DatanodeBuilder::new(dn_opts, plugins.clone())
+            .with_kv_backend(kv_backend.clone())
+            .build()
+            .await
+            .context(StartDatanodeSnafu)?;
+
+        let information_extension = Arc::new(StandaloneInformationExtension::new(
+            datanode.region_server(),
+            procedure_manager.clone(),
+        ));
         let catalog_manager = KvBackendCatalogManager::new(
-            dn_opts.mode,
-            None,
+            information_extension,
             kv_backend.clone(),
             layered_cache_registry.clone(),
             Some(procedure_manager.clone()),
@@ -489,12 +504,6 @@ impl StartCommand {
         let table_metadata_manager =
             Self::create_table_metadata_manager(kv_backend.clone()).await?;
 
-        let datanode = DatanodeBuilder::new(dn_opts, plugins.clone())
-            .with_kv_backend(kv_backend.clone())
-            .build()
-            .await
-            .context(StartDatanodeSnafu)?;
-
         let flow_metadata_manager = Arc::new(FlowMetadataManager::new(kv_backend.clone()));
         let flow_builder = FlownodeBuilder::new(
             Default::default(),
@@ -644,6 +653,91 @@ impl StartCommand {
     }
 }
 
+struct StandaloneInformationExtension {
+    region_server: RegionServer,
+    procedure_manager: ProcedureManagerRef,
+    start_time_ms: u64,
+}
+
+impl StandaloneInformationExtension {
+    pub fn new(region_server: RegionServer, procedure_manager: ProcedureManagerRef) -> Self {
+        Self {
+            region_server,
+            procedure_manager,
+            start_time_ms: common_time::util::current_time_millis() as u64,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl InformationExtension for StandaloneInformationExtension {
+    type Error = catalog::error::Error;
+
+    async fn nodes(&self) -> std::result::Result<Vec<NodeInfo>, Self::Error> {
+        let build_info = common_version::build_info();
+        let node_info = NodeInfo {
+            // For the standalone:
+            // - id always 0
+            // - empty string for peer_addr
+            peer: Peer {
+                id: 0,
+                addr: "".to_string(),
+            },
+            last_activity_ts: -1,
+            status: NodeStatus::Standalone,
+            version: build_info.version.to_string(),
+            git_commit: build_info.commit_short.to_string(),
+            // Use `self.start_time_ms` instead.
+            // It's not precise but enough.
+            start_time_ms: self.start_time_ms,
+        };
+        Ok(vec![node_info])
+    }
+
+    async fn procedures(&self) -> std::result::Result<Vec<(String, ProcedureInfo)>, Self::Error> {
+        self.procedure_manager
+            .list_procedures()
+            .await
+            .map_err(BoxedError::new)
+            .map(|procedures| {
+                procedures
+                    .into_iter()
+                    .map(|procedure| {
+                        let status = procedure.state.as_str_name().to_string();
+                        (status, procedure)
+                    })
+                    .collect::<Vec<_>>()
+            })
+            .context(catalog::error::ListProceduresSnafu)
+    }
+
+    async fn region_stats(&self) -> std::result::Result<Vec<RegionStat>, Self::Error> {
+        let stats = self
+            .region_server
+            .reportable_regions()
+            .into_iter()
+            .map(|stat| {
+                let region_stat = self
+                    .region_server
+                    .region_statistic(stat.region_id)
+                    .unwrap_or_default();
+                RegionStat {
+                    id: stat.region_id,
+                    rcus: 0,
+                    wcus: 0,
+                    approximate_bytes: region_stat.estimated_disk_size() as i64,
+                    engine: stat.engine,
+                    role: RegionRole::from(stat.role).into(),
+                    memtable_size: region_stat.memtable_size,
+                    manifest_size: region_stat.manifest_size,
+                    sst_size: region_stat.sst_size,
+                }
+            })
+            .collect::<Vec<_>>();
+        Ok(stats)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::default::Default;
diff --git a/tests-integration/src/cluster.rs b/tests-integration/src/cluster.rs
index ad2c3e369f..44e5946769 100644
--- a/tests-integration/src/cluster.rs
+++ b/tests-integration/src/cluster.rs
@@ -23,6 +23,7 @@ use cache::{build_fundamental_cache_registry, with_default_composite_cache_regis
 use catalog::kvbackend::{CachedMetaKvBackendBuilder, KvBackendCatalogManager, MetaKvBackend};
 use client::client_manager::NodeClients;
 use client::Client;
+use cmd::DistributedInformationExtension;
 use common_base::Plugins;
 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
 use common_meta::cache::{CacheRegistryBuilder, LayeredCacheRegistryBuilder};
@@ -366,9 +367,10 @@ impl GreptimeDbClusterBuilder {
             .build(),
         );
 
+        let information_extension =
+            Arc::new(DistributedInformationExtension::new(meta_client.clone()));
         let catalog_manager = KvBackendCatalogManager::new(
-            Mode::Distributed,
-            Some(meta_client.clone()),
+            information_extension,
             cached_meta_backend.clone(),
             cache_registry.clone(),
             None,
diff --git a/tests-integration/src/standalone.rs b/tests-integration/src/standalone.rs
index 123614e436..47095ecc65 100644
--- a/tests-integration/src/standalone.rs
+++ b/tests-integration/src/standalone.rs
@@ -15,6 +15,7 @@
 use std::sync::Arc;
 
 use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry};
+use catalog::information_schema::NoopInformationExtension;
 use catalog::kvbackend::KvBackendCatalogManager;
 use cmd::error::StartFlownodeSnafu;
 use cmd::standalone::StandaloneOptions;
@@ -146,8 +147,7 @@ impl GreptimeDbStandaloneBuilder {
         );
 
         let catalog_manager = KvBackendCatalogManager::new(
-            Mode::Standalone,
-            None,
+            Arc::new(NoopInformationExtension),
             kv_backend.clone(),
             cache_registry.clone(),
             Some(procedure_manager.clone()),

From d168bde22622f8f7ac90ca8e0a31a05cc03e5aae Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Fri, 11 Oct 2024 12:16:37 +0800
Subject: [PATCH 065/128] feat!: move v1/prof API to debug/prof (#4810)

* feat!: move v1/prof to debug/prof

* docs: update readme

* docs: move prof docs to docs dir

* chore: update message

* feat!: remove v1/prof

* docs: update mem prof docs
---
 .../how-to/how-to-profile-cpu.md              |  6 ++---
 .../how-to/how-to-profile-memory.md           | 13 ++++++----
 src/servers/src/http.rs                       | 25 ++++++++++---------
 3 files changed, 24 insertions(+), 20 deletions(-)
 rename src/servers/src/http/pprof/README.md => docs/how-to/how-to-profile-cpu.md (71%)
 rename src/common/mem-prof/README.md => docs/how-to/how-to-profile-memory.md (67%)

diff --git a/src/servers/src/http/pprof/README.md b/docs/how-to/how-to-profile-cpu.md
similarity index 71%
rename from src/servers/src/http/pprof/README.md
rename to docs/how-to/how-to-profile-cpu.md
index b8393ae3f5..b73c85ea2f 100644
--- a/src/servers/src/http/pprof/README.md
+++ b/docs/how-to/how-to-profile-cpu.md
@@ -9,7 +9,7 @@ cargo build --features=pprof
 ## HTTP API
 Sample at 99 Hertz, for 5 seconds, output report in [protobuf format](https://github.com/google/pprof/blob/master/proto/profile.proto).
 ```bash
-curl -s '0:4000/v1/prof/cpu' > /tmp/pprof.out
+curl -s '0:4000/debug/prof/cpu' > /tmp/pprof.out
 ```
 
 Then you can use `pprof` command with the protobuf file.
@@ -19,10 +19,10 @@ go tool pprof -top /tmp/pprof.out
 
 Sample at 99 Hertz, for 60 seconds, output report in flamegraph format.
 ```bash
-curl -s '0:4000/v1/prof/cpu?seconds=60&output=flamegraph' > /tmp/pprof.svg
+curl -s '0:4000/debug/prof/cpu?seconds=60&output=flamegraph' > /tmp/pprof.svg
 ```
 
 Sample at 49 Hertz, for 10 seconds, output report in text format.
 ```bash
-curl -s '0:4000/v1/prof/cpu?seconds=10&frequency=49&output=text' > /tmp/pprof.txt
+curl -s '0:4000/debug/prof/cpu?seconds=10&frequency=49&output=text' > /tmp/pprof.txt
 ```
diff --git a/src/common/mem-prof/README.md b/docs/how-to/how-to-profile-memory.md
similarity index 67%
rename from src/common/mem-prof/README.md
rename to docs/how-to/how-to-profile-memory.md
index da1bbae9f0..7211683190 100644
--- a/src/common/mem-prof/README.md
+++ b/docs/how-to/how-to-profile-memory.md
@@ -12,10 +12,10 @@ brew install jemalloc
 sudo apt install libjemalloc-dev
 ```
 
-### [flamegraph](https://github.com/brendangregg/FlameGraph) 
+### [flamegraph](https://github.com/brendangregg/FlameGraph)
 
 ```bash
-curl https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl > ./flamegraph.pl 
+curl https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl > ./flamegraph.pl
 ```
 
 ### Build GreptimeDB with `mem-prof` feature.
@@ -35,7 +35,7 @@ MALLOC_CONF=prof:true,lg_prof_interval:28 ./target/debug/greptime standalone sta
 Dump memory profiling data through HTTP API:
 
 ```bash
-curl localhost:4000/v1/prof/mem > greptime.hprof
+curl localhost:4000/debug/prof/mem > greptime.hprof
 ```
 
 You can periodically dump profiling data and compare them to find the delta memory usage.
@@ -45,6 +45,9 @@ You can periodically dump profiling data and compare them to find the delta memo
 To create flamegraph according to dumped profiling data:
 
 ```bash
-jeprof --svg <path_to_greptimedb_binary> --base=<baseline_prof> <profile_data> > output.svg
-```
+sudo apt install -y libjemalloc-dev
 
+jeprof <path_to_greptime_binary> <profile_data> --collapse | ./flamegraph.pl > mem-prof.svg
+
+jeprof <path_to_greptime_binary> --base <baseline_prof> <profile_data> --collapse | ./flamegraph.pl > output.svg
+```
diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs
index 953ff9e73a..67309b7244 100644
--- a/src/servers/src/http.rs
+++ b/src/servers/src/http.rs
@@ -730,6 +730,7 @@ impl HttpServer {
                         authorize::check_http_auth,
                     )),
             )
+            // Handlers for debug, we don't expect a timeout.
             .nest(
                 "/debug",
                 Router::new()
@@ -737,19 +738,19 @@ impl HttpServer {
                     .route(
                         "/log_level",
                         routing::get(dyn_log::dyn_log_handler).post(dyn_log::dyn_log_handler),
-                    ),
-            )
-            // Handlers for debug, we don't expect a timeout.
-            .nest(
-                &format!("/{HTTP_API_VERSION}/prof"),
-                Router::new()
-                    .route(
-                        "/cpu",
-                        routing::get(pprof::pprof_handler).post(pprof::pprof_handler),
                     )
-                    .route(
-                        "/mem",
-                        routing::get(mem_prof::mem_prof_handler).post(mem_prof::mem_prof_handler),
+                    .nest(
+                        "/prof",
+                        Router::new()
+                            .route(
+                                "/cpu",
+                                routing::get(pprof::pprof_handler).post(pprof::pprof_handler),
+                            )
+                            .route(
+                                "/mem",
+                                routing::get(mem_prof::mem_prof_handler)
+                                    .post(mem_prof::mem_prof_handler),
+                            ),
                     ),
             )
     }

From 7dd0e3ab37264d11a7deec619ecb3845eb249705 Mon Sep 17 00:00:00 2001
From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com>
Date: Fri, 11 Oct 2024 16:23:01 +0800
Subject: [PATCH 066/128] fix: Panic in UNION ALL queries (#4796)

* fix/union_all_panic:
 Improve MetricCollector by incrementing level and fix underflow issue; add tests for UNION ALL queries

* chore: remove useless documentation

* fix/union_all_panic: Add order by clause to UNION ALL select queries in tests
---
 src/common/recordbatch/src/adapter.rs                 |  4 ++--
 tests/cases/standalone/common/select/union_all.result | 10 ++++++++++
 tests/cases/standalone/common/select/union_all.sql    |  1 +
 3 files changed, 13 insertions(+), 2 deletions(-)
 create mode 100644 tests/cases/standalone/common/select/union_all.result
 create mode 100644 tests/cases/standalone/common/select/union_all.sql

diff --git a/src/common/recordbatch/src/adapter.rs b/src/common/recordbatch/src/adapter.rs
index 85236381b2..98f4d0a38a 100644
--- a/src/common/recordbatch/src/adapter.rs
+++ b/src/common/recordbatch/src/adapter.rs
@@ -329,6 +329,7 @@ impl ExecutionPlanVisitor for MetricCollector {
                 level: self.current_level,
                 metrics: vec![],
             });
+            self.current_level += 1;
             return Ok(true);
         };
 
@@ -365,8 +366,7 @@ impl ExecutionPlanVisitor for MetricCollector {
     }
 
     fn post_visit(&mut self, _plan: &dyn ExecutionPlan) -> std::result::Result<bool, Self::Error> {
-        // the last minus will underflow
-        self.current_level = self.current_level.wrapping_sub(1);
+        self.current_level -= 1;
         Ok(true)
     }
 }
diff --git a/tests/cases/standalone/common/select/union_all.result b/tests/cases/standalone/common/select/union_all.result
new file mode 100644
index 0000000000..54332e1254
--- /dev/null
+++ b/tests/cases/standalone/common/select/union_all.result
@@ -0,0 +1,10 @@
+SELECT 123 as a, 'h' as b UNION ALL SELECT 456 as a, 'e' as b UNION ALL SELECT 789 as a, 'l' as b order by a;
+
++-----+---+
+| a   | b |
++-----+---+
+| 123 | h |
+| 456 | e |
+| 789 | l |
++-----+---+
+
diff --git a/tests/cases/standalone/common/select/union_all.sql b/tests/cases/standalone/common/select/union_all.sql
new file mode 100644
index 0000000000..207d8463fc
--- /dev/null
+++ b/tests/cases/standalone/common/select/union_all.sql
@@ -0,0 +1 @@
+SELECT 123 as a, 'h' as b UNION ALL SELECT 456 as a, 'e' as b UNION ALL SELECT 789 as a, 'l' as b order by a;

From a61c0bd1d8a16b208266bbac92f3b8fe2cbdf594 Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Fri, 11 Oct 2024 18:02:45 +0800
Subject: [PATCH 067/128] fix: error in admin function is not formatted
 properly (#4820)

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 src/operator/src/statement/admin.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/operator/src/statement/admin.rs b/src/operator/src/statement/admin.rs
index 37e6acf966..6c0413c2aa 100644
--- a/src/operator/src/statement/admin.rs
+++ b/src/operator/src/statement/admin.rs
@@ -59,7 +59,7 @@ impl StatementExecutor {
             .map(|arg| {
                 let FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Value(value))) = arg else {
                     return error::BuildAdminFunctionArgsSnafu {
-                        msg: "unsupported function arg {arg}",
+                        msg: format!("unsupported function arg {arg}"),
                     }
                     .fail();
                 };
@@ -200,7 +200,7 @@ fn values_to_vectors_by_valid_types(
             }
 
             error::BuildAdminFunctionArgsSnafu {
-                msg: "failed to cast {value}",
+                msg: format!("failed to cast {value}"),
             }
             .fail()
         })

From 0f907ef99e5d2cc2a8ea40ea72b564173ad3a0d2 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Fri, 11 Oct 2024 19:32:15 +0800
Subject: [PATCH 068/128] fix: correct table name formatting (#4819)

---
 src/common/meta/src/ddl/utils.rs      |  9 +++++----
 src/common/meta/src/key/table_name.rs | 11 +++++++++++
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/common/meta/src/ddl/utils.rs b/src/common/meta/src/ddl/utils.rs
index fd1737a3e6..b1608d40e3 100644
--- a/src/common/meta/src/ddl/utils.rs
+++ b/src/common/meta/src/ddl/utils.rs
@@ -18,6 +18,7 @@ use common_procedure::error::Error as ProcedureError;
 use snafu::{ensure, OptionExt, ResultExt};
 use store_api::metric_engine_consts::LOGICAL_TABLE_METADATA_KEY;
 use table::metadata::TableId;
+use table::table_reference::TableReference;
 
 use crate::ddl::DetectingRegion;
 use crate::error::{Error, OperateDatanodeSnafu, Result, TableNotFoundSnafu, UnsupportedSnafu};
@@ -109,8 +110,8 @@ pub async fn check_and_get_physical_table_id(
         .table_name_manager()
         .get(physical_table_name)
         .await?
-        .context(TableNotFoundSnafu {
-            table_name: physical_table_name.to_string(),
+        .with_context(|| TableNotFoundSnafu {
+            table_name: TableReference::from(physical_table_name).to_string(),
         })
         .map(|table| table.table_id())
 }
@@ -123,8 +124,8 @@ pub async fn get_physical_table_id(
         .table_name_manager()
         .get(logical_table_name)
         .await?
-        .context(TableNotFoundSnafu {
-            table_name: logical_table_name.to_string(),
+        .with_context(|| TableNotFoundSnafu {
+            table_name: TableReference::from(logical_table_name).to_string(),
         })
         .map(|table| table.table_id())?;
 
diff --git a/src/common/meta/src/key/table_name.rs b/src/common/meta/src/key/table_name.rs
index a632e3a233..e508c5e877 100644
--- a/src/common/meta/src/key/table_name.rs
+++ b/src/common/meta/src/key/table_name.rs
@@ -21,6 +21,7 @@ use serde::{Deserialize, Serialize};
 use snafu::OptionExt;
 use table::metadata::TableId;
 use table::table_name::TableName;
+use table::table_reference::TableReference;
 
 use super::{MetadataKey, MetadataValue, TABLE_NAME_KEY_PATTERN, TABLE_NAME_KEY_PREFIX};
 use crate::error::{Error, InvalidMetadataSnafu, Result};
@@ -122,6 +123,16 @@ impl From<TableNameKey<'_>> for TableName {
     }
 }
 
+impl<'a> From<TableNameKey<'a>> for TableReference<'a> {
+    fn from(value: TableNameKey<'a>) -> Self {
+        Self {
+            catalog: value.catalog,
+            schema: value.schema,
+            table: value.table,
+        }
+    }
+}
+
 impl<'a> TryFrom<&'a str> for TableNameKey<'a> {
     type Error = Error;
 

From 4bb1f4f18447685fd1536e4fddafb0c9a6bb116e Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Fri, 11 Oct 2024 20:48:53 +0800
Subject: [PATCH 069/128] feat: introduce `LeadershipChangeNotifier` and
 `LeadershipChangeListener` (#4817)

* feat: introduce `LeadershipChangeNotifier`

* refactor: use `LeadershipChangeNotifier`

* chore: apply suggestions from CR

* chore: apply suggestions from CR

* chore: adjust log styling
---
 src/common/meta/src/error.rs                 |  18 ++-
 src/common/meta/src/leadership_notifier.rs   | 156 +++++++++++++++++++
 src/common/meta/src/lib.rs                   |   1 +
 src/common/meta/src/wal_options_allocator.rs |  17 ++
 src/meta-srv/src/lib.rs                      |   1 -
 src/meta-srv/src/metasrv.rs                  |  62 ++++----
 src/meta-srv/src/procedure.rs                |  30 ++++
 src/meta-srv/src/region/supervisor.rs        |  19 +++
 8 files changed, 270 insertions(+), 34 deletions(-)
 create mode 100644 src/common/meta/src/leadership_notifier.rs

diff --git a/src/common/meta/src/error.rs b/src/common/meta/src/error.rs
index 849ee28948..0e7709df0b 100644
--- a/src/common/meta/src/error.rs
+++ b/src/common/meta/src/error.rs
@@ -147,6 +147,20 @@ pub enum Error {
         source: common_procedure::Error,
     },
 
+    #[snafu(display("Failed to start procedure manager"))]
+    StartProcedureManager {
+        #[snafu(implicit)]
+        location: Location,
+        source: common_procedure::Error,
+    },
+
+    #[snafu(display("Failed to stop procedure manager"))]
+    StopProcedureManager {
+        #[snafu(implicit)]
+        location: Location,
+        source: common_procedure::Error,
+    },
+
     #[snafu(display(
         "Failed to get procedure output, procedure id: {procedure_id}, error: {err_msg}"
     ))]
@@ -715,7 +729,9 @@ impl ErrorExt for Error {
 
             SubmitProcedure { source, .. }
             | QueryProcedure { source, .. }
-            | WaitProcedure { source, .. } => source.status_code(),
+            | WaitProcedure { source, .. }
+            | StartProcedureManager { source, .. }
+            | StopProcedureManager { source, .. } => source.status_code(),
             RegisterProcedureLoader { source, .. } => source.status_code(),
             External { source, .. } => source.status_code(),
             OperateDatanode { source, .. } => source.status_code(),
diff --git a/src/common/meta/src/leadership_notifier.rs b/src/common/meta/src/leadership_notifier.rs
new file mode 100644
index 0000000000..4ba65d1adc
--- /dev/null
+++ b/src/common/meta/src/leadership_notifier.rs
@@ -0,0 +1,156 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use common_telemetry::error;
+
+use crate::error::Result;
+
+pub type LeadershipChangeNotifierCustomizerRef = Arc<dyn LeadershipChangeNotifierCustomizer>;
+
+/// A trait for customizing the leadership change notifier.
+pub trait LeadershipChangeNotifierCustomizer: Send + Sync {
+    fn customize(&self, notifier: &mut LeadershipChangeNotifier);
+}
+
+/// A trait for handling leadership change events in a distributed system.
+#[async_trait]
+pub trait LeadershipChangeListener: Send + Sync {
+    /// Returns the listener name.
+    fn name(&self) -> &str;
+
+    /// Called when the node transitions to the leader role.
+    async fn on_leader_start(&self) -> Result<()>;
+
+    /// Called when the node transitions to the follower role.
+    async fn on_leader_stop(&self) -> Result<()>;
+}
+
+/// A notifier for leadership change events.
+#[derive(Default)]
+pub struct LeadershipChangeNotifier {
+    listeners: Vec<Arc<dyn LeadershipChangeListener>>,
+}
+
+impl LeadershipChangeNotifier {
+    /// Adds a listener to the notifier.
+    pub fn add_listener(&mut self, listener: Arc<dyn LeadershipChangeListener>) {
+        self.listeners.push(listener);
+    }
+
+    /// Notify all listeners that the node has become a leader.
+    pub async fn notify_on_leader_start(&self) {
+        for listener in &self.listeners {
+            if let Err(err) = listener.on_leader_start().await {
+                error!(
+                    err;
+                    "Failed to notify listener: {}, event 'on_leader_start'",
+                    listener.name()
+                );
+            }
+        }
+    }
+
+    /// Notify all listeners that the node has become a follower.
+    pub async fn notify_on_leader_stop(&self) {
+        for listener in &self.listeners {
+            if let Err(err) = listener.on_leader_stop().await {
+                error!(
+                    err;
+                    "Failed to notify listener: {}, event: 'on_follower_start'",
+                    listener.name()
+                );
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::Arc;
+
+    use super::*;
+
+    struct MockListener {
+        name: String,
+        on_leader_start_fn: Option<Box<dyn Fn() -> Result<()> + Send + Sync>>,
+        on_follower_start_fn: Option<Box<dyn Fn() -> Result<()> + Send + Sync>>,
+    }
+
+    #[async_trait::async_trait]
+    impl LeadershipChangeListener for MockListener {
+        fn name(&self) -> &str {
+            &self.name
+        }
+
+        async fn on_leader_start(&self) -> Result<()> {
+            if let Some(f) = &self.on_leader_start_fn {
+                return f();
+            }
+            Ok(())
+        }
+
+        async fn on_leader_stop(&self) -> Result<()> {
+            if let Some(f) = &self.on_follower_start_fn {
+                return f();
+            }
+            Ok(())
+        }
+    }
+
+    #[tokio::test]
+    async fn test_leadership_change_notifier() {
+        let mut notifier = LeadershipChangeNotifier::default();
+        let listener1 = Arc::new(MockListener {
+            name: "listener1".to_string(),
+            on_leader_start_fn: None,
+            on_follower_start_fn: None,
+        });
+        let called_on_leader_start = Arc::new(AtomicBool::new(false));
+        let called_on_follower_start = Arc::new(AtomicBool::new(false));
+        let called_on_leader_start_moved = called_on_leader_start.clone();
+        let called_on_follower_start_moved = called_on_follower_start.clone();
+        let listener2 = Arc::new(MockListener {
+            name: "listener2".to_string(),
+            on_leader_start_fn: Some(Box::new(move || {
+                called_on_leader_start_moved.store(true, Ordering::Relaxed);
+                Ok(())
+            })),
+            on_follower_start_fn: Some(Box::new(move || {
+                called_on_follower_start_moved.store(true, Ordering::Relaxed);
+                Ok(())
+            })),
+        });
+
+        notifier.add_listener(listener1);
+        notifier.add_listener(listener2);
+
+        let listener1 = notifier.listeners.first().unwrap();
+        let listener2 = notifier.listeners.get(1).unwrap();
+
+        assert_eq!(listener1.name(), "listener1");
+        assert_eq!(listener2.name(), "listener2");
+
+        notifier.notify_on_leader_start().await;
+        assert!(!called_on_follower_start.load(Ordering::Relaxed));
+        assert!(called_on_leader_start.load(Ordering::Relaxed));
+
+        notifier.notify_on_leader_stop().await;
+        assert!(called_on_follower_start.load(Ordering::Relaxed));
+        assert!(called_on_leader_start.load(Ordering::Relaxed));
+    }
+}
diff --git a/src/common/meta/src/lib.rs b/src/common/meta/src/lib.rs
index 1dd658890c..158350bc32 100644
--- a/src/common/meta/src/lib.rs
+++ b/src/common/meta/src/lib.rs
@@ -32,6 +32,7 @@ pub mod heartbeat;
 pub mod instruction;
 pub mod key;
 pub mod kv_backend;
+pub mod leadership_notifier;
 pub mod lock_key;
 pub mod metrics;
 pub mod node_manager;
diff --git a/src/common/meta/src/wal_options_allocator.rs b/src/common/meta/src/wal_options_allocator.rs
index ba0c6f407f..283f43b9a8 100644
--- a/src/common/meta/src/wal_options_allocator.rs
+++ b/src/common/meta/src/wal_options_allocator.rs
@@ -17,6 +17,7 @@ pub mod kafka;
 use std::collections::HashMap;
 use std::sync::Arc;
 
+use async_trait::async_trait;
 use common_wal::config::MetasrvWalConfig;
 use common_wal::options::{KafkaWalOptions, WalOptions, WAL_OPTIONS_KEY};
 use snafu::ResultExt;
@@ -24,6 +25,7 @@ use store_api::storage::{RegionId, RegionNumber};
 
 use crate::error::{EncodeWalOptionsSnafu, Result};
 use crate::kv_backend::KvBackendRef;
+use crate::leadership_notifier::LeadershipChangeListener;
 use crate::wal_options_allocator::kafka::topic_manager::TopicManager as KafkaTopicManager;
 
 /// Allocates wal options in region granularity.
@@ -94,6 +96,21 @@ impl WalOptionsAllocator {
     }
 }
 
+#[async_trait]
+impl LeadershipChangeListener for WalOptionsAllocator {
+    fn name(&self) -> &str {
+        "WalOptionsAllocator"
+    }
+
+    async fn on_leader_start(&self) -> Result<()> {
+        self.start().await
+    }
+
+    async fn on_leader_stop(&self) -> Result<()> {
+        Ok(())
+    }
+}
+
 /// Allocates a wal options for each region. The allocated wal options is encoded immediately.
 pub fn allocate_region_wal_options(
     regions: Vec<RegionNumber>,
diff --git a/src/meta-srv/src/lib.rs b/src/meta-srv/src/lib.rs
index 717edc92b2..01b48f1da0 100644
--- a/src/meta-srv/src/lib.rs
+++ b/src/meta-srv/src/lib.rs
@@ -40,7 +40,6 @@ pub mod selector;
 pub mod service;
 pub mod state;
 pub mod table_meta_alloc;
-
 pub use crate::error::Result;
 
 mod greptimedb_telemetry;
diff --git a/src/meta-srv/src/metasrv.rs b/src/meta-srv/src/metasrv.rs
index de7d54aa65..3cdb64e1e0 100644
--- a/src/meta-srv/src/metasrv.rs
+++ b/src/meta-srv/src/metasrv.rs
@@ -29,6 +29,9 @@ use common_meta::cache_invalidator::CacheInvalidatorRef;
 use common_meta::ddl::ProcedureExecutorRef;
 use common_meta::key::TableMetadataManagerRef;
 use common_meta::kv_backend::{KvBackendRef, ResettableKvBackend, ResettableKvBackendRef};
+use common_meta::leadership_notifier::{
+    LeadershipChangeNotifier, LeadershipChangeNotifierCustomizerRef,
+};
 use common_meta::peer::Peer;
 use common_meta::region_keeper::MemoryRegionKeeperRef;
 use common_meta::wal_options_allocator::WalOptionsAllocatorRef;
@@ -56,6 +59,7 @@ use crate::handler::HeartbeatHandlerGroupRef;
 use crate::lease::lookup_datanode_peer;
 use crate::lock::DistLockRef;
 use crate::procedure::region_migration::manager::RegionMigrationManagerRef;
+use crate::procedure::ProcedureManagerListenerAdapter;
 use crate::pubsub::{PublisherRef, SubscriptionManagerRef};
 use crate::region::supervisor::RegionSupervisorTickerRef;
 use crate::selector::{Selector, SelectorType};
@@ -291,17 +295,15 @@ pub type SelectorRef = Arc<dyn Selector<Context = SelectorContext, Output = Vec<
 pub type ElectionRef = Arc<dyn Election<Leader = LeaderValue>>;
 
 pub struct MetaStateHandler {
-    procedure_manager: ProcedureManagerRef,
-    wal_options_allocator: WalOptionsAllocatorRef,
     subscribe_manager: Option<SubscriptionManagerRef>,
     greptimedb_telemetry_task: Arc<GreptimeDBTelemetryTask>,
     leader_cached_kv_backend: Arc<LeaderCachedKvBackend>,
-    region_supervisor_ticker: Option<RegionSupervisorTickerRef>,
+    leadership_change_notifier: LeadershipChangeNotifier,
     state: StateRef,
 }
 
 impl MetaStateHandler {
-    pub async fn on_become_leader(&self) {
+    pub async fn on_leader_start(&self) {
         self.state.write().unwrap().next_state(become_leader(false));
 
         if let Err(e) = self.leader_cached_kv_backend.load().await {
@@ -310,33 +312,19 @@ impl MetaStateHandler {
             self.state.write().unwrap().next_state(become_leader(true));
         }
 
-        if let Some(ticker) = self.region_supervisor_ticker.as_ref() {
-            ticker.start();
-        }
-
-        if let Err(e) = self.procedure_manager.start().await {
-            error!(e; "Failed to start procedure manager");
-        }
-
-        if let Err(e) = self.wal_options_allocator.start().await {
-            error!(e; "Failed to start wal options allocator");
-        }
+        self.leadership_change_notifier
+            .notify_on_leader_start()
+            .await;
 
         self.greptimedb_telemetry_task.should_report(true);
     }
 
-    pub async fn on_become_follower(&self) {
+    pub async fn on_leader_stop(&self) {
         self.state.write().unwrap().next_state(become_follower());
 
-        // Stops the procedures.
-        if let Err(e) = self.procedure_manager.stop().await {
-            error!(e; "Failed to stop procedure manager");
-        }
-
-        if let Some(ticker) = self.region_supervisor_ticker.as_ref() {
-            // Stops the supervisor ticker.
-            ticker.stop();
-        }
+        self.leadership_change_notifier
+            .notify_on_leader_stop()
+            .await;
 
         // Suspends reporting.
         self.greptimedb_telemetry_task.should_report(false);
@@ -410,15 +398,25 @@ impl Metasrv {
             greptimedb_telemetry_task
                 .start()
                 .context(StartTelemetryTaskSnafu)?;
-            let region_supervisor_ticker = self.region_supervisor_ticker.clone();
+
+            // Builds leadership change notifier.
+            let mut leadership_change_notifier = LeadershipChangeNotifier::default();
+            leadership_change_notifier.add_listener(self.wal_options_allocator.clone());
+            leadership_change_notifier
+                .add_listener(Arc::new(ProcedureManagerListenerAdapter(procedure_manager)));
+            if let Some(region_supervisor_ticker) = &self.region_supervisor_ticker {
+                leadership_change_notifier.add_listener(region_supervisor_ticker.clone() as _);
+            }
+            if let Some(customizer) = self.plugins.get::<LeadershipChangeNotifierCustomizerRef>() {
+                customizer.customize(&mut leadership_change_notifier);
+            }
+
             let state_handler = MetaStateHandler {
                 greptimedb_telemetry_task,
                 subscribe_manager,
-                procedure_manager,
-                wal_options_allocator: self.wal_options_allocator.clone(),
                 state: self.state.clone(),
                 leader_cached_kv_backend: leader_cached_kv_backend.clone(),
-                region_supervisor_ticker,
+                leadership_change_notifier,
             };
             let _handle = common_runtime::spawn_global(async move {
                 loop {
@@ -429,12 +427,12 @@ impl Metasrv {
                             info!("Leader's cache has bean cleared on leader change: {msg}");
                             match msg {
                                 LeaderChangeMessage::Elected(_) => {
-                                    state_handler.on_become_leader().await;
+                                    state_handler.on_leader_start().await;
                                 }
                                 LeaderChangeMessage::StepDown(leader) => {
                                     error!("Leader :{:?} step down", leader);
 
-                                    state_handler.on_become_follower().await;
+                                    state_handler.on_leader_stop().await;
                                 }
                             }
                         }
@@ -448,7 +446,7 @@ impl Metasrv {
                     }
                 }
 
-                state_handler.on_become_follower().await;
+                state_handler.on_leader_stop().await;
             });
 
             // Register candidate and keep lease in background.
diff --git a/src/meta-srv/src/procedure.rs b/src/meta-srv/src/procedure.rs
index 1f430654d2..8e696723e8 100644
--- a/src/meta-srv/src/procedure.rs
+++ b/src/meta-srv/src/procedure.rs
@@ -12,7 +12,37 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use async_trait::async_trait;
+use common_meta::error::{self, Result};
+use common_meta::leadership_notifier::LeadershipChangeListener;
+use common_procedure::ProcedureManagerRef;
+use snafu::ResultExt;
+
 pub mod region_migration;
 #[cfg(test)]
 mod tests;
 pub mod utils;
+
+#[derive(Clone)]
+pub struct ProcedureManagerListenerAdapter(pub ProcedureManagerRef);
+
+#[async_trait]
+impl LeadershipChangeListener for ProcedureManagerListenerAdapter {
+    fn name(&self) -> &str {
+        "ProcedureManager"
+    }
+
+    async fn on_leader_start(&self) -> Result<()> {
+        self.0
+            .start()
+            .await
+            .context(error::StartProcedureManagerSnafu)
+    }
+
+    async fn on_leader_stop(&self) -> Result<()> {
+        self.0
+            .stop()
+            .await
+            .context(error::StopProcedureManagerSnafu)
+    }
+}
diff --git a/src/meta-srv/src/region/supervisor.rs b/src/meta-srv/src/region/supervisor.rs
index 80f6c84865..32da666f2b 100644
--- a/src/meta-srv/src/region/supervisor.rs
+++ b/src/meta-srv/src/region/supervisor.rs
@@ -16,10 +16,12 @@ use std::fmt::Debug;
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
+use async_trait::async_trait;
 use common_meta::datanode::Stat;
 use common_meta::ddl::{DetectingRegion, RegionFailureDetectorController};
 use common_meta::key::MAINTENANCE_KEY;
 use common_meta::kv_backend::KvBackendRef;
+use common_meta::leadership_notifier::LeadershipChangeListener;
 use common_meta::peer::PeerLookupServiceRef;
 use common_meta::{ClusterId, DatanodeId};
 use common_runtime::JoinHandle;
@@ -129,6 +131,23 @@ pub struct RegionSupervisorTicker {
     sender: Sender<Event>,
 }
 
+#[async_trait]
+impl LeadershipChangeListener for RegionSupervisorTicker {
+    fn name(&self) -> &'static str {
+        "RegionSupervisorTicker"
+    }
+
+    async fn on_leader_start(&self) -> common_meta::error::Result<()> {
+        self.start();
+        Ok(())
+    }
+
+    async fn on_leader_stop(&self) -> common_meta::error::Result<()> {
+        self.stop();
+        Ok(())
+    }
+}
+
 impl RegionSupervisorTicker {
     pub(crate) fn new(tick_interval: Duration, sender: Sender<Event>) -> Self {
         Self {

From aaa9b329085de638e5122bf132c239359dba65d6 Mon Sep 17 00:00:00 2001
From: Ning Sun <sunng@protonmail.com>
Date: Fri, 11 Oct 2024 10:57:54 -0700
Subject: [PATCH 070/128] feat: add more h3 functions (#4770)

* feat: add more h3 grid functions

* feat: add more traversal functions

* refactor: update some function definitions

* style: format

* refactor: avoid creating slice in nested loop

* feat: ensure column number and length

* refactor: fix lint warnings

* refactor: merge main

* Apply suggestions from code review

Co-authored-by: LFC <990479+MichaelScofield@users.noreply.github.com>

* Update src/common/function/src/scalars/geo/h3.rs

Co-authored-by: LFC <990479+MichaelScofield@users.noreply.github.com>

* style: format

---------

Co-authored-by: LFC <990479+MichaelScofield@users.noreply.github.com>
---
 src/common/function/src/scalars/geo.rs        |  29 +-
 src/common/function/src/scalars/geo/h3.rs     | 728 +++++++++++++-----
 .../standalone/common/function/geo.result     |  59 +-
 .../cases/standalone/common/function/geo.sql  |  22 +-
 4 files changed, 595 insertions(+), 243 deletions(-)

diff --git a/src/common/function/src/scalars/geo.rs b/src/common/function/src/scalars/geo.rs
index 37b6c0704b..e47a1de9f2 100644
--- a/src/common/function/src/scalars/geo.rs
+++ b/src/common/function/src/scalars/geo.rs
@@ -29,18 +29,31 @@ impl GeoFunctions {
         // geohash
         registry.register(Arc::new(GeohashFunction));
         registry.register(Arc::new(GeohashNeighboursFunction));
-        // h3 family
+
+        // h3 index
         registry.register(Arc::new(h3::H3LatLngToCell));
         registry.register(Arc::new(h3::H3LatLngToCellString));
+
+        // h3 index inspection
         registry.register(Arc::new(h3::H3CellBase));
-        registry.register(Arc::new(h3::H3CellCenterChild));
-        registry.register(Arc::new(h3::H3CellCenterLat));
-        registry.register(Arc::new(h3::H3CellCenterLng));
         registry.register(Arc::new(h3::H3CellIsPentagon));
-        registry.register(Arc::new(h3::H3CellParent));
-        registry.register(Arc::new(h3::H3CellResolution));
-        registry.register(Arc::new(h3::H3CellToString));
-        registry.register(Arc::new(h3::H3IsNeighbour));
         registry.register(Arc::new(h3::H3StringToCell));
+        registry.register(Arc::new(h3::H3CellToString));
+        registry.register(Arc::new(h3::H3CellCenterLatLng));
+        registry.register(Arc::new(h3::H3CellResolution));
+
+        // h3 hierarchical grid
+        registry.register(Arc::new(h3::H3CellCenterChild));
+        registry.register(Arc::new(h3::H3CellParent));
+        registry.register(Arc::new(h3::H3CellToChildren));
+        registry.register(Arc::new(h3::H3CellToChildrenSize));
+        registry.register(Arc::new(h3::H3CellToChildPos));
+        registry.register(Arc::new(h3::H3ChildPosToCell));
+
+        // h3 grid traversal
+        registry.register(Arc::new(h3::H3GridDisk));
+        registry.register(Arc::new(h3::H3GridDiskDistances));
+        registry.register(Arc::new(h3::H3GridDistance));
+        registry.register(Arc::new(h3::H3GridPathCells));
     }
 }
diff --git a/src/common/function/src/scalars/geo/h3.rs b/src/common/function/src/scalars/geo/h3.rs
index 672fbfd714..00c567d8d7 100644
--- a/src/common/function/src/scalars/geo/h3.rs
+++ b/src/common/function/src/scalars/geo/h3.rs
@@ -20,18 +20,71 @@ use common_query::error::{self, InvalidFuncArgsSnafu, Result};
 use common_query::prelude::{Signature, TypeSignature};
 use datafusion::logical_expr::Volatility;
 use datatypes::prelude::ConcreteDataType;
-use datatypes::scalars::ScalarVectorBuilder;
-use datatypes::value::Value;
+use datatypes::scalars::{Scalar, ScalarVectorBuilder};
+use datatypes::value::{ListValue, Value};
 use datatypes::vectors::{
-    BooleanVectorBuilder, Float64VectorBuilder, MutableVector, StringVectorBuilder,
-    UInt64VectorBuilder, UInt8VectorBuilder, VectorRef,
+    BooleanVectorBuilder, Int32VectorBuilder, ListVectorBuilder, MutableVector,
+    StringVectorBuilder, UInt64VectorBuilder, UInt8VectorBuilder, VectorRef,
 };
 use derive_more::Display;
 use h3o::{CellIndex, LatLng, Resolution};
+use once_cell::sync::Lazy;
 use snafu::{ensure, ResultExt};
 
+use super::helpers::{ensure_columns_len, ensure_columns_n};
 use crate::function::{Function, FunctionContext};
 
+static CELL_TYPES: Lazy<Vec<ConcreteDataType>> = Lazy::new(|| {
+    vec![
+        ConcreteDataType::int64_datatype(),
+        ConcreteDataType::uint64_datatype(),
+    ]
+});
+
+static COORDINATE_TYPES: Lazy<Vec<ConcreteDataType>> = Lazy::new(|| {
+    vec![
+        ConcreteDataType::float32_datatype(),
+        ConcreteDataType::float64_datatype(),
+    ]
+});
+static RESOLUTION_TYPES: Lazy<Vec<ConcreteDataType>> = Lazy::new(|| {
+    vec![
+        ConcreteDataType::int8_datatype(),
+        ConcreteDataType::int16_datatype(),
+        ConcreteDataType::int32_datatype(),
+        ConcreteDataType::int64_datatype(),
+        ConcreteDataType::uint8_datatype(),
+        ConcreteDataType::uint16_datatype(),
+        ConcreteDataType::uint32_datatype(),
+        ConcreteDataType::uint64_datatype(),
+    ]
+});
+static DISTANCE_TYPES: Lazy<Vec<ConcreteDataType>> = Lazy::new(|| {
+    vec![
+        ConcreteDataType::int8_datatype(),
+        ConcreteDataType::int16_datatype(),
+        ConcreteDataType::int32_datatype(),
+        ConcreteDataType::int64_datatype(),
+        ConcreteDataType::uint8_datatype(),
+        ConcreteDataType::uint16_datatype(),
+        ConcreteDataType::uint32_datatype(),
+        ConcreteDataType::uint64_datatype(),
+    ]
+});
+
+static POSITION_TYPES: Lazy<Vec<ConcreteDataType>> = Lazy::new(|| {
+    vec![
+        ConcreteDataType::int8_datatype(),
+        ConcreteDataType::int16_datatype(),
+        ConcreteDataType::int32_datatype(),
+        ConcreteDataType::int64_datatype(),
+        ConcreteDataType::uint8_datatype(),
+        ConcreteDataType::uint16_datatype(),
+        ConcreteDataType::uint32_datatype(),
+        ConcreteDataType::uint64_datatype(),
+    ]
+});
+
 /// Function that returns [h3] encoding cellid for a given geospatial coordinate.
 ///
 /// [h3]: https://h3geo.org/
@@ -50,20 +103,8 @@ impl Function for H3LatLngToCell {
 
     fn signature(&self) -> Signature {
         let mut signatures = Vec::new();
-        for coord_type in &[
-            ConcreteDataType::float32_datatype(),
-            ConcreteDataType::float64_datatype(),
-        ] {
-            for resolution_type in &[
-                ConcreteDataType::int8_datatype(),
-                ConcreteDataType::int16_datatype(),
-                ConcreteDataType::int32_datatype(),
-                ConcreteDataType::int64_datatype(),
-                ConcreteDataType::uint8_datatype(),
-                ConcreteDataType::uint16_datatype(),
-                ConcreteDataType::uint32_datatype(),
-                ConcreteDataType::uint64_datatype(),
-            ] {
+        for coord_type in COORDINATE_TYPES.as_slice() {
+            for resolution_type in RESOLUTION_TYPES.as_slice() {
                 signatures.push(TypeSignature::Exact(vec![
                     // latitude
                     coord_type.clone(),
@@ -78,15 +119,7 @@ impl Function for H3LatLngToCell {
     }
 
     fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        ensure!(
-            columns.len() == 3,
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not correct, expect 3, provided : {}",
-                    columns.len()
-                ),
-            }
-        );
+        ensure_columns_n!(columns, 3);
 
         let lat_vec = &columns[0];
         let lon_vec = &columns[1];
@@ -142,20 +175,8 @@ impl Function for H3LatLngToCellString {
 
     fn signature(&self) -> Signature {
         let mut signatures = Vec::new();
-        for coord_type in &[
-            ConcreteDataType::float32_datatype(),
-            ConcreteDataType::float64_datatype(),
-        ] {
-            for resolution_type in &[
-                ConcreteDataType::int8_datatype(),
-                ConcreteDataType::int16_datatype(),
-                ConcreteDataType::int32_datatype(),
-                ConcreteDataType::int64_datatype(),
-                ConcreteDataType::uint8_datatype(),
-                ConcreteDataType::uint16_datatype(),
-                ConcreteDataType::uint32_datatype(),
-                ConcreteDataType::uint64_datatype(),
-            ] {
+        for coord_type in COORDINATE_TYPES.as_slice() {
+            for resolution_type in RESOLUTION_TYPES.as_slice() {
                 signatures.push(TypeSignature::Exact(vec![
                     // latitude
                     coord_type.clone(),
@@ -170,15 +191,7 @@ impl Function for H3LatLngToCellString {
     }
 
     fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        ensure!(
-            columns.len() == 3,
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not correct, expect 3, provided : {}",
-                    columns.len()
-                ),
-            }
-        );
+        ensure_columns_n!(columns, 3);
 
         let lat_vec = &columns[0];
         let lon_vec = &columns[1];
@@ -234,15 +247,7 @@ impl Function for H3CellToString {
     }
 
     fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        ensure!(
-            columns.len() == 1,
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not correct, expect 1, provided : {}",
-                    columns.len()
-                ),
-            }
-        );
+        ensure_columns_n!(columns, 1);
 
         let cell_vec = &columns[0];
         let size = cell_vec.len();
@@ -280,15 +285,7 @@ impl Function for H3StringToCell {
     }
 
     fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        ensure!(
-            columns.len() == 1,
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not correct, expect 1, provided : {}",
-                    columns.len()
-                ),
-            }
-        );
+        ensure_columns_n!(columns, 1);
 
         let string_vec = &columns[0];
         let size = string_vec.len();
@@ -319,18 +316,20 @@ impl Function for H3StringToCell {
     }
 }
 
-/// Function that returns centroid latitude of given cell id
+/// Function that returns centroid latitude and longitude of given cell id
 #[derive(Clone, Debug, Default, Display)]
 #[display("{}", self.name())]
-pub struct H3CellCenterLat;
+pub struct H3CellCenterLatLng;
 
-impl Function for H3CellCenterLat {
+impl Function for H3CellCenterLatLng {
     fn name(&self) -> &str {
-        "h3_cell_center_lat"
+        "h3_cell_center_latlng"
     }
 
     fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
-        Ok(ConcreteDataType::float64_datatype())
+        Ok(ConcreteDataType::list_datatype(
+            ConcreteDataType::float64_datatype(),
+        ))
     }
 
     fn signature(&self) -> Signature {
@@ -338,69 +337,26 @@ impl Function for H3CellCenterLat {
     }
 
     fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        ensure!(
-            columns.len() == 1,
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not correct, expect 1, provided : {}",
-                    columns.len()
-                ),
-            }
-        );
+        ensure_columns_n!(columns, 1);
 
         let cell_vec = &columns[0];
         let size = cell_vec.len();
-        let mut results = Float64VectorBuilder::with_capacity(size);
+        let mut results =
+            ListVectorBuilder::with_type_capacity(ConcreteDataType::float64_datatype(), size);
 
         for i in 0..size {
             let cell = cell_from_value(cell_vec.get(i))?;
-            let lat = cell.map(|cell| LatLng::from(cell).lat());
+            let latlng = cell.map(LatLng::from);
 
-            results.push(lat);
-        }
-
-        Ok(results.to_vector())
-    }
-}
-
-/// Function that returns centroid longitude of given cell id
-#[derive(Clone, Debug, Default, Display)]
-#[display("{}", self.name())]
-pub struct H3CellCenterLng;
-
-impl Function for H3CellCenterLng {
-    fn name(&self) -> &str {
-        "h3_cell_center_lng"
-    }
-
-    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
-        Ok(ConcreteDataType::float64_datatype())
-    }
-
-    fn signature(&self) -> Signature {
-        signature_of_cell()
-    }
-
-    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        ensure!(
-            columns.len() == 1,
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not correct, expect 1, provided : {}",
-                    columns.len()
-                ),
+            if let Some(latlng) = latlng {
+                let result = ListValue::new(
+                    vec![latlng.lat().into(), latlng.lng().into()],
+                    ConcreteDataType::float64_datatype(),
+                );
+                results.push(Some(result.as_scalar_ref()));
+            } else {
+                results.push(None);
             }
-        );
-
-        let cell_vec = &columns[0];
-        let size = cell_vec.len();
-        let mut results = Float64VectorBuilder::with_capacity(size);
-
-        for i in 0..size {
-            let cell = cell_from_value(cell_vec.get(i))?;
-            let lat = cell.map(|cell| LatLng::from(cell).lng());
-
-            results.push(lat);
         }
 
         Ok(results.to_vector())
@@ -470,15 +426,7 @@ impl Function for H3CellBase {
     }
 
     fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        ensure!(
-            columns.len() == 1,
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not correct, expect 1, provided : {}",
-                    columns.len()
-                ),
-            }
-        );
+        ensure_columns_n!(columns, 1);
 
         let cell_vec = &columns[0];
         let size = cell_vec.len();
@@ -514,15 +462,7 @@ impl Function for H3CellIsPentagon {
     }
 
     fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        ensure!(
-            columns.len() == 1,
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not correct, expect 1, provided : {}",
-                    columns.len()
-                ),
-            }
-        );
+        ensure_columns_n!(columns, 1);
 
         let cell_vec = &columns[0];
         let size = cell_vec.len();
@@ -558,15 +498,7 @@ impl Function for H3CellCenterChild {
     }
 
     fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        ensure!(
-            columns.len() == 2,
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not correct, expect 2, provided : {}",
-                    columns.len()
-                ),
-            }
-        );
+        ensure_columns_n!(columns, 2);
 
         let cell_vec = &columns[0];
         let res_vec = &columns[1];
@@ -606,15 +538,7 @@ impl Function for H3CellParent {
     }
 
     fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        ensure!(
-            columns.len() == 2,
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not correct, expect 2, provided : {}",
-                    columns.len()
-                ),
-            }
-        );
+        ensure_columns_n!(columns, 2);
 
         let cell_vec = &columns[0];
         let res_vec = &columns[1];
@@ -633,48 +557,323 @@ impl Function for H3CellParent {
     }
 }
 
-/// Function that checks if two cells are neighbour
+/// Function that returns children cell list
 #[derive(Clone, Debug, Default, Display)]
 #[display("{}", self.name())]
-pub struct H3IsNeighbour;
+pub struct H3CellToChildren;
 
-impl Function for H3IsNeighbour {
+impl Function for H3CellToChildren {
     fn name(&self) -> &str {
-        "h3_is_neighbour"
+        "h3_cell_to_children"
     }
 
     fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
-        Ok(ConcreteDataType::boolean_datatype())
+        Ok(ConcreteDataType::list_datatype(
+            ConcreteDataType::uint64_datatype(),
+        ))
     }
 
     fn signature(&self) -> Signature {
-        signature_of_double_cell()
+        signature_of_cell_and_resolution()
     }
 
     fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        ensure!(
-            columns.len() == 2,
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not correct, expect 2, provided : {}",
-                    columns.len()
-                ),
-            }
-        );
+        ensure_columns_n!(columns, 2);
 
         let cell_vec = &columns[0];
-        let cell2_vec = &columns[1];
+        let res_vec = &columns[1];
         let size = cell_vec.len();
-        let mut results = BooleanVectorBuilder::with_capacity(size);
+        let mut results =
+            ListVectorBuilder::with_type_capacity(ConcreteDataType::uint64_datatype(), size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i))?;
+            let res = value_to_resolution(res_vec.get(i))?;
+            let result = cell.map(|cell| {
+                let children: Vec<Value> = cell
+                    .children(res)
+                    .map(|child| Value::from(u64::from(child)))
+                    .collect();
+                ListValue::new(children, ConcreteDataType::uint64_datatype())
+            });
+
+            if let Some(list_value) = result {
+                results.push(Some(list_value.as_scalar_ref()));
+            } else {
+                results.push(None);
+            }
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that returns children cell count
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3CellToChildrenSize;
+
+impl Function for H3CellToChildrenSize {
+    fn name(&self) -> &str {
+        "h3_cell_to_children_size"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::uint64_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell_and_resolution()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure_columns_n!(columns, 2);
+
+        let cell_vec = &columns[0];
+        let res_vec = &columns[1];
+        let size = cell_vec.len();
+        let mut results = UInt64VectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i))?;
+            let res = value_to_resolution(res_vec.get(i))?;
+            let result = cell.map(|cell| cell.children_count(res));
+            results.push(result);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that returns the cell position if its parent at given resolution
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3CellToChildPos;
+
+impl Function for H3CellToChildPos {
+    fn name(&self) -> &str {
+        "h3_cell_to_child_pos"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::uint64_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell_and_resolution()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure_columns_n!(columns, 2);
+
+        let cell_vec = &columns[0];
+        let res_vec = &columns[1];
+        let size = cell_vec.len();
+        let mut results = UInt64VectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i))?;
+            let res = value_to_resolution(res_vec.get(i))?;
+            let result = cell.and_then(|cell| cell.child_position(res));
+            results.push(result);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that returns the cell at given position of the parent at given resolution
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3ChildPosToCell;
+
+impl Function for H3ChildPosToCell {
+    fn name(&self) -> &str {
+        "h3_child_pos_to_cell"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::uint64_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        let mut signatures =
+            Vec::with_capacity(POSITION_TYPES.len() * CELL_TYPES.len() * RESOLUTION_TYPES.len());
+        for position_type in POSITION_TYPES.as_slice() {
+            for cell_type in CELL_TYPES.as_slice() {
+                for resolution_type in RESOLUTION_TYPES.as_slice() {
+                    signatures.push(TypeSignature::Exact(vec![
+                        position_type.clone(),
+                        cell_type.clone(),
+                        resolution_type.clone(),
+                    ]));
+                }
+            }
+        }
+        Signature::one_of(signatures, Volatility::Stable)
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure_columns_n!(columns, 3);
+
+        let pos_vec = &columns[0];
+        let cell_vec = &columns[1];
+        let res_vec = &columns[2];
+        let size = cell_vec.len();
+        let mut results = UInt64VectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i))?;
+            let pos = value_to_position(pos_vec.get(i))?;
+            let res = value_to_resolution(res_vec.get(i))?;
+            let result = cell.and_then(|cell| cell.child_at(pos, res).map(u64::from));
+            results.push(result);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that returns cells with k distances of given cell
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3GridDisk;
+
+impl Function for H3GridDisk {
+    fn name(&self) -> &str {
+        "h3_grid_disk"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::list_datatype(
+            ConcreteDataType::uint64_datatype(),
+        ))
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell_and_distance()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure_columns_n!(columns, 2);
+
+        let cell_vec = &columns[0];
+        let k_vec = &columns[1];
+        let size = cell_vec.len();
+        let mut results =
+            ListVectorBuilder::with_type_capacity(ConcreteDataType::uint64_datatype(), size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i))?;
+            let k = value_to_distance(k_vec.get(i))?;
+
+            let result = cell.map(|cell| {
+                let children: Vec<Value> = cell
+                    .grid_disk::<Vec<_>>(k)
+                    .into_iter()
+                    .map(|child| Value::from(u64::from(child)))
+                    .collect();
+                ListValue::new(children, ConcreteDataType::uint64_datatype())
+            });
+
+            if let Some(list_value) = result {
+                results.push(Some(list_value.as_scalar_ref()));
+            } else {
+                results.push(None);
+            }
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that returns all cells within k distances of given cell
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3GridDiskDistances;
+
+impl Function for H3GridDiskDistances {
+    fn name(&self) -> &str {
+        "h3_grid_disk_distances"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::list_datatype(
+            ConcreteDataType::uint64_datatype(),
+        ))
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell_and_distance()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure_columns_n!(columns, 2);
+
+        let cell_vec = &columns[0];
+        let k_vec = &columns[1];
+        let size = cell_vec.len();
+        let mut results =
+            ListVectorBuilder::with_type_capacity(ConcreteDataType::uint64_datatype(), size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i))?;
+            let k = value_to_distance(k_vec.get(i))?;
+
+            let result = cell.map(|cell| {
+                let children: Vec<Value> = cell
+                    .grid_disk_distances::<Vec<_>>(k)
+                    .into_iter()
+                    .map(|(child, _distance)| Value::from(u64::from(child)))
+                    .collect();
+                ListValue::new(children, ConcreteDataType::uint64_datatype())
+            });
+
+            if let Some(list_value) = result {
+                results.push(Some(list_value.as_scalar_ref()));
+            } else {
+                results.push(None);
+            }
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Function that returns distance between two cells
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3GridDistance;
+
+impl Function for H3GridDistance {
+    fn name(&self) -> &str {
+        "h3_grid_distance"
+    }
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::int32_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_double_cells()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure_columns_n!(columns, 2);
+
+        let cell_this_vec = &columns[0];
+        let cell_that_vec = &columns[1];
+        let size = cell_this_vec.len();
+
+        let mut results = Int32VectorBuilder::with_capacity(size);
 
         for i in 0..size {
             let result = match (
-                cell_from_value(cell_vec.get(i))?,
-                cell_from_value(cell2_vec.get(i))?,
+                cell_from_value(cell_this_vec.get(i))?,
+                cell_from_value(cell_that_vec.get(i))?,
             ) {
                 (Some(cell_this), Some(cell_that)) => {
-                    let is_neighbour = cell_this
-                        .is_neighbor_with(cell_that)
+                    let dist = cell_this
+                        .grid_distance(cell_that)
                         .map_err(|e| {
                             BoxedError::new(PlainError::new(
                                 format!("H3 error: {}", e),
@@ -682,7 +881,7 @@ impl Function for H3IsNeighbour {
                             ))
                         })
                         .context(error::ExecuteSnafu)?;
-                    Some(is_neighbour)
+                    Some(dist)
                 }
                 _ => None,
             };
@@ -694,6 +893,73 @@ impl Function for H3IsNeighbour {
     }
 }
 
+/// Function that returns path cells between two cells
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct H3GridPathCells;
+
+impl Function for H3GridPathCells {
+    fn name(&self) -> &str {
+        "h3_grid_path_cells"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::list_datatype(
+            ConcreteDataType::uint64_datatype(),
+        ))
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_double_cells()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure_columns_n!(columns, 2);
+
+        let cell_this_vec = &columns[0];
+        let cell_that_vec = &columns[1];
+        let size = cell_this_vec.len();
+        let mut results =
+            ListVectorBuilder::with_type_capacity(ConcreteDataType::uint64_datatype(), size);
+
+        for i in 0..size {
+            let result = match (
+                cell_from_value(cell_this_vec.get(i))?,
+                cell_from_value(cell_that_vec.get(i))?,
+            ) {
+                (Some(cell_this), Some(cell_that)) => {
+                    let cells = cell_this
+                        .grid_path_cells(cell_that)
+                        .and_then(|x| x.collect::<std::result::Result<Vec<CellIndex>, _>>())
+                        .map_err(|e| {
+                            BoxedError::new(PlainError::new(
+                                format!("H3 error: {}", e),
+                                StatusCode::EngineExecuteQuery,
+                            ))
+                        })
+                        .context(error::ExecuteSnafu)?;
+                    Some(ListValue::new(
+                        cells
+                            .into_iter()
+                            .map(|c| Value::from(u64::from(c)))
+                            .collect(),
+                        ConcreteDataType::uint64_datatype(),
+                    ))
+                }
+                _ => None,
+            };
+
+            if let Some(list_value) = result {
+                results.push(Some(list_value.as_scalar_ref()));
+            } else {
+                results.push(None);
+            }
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
 fn value_to_resolution(v: Value) -> Result<Resolution> {
     let r = match v {
         Value::Int8(v) => v as u8,
@@ -716,26 +982,59 @@ fn value_to_resolution(v: Value) -> Result<Resolution> {
         .context(error::ExecuteSnafu)
 }
 
+macro_rules! ensure_and_coerce {
+    ($compare:expr, $coerce:expr) => {{
+        ensure!(
+            $compare,
+            InvalidFuncArgsSnafu {
+                err_msg: "Argument was outside of acceptable range "
+            }
+        );
+        Ok($coerce)
+    }};
+}
+
+fn value_to_position(v: Value) -> Result<u64> {
+    match v {
+        Value::Int8(v) => ensure_and_coerce!(v >= 0, v as u64),
+        Value::Int16(v) => ensure_and_coerce!(v >= 0, v as u64),
+        Value::Int32(v) => ensure_and_coerce!(v >= 0, v as u64),
+        Value::Int64(v) => ensure_and_coerce!(v >= 0, v as u64),
+        Value::UInt8(v) => Ok(v as u64),
+        Value::UInt16(v) => Ok(v as u64),
+        Value::UInt32(v) => Ok(v as u64),
+        Value::UInt64(v) => Ok(v),
+        _ => unreachable!(),
+    }
+}
+
+fn value_to_distance(v: Value) -> Result<u32> {
+    match v {
+        Value::Int8(v) => ensure_and_coerce!(v >= 0, v as u32),
+        Value::Int16(v) => ensure_and_coerce!(v >= 0, v as u32),
+        Value::Int32(v) => ensure_and_coerce!(v >= 0, v as u32),
+        Value::Int64(v) => ensure_and_coerce!(v >= 0, v as u32),
+        Value::UInt8(v) => Ok(v as u32),
+        Value::UInt16(v) => Ok(v as u32),
+        Value::UInt32(v) => Ok(v),
+        Value::UInt64(v) => Ok(v as u32),
+        _ => unreachable!(),
+    }
+}
+
 fn signature_of_cell() -> Signature {
-    let mut signatures = Vec::new();
-    for cell_type in &[
-        ConcreteDataType::uint64_datatype(),
-        ConcreteDataType::int64_datatype(),
-    ] {
+    let mut signatures = Vec::with_capacity(CELL_TYPES.len());
+    for cell_type in CELL_TYPES.as_slice() {
         signatures.push(TypeSignature::Exact(vec![cell_type.clone()]));
     }
 
     Signature::one_of(signatures, Volatility::Stable)
 }
 
-fn signature_of_double_cell() -> Signature {
-    let mut signatures = Vec::new();
-    let cell_types = &[
-        ConcreteDataType::uint64_datatype(),
-        ConcreteDataType::int64_datatype(),
-    ];
-    for cell_type in cell_types {
-        for cell_type2 in cell_types {
+fn signature_of_double_cells() -> Signature {
+    let mut signatures = Vec::with_capacity(CELL_TYPES.len() * CELL_TYPES.len());
+    for cell_type in CELL_TYPES.as_slice() {
+        for cell_type2 in CELL_TYPES.as_slice() {
             signatures.push(TypeSignature::Exact(vec![
                 cell_type.clone(),
                 cell_type2.clone(),
@@ -747,21 +1046,9 @@ fn signature_of_double_cell() -> Signature {
 }
 
 fn signature_of_cell_and_resolution() -> Signature {
-    let mut signatures = Vec::new();
-    for cell_type in &[
-        ConcreteDataType::uint64_datatype(),
-        ConcreteDataType::int64_datatype(),
-    ] {
-        for resolution_type in &[
-            ConcreteDataType::int8_datatype(),
-            ConcreteDataType::int16_datatype(),
-            ConcreteDataType::int32_datatype(),
-            ConcreteDataType::int64_datatype(),
-            ConcreteDataType::uint8_datatype(),
-            ConcreteDataType::uint16_datatype(),
-            ConcreteDataType::uint32_datatype(),
-            ConcreteDataType::uint64_datatype(),
-        ] {
+    let mut signatures = Vec::with_capacity(CELL_TYPES.len() * RESOLUTION_TYPES.len());
+    for cell_type in CELL_TYPES.as_slice() {
+        for resolution_type in RESOLUTION_TYPES.as_slice() {
             signatures.push(TypeSignature::Exact(vec![
                 cell_type.clone(),
                 resolution_type.clone(),
@@ -771,6 +1058,19 @@ fn signature_of_cell_and_resolution() -> Signature {
     Signature::one_of(signatures, Volatility::Stable)
 }
 
+fn signature_of_cell_and_distance() -> Signature {
+    let mut signatures = Vec::with_capacity(CELL_TYPES.len() * DISTANCE_TYPES.len());
+    for cell_type in CELL_TYPES.as_slice() {
+        for distance_type in DISTANCE_TYPES.as_slice() {
+            signatures.push(TypeSignature::Exact(vec![
+                cell_type.clone(),
+                distance_type.clone(),
+            ]));
+        }
+    }
+    Signature::one_of(signatures, Volatility::Stable)
+}
+
 fn cell_from_value(v: Value) -> Result<Option<CellIndex>> {
     let cell = match v {
         Value::Int64(v) => Some(
diff --git a/tests/cases/standalone/common/function/geo.result b/tests/cases/standalone/common/function/geo.result
index 4f9d168ac0..75caeb886b 100644
--- a/tests/cases/standalone/common/function/geo.result
+++ b/tests/cases/standalone/common/function/geo.result
@@ -102,35 +102,58 @@ SELECT h3_cell_to_string(h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64)) AS c
 | 88283082e7fffff | 613196570438926335 |
 +-----------------+--------------------+
 
-SELECT h3_cell_center_lat(h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64)) AS cell_center_lat, h3_cell_center_lng(h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64)) AS cell_center_lng;
+SELECT h3_cell_center_latlng(h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64)) AS cell_center;
 
-+-------------------+---------------------+
-| cell_center_lat   | cell_center_lng     |
-+-------------------+---------------------+
-| 37.77246152245501 | -122.39010997087324 |
-+-------------------+---------------------+
++------------------------------------------+
+| cell_center                              |
++------------------------------------------+
+| [37.77246152245501, -122.39010997087324] |
++------------------------------------------+
 
 SELECT
     h3_cell_resolution(cell) AS resolution,
     h3_cell_base(cell) AS base,
     h3_cell_is_pentagon(cell) AS pentagon,
     h3_cell_parent(cell, 6::UInt64) AS parent,
+    h3_cell_to_children(cell, 10::UInt64) AS children,
+    h3_cell_to_children_size(cell, 10) AS children_count,
+    h3_cell_to_child_pos(cell, 6) AS child_pos,
+    h3_child_pos_to_cell(25, cell, 11) AS child
 FROM (SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64) AS cell);
 
-+------------+------+----------+--------------------+
-| resolution | base | pentagon | parent             |
-+------------+------+----------+--------------------+
-| 8          | 20   | false    | 604189371209351167 |
-+------------+------+----------+--------------------+
++------------+------+----------+--------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------+-----------+--------------------+
+| resolution | base | pentagon | parent             | children                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             | children_count | child_pos | child              |
++------------+------+----------+--------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------+-----------+--------------------+
+| 8          | 20   | false    | 604189371209351167 | [622203769691602943, 622203769691635711, 622203769691668479, 622203769691701247, 622203769691734015, 622203769691766783, 622203769691799551, 622203769691865087, 622203769691897855, 622203769691930623, 622203769691963391, 622203769691996159, 622203769692028927, 622203769692061695, 622203769692127231, 622203769692159999, 622203769692192767, 622203769692225535, 622203769692258303, 622203769692291071, 622203769692323839, 622203769692389375, 622203769692422143, 622203769692454911, 622203769692487679, 622203769692520447, 622203769692553215, 622203769692585983, 622203769692651519, 622203769692684287, 622203769692717055, 622203769692749823, 622203769692782591, 622203769692815359, 622203769692848127, 622203769692913663, 622203769692946431, 622203769692979199, 622203769693011967, 622203769693044735, 622203769693077503, 622203769693110271, 622203769693175807, 622203769693208575, 622203769693241343, 622203769693274111, 622203769693306879, 622203769693339647, 622203769693372415] | 49             | 45        | 626707369319059455 |
++------------+------+----------+--------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------+-----------+--------------------+
 
-SELECT h3_is_neighbour(cell1, cell2)
-FROM (SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64) AS cell1, h3_latlng_to_cell(36.76938, -122.3889, 8::UInt64) AS cell2);
+SELECT
+    h3_grid_disk(cell, 0) AS current_cell,
+    h3_grid_disk(cell, 3) AS grids,
+    h3_grid_disk_distances(cell, 3) AS all_grids,
+FROM (SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64) AS cell);
 
-+------------------------------+
-| h3_is_neighbour(cell1,cell2) |
-+------------------------------+
-| false                        |
-+------------------------------+
++----------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| current_cell         | grids                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | all_grids                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |
++----------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| [613196570438926335] | [613196570438926335, 613196570436829183, 613196569755254783, 613196570378108927, 613196570373914623, 613196570434732031, 613196570432634879, 613196570445217791, 613196570250182655, 613196569753157631, 613196569744769023, 613196569746866175, 613196570369720319, 613196570365526015, 613196570376011775, 613196570336165887, 613196570344554495, 613196570443120639, 613196570441023487, 613196570220822527, 613196570258571263, 613196570248085503, 613196570254376959, 613196569757351935, 613196569748963327, 613196569751060479, 613196569686048767, 613196569688145919, 613196570371817471, 613196570367623167, 613196570394886143, 613196570338263039, 613196570331971583, 613196570340360191, 613196570405371903, 613196570403274751, 613196570216628223] | [613196570438926335, 613196570436829183, 613196569755254783, 613196570378108927, 613196570373914623, 613196570434732031, 613196570432634879, 613196570445217791, 613196570250182655, 613196569753157631, 613196569744769023, 613196569746866175, 613196570369720319, 613196570365526015, 613196570376011775, 613196570336165887, 613196570344554495, 613196570443120639, 613196570441023487, 613196570220822527, 613196570258571263, 613196570248085503, 613196570254376959, 613196569757351935, 613196569748963327, 613196569751060479, 613196569686048767, 613196569688145919, 613196570371817471, 613196570367623167, 613196570394886143, 613196570338263039, 613196570331971583, 613196570340360191, 613196570405371903, 613196570403274751, 613196570216628223] |
++----------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+
+SELECT
+    h3_grid_distance(cell1, cell2) AS distance,
+    h3_grid_path_cells(cell1, cell2) AS path_cells,
+FROM
+    (
+      SELECT
+          h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64) AS cell1,
+          h3_latlng_to_cell(39.634, -104.999, 8::UInt64) AS cell2
+    );
+
++----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| distance | path_cells                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
++----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+| 1612     | [613196570438926335, 613196569755254783, 613196569744769023, 613196569748963327, 613196569669271551, 613196569673465855, 613196569763643391, 613196569767837695, 613196570023690239, 613196570021593087, 613196570025787391, 613196569998524415, 613196570002718719, 613196570040467455, 613196570029981695, 613196570034175999, 613196572437512191, 613196572441706495, 613196572414443519, 613196572418637823, 613196572456386559, 613196572445900799, 613196572450095103, 613196572705947647, 613196572710141951, 613196572691267583, 613196572680781823, 613196572684976127, 613196572722724863, 613196572726919167, 613196592932978687, 613196592937172991, 613196592421273599, 613196592410787839, 613196592414982143, 613196592452730879, 613196592456925183, 613196592261890047, 613196592266084351, 613196592689709055, 613196592679223295, 613196592683417599, 613196592496771071, 613196592500965375, 613196592538714111, 613196592528228351, 613196592532422655, 613196587587338239, 613196587591532543, 613196587396497407, 613196587400691711, 613196587438440447, 613196587427954687, 613196587432148991, 613196586916249599, 613196586920443903, 613196587664932863, 613196587669127167, 613196587706875903, 613196587704778751, 613196587708973055, 613196587681710079, 613196587685904383, 613196593444683775, 613196593434198015, 613196593438392319, 613196593476141055, 613196593480335359, 613196593453072383, 613196593457266687, 613196593713119231, 613196593702633471, 613196593706827775, 613196593744576511, 613196593748770815, 613196593729896447, 613196593719410687, 613196593723604991, 613196962272903167, 613196962277097471, 613196962249834495, 613196962254028799, 613196962291777535, 613196962281291775, 613196962285486079, 613196961601814527, 613196961606008831, 613196961578745855, 613196961582940159, 613196961620688895, 613196961610203135, 613196961614397439, 613196961931067391, 613196961935261695, 613196961855569919, 613196961845084159, 613196961849278463, 613196961887027199, 613196961891221503, 613196956830793727, 613196956834988031, 613196956755296255, 613196956744810495, 613196956749004799, 613196956786753535, 613196956790947839, 613196957099229183, 613196957103423487, 613196957023731711, 613196957021634559, 613196957025828863, 613196957116006399, 613196957120200703, 613196962878980095, 613196962868494335, 613196962872688639, 613196962792996863, 613196962797191167, 613196962887368703, 613196962891563007, 613196963147415551, 613196963136929791, 613196963141124095, 613196963061432319, 613196963065626623, 613196963164192767, 613196963153707007, 613196963157901311, 613196979519881215, 613196979524075519, 613196979496812543, 613196979501006847, 613196979538755583, 613196979528269823, 613196979532464127, 613196978848792575, 613196978852986879, 613196978825723903, 613196978829918207, 613196978867666943, 613196978857181183, 613196978861375487, 613196979614253055, 613196979618447359, 613196979102547967, 613196979092062207, 613196979096256511, 613196979134005247, 613196979138199551, 613196977869422591, 613196977873616895, 613196974002274303, 613196973991788543, 613196973995982847, 613196974033731583, 613196974037925887, 613196978137858047, 613196978142052351, 613196978139955199, 613196974268612607, 613196974272806911, 613196980520222719, 613196980524417023, 613196980562165759, 613196980551679999, 613196980555874303, 613196980039974911, 613196980044169215, 613196979849134079, 613196979853328383, 613196979891077119, 613196979880591359, 613196979884785663, 613196980308410367, 613196980312604671, 613196980125958143, 613196980115472383, 613196980119666687, 613196980157415423, 613196980161609727, 613196980134346751, 613196980138541055, 613220567281041407, 613220567270555647, 613220567274749951, 613220567312498687, 613220567316692991, 613220567289430015, 613220567293624319, 613220567549476863, 613220567538991103, 613220567543185407, 613220567524311039, 613220567528505343, 613220567566254079, 613220567555768319, 613220567559962623, 613220565802549247, 613220565806743551, 613220565779480575, 613220565783674879, 613220565821423615, 613220565810937855, 613220565815132159, 613220566070984703, 613220566075179007, 613220566047916031, 613220566052110335, 613220566050013183, 613220566087761919, 613220566091956223, 613220568547721215, 613220568551915519, 613220568472223743, 613220568461737983, 613220568465932287, 613220568503681023, 613220568507875327, 613220567876632575, 613220567880826879, 613220567801135103, 613220567790649343, 613220567794843647, 613220567832592383, 613220567836786687, 613220568153456639, 613220568142970879, 613220568147165183, 613220568067473407, 613220568071667711, 613220568161845247, 613220568166039551, 613220524398477311, 613220524387991551, 613220524392185855, 613220524312494079, 613220524316688383, 613220524406865919, 613220524411060223, 613220524666912767, 613220524656427007, 613220524669009919, 613220524641746943, 613220524645941247, 613220524683689983, 613220524673204223, 613220524677398527, 613220522919985151, 613220522924179455, 613220522896916479, 613220522901110783, 613220522938859519, 613220522928373759, 613220522932568063, 613220523188420607, 613220523192614911, 613220523165351935, 613220523169546239, 613220523167449087, 613220523205197823, 613220523209392127, 613220523014356991, 613220523018551295, 613220525589659647, 613220525579173887, 613220525583368191, 613220525621116927, 613220525625311231, 613220525430276095, 613220525434470399, 613220524918571007, 613220524908085247, 613220524912279551, 613220524950028287, 613220524954222591, 613220525707100159, 613220525696614399, 613220525700808703, 613220525184909311, 613220525189103615, 613220573312450559, 613220573316644863, 613220573354393599, 613220573343907839, 613220573348102143, 613220541559472127, 613220541563666431, 613220573580886015, 613220573585080319, 613220573622829055, 613220573612343295, 613220573624926207, 613220573597663231, 613220573601857535, 613220540571713535, 613220540561227775, 613220540565422079, 613220540603170815, 613220540607365119, 613220540580102143, 613220540584296447, 613220539900624895, 613220539890139135, 613220539894333439, 613220539932082175, 613220539936276479, 613220539909013503, 613220539913207807, 613220539911110655, 613220540166963199, 613220540171157503, 613220540143894527, 613220540148088831, 613220540185837567, 613220540175351807, 613220540179546111, 613220542582882303, 613220542587076607, 613220542559813631, 613220542564007935, 613220542601756671, 613220542591270911, 613220542595465215, 613220542851317759, 613220542855512063, 613220542836637695, 613220542826151935, 613220542830346239, 613220542868094975, 613220542872289279, 613220461710409727, 613220461714604031, 613220461634912255, 613220461624426495, 613220461628620799, 613220461666369535, 613220461670563839, 613220461978845183, 613220461983039487, 613220461903347711, 613220461892861951, 613220461905444863, 613220461995622399, 613220461999816703, 613220214221307903, 613220214210822143, 613220214215016447, 613220214135324671, 613220214139518975, 613220214229696511, 613220214233890815, 613220213550219263, 613220213539733503, 613220213543927807, 613220213464236031, 613220213468430335, 613220213558607871, 613220213562802175, 613220213560705023, 613220213816557567, 613220213820751871, 613220213793488895, 613220213797683199, 613220213835431935, 613220213824946175, 613220213829140479, 613220216232476671, 613220216236670975, 613220216209407999, 613220216213602303, 613220216251351039, 613220216240865279, 613220216245059583, 613220216500912127, 613220476981870591, 613220216486232063, 613220216475746303, 613220216479940607, 613220216517689343, 613220216521883647, 613220479393595391, 613220479397789695, 613220478881890303, 613220478871404543, 613220478875598847, 613220478913347583, 613220478917541887, 613220478722506751, 613220478726701055, 613220479150325759, 613220479139839999, 613220479152422911, 613220478957387775, 613220478961582079, 613220478999330815, 613220478988845055, 613220478993039359, 613220231382302719, 613220231386497023, 613220231191461887, 613220231195656191, 613220231233404927, 613220231222919167, 613220231227113471, 613220230711214079, 613220230715408383, 613220231459897343, 613220231464091647, 613220231461994495, 613220231499743231, 613220231503937535, 613220231476674559, 613220231480868863, 613220631302897663, 613220631292411903, 613220631296606207, 613220631334354943, 613220631338549247, 613220631311286271, 613220631315480575, 613220631571333119, 613220631560847359, 613220631565041663, 613220631602790399, 613220631550361599, 613220631588110335, 613220631577624575, 613220631581818879, 613220633985155071, 613220633989349375, 613220633962086399, 613220633966280703, 613220634004029439, 613220633993543679, 613220633997737983, 613220633314066431, 613220633318260735, 613220633290997759, 613220633295192063, 613220633332940799, 613220633322455039, 613220633335037951, 613220633643319295, 613220633647513599, 613220633567821823, 613220633557336063, 613220633561530367, 613220633599279103, 613220633603473407, 613220600625758207, 613220600629952511, 613220600550260735, 613220600539774975, 613220600543969279, 613220600581718015, 613220600585912319, 613220600894193663, 613220600898387967, 613220600896290815, 613220600816599039, 613220600820793343, 613220600910970879, 613220600915165183, 613220588420333567, 613220588409847807, 613220588414042111, 613220588334350335, 613220588338544639, 613220588428722175, 613220588432916479, 613220588688769023, 613220588678283263, 613220588682477567, 613220588602785791, 613220588667797503, 613220588705546239, 613220588695060479, 613220588699254783, 613220591102590975, 613220591106785279, 613220591079522303, 613220591083716607, 613220591121465343, 613220591110979583, 613220591115173887, 613220590431502335, 613220590435696639, 613220590408433663, 613220590412627967, 613220590450376703, 613220590439890943, 613220590452473855, 613220591196962815, 613220591201157119, 613220590685257727, 613220590674771967, 613220590678966271, 613220590716715007, 613220590720909311, 613220589452132351, 613220589456326655, 613220617797238783, 613220617786753023, 613220617790947327, 613220617828696063, 613220617832890367, 613220589720567807, 613220589718470655, 613220589722664959, 613220618063577087, 613220618067771391, 613220606061576191, 613220606065770495, 613220606103519231, 613220606093033471, 613220606097227775, 613220605581328383, 613220605585522687, 613220605390487551, 613220605394681855, 613220605432430591, 613220605421944831, 613220605426139135, 613220605849763839, 613220605411459071, 613220605667311615, 613220605656825855, 613220605661020159, 613220605698768895, 613220605702963199, 613220605675700223, 613220605679894527, 613220608083230719, 613220608072744959, 613220608076939263, 613220608114687999, 613220608118882303, 613220608091619327, 613220608095813631, 613220608351666175, 613220608341180415, 613220608353763327, 613220608326500351, 613220608330694655, 613220608368443391, 613220608357957631, 613220608362151935, 613220606604738559, 613220606608932863, 613220606581669887, 613220606585864191, 613220606623612927, 613220606613127167, 613220606617321471, 613220606873174015, 613220606877368319, 613220606850105343, 613220606848008191, 613220606852202495, 613220606889951231, 613220606894145535, 613221654100705279, 613221654104899583, 613221654025207807, 613221654014722047, 613221654018916351, 613221654056665087, 613221654060859391, 613221653429616639, 613221653433810943, 613221653354119167, 613221653343633407, 613221653347827711, 613221653385576447, 613221653450588159, 613221653706440703, 613221653695954943, 613221653700149247, 613221653620457471, 613221653624651775, 613221653714829311, 613221653719023615, 613221656122359807, 613221656111874047, 613221656116068351, 613221656036376575, 613221656040570879, 613221656130748415, 613221656134942719, 613221656390795263, 613221656388698111, 613221656392892415, 613221656365629439, 613221656369823743, 613221656407572479, 613221656397086719, 613221656401281023, 613221654643867647, 613221654648061951, 613221654620798975, 613221654624993279, 613221654662742015, 613221654652256255, 613221654656450559, 613221654912303103, 613221654916497407, 613221654889234431, 613221654887137279, 613221654891331583, 613221654929080319, 613221654933274623, 613221654738239487, 613221654742433791, 613221671272185855, 613221671261700095, 613221671265894399, 613221671303643135, 613221671307837439, 613221671112802303, 613221671116996607, 613221670601097215, 613221670590611455, 613221670594805759, 613221670632554495, 613221671351877631, 613221671389626367, 613221671379140607, 613221671383334911, 613221670867435519, 613221670871629823, 613221669602852863, 613221669607047167, 613221669644795903, 613221669634310143, 613221669638504447, 613221673283354623, 613221673287548927, 613221669871288319, 613221669875482623, 613221669913231359, 613221669911134207, 613221669915328511, 613221669888065535, 613221669892259839, 613221672295596031, 613221672285110271, 613221672289304575, 613221672327053311, 613221672331247615, 613221672303984639, 613221672308178943, 613221671624507391, 613221671614021631, 613221671618215935, 613221671655964671, 613221671660158975, 613221671632895999, 613221671630798847, 613221671634993151, 613221671890845695, 613221671895039999, 613221671867777023, 613221671871971327, 613221671909720063, 613221671899234303, 613221671903428607, 613221559416389631, 613221559420583935, 613221559393320959, 613221559397515263, 613221559435263999, 613221559424778239, 613221559428972543, 613221559684825087, 613221559749836799, 613221559670145023, 613221559659659263, 613221559663853567, 613221559701602303, 613221559705796607, 613221558000812031, 613221558005006335, 613221557925314559, 613221557914828799, 613221557919023103, 613221557956771839, 613221557960966143, 613221558269247487, 613221558273441791, 613221558193750015, 613221558191652863, 613221558195847167, 613221558286024703, 613221558290219007, 613221560693555199, 613221560683069439, 613221560687263743, 613221560607571967, 613221560611766271, 613221560701943807, 613221560706138111, 613221560022466559, 613221560011980799, 613221560016175103, 613221559936483327, 613221559940677631, 613221560030855167, 613221560028758015, 613221560032952319, 613221560288804863, 613221560292999167, 613221560265736191, 613221560269930495, 613221560307679231, 613221560297193471, 613221560301387775, 613221576663367679, 613221576667561983, 613221576640299007, 613221576644493311, 613221576682242047, 613221576671756287, 613221576675950591, 613221580784271359, 613221580788465663, 613221576917123071, 613221576906637311, 613221576910831615, 613221576948580351, 613221576952774655, 613221575683997695, 613221575688191999, 613221575172292607, 613221575161806847, 613221575166001151, 613221575203749887, 613221575207944191, 613221575012909055, 613221575017103359, 613221575440728063, 613221575438630911, 613221575442825215, 613221575247790079, 613221575251984383, 613221575289733119, 613221575279247359, 613221575283441663, 613221577854550015, 613221577858744319, 613221577663709183, 613221577667903487, 613221577705652223, 613221577695166463, 613221577699360767, 613221577183461375, 613221577187655679, 613221577932144639, 613221577930047487, 613221577934241791, 613221577971990527, 613221577976184831, 613221577948921855, 613221577953116159, 613221735109492735, 613221735099006975, 613221735103201279, 613221735140950015, 613221735145144319, 613221735117881343, 613221735122075647, 613221735377928191, 613221735367442431, 613221735371636735, 613221735352762367, 613221735356956671, 613221735394705407, 613221735384219647, 613221735388413951, 613221730275557375, 613221730279751679, 613221730252488703, 613221730256683007, 613221730294431743, 613221730283945983, 613221730288140287, 613221729604468735, 613221729608663039, 613221729581400063, 613221729585594367, 613221729623343103, 613221729621245951, 613221729625440255, 613221729933721599, 613221729937915903, 613221729858224127, 613221729847738367, 613221729851932671, 613221729889681407, 613221729893875711, 613221732349640703, 613221732353835007, 613221732274143231, 613221732263657471, 613221732267851775, 613221732305600511, 613221732309794815, 613221732626464767, 613221732615979007, 613221732620173311, 613221732540481535, 613221732544675839, 613221732634853375, 613221732639047679, 613221692226928639, 613221692216442879, 613221692220637183, 613221692140945407, 613221692145139711, 613221692235317247, 613221692239511551, 613221692495364095, 613221692484878335, 613221692489072639, 613221692470198271, 613221692474392575, 613221692512141311, 613221692501655551, 613221692505849855, 613221687392993279, 613221687397187583, 613221687369924607, 613221687374118911, 613221687411867647, 613221687401381887, 613221687405576191, 613221686721904639, 613221686726098943, 613221686698835967, 613221686703030271, 613221686740779007, 613221686738681855, 613221686742876159, 613221687487365119, 613221687491559423, 613221686975660031, 613221686965174271, 613221686969368575, 613221687007117311, 613221687011311615, 613221693258727423, 613221693262921727, 613221689391579135, 613221689381093375, 613221689385287679, 613221689423036415, 613221689427230719, 613221693535551487, 613221693525065727, 613221693529260031, 613221689657917439, 613221689662111743, 613221709868171263, 613221709872365567, 613221709910114303, 613221709899628543, 613221709903822847, 613221709387923455, 613221709392117759, 613221709197082623, 613221709201276927, 613221709239025663, 613221709228539903, 613221709232734207, 613221709213859839, 613221709218054143, 613221709473906687, 613221709463420927, 613221709467615231, 613221709505363967, 613221709509558271, 613221709482295295, 613221709486489599, 613221704373633023, 613221704363147263, 613221704367341567, 613221704405090303, 613221704409284607, 613221704382021631, 613221704386215935, 613221704642068479, 613221704639971327, 613221704644165631, 613221704616902655, 613221704621096959, 613221704658845695, 613221704648359935, 613221704652554239, 613221710411333631, 613221710415527935, 613221710388264959, 613221710392459263, 613221710430207999, 613221710419722239, 613221710423916543, 613221710679769087, 613221710683963391, 613221710665089023, 613221710654603263, 613221710658797567, 613221710696546303, 613221710700740607, 613168113669636095, 613168113665441791, 613168113627693055, 613168113623498751, 613168113633984511, 613168113713676287, 613168113709481983, 613168113401200639, 613168113397006335, 613168113359257599, 613168113355063295, 613168113365549055, 613168113384423423, 613168113380229119, 613168114063900671, 613168114059706367, 613168114070192127, 613168114032443391, 613168114028249087, 613168114055512063, 613168114051317759, 613168111647981567, 613168111643787263, 613168111654273023, 613168111616524287, 613168111612329983, 613168111639592959, 613168111635398655, 613168111637495807, 613168111381643263, 613168111377448959, 613168111404711935, 613168111400517631, 613168111362768895, 613168111358574591, 613168111369060351, 613168113126473727, 613168113122279423, 613168113149542399, 613168113145348095, 613168113107599359, 613168113103405055, 613168113113890815, 613168112858038271, 613168112853843967, 613168112872718335, 613168112868524031, 613168112879009791, 613168112841261055, 613168112837066751, 613168113032101887, 613168113027907583, 613168096498155519, 613168096493961215, 613168096504446975, 613168096466698239, 613168096462503935, 613168096657539071, 613168096653344767, 613168096229720063, 613168096225525759, 613168096236011519, 613168096422658047, 613168096418463743, 613168096380715007, 613168096376520703, 613168096387006463, 613168096902905855, 613168096898711551, 613168098167488511, 613168098163294207, 613168098125545471, 613168098121351167, 613168098131836927, 613168094486986751, 613168094482792447, 613168097899053055, 613168097894858751, 613168097896955903, 613168097859207167, 613168097855012863, 613168097882275839, 613168097878081535, 613168095474745343, 613168095470551039, 613168095481036799, 613168095443288063, 613168095439093759, 613168095466356735, 613168095462162431, 613168095206309887, 613168095202115583, 613168095212601343, 613168095174852607, 613168095170658303, 613168095189532671, 613168095185338367, 613168095195824127, 613168095879495679, 613168095875301375, 613168095785123839, 613168095780929535, 613168095860621311, 613168095856427007, 613168095866912767, 613168208353951743, 613168208349757439, 613168208259579903, 613168208255385599, 613168208335077375, 613168208330883071, 613168208341368831, 613168208024698879, 613168208020504575, 613168207982755839, 613168207978561535, 613168207989047295, 613168208068739071, 613168208064544767, 613168209769529343, 613168209765335039, 613168209727586303, 613168209723391999, 613168209733877759, 613168209813569535, 613168209809375231, 613168209501093887, 613168209496899583, 613168209498996735, 613168209461247999, 613168209457053695, 613168209484316671, 613168209480122367, 613168207076786175, 613168207072591871, 613168207083077631, 613168207045328895, 613168207041134591, 613168207068397567, 613168207064203263, 613168206808350719, 613168206804156415, 613168206814642175, 613168206776893439, 613168206772699135, 613168206791573503, 613168206787379199, 613168206797864959, 613168207481536511, 613168207477342207, 613168207504605183, 613168207500410879, 613168207462662143, 613168207458467839, 613168207468953599, 613168191106973695, 613168191102779391, 613168191130042367, 613168191125848063, 613168191088099327, 613168191083905023, 613168191086002175, 613168159068782591, 613168159064588287, 613168190853218303, 613168190849023999, 613168190859509759, 613168190821761023, 613168190817566719, 613168192086343679, 613168192082149375, 613168192598048767, 613168192593854463, 613168192604340223, 613168192566591487, 613168192562397183, 613168191817908223, 613168191813713919, 613168191815811071, 613168192331710463, 613168192327516159, 613168192522551295, 613168192518356991, 613168192480608255, 613168192476413951, 613168192486899711, 613168189915791359, 613168189911597055, 613168190106632191, 613168190102437887, 613168190064689151, 613168190060494847, 613168190070980607, 613168189647355903, 613168189643161599, 613168189829808127, 613168189825613823, 613168189836099583, 613168189798350847, 613168189794156543, 613168189821419519, 613168189817225215, 613168202312056831, 613168202307862527, 613168202318348287, 613168202280599551, 613168202276405247, 613168202303668223, 613168202299473919, 613168202043621375, 613168202039427071, 613168202041524223, 613168201951346687, 613168201947152383, 613168202026844159, 613168202022649855, 613168202033135615, 613168037494783999, 613168037490589695, 613168037400412159, 613168037396217855, 613168037475909631, 613168037471715327, 613168037482201087, 613168037226348543, 613168037222154239, 613168037131976703, 613168037127782399, 613168037129879551, 613168037209571327, 613168037205377023, 613168037836619775, 613168037832425471, 613168037794676735, 613168037790482431, 613168037800968191, 613168037880659967, 613168037876465663, 613168035420700671, 613168035416506367, 613168035378757631, 613168035374563327, 613168035385049087, 613168035464740863, 613168035399729151, 613168035143876607, 613168035139682303, 613168035150168063, 613168035112419327, 613168035108225023, 613168035135487999, 613168035131293695, 613168047626125311, 613168047621931007, 613168047632416767, 613168047594668031, 613168047590473727, 613168047617736703, 613168047613542399, 613168047357689855, 613168047353495551, 613168047355592703, 613168047382855679, 613168047378661375, 613168047340912639, 613168047336718335, 613168047347204095, 613168020247805951, 613168020243611647, 613168020270874623, 613168020266680319, 613168020228931583, 613168020224737279, 613168020235223039, 613168019979370495, 613168019975176191, 613168020002439167, 613168019998244863, 613168020000342015, 613168019962593279, 613168019958398975, 613168020153434111, 613168020149239807, 613168020665139199, 613168020660944895, 613168020671430655, 613168020633681919, 613168020629487615, 613168032635682815, 613168032631488511, 613168018249220095, 613168018245025791, 613168018255511551, 613168018217762815, 613168032396607487, 613168032358858751, 613168032354664447, 613168032365150207, 613168017982881791, 613168017978687487, 613168029984882687, 613168029980688383, 613168029942939647, 613168029938745343, 613168029949231103, 613168030465130495, 613168030460936191, 613168029716447231, 613168029712252927, 613168029674504191, 613168029670309887, 613168029672407039, 613168029699670015, 613168029695475711, 613168030379147263, 613168030374952959, 613168030385438719, 613168030347689983, 613168030343495679, 613168030370758655, 613168030366564351, 613168063396708351, 613168063392514047, 613168063402999807, 613168063365251071, 613168063361056767, 613168063388319743, 613168063384125439, 613168063386222591, 613168063130370047, 613168063126175743, 613168063035998207, 613168063031803903, 613168063111495679, 613168063107301375, 613168063117787135, 613168831526862847, 613168831522668543, 613168831432491007, 613168831428296703, 613168831507988479, 613168831503794175, 613168831514279935, 613168831258427391, 613168831193415679, 613168831155666943, 613168831151472639, 613168831161958399, 613168831241650175, 613168831237455871, 613168828781690879, 613168828777496575, 613168828739747839, 613168828735553535, 613168828746039295, 613168828825731071, 613168828821536767, 613168828513255423, 613168828509061119, 613168828471312383, 613168828467118079, 613168828469215231, 613168828496478207, 613168828492283903, 613168829175955455, 613168829171761151, 613168829182246911, 613168829144498175, 613168829140303871, 613168829167566847, 613168829163372543, 613167015357579263, 613167015353384959, 613167015363870719, 613167015326121983, 613167015321927679, 613167015349190655, 613167015336607743, 613167015347093503, 613167015091240959, 613167015087046655, 613167015114309631, 613167015110115327, 613167015072366591, 613167015068172287, 613167015078658047, 613168814279884799, 613168814275690495, 613168814302953471, 613168814298759167, 613168814261010431, 613168814256816127, 613168814267301887, 613168814011449343, 613168813510230015, 613168814026129407, 613168814021935103, 613168814032420863, 613168813994672127, 613168813990477823, 613168814185512959, 613168814181318655, 613168811610210303, 613168811606015999, 613168811616501759, 613168811578753023, 613168811574558719, 613168811769593855, 613168811765399551, 613168811341774847, 613168811337580543, 613168811339677695, 613168811534712831, 613168811530518527, 613168811492769791, 613168811488575487, 613168811499061247, 613168812014960639, 613168812010766335, 613168813279543295, 613168813275348991, 613168813237600255, 613168813233405951, 613168813243891711, 613166998196584447, 613166998192390143, 613168813011107839, 613168812998524927, 613168813009010687, 613168812971261951, 613168812967067647, 613168812994330623, 613168812990136319, 613167078641238015, 613167078637043711, 613167078647529471, 613167078609780735, 613167078605586431, 613167078632849407, 613167078628655103, 613167078372802559, 613167078368608255, 613167078379094015, 613167078341345279, 613167078276333567, 613167078356025343, 613167078351831039, 613167078362316799, 613167079045988351, 613167079041794047, 613167078951616511, 613167078947422207, 613167079027113983, 613167079022919679, 613167079033405439, 613167076630069247, 613167076625874943, 613167076535697407, 613167076531503103, 613167076611194879, 613167076607000575, 613167076609097727, 613167076300816383, 613167076296622079, 613167076258873343, 613167076254679039, 613167076265164799, 613167076344856575, 613167076340662271, 613167078045646847, 613167078041452543, 613167078003703807, 613167077999509503, 613167078009995263, 613167078089687039, 613167078085492735, 613167077777211391, 613167077764628479, 613167077775114239, 613167077737365503, 613167077733171199, 613167077760434175, 613167077756239871, 613167061394259967, 613167061390065663, 613167061400551423, 613167061362802687, 613167061358608383, 613167061385871359, 613167061381677055, 613167061125824511, 613167061121630207, 613167061132115967, 613167061094367231, 613167061146796031, 613167061109047295, 613167061104852991, 613167061115338751, 613167061799010303, 613167061794815999, 613167061822078975, 613167061817884671, 613167061780135935, 613167061775941631, 613167061786427391, 613167059383091199, 613167059378896895, 613167059406159871, 613167059401965567, 613167059364216831, 613167059366313983, 613167059362119679, 613167055262187519, 613167055257993215, 613167059129335807, 613167059125141503, 613167059135627263, 613167059097878527, 613167059093684223, 613167060362461183, 613167060358266879, 613167060874166271, 613167060869971967, 613167060880457727, 613167060842708991, 613167060838514687, 613167060094025727, 613167060081442815, 613167060091928575, 613167060607827967, 613167060603633663, 613167060798668799, 613167060794474495, 613167060756725759, 613167060752531455, 613167060763017215, 613167104362807295, 613167104358612991, 613167104553648127, 613167104549453823, 613167104511705087, 613167104507510783, 613167104517996543, 613167104094371839, 613167104532676607, 613167104276824063, 613167104272629759, 613167104283115519, 613167104245366783, 613167104241172479, 613167104268435455, 613167104264241151, 613167098505461759, 613167098501267455, 613167098511753215, 613167098474004479, 613167098469810175, 613167098497073151, 613167098492878847, 613167098237026303, 613167098239123455, 613167098234929151, 613167098144751615, 613167098140557311, 613167098220249087, 613167098216054783, 613167098226540543, 613167103339397119, 613167103335202815, 613167103245025279, 613167103240830975, 613167103320522751, 613167103316328447, 613167103326814207, 613167103070961663, 613167103066767359, 613167102976589823, 613167102964006911, 613167102974492671, 613167103054184447, 613167103049990143, 613167103681232895, 613167103677038591, 613167103639289855, 613167103635095551, 613167103645581311, 613167103725273087, 613167103721078783, 613166949867716607, 613166949863522303, 613166949825773567, 613166949821579263, 613166949832065023, 613166949850939391, 613166949846745087, 613166949590892543, 613166949586698239, 613166949597183999, 613166949559435263, 613166949555240959, 613166949582503935, 613166949578309631, 613166943819530239, 613166943815335935, 613166943825821695, 613166943788072959, 613166943783878655, 613166943811141631, 613166943806947327, 613166943551094783, 613166943553191935, 613166943548997631, 613166943576260607, 613166943572066303, 613166943534317567, 613166943530123263, 613166943540609023, 613166948653465599, 613166948649271295, 613166948676534271, 613166948672339967, 613166948634591231, 613166948630396927, 613166948640882687, 613166948385030143, 613166948380835839, 613166948408098815, 613166948395515903, 613166948406001663, 613166948368252927, 613166948364058623, 613166948559093759, 613166948554899455, 613166949070798847, 613166949066604543, 613166949077090303, 613166949039341567, 613166949035147263, 613166928829087743, 613166928824893439, 613166932696236031, 613166932692041727, 613166932702527487, 613166928594206719, 613166928590012415, 613166928552263679, 613166928548069375, 613166928558555135, 613166932429897727, 613166932425703423, 613166926178287615, 613166926174093311, 613166926136344575, 613166926132150271, 613166926142636031, 613166926658535423, 613166926654341119, 613166925909852159, 613166925905657855, 613166925867909119, 613166925870006271, 613166925865811967, 613166925893074943, 613166925888880639, 613166926572552191, 613166926568357887, 613166926578843647, 613166926541094911, 613166926536900607, 613166926564163583, 613166926559969279, 613166931672825855, 613166931668631551, 613166931679117311, 613166931641368575, 613166931637174271, 613166931664437247, 613166931651854335, 613166931662340095, 613166931406487551, 613166931402293247, 613166931312115711, 613166931307921407, 613166931387613183, 613166931383418879, 613166931393904639, 613167727720267775, 613167727716073471, 613167727625895935, 613167727621701631, 613167727701393407, 613167727697199103, 613167727707684863, 613167727391014911, 613167727386820607, 613167727349071871, 613167727344877567, 613167727355363327, 613167727435055103, 613167727430860799, 613167724975095807, 613167724970901503] |
++----------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
 
 SELECT geohash(37.76938, -122.3889, 9);
 
diff --git a/tests/cases/standalone/common/function/geo.sql b/tests/cases/standalone/common/function/geo.sql
index cd9d403e6e..af2a16517d 100644
--- a/tests/cases/standalone/common/function/geo.sql
+++ b/tests/cases/standalone/common/function/geo.sql
@@ -26,18 +26,34 @@ SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64), h3_latlng_to_cell_stri
 
 SELECT h3_cell_to_string(h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64)) AS cell_str, h3_string_to_cell(h3_latlng_to_cell_string(37.76938, -122.3889, 8::UInt64)) AS cell_index;
 
-SELECT h3_cell_center_lat(h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64)) AS cell_center_lat, h3_cell_center_lng(h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64)) AS cell_center_lng;
+SELECT h3_cell_center_latlng(h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64)) AS cell_center;
 
 SELECT
     h3_cell_resolution(cell) AS resolution,
     h3_cell_base(cell) AS base,
     h3_cell_is_pentagon(cell) AS pentagon,
     h3_cell_parent(cell, 6::UInt64) AS parent,
+    h3_cell_to_children(cell, 10::UInt64) AS children,
+    h3_cell_to_children_size(cell, 10) AS children_count,
+    h3_cell_to_child_pos(cell, 6) AS child_pos,
+    h3_child_pos_to_cell(25, cell, 11) AS child
 FROM (SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64) AS cell);
 
-SELECT h3_is_neighbour(cell1, cell2)
-FROM (SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64) AS cell1, h3_latlng_to_cell(36.76938, -122.3889, 8::UInt64) AS cell2);
+SELECT
+    h3_grid_disk(cell, 0) AS current_cell,
+    h3_grid_disk(cell, 3) AS grids,
+    h3_grid_disk_distances(cell, 3) AS all_grids,
+FROM (SELECT h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64) AS cell);
 
+SELECT
+    h3_grid_distance(cell1, cell2) AS distance,
+    h3_grid_path_cells(cell1, cell2) AS path_cells,
+FROM
+    (
+      SELECT
+          h3_latlng_to_cell(37.76938, -122.3889, 8::UInt64) AS cell1,
+          h3_latlng_to_cell(39.634, -104.999, 8::UInt64) AS cell2
+    );
 
 SELECT geohash(37.76938, -122.3889, 9);
 

From 856c0280f53d3d70ee6f5b0e71568f549f6a5479 Mon Sep 17 00:00:00 2001
From: jeremyhi <jiachun_feng@proton.me>
Date: Sat, 12 Oct 2024 17:04:22 +0800
Subject: [PATCH 071/128] feat: remove the distributed lock (#4825)

* feat: remove the distributed lock as we do not need it any more

* chore: delete todo comment

* chore: remove unused error
---
 src/meta-client/examples/lock.rs    | 129 --------------------
 src/meta-client/src/client.rs       |  35 ------
 src/meta-client/src/client/lock.rs  | 178 ----------------------------
 src/meta-srv/src/bootstrap.rs       |  22 +---
 src/meta-srv/src/error.rs           |  27 -----
 src/meta-srv/src/lib.rs             |   1 -
 src/meta-srv/src/lock.rs            |  99 ----------------
 src/meta-srv/src/lock/etcd.rs       |  93 ---------------
 src/meta-srv/src/lock/memory.rs     | 112 -----------------
 src/meta-srv/src/metasrv.rs         |   6 -
 src/meta-srv/src/metasrv/builder.rs |  12 --
 src/meta-srv/src/service.rs         |   1 -
 src/meta-srv/src/service/lock.rs    |  51 --------
 13 files changed, 4 insertions(+), 762 deletions(-)
 delete mode 100644 src/meta-client/examples/lock.rs
 delete mode 100644 src/meta-client/src/client/lock.rs
 delete mode 100644 src/meta-srv/src/lock.rs
 delete mode 100644 src/meta-srv/src/lock/etcd.rs
 delete mode 100644 src/meta-srv/src/lock/memory.rs
 delete mode 100644 src/meta-srv/src/service/lock.rs

diff --git a/src/meta-client/examples/lock.rs b/src/meta-client/examples/lock.rs
deleted file mode 100644
index c8a8b61d60..0000000000
--- a/src/meta-client/examples/lock.rs
+++ /dev/null
@@ -1,129 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-use std::time::Duration;
-
-use api::v1::meta::Role;
-use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
-use common_meta::rpc::lock::{LockRequest, UnlockRequest};
-use meta_client::client::MetaClientBuilder;
-use meta_client::MetaClientRef;
-use tracing::{info, subscriber};
-use tracing_subscriber::FmtSubscriber;
-
-fn main() {
-    subscriber::set_global_default(FmtSubscriber::builder().finish()).unwrap();
-    run();
-}
-
-#[tokio::main]
-async fn run() {
-    let id = (1000u64, 2000u64);
-    let config = ChannelConfig::new()
-        .timeout(Duration::from_secs(30))
-        .connect_timeout(Duration::from_secs(5))
-        .tcp_nodelay(true);
-    let channel_manager = ChannelManager::with_config(config);
-    let mut meta_client = MetaClientBuilder::new(id.0, id.1, Role::Datanode)
-        .enable_lock()
-        .channel_manager(channel_manager)
-        .build();
-    meta_client.start(&["127.0.0.1:3002"]).await.unwrap();
-    let meta_client = Arc::new(meta_client);
-
-    run_normal(meta_client.clone()).await;
-
-    run_multi_thread(meta_client.clone()).await;
-
-    run_multi_thread_with_one_timeout(meta_client).await;
-}
-
-async fn run_normal(meta_client: MetaClientRef) {
-    let name = "lock_name".as_bytes().to_vec();
-    let expire_secs = 60;
-
-    let lock_req = LockRequest { name, expire_secs };
-
-    let lock_result = meta_client.lock(lock_req).await.unwrap();
-    let key = lock_result.key;
-    info!(
-        "lock success! Returned key: {}",
-        String::from_utf8(key.clone()).unwrap()
-    );
-
-    // It is recommended that time of holding lock is less than the timeout of the grpc channel
-    info!("do some work, take 3 seconds");
-    tokio::time::sleep(Duration::from_secs(3)).await;
-
-    let unlock_req = UnlockRequest { key };
-
-    meta_client.unlock(unlock_req).await.unwrap();
-    info!("unlock success!");
-}
-
-async fn run_multi_thread(meta_client: MetaClientRef) {
-    let meta_client_clone = meta_client.clone();
-    let join1 = tokio::spawn(async move {
-        run_normal(meta_client_clone.clone()).await;
-    });
-
-    tokio::time::sleep(Duration::from_secs(1)).await;
-
-    let join2 = tokio::spawn(async move {
-        run_normal(meta_client).await;
-    });
-
-    join1.await.unwrap();
-    join2.await.unwrap();
-}
-
-async fn run_multi_thread_with_one_timeout(meta_client: MetaClientRef) {
-    let meta_client_clone = meta_client.clone();
-    let join1 = tokio::spawn(async move {
-        run_with_timeout(meta_client_clone.clone()).await;
-    });
-
-    tokio::time::sleep(Duration::from_secs(1)).await;
-
-    let join2 = tokio::spawn(async move {
-        run_normal(meta_client).await;
-    });
-
-    join1.await.unwrap();
-    join2.await.unwrap();
-}
-
-async fn run_with_timeout(meta_client: MetaClientRef) {
-    let name = "lock_name".as_bytes().to_vec();
-    let expire_secs = 5;
-
-    let lock_req = LockRequest { name, expire_secs };
-
-    let lock_result = meta_client.lock(lock_req).await.unwrap();
-    let key = lock_result.key;
-    info!(
-        "lock success! Returned key: {}",
-        String::from_utf8(key.clone()).unwrap()
-    );
-
-    // It is recommended that time of holding lock is less than the timeout of the grpc channel
-    info!("do some work, take 20 seconds");
-    tokio::time::sleep(Duration::from_secs(20)).await;
-
-    let unlock_req = UnlockRequest { key };
-
-    meta_client.unlock(unlock_req).await.unwrap();
-    info!("unlock success!");
-}
diff --git a/src/meta-client/src/client.rs b/src/meta-client/src/client.rs
index 1c5b96a684..ed6fdf13fb 100644
--- a/src/meta-client/src/client.rs
+++ b/src/meta-client/src/client.rs
@@ -15,7 +15,6 @@
 mod ask_leader;
 mod heartbeat;
 mod load_balance;
-mod lock;
 mod procedure;
 
 mod cluster;
@@ -33,7 +32,6 @@ use common_meta::datanode::{DatanodeStatKey, DatanodeStatValue, RegionStat};
 use common_meta::ddl::{ExecutorContext, ProcedureExecutor};
 use common_meta::error::{self as meta_error, Result as MetaResult};
 use common_meta::rpc::ddl::{SubmitDdlTaskRequest, SubmitDdlTaskResponse};
-use common_meta::rpc::lock::{LockRequest, LockResponse, UnlockRequest};
 use common_meta::rpc::procedure::{
     MigrateRegionRequest, MigrateRegionResponse, ProcedureStateResponse,
 };
@@ -45,7 +43,6 @@ use common_meta::rpc::store::{
 use common_meta::ClusterId;
 use common_telemetry::info;
 use heartbeat::Client as HeartbeatClient;
-use lock::Client as LockClient;
 use procedure::Client as ProcedureClient;
 use snafu::{OptionExt, ResultExt};
 use store::Client as StoreClient;
@@ -67,7 +64,6 @@ pub struct MetaClientBuilder {
     role: Role,
     enable_heartbeat: bool,
     enable_store: bool,
-    enable_lock: bool,
     enable_procedure: bool,
     enable_access_cluster_info: bool,
     channel_manager: Option<ChannelManager>,
@@ -123,13 +119,6 @@ impl MetaClientBuilder {
         }
     }
 
-    pub fn enable_lock(self) -> Self {
-        Self {
-            enable_lock: true,
-            ..self
-        }
-    }
-
     pub fn enable_procedure(self) -> Self {
         Self {
             enable_procedure: true,
@@ -188,10 +177,6 @@ impl MetaClientBuilder {
             client.store = Some(StoreClient::new(self.id, self.role, mgr.clone()));
         }
 
-        if self.enable_lock {
-            client.lock = Some(LockClient::new(self.id, self.role, mgr.clone()));
-        }
-
         if self.enable_procedure {
             let mgr = self.ddl_channel_manager.unwrap_or(mgr.clone());
             client.procedure = Some(ProcedureClient::new(
@@ -221,7 +206,6 @@ pub struct MetaClient {
     channel_manager: ChannelManager,
     heartbeat: Option<HeartbeatClient>,
     store: Option<StoreClient>,
-    lock: Option<LockClient>,
     procedure: Option<ProcedureClient>,
     cluster: Option<ClusterClient>,
 }
@@ -383,10 +367,6 @@ impl MetaClient {
             client.start(urls.clone()).await?;
             info!("Store client started");
         }
-        if let Some(client) = &mut self.lock {
-            client.start(urls.clone()).await?;
-            info!("Lock client started");
-        }
         if let Some(client) = &mut self.procedure {
             client.start(urls.clone()).await?;
             info!("DDL client started");
@@ -482,15 +462,6 @@ impl MetaClient {
             .context(ConvertMetaResponseSnafu)
     }
 
-    pub async fn lock(&self, req: LockRequest) -> Result<LockResponse> {
-        self.lock_client()?.lock(req.into()).await.map(Into::into)
-    }
-
-    pub async fn unlock(&self, req: UnlockRequest) -> Result<()> {
-        let _ = self.lock_client()?.unlock(req.into()).await?;
-        Ok(())
-    }
-
     /// Query the procedure state by its id.
     pub async fn query_procedure_state(&self, pid: &str) -> Result<ProcedureStateResponse> {
         self.procedure_client()?.query_procedure_state(pid).await
@@ -538,12 +509,6 @@ impl MetaClient {
         })
     }
 
-    pub fn lock_client(&self) -> Result<LockClient> {
-        self.lock.clone().context(NotStartedSnafu {
-            name: "lock_client",
-        })
-    }
-
     pub fn procedure_client(&self) -> Result<ProcedureClient> {
         self.procedure.clone().context(NotStartedSnafu {
             name: "procedure_client",
diff --git a/src/meta-client/src/client/lock.rs b/src/meta-client/src/client/lock.rs
deleted file mode 100644
index 66fe077c22..0000000000
--- a/src/meta-client/src/client/lock.rs
+++ /dev/null
@@ -1,178 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashSet;
-use std::sync::Arc;
-
-use api::v1::meta::lock_client::LockClient;
-use api::v1::meta::{LockRequest, LockResponse, Role, UnlockRequest, UnlockResponse};
-use common_grpc::channel_manager::ChannelManager;
-use common_telemetry::tracing_context::TracingContext;
-use snafu::{ensure, OptionExt, ResultExt};
-use tokio::sync::RwLock;
-use tonic::transport::Channel;
-
-use crate::client::{load_balance, Id};
-use crate::error;
-use crate::error::Result;
-
-#[derive(Clone, Debug)]
-pub struct Client {
-    inner: Arc<RwLock<Inner>>,
-}
-
-impl Client {
-    pub fn new(id: Id, role: Role, channel_manager: ChannelManager) -> Self {
-        let inner = Arc::new(RwLock::new(Inner {
-            id,
-            role,
-            channel_manager,
-            peers: vec![],
-        }));
-
-        Self { inner }
-    }
-
-    pub async fn start<U, A>(&mut self, urls: A) -> Result<()>
-    where
-        U: AsRef<str>,
-        A: AsRef<[U]>,
-    {
-        let mut inner = self.inner.write().await;
-        inner.start(urls).await
-    }
-
-    pub async fn lock(&self, req: LockRequest) -> Result<LockResponse> {
-        let inner = self.inner.read().await;
-        inner.lock(req).await
-    }
-
-    pub async fn unlock(&self, req: UnlockRequest) -> Result<UnlockResponse> {
-        let inner = self.inner.read().await;
-        inner.unlock(req).await
-    }
-}
-
-#[derive(Debug)]
-struct Inner {
-    id: Id,
-    role: Role,
-    channel_manager: ChannelManager,
-    peers: Vec<String>,
-}
-
-impl Inner {
-    async fn start<U, A>(&mut self, urls: A) -> Result<()>
-    where
-        U: AsRef<str>,
-        A: AsRef<[U]>,
-    {
-        ensure!(
-            !self.is_started(),
-            error::IllegalGrpcClientStateSnafu {
-                err_msg: "Lock client already started",
-            }
-        );
-
-        self.peers = urls
-            .as_ref()
-            .iter()
-            .map(|url| url.as_ref().to_string())
-            .collect::<HashSet<_>>()
-            .drain()
-            .collect::<Vec<_>>();
-
-        Ok(())
-    }
-
-    fn random_client(&self) -> Result<LockClient<Channel>> {
-        let len = self.peers.len();
-        let peer = load_balance::random_get(len, |i| Some(&self.peers[i])).context(
-            error::IllegalGrpcClientStateSnafu {
-                err_msg: "Empty peers, lock client may not start yet",
-            },
-        )?;
-
-        self.make_client(peer)
-    }
-
-    fn make_client(&self, addr: impl AsRef<str>) -> Result<LockClient<Channel>> {
-        let channel = self
-            .channel_manager
-            .get(addr)
-            .context(error::CreateChannelSnafu)?;
-
-        Ok(LockClient::new(channel))
-    }
-
-    #[inline]
-    fn is_started(&self) -> bool {
-        !self.peers.is_empty()
-    }
-
-    async fn lock(&self, mut req: LockRequest) -> Result<LockResponse> {
-        let mut client = self.random_client()?;
-        req.set_header(
-            self.id,
-            self.role,
-            TracingContext::from_current_span().to_w3c(),
-        );
-        let res = client.lock(req).await.map_err(error::Error::from)?;
-
-        Ok(res.into_inner())
-    }
-
-    async fn unlock(&self, mut req: UnlockRequest) -> Result<UnlockResponse> {
-        let mut client = self.random_client()?;
-        req.set_header(
-            self.id,
-            self.role,
-            TracingContext::from_current_span().to_w3c(),
-        );
-        let res = client.unlock(req).await.map_err(error::Error::from)?;
-
-        Ok(res.into_inner())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[tokio::test]
-    async fn test_already_start() {
-        let mut client = Client::new((0, 0), Role::Datanode, ChannelManager::default());
-        client
-            .start(&["127.0.0.1:1000", "127.0.0.1:1001"])
-            .await
-            .unwrap();
-        let res = client.start(&["127.0.0.1:1002"]).await;
-        assert!(res.is_err());
-        assert!(matches!(
-            res.err(),
-            Some(error::Error::IllegalGrpcClientState { .. })
-        ));
-    }
-
-    #[tokio::test]
-    async fn test_start_with_duplicate_peers() {
-        let mut client = Client::new((0, 0), Role::Datanode, ChannelManager::default());
-        client
-            .start(&["127.0.0.1:1000", "127.0.0.1:1000", "127.0.0.1:1000"])
-            .await
-            .unwrap();
-
-        assert_eq!(1, client.inner.write().await.peers.len());
-    }
-}
diff --git a/src/meta-srv/src/bootstrap.rs b/src/meta-srv/src/bootstrap.rs
index dcfac253f3..d10b22c610 100644
--- a/src/meta-srv/src/bootstrap.rs
+++ b/src/meta-srv/src/bootstrap.rs
@@ -16,7 +16,6 @@ use std::sync::Arc;
 
 use api::v1::meta::cluster_server::ClusterServer;
 use api::v1::meta::heartbeat_server::HeartbeatServer;
-use api::v1::meta::lock_server::LockServer;
 use api::v1::meta::procedure_service_server::ProcedureServiceServer;
 use api::v1::meta::store_server::StoreServer;
 use common_base::Plugins;
@@ -48,8 +47,6 @@ use crate::election::etcd::EtcdElection;
 #[cfg(feature = "pg_kvbackend")]
 use crate::error::InvalidArgumentsSnafu;
 use crate::error::{InitExportMetricsTaskSnafu, TomlFormatSnafu};
-use crate::lock::etcd::EtcdLock;
-use crate::lock::memory::MemLock;
 use crate::metasrv::builder::MetasrvBuilder;
 use crate::metasrv::{BackendImpl, Metasrv, MetasrvOptions, SelectorRef};
 use crate::selector::lease_based::LeaseBasedSelector;
@@ -183,7 +180,6 @@ pub fn router(metasrv: Metasrv) -> Router {
         .add_service(HeartbeatServer::new(metasrv.clone()))
         .add_service(StoreServer::new(metasrv.clone()))
         .add_service(ClusterServer::new(metasrv.clone()))
-        .add_service(LockServer::new(metasrv.clone()))
         .add_service(ProcedureServiceServer::new(metasrv.clone()))
         .add_service(admin::make_admin_service(metasrv))
 }
@@ -193,13 +189,9 @@ pub async fn metasrv_builder(
     plugins: Plugins,
     kv_backend: Option<KvBackendRef>,
 ) -> Result<MetasrvBuilder> {
-    let (kv_backend, election, lock) = match (kv_backend, &opts.backend) {
-        (Some(kv_backend), _) => (kv_backend, None, Some(Arc::new(MemLock::default()) as _)),
-        (None, BackendImpl::MemoryStore) => (
-            Arc::new(MemoryKvBackend::new()) as _,
-            None,
-            Some(Arc::new(MemLock::default()) as _),
-        ),
+    let (kv_backend, election) = match (kv_backend, &opts.backend) {
+        (Some(kv_backend), _) => (kv_backend, None),
+        (None, BackendImpl::MemoryStore) => (Arc::new(MemoryKvBackend::new()) as _, None),
         (None, BackendImpl::EtcdStore) => {
             let etcd_client = create_etcd_client(opts).await?;
             let kv_backend = {
@@ -224,18 +216,13 @@ pub async fn metasrv_builder(
                     )
                     .await?,
                 ),
-                Some(EtcdLock::with_etcd_client(
-                    etcd_client,
-                    opts.store_key_prefix.clone(),
-                )?),
             )
         }
         #[cfg(feature = "pg_kvbackend")]
         (None, BackendImpl::PostgresStore) => {
             let pg_client = create_postgres_client(opts).await?;
             let kv_backend = PgStore::with_pg_client(pg_client).await.unwrap();
-            // TODO: implement locking and leader election for pg backend.
-            (kv_backend, None, None)
+            (kv_backend, None)
         }
     };
 
@@ -253,7 +240,6 @@ pub async fn metasrv_builder(
         .in_memory(in_memory)
         .selector(selector)
         .election(election)
-        .lock(lock)
         .plugins(plugins))
 }
 
diff --git a/src/meta-srv/src/error.rs b/src/meta-srv/src/error.rs
index 4c3d974c0a..6bcac5db2a 100644
--- a/src/meta-srv/src/error.rs
+++ b/src/meta-srv/src/error.rs
@@ -448,30 +448,6 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to lock based on etcd"))]
-    Lock {
-        #[snafu(source)]
-        error: etcd_client::Error,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Failed to unlock based on etcd"))]
-    Unlock {
-        #[snafu(source)]
-        error: etcd_client::Error,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Failed to grant lease"))]
-    LeaseGrant {
-        #[snafu(source)]
-        error: etcd_client::Error,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
     #[snafu(display("Invalid utf-8 value"))]
     InvalidUtf8Value {
         #[snafu(source)]
@@ -770,9 +746,6 @@ impl ErrorExt for Error {
             | Error::ResponseHeaderNotFound { .. }
             | Error::IsNotLeader { .. }
             | Error::InvalidHttpBody { .. }
-            | Error::Lock { .. }
-            | Error::Unlock { .. }
-            | Error::LeaseGrant { .. }
             | Error::ExceededRetryLimit { .. }
             | Error::SendShutdownSignal { .. }
             | Error::PusherNotFound { .. }
diff --git a/src/meta-srv/src/lib.rs b/src/meta-srv/src/lib.rs
index 01b48f1da0..a438f4255f 100644
--- a/src/meta-srv/src/lib.rs
+++ b/src/meta-srv/src/lib.rs
@@ -28,7 +28,6 @@ pub mod flow_meta_alloc;
 pub mod handler;
 pub mod key;
 pub mod lease;
-pub mod lock;
 pub mod metasrv;
 mod metrics;
 #[cfg(feature = "mock")]
diff --git a/src/meta-srv/src/lock.rs b/src/meta-srv/src/lock.rs
deleted file mode 100644
index 53451591da..0000000000
--- a/src/meta-srv/src/lock.rs
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-pub mod etcd;
-pub(crate) mod memory;
-
-use std::sync::Arc;
-
-use common_telemetry::error;
-
-use crate::error::Result;
-
-pub type Key = Vec<u8>;
-
-pub const DEFAULT_EXPIRE_TIME_SECS: u64 = 10;
-
-pub struct Opts {
-    // If the expiration time is exceeded and currently holds the lock, the lock is
-    // automatically released.
-    pub expire_secs: Option<u64>,
-}
-
-impl Default for Opts {
-    fn default() -> Self {
-        Opts {
-            expire_secs: Some(DEFAULT_EXPIRE_TIME_SECS),
-        }
-    }
-}
-
-#[async_trait::async_trait]
-pub trait DistLock: Send + Sync {
-    // Lock acquires a distributed shared lock on a given named lock. On success, it
-    // will return a unique key that exists so long as the lock is held by the caller.
-    async fn lock(&self, name: Vec<u8>, opts: Opts) -> Result<Key>;
-
-    // Unlock takes a key returned by Lock and releases the hold on lock.
-    async fn unlock(&self, key: Vec<u8>) -> Result<()>;
-}
-
-pub type DistLockRef = Arc<dyn DistLock>;
-
-pub struct DistLockGuard<'a> {
-    lock: &'a DistLockRef,
-    name: Vec<u8>,
-    key: Option<Key>,
-}
-
-impl<'a> DistLockGuard<'a> {
-    pub fn new(lock: &'a DistLockRef, name: Vec<u8>) -> Self {
-        Self {
-            lock,
-            name,
-            key: None,
-        }
-    }
-
-    pub async fn lock(&mut self) -> Result<()> {
-        if self.key.is_some() {
-            return Ok(());
-        }
-        let key = self
-            .lock
-            .lock(
-                self.name.clone(),
-                Opts {
-                    expire_secs: Some(2),
-                },
-            )
-            .await?;
-        self.key = Some(key);
-        Ok(())
-    }
-}
-
-impl Drop for DistLockGuard<'_> {
-    fn drop(&mut self) {
-        if let Some(key) = self.key.take() {
-            let lock = self.lock.clone();
-            let name = self.name.clone();
-            let _handle = common_runtime::spawn_global(async move {
-                if let Err(e) = lock.unlock(key).await {
-                    error!(e; "Failed to unlock '{}'", String::from_utf8_lossy(&name));
-                }
-            });
-        }
-    }
-}
diff --git a/src/meta-srv/src/lock/etcd.rs b/src/meta-srv/src/lock/etcd.rs
deleted file mode 100644
index 3f53b40e15..0000000000
--- a/src/meta-srv/src/lock/etcd.rs
+++ /dev/null
@@ -1,93 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-
-use etcd_client::{Client, LockOptions};
-use snafu::ResultExt;
-
-use super::{DistLock, DistLockRef, Opts, DEFAULT_EXPIRE_TIME_SECS};
-use crate::error;
-use crate::error::Result;
-
-/// A implementation of distributed lock based on etcd. The Clone of EtcdLock is cheap.
-#[derive(Clone)]
-pub struct EtcdLock {
-    client: Client,
-    store_key_prefix: String,
-}
-
-impl EtcdLock {
-    pub async fn with_endpoints<E, S>(endpoints: S, store_key_prefix: String) -> Result<DistLockRef>
-    where
-        E: AsRef<str>,
-        S: AsRef<[E]>,
-    {
-        let client = Client::connect(endpoints, None)
-            .await
-            .context(error::ConnectEtcdSnafu)?;
-
-        Self::with_etcd_client(client, store_key_prefix)
-    }
-
-    pub fn with_etcd_client(client: Client, store_key_prefix: String) -> Result<DistLockRef> {
-        Ok(Arc::new(EtcdLock {
-            client,
-            store_key_prefix,
-        }))
-    }
-
-    fn lock_key(&self, key: Vec<u8>) -> Vec<u8> {
-        if self.store_key_prefix.is_empty() {
-            key
-        } else {
-            let mut prefix = self.store_key_prefix.as_bytes().to_vec();
-            prefix.extend_from_slice(&key);
-            prefix
-        }
-    }
-}
-
-#[async_trait::async_trait]
-impl DistLock for EtcdLock {
-    async fn lock(&self, key: Vec<u8>, opts: Opts) -> Result<Vec<u8>> {
-        let expire = opts.expire_secs.unwrap_or(DEFAULT_EXPIRE_TIME_SECS) as i64;
-
-        let mut client = self.client.clone();
-
-        let resp = client
-            .lease_grant(expire, None)
-            .await
-            .context(error::LeaseGrantSnafu)?;
-
-        let lease_id = resp.id();
-        let lock_opts = LockOptions::new().with_lease(lease_id);
-
-        let resp = client
-            .lock(self.lock_key(key), Some(lock_opts))
-            .await
-            .context(error::LockSnafu)?;
-
-        Ok(resp.key().to_vec())
-    }
-
-    async fn unlock(&self, key: Vec<u8>) -> Result<()> {
-        let mut client = self.client.clone();
-        let _ = client
-            .unlock(self.lock_key(key))
-            .await
-            .context(error::UnlockSnafu)?;
-        Ok(())
-    }
-}
diff --git a/src/meta-srv/src/lock/memory.rs b/src/meta-srv/src/lock/memory.rs
deleted file mode 100644
index d312700d12..0000000000
--- a/src/meta-srv/src/lock/memory.rs
+++ /dev/null
@@ -1,112 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use dashmap::DashMap;
-use tokio::sync::{Mutex, OwnedMutexGuard};
-
-use crate::error::Result;
-use crate::lock::{DistLock, Key, Opts};
-
-#[derive(Default)]
-pub(crate) struct MemLock {
-    mutexes: DashMap<Key, Arc<Mutex<()>>>,
-    guards: DashMap<Key, OwnedMutexGuard<()>>,
-}
-
-#[async_trait]
-impl DistLock for MemLock {
-    async fn lock(&self, key: Vec<u8>, _opts: Opts) -> Result<Key> {
-        let mutex = self
-            .mutexes
-            .entry(key.clone())
-            .or_insert_with(|| Arc::new(Mutex::new(())))
-            .clone();
-
-        let guard = mutex.lock_owned().await;
-
-        let _ = self.guards.insert(key.clone(), guard);
-        Ok(key)
-    }
-
-    async fn unlock(&self, key: Vec<u8>) -> Result<()> {
-        // drop the guard, so that the mutex can be unlocked,
-        // effectively make the `mutex.lock_owned` in `lock` method to proceed
-        let _ = self.guards.remove(&key);
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::collections::HashMap;
-    use std::sync::atomic::{AtomicU32, Ordering};
-
-    use rand::seq::SliceRandom;
-
-    use super::*;
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test_mem_lock_concurrently() {
-        let lock = Arc::new(MemLock::default());
-
-        let keys = (0..10)
-            .map(|i| format!("my-lock-{i}").into_bytes())
-            .collect::<Vec<Key>>();
-        let counters: [(Key, AtomicU32); 10] = keys
-            .iter()
-            .map(|x| (x.clone(), AtomicU32::new(0)))
-            .collect::<Vec<_>>()
-            .try_into()
-            .unwrap();
-        let counters = Arc::new(HashMap::from(counters));
-
-        let tasks = (0..100)
-            .map(|_| {
-                let mut keys = keys.clone();
-                keys.shuffle(&mut rand::thread_rng());
-
-                let lock_clone = lock.clone();
-                let counters_clone = counters.clone();
-                tokio::spawn(async move {
-                    // every key counter will be added by 1 for 10 times
-                    for i in 0..100 {
-                        let key = &keys[i % keys.len()];
-                        assert!(lock_clone
-                            .lock(key.clone(), Opts { expire_secs: None })
-                            .await
-                            .is_ok());
-
-                        // Intentionally create a critical section:
-                        // if our MemLock is flawed, the resulting counter is wrong.
-                        //
-                        // Note that AtomicU32 is only used to enable the updates from multiple tasks,
-                        // does not make any guarantee about the correctness of the result.
-
-                        let counter = counters_clone.get(key).unwrap();
-                        let v = counter.load(Ordering::Relaxed);
-                        counter.store(v + 1, Ordering::Relaxed);
-
-                        lock_clone.unlock(key.clone()).await.unwrap();
-                    }
-                })
-            })
-            .collect::<Vec<_>>();
-        let _ = futures::future::join_all(tasks).await;
-
-        assert!(counters.values().all(|x| x.load(Ordering::Relaxed) == 1000));
-    }
-}
diff --git a/src/meta-srv/src/metasrv.rs b/src/meta-srv/src/metasrv.rs
index 3cdb64e1e0..c43eec60d4 100644
--- a/src/meta-srv/src/metasrv.rs
+++ b/src/meta-srv/src/metasrv.rs
@@ -57,7 +57,6 @@ use crate::error::{
 use crate::failure_detector::PhiAccrualFailureDetectorOptions;
 use crate::handler::HeartbeatHandlerGroupRef;
 use crate::lease::lookup_datanode_peer;
-use crate::lock::DistLockRef;
 use crate::procedure::region_migration::manager::RegionMigrationManagerRef;
 use crate::procedure::ProcedureManagerListenerAdapter;
 use crate::pubsub::{PublisherRef, SubscriptionManagerRef};
@@ -356,7 +355,6 @@ pub struct Metasrv {
     flow_selector: SelectorRef,
     handler_group: HeartbeatHandlerGroupRef,
     election: Option<ElectionRef>,
-    lock: DistLockRef,
     procedure_manager: ProcedureManagerRef,
     mailbox: MailboxRef,
     procedure_executor: ProcedureExecutorRef,
@@ -568,10 +566,6 @@ impl Metasrv {
         self.election.as_ref()
     }
 
-    pub fn lock(&self) -> &DistLockRef {
-        &self.lock
-    }
-
     pub fn mailbox(&self) -> &MailboxRef {
         &self.mailbox
     }
diff --git a/src/meta-srv/src/metasrv/builder.rs b/src/meta-srv/src/metasrv/builder.rs
index 050f825174..db01b3ec9d 100644
--- a/src/meta-srv/src/metasrv/builder.rs
+++ b/src/meta-srv/src/metasrv/builder.rs
@@ -52,8 +52,6 @@ use crate::handler::{
     HeartbeatHandlerGroup, HeartbeatHandlerGroupBuilder, HeartbeatMailbox, Pushers,
 };
 use crate::lease::MetaPeerLookupService;
-use crate::lock::memory::MemLock;
-use crate::lock::DistLockRef;
 use crate::metasrv::{
     ElectionRef, Metasrv, MetasrvInfo, MetasrvOptions, SelectorContext, SelectorRef, TABLE_ID_SEQ,
 };
@@ -79,7 +77,6 @@ pub struct MetasrvBuilder {
     handler_group: Option<HeartbeatHandlerGroup>,
     election: Option<ElectionRef>,
     meta_peer_client: Option<MetaPeerClientRef>,
-    lock: Option<DistLockRef>,
     node_manager: Option<NodeManagerRef>,
     plugins: Option<Plugins>,
     table_metadata_allocator: Option<TableMetadataAllocatorRef>,
@@ -95,7 +92,6 @@ impl MetasrvBuilder {
             meta_peer_client: None,
             election: None,
             options: None,
-            lock: None,
             node_manager: None,
             plugins: None,
             table_metadata_allocator: None,
@@ -137,11 +133,6 @@ impl MetasrvBuilder {
         self
     }
 
-    pub fn lock(mut self, lock: Option<DistLockRef>) -> Self {
-        self.lock = lock;
-        self
-    }
-
     pub fn node_manager(mut self, node_manager: NodeManagerRef) -> Self {
         self.node_manager = Some(node_manager);
         self
@@ -171,7 +162,6 @@ impl MetasrvBuilder {
             in_memory,
             selector,
             handler_group,
-            lock,
             node_manager,
             plugins,
             table_metadata_allocator,
@@ -205,7 +195,6 @@ impl MetasrvBuilder {
         let flow_metadata_manager = Arc::new(FlowMetadataManager::new(
             leader_cached_kv_backend.clone() as _,
         ));
-        let lock = lock.unwrap_or_else(|| Arc::new(MemLock::default()));
         let selector_ctx = SelectorContext {
             server_addr: options.server_addr.clone(),
             datanode_lease_secs: distributed_time_constants::DATANODE_LEASE_SECS,
@@ -384,7 +373,6 @@ impl MetasrvBuilder {
             flow_selector: Arc::new(RoundRobinSelector::new(SelectTarget::Flownode)),
             handler_group: Arc::new(handler_group),
             election,
-            lock,
             procedure_manager,
             mailbox,
             procedure_executor: ddl_manager,
diff --git a/src/meta-srv/src/service.rs b/src/meta-srv/src/service.rs
index c3e2e781e8..e260b8b980 100644
--- a/src/meta-srv/src/service.rs
+++ b/src/meta-srv/src/service.rs
@@ -20,7 +20,6 @@ use tonic::{Response, Status};
 pub mod admin;
 pub mod cluster;
 mod heartbeat;
-pub mod lock;
 pub mod mailbox;
 pub mod procedure;
 pub mod store;
diff --git a/src/meta-srv/src/service/lock.rs b/src/meta-srv/src/service/lock.rs
deleted file mode 100644
index 4334bdfc37..0000000000
--- a/src/meta-srv/src/service/lock.rs
+++ /dev/null
@@ -1,51 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use api::v1::meta::{lock_server, LockRequest, LockResponse, UnlockRequest, UnlockResponse};
-use tonic::{Request, Response};
-
-use super::GrpcResult;
-use crate::lock::Opts;
-use crate::metasrv::Metasrv;
-
-#[async_trait::async_trait]
-impl lock_server::Lock for Metasrv {
-    async fn lock(&self, request: Request<LockRequest>) -> GrpcResult<LockResponse> {
-        let LockRequest {
-            name, expire_secs, ..
-        } = request.into_inner();
-        let expire_secs = Some(expire_secs as u64);
-
-        let key = self.lock().lock(name, Opts { expire_secs }).await?;
-
-        let resp = LockResponse {
-            key,
-            ..Default::default()
-        };
-
-        Ok(Response::new(resp))
-    }
-
-    async fn unlock(&self, request: Request<UnlockRequest>) -> GrpcResult<UnlockResponse> {
-        let UnlockRequest { key, .. } = request.into_inner();
-
-        let _ = self.lock().unlock(key).await?;
-
-        let resp = UnlockResponse {
-            ..Default::default()
-        };
-
-        Ok(Response::new(resp))
-    }
-}

From 2f2b4b306cb480c1396f452f2c27dc2206705d38 Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Mon, 14 Oct 2024 11:09:03 +0800
Subject: [PATCH 072/128] feat!: implement interval type by multiple structs
 (#4772)

* define structs and methods

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* feat: re-implement interval types in time crate

* feat: use new

* feat: interval value

* feat: query crate interval

* feat: pg and mysql interval

* chore: remove unused imports

* chore: remove commented codes

* feat: make flow compile but may not work

* feat: flow datetime

* test: fix some tests

* test: fix some flow tests(WIP)

* chore: some fix test&docs

* fix: change interval order

* chore: remove unused codes

* chore: fix cilppy

* chore: now signature change

* chore: remove todo

* feat: update error message

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
Co-authored-by: Ruihang Xia <waynestxia@gmail.com>
Co-authored-by: discord9 <discord9@163.com>
---
 src/api/src/helper.rs                         | 108 ++-
 .../function/src/scalars/date/date_add.rs     | 126 +--
 .../function/src/scalars/date/date_sub.rs     | 128 +---
 src/common/grpc/src/select.rs                 |   4 +-
 src/common/time/src/date.rs                   |  71 +-
 src/common/time/src/datetime.rs               |  86 ++-
 src/common/time/src/interval.rs               | 724 +++++++-----------
 src/common/time/src/lib.rs                    |   2 +-
 src/common/time/src/timestamp.rs              |  98 ++-
 src/datatypes/src/interval.rs                 |  84 +-
 src/datatypes/src/types/interval_type.rs      |  10 +-
 src/datatypes/src/types/primitive_type.rs     |  14 +-
 src/datatypes/src/types/string_type.rs        |  10 +-
 src/datatypes/src/value.rs                    | 199 +++--
 src/datatypes/src/vectors/helper.rs           |   7 +-
 src/flow/src/compute/render/reduce.rs         |  41 +-
 src/flow/src/expr/df_func.rs                  |  19 +-
 src/flow/src/expr/func.rs                     | 171 +++--
 src/flow/src/repr.rs                          |  10 +-
 src/flow/src/transform.rs                     |  18 +-
 src/flow/src/transform/aggr.rs                | 120 ++-
 src/flow/src/transform/literal.rs             |  21 +-
 src/mito2/src/row_converter.rs                |  55 +-
 src/query/src/range_select/plan_rewrite.rs    | 117 +--
 src/servers/src/mysql/writer.rs               |   6 +-
 src/servers/src/postgres/types.rs             |  86 ++-
 src/servers/src/postgres/types/interval.rs    |  50 +-
 src/sql/src/statements.rs                     |   4 +-
 .../standalone/common/range/interval.result   |  27 +-
 .../standalone/common/range/interval.sql      |   2 +-
 30 files changed, 1190 insertions(+), 1228 deletions(-)

diff --git a/src/api/src/helper.rs b/src/api/src/helper.rs
index 101cae8802..4281201c42 100644
--- a/src/api/src/helper.rs
+++ b/src/api/src/helper.rs
@@ -17,10 +17,11 @@ use std::sync::Arc;
 use common_base::BitVec;
 use common_decimal::decimal128::{DECIMAL128_DEFAULT_SCALE, DECIMAL128_MAX_PRECISION};
 use common_decimal::Decimal128;
-use common_time::interval::IntervalUnit;
 use common_time::time::Time;
 use common_time::timestamp::TimeUnit;
-use common_time::{Date, DateTime, Interval, Timestamp};
+use common_time::{
+    Date, DateTime, IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth, Timestamp,
+};
 use datatypes::prelude::{ConcreteDataType, ValueRef};
 use datatypes::scalars::ScalarVector;
 use datatypes::types::{
@@ -456,13 +457,11 @@ pub fn push_vals(column: &mut Column, origin_count: usize, vector: VectorRef) {
             TimeUnit::Microsecond => values.time_microsecond_values.push(val.value()),
             TimeUnit::Nanosecond => values.time_nanosecond_values.push(val.value()),
         },
-        Value::Interval(val) => match val.unit() {
-            IntervalUnit::YearMonth => values.interval_year_month_values.push(val.to_i32()),
-            IntervalUnit::DayTime => values.interval_day_time_values.push(val.to_i64()),
-            IntervalUnit::MonthDayNano => values
-                .interval_month_day_nano_values
-                .push(convert_i128_to_interval(val.to_i128())),
-        },
+        Value::IntervalYearMonth(val) => values.interval_year_month_values.push(val.to_i32()),
+        Value::IntervalDayTime(val) => values.interval_day_time_values.push(val.to_i64()),
+        Value::IntervalMonthDayNano(val) => values
+            .interval_month_day_nano_values
+            .push(convert_month_day_nano_to_pb(val)),
         Value::Decimal128(val) => values.decimal128_values.push(convert_to_pb_decimal128(val)),
         Value::List(_) | Value::Duration(_) => unreachable!(),
     });
@@ -507,14 +506,12 @@ fn ddl_request_type(request: &DdlRequest) -> &'static str {
     }
 }
 
-/// Converts an i128 value to google protobuf type [IntervalMonthDayNano].
-pub fn convert_i128_to_interval(v: i128) -> v1::IntervalMonthDayNano {
-    let interval = Interval::from_i128(v);
-    let (months, days, nanoseconds) = interval.to_month_day_nano();
+/// Converts an interval to google protobuf type [IntervalMonthDayNano].
+pub fn convert_month_day_nano_to_pb(v: IntervalMonthDayNano) -> v1::IntervalMonthDayNano {
     v1::IntervalMonthDayNano {
-        months,
-        days,
-        nanoseconds,
+        months: v.months,
+        days: v.days,
+        nanoseconds: v.nanoseconds,
     }
 }
 
@@ -562,11 +559,15 @@ pub fn pb_value_to_value_ref<'a>(
         ValueData::TimeMillisecondValue(t) => ValueRef::Time(Time::new_millisecond(*t)),
         ValueData::TimeMicrosecondValue(t) => ValueRef::Time(Time::new_microsecond(*t)),
         ValueData::TimeNanosecondValue(t) => ValueRef::Time(Time::new_nanosecond(*t)),
-        ValueData::IntervalYearMonthValue(v) => ValueRef::Interval(Interval::from_i32(*v)),
-        ValueData::IntervalDayTimeValue(v) => ValueRef::Interval(Interval::from_i64(*v)),
+        ValueData::IntervalYearMonthValue(v) => {
+            ValueRef::IntervalYearMonth(IntervalYearMonth::from_i32(*v))
+        }
+        ValueData::IntervalDayTimeValue(v) => {
+            ValueRef::IntervalDayTime(IntervalDayTime::from_i64(*v))
+        }
         ValueData::IntervalMonthDayNanoValue(v) => {
-            let interval = Interval::from_month_day_nano(v.months, v.days, v.nanoseconds);
-            ValueRef::Interval(interval)
+            let interval = IntervalMonthDayNano::new(v.months, v.days, v.nanoseconds);
+            ValueRef::IntervalMonthDayNano(interval)
         }
         ValueData::Decimal128Value(v) => {
             // get precision and scale from datatype_extension
@@ -657,7 +658,7 @@ pub fn pb_values_to_vector_ref(data_type: &ConcreteDataType, values: Values) ->
             IntervalType::MonthDayNano(_) => {
                 Arc::new(IntervalMonthDayNanoVector::from_iter_values(
                     values.interval_month_day_nano_values.iter().map(|x| {
-                        Interval::from_month_day_nano(x.months, x.days, x.nanoseconds).to_i128()
+                        IntervalMonthDayNano::new(x.months, x.days, x.nanoseconds).to_i128()
                     }),
                 ))
             }
@@ -802,18 +803,18 @@ pub fn pb_values_to_values(data_type: &ConcreteDataType, values: Values) -> Vec<
         ConcreteDataType::Interval(IntervalType::YearMonth(_)) => values
             .interval_year_month_values
             .into_iter()
-            .map(|v| Value::Interval(Interval::from_i32(v)))
+            .map(|v| Value::IntervalYearMonth(IntervalYearMonth::from_i32(v)))
             .collect(),
         ConcreteDataType::Interval(IntervalType::DayTime(_)) => values
             .interval_day_time_values
             .into_iter()
-            .map(|v| Value::Interval(Interval::from_i64(v)))
+            .map(|v| Value::IntervalDayTime(IntervalDayTime::from_i64(v)))
             .collect(),
         ConcreteDataType::Interval(IntervalType::MonthDayNano(_)) => values
             .interval_month_day_nano_values
             .into_iter()
             .map(|v| {
-                Value::Interval(Interval::from_month_day_nano(
+                Value::IntervalMonthDayNano(IntervalMonthDayNano::new(
                     v.months,
                     v.days,
                     v.nanoseconds,
@@ -941,18 +942,16 @@ pub fn to_proto_value(value: Value) -> Option<v1::Value> {
                 value_data: Some(ValueData::TimeNanosecondValue(v.value())),
             },
         },
-        Value::Interval(v) => match v.unit() {
-            IntervalUnit::YearMonth => v1::Value {
-                value_data: Some(ValueData::IntervalYearMonthValue(v.to_i32())),
-            },
-            IntervalUnit::DayTime => v1::Value {
-                value_data: Some(ValueData::IntervalDayTimeValue(v.to_i64())),
-            },
-            IntervalUnit::MonthDayNano => v1::Value {
-                value_data: Some(ValueData::IntervalMonthDayNanoValue(
-                    convert_i128_to_interval(v.to_i128()),
-                )),
-            },
+        Value::IntervalYearMonth(v) => v1::Value {
+            value_data: Some(ValueData::IntervalYearMonthValue(v.to_i32())),
+        },
+        Value::IntervalDayTime(v) => v1::Value {
+            value_data: Some(ValueData::IntervalDayTimeValue(v.to_i64())),
+        },
+        Value::IntervalMonthDayNano(v) => v1::Value {
+            value_data: Some(ValueData::IntervalMonthDayNanoValue(
+                convert_month_day_nano_to_pb(v),
+            )),
         },
         Value::Decimal128(v) => v1::Value {
             value_data: Some(ValueData::Decimal128Value(convert_to_pb_decimal128(v))),
@@ -1044,13 +1043,11 @@ pub fn value_to_grpc_value(value: Value) -> GrpcValue {
                 TimeUnit::Microsecond => ValueData::TimeMicrosecondValue(v.value()),
                 TimeUnit::Nanosecond => ValueData::TimeNanosecondValue(v.value()),
             }),
-            Value::Interval(v) => Some(match v.unit() {
-                IntervalUnit::YearMonth => ValueData::IntervalYearMonthValue(v.to_i32()),
-                IntervalUnit::DayTime => ValueData::IntervalDayTimeValue(v.to_i64()),
-                IntervalUnit::MonthDayNano => {
-                    ValueData::IntervalMonthDayNanoValue(convert_i128_to_interval(v.to_i128()))
-                }
-            }),
+            Value::IntervalYearMonth(v) => Some(ValueData::IntervalYearMonthValue(v.to_i32())),
+            Value::IntervalDayTime(v) => Some(ValueData::IntervalDayTimeValue(v.to_i64())),
+            Value::IntervalMonthDayNano(v) => Some(ValueData::IntervalMonthDayNanoValue(
+                convert_month_day_nano_to_pb(v),
+            )),
             Value::Decimal128(v) => Some(ValueData::Decimal128Value(convert_to_pb_decimal128(v))),
             Value::List(_) | Value::Duration(_) => unreachable!(),
         },
@@ -1061,6 +1058,7 @@ pub fn value_to_grpc_value(value: Value) -> GrpcValue {
 mod tests {
     use std::sync::Arc;
 
+    use common_time::interval::IntervalUnit;
     use datatypes::types::{
         Int32Type, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalYearMonthType,
         TimeMillisecondType, TimeSecondType, TimestampMillisecondType, TimestampSecondType,
@@ -1506,11 +1504,11 @@ mod tests {
 
     #[test]
     fn test_convert_i128_to_interval() {
-        let i128_val = 3000;
-        let interval = convert_i128_to_interval(i128_val);
+        let i128_val = 3;
+        let interval = convert_month_day_nano_to_pb(IntervalMonthDayNano::from_i128(i128_val));
         assert_eq!(interval.months, 0);
         assert_eq!(interval.days, 0);
-        assert_eq!(interval.nanoseconds, 3000);
+        assert_eq!(interval.nanoseconds, 3);
     }
 
     #[test]
@@ -1590,9 +1588,9 @@ mod tests {
             },
         );
         let expect = vec![
-            Value::Interval(Interval::from_year_month(1_i32)),
-            Value::Interval(Interval::from_year_month(2_i32)),
-            Value::Interval(Interval::from_year_month(3_i32)),
+            Value::IntervalYearMonth(IntervalYearMonth::new(1_i32)),
+            Value::IntervalYearMonth(IntervalYearMonth::new(2_i32)),
+            Value::IntervalYearMonth(IntervalYearMonth::new(3_i32)),
         ];
         assert_eq!(expect, actual);
 
@@ -1605,9 +1603,9 @@ mod tests {
             },
         );
         let expect = vec![
-            Value::Interval(Interval::from_i64(1_i64)),
-            Value::Interval(Interval::from_i64(2_i64)),
-            Value::Interval(Interval::from_i64(3_i64)),
+            Value::IntervalDayTime(IntervalDayTime::from_i64(1_i64)),
+            Value::IntervalDayTime(IntervalDayTime::from_i64(2_i64)),
+            Value::IntervalDayTime(IntervalDayTime::from_i64(3_i64)),
         ];
         assert_eq!(expect, actual);
 
@@ -1636,9 +1634,9 @@ mod tests {
             },
         );
         let expect = vec![
-            Value::Interval(Interval::from_month_day_nano(1, 2, 3)),
-            Value::Interval(Interval::from_month_day_nano(5, 6, 7)),
-            Value::Interval(Interval::from_month_day_nano(9, 10, 11)),
+            Value::IntervalMonthDayNano(IntervalMonthDayNano::new(1, 2, 3)),
+            Value::IntervalMonthDayNano(IntervalMonthDayNano::new(5, 6, 7)),
+            Value::IntervalMonthDayNano(IntervalMonthDayNano::new(9, 10, 11)),
         ];
         assert_eq!(expect, actual);
     }
diff --git a/src/common/function/src/scalars/date/date_add.rs b/src/common/function/src/scalars/date/date_add.rs
index 1052acb868..2307d2caab 100644
--- a/src/common/function/src/scalars/date/date_add.rs
+++ b/src/common/function/src/scalars/date/date_add.rs
@@ -14,18 +14,19 @@
 
 use std::fmt;
 
-use common_query::error::{InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu};
+use common_query::error::{ArrowComputeSnafu, IntoVectorSnafu, InvalidFuncArgsSnafu, Result};
 use common_query::prelude::Signature;
-use datatypes::data_type::DataType;
+use datatypes::arrow::compute::kernels::numeric;
 use datatypes::prelude::ConcreteDataType;
-use datatypes::value::ValueRef;
-use datatypes::vectors::VectorRef;
-use snafu::ensure;
+use datatypes::vectors::{Helper, VectorRef};
+use snafu::{ensure, ResultExt};
 
 use crate::function::{Function, FunctionContext};
 use crate::helper;
 
-/// A function adds an interval value to Timestamp, Date or DateTime, and return the result.
+/// A function adds an interval value to Timestamp, Date, and return the result.
+/// The implementation of datetime type is based on Date64 which is incorrect so this function
+/// doesn't support the datetime type.
 #[derive(Clone, Debug, Default)]
 pub struct DateAddFunction;
 
@@ -44,7 +45,6 @@ impl Function for DateAddFunction {
         helper::one_of_sigs2(
             vec![
                 ConcreteDataType::date_datatype(),
-                ConcreteDataType::datetime_datatype(),
                 ConcreteDataType::timestamp_second_datatype(),
                 ConcreteDataType::timestamp_millisecond_datatype(),
                 ConcreteDataType::timestamp_microsecond_datatype(),
@@ -69,64 +69,14 @@ impl Function for DateAddFunction {
             }
         );
 
-        let left = &columns[0];
-        let right = &columns[1];
+        let left = columns[0].to_arrow_array();
+        let right = columns[1].to_arrow_array();
 
-        let size = left.len();
-        let left_datatype = columns[0].data_type();
-        match left_datatype {
-            ConcreteDataType::Timestamp(_) => {
-                let mut result = left_datatype.create_mutable_vector(size);
-                for i in 0..size {
-                    let ts = left.get(i).as_timestamp();
-                    let interval = right.get(i).as_interval();
-
-                    let new_ts = match (ts, interval) {
-                        (Some(ts), Some(interval)) => ts.add_interval(interval),
-                        _ => ts,
-                    };
-
-                    result.push_value_ref(ValueRef::from(new_ts));
-                }
-
-                Ok(result.to_vector())
-            }
-            ConcreteDataType::Date(_) => {
-                let mut result = left_datatype.create_mutable_vector(size);
-                for i in 0..size {
-                    let date = left.get(i).as_date();
-                    let interval = right.get(i).as_interval();
-                    let new_date = match (date, interval) {
-                        (Some(date), Some(interval)) => date.add_interval(interval),
-                        _ => date,
-                    };
-
-                    result.push_value_ref(ValueRef::from(new_date));
-                }
-
-                Ok(result.to_vector())
-            }
-            ConcreteDataType::DateTime(_) => {
-                let mut result = left_datatype.create_mutable_vector(size);
-                for i in 0..size {
-                    let datetime = left.get(i).as_datetime();
-                    let interval = right.get(i).as_interval();
-                    let new_datetime = match (datetime, interval) {
-                        (Some(datetime), Some(interval)) => datetime.add_interval(interval),
-                        _ => datetime,
-                    };
-
-                    result.push_value_ref(ValueRef::from(new_datetime));
-                }
-
-                Ok(result.to_vector())
-            }
-            _ => UnsupportedInputDataTypeSnafu {
-                function: NAME,
-                datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
-            }
-            .fail(),
-        }
+        let result = numeric::add(&left, &right).context(ArrowComputeSnafu)?;
+        let arrow_type = result.data_type().clone();
+        Helper::try_into_vector(result).context(IntoVectorSnafu {
+            data_type: arrow_type,
+        })
     }
 }
 
@@ -144,8 +94,7 @@ mod tests {
     use datatypes::prelude::ConcreteDataType;
     use datatypes::value::Value;
     use datatypes::vectors::{
-        DateTimeVector, DateVector, IntervalDayTimeVector, IntervalYearMonthVector,
-        TimestampSecondVector,
+        DateVector, IntervalDayTimeVector, IntervalYearMonthVector, TimestampSecondVector,
     };
 
     use super::{DateAddFunction, *};
@@ -168,16 +117,15 @@ mod tests {
             ConcreteDataType::date_datatype(),
             f.return_type(&[ConcreteDataType::date_datatype()]).unwrap()
         );
-        assert_eq!(
-            ConcreteDataType::datetime_datatype(),
-            f.return_type(&[ConcreteDataType::datetime_datatype()])
-                .unwrap()
-        );
-        assert!(matches!(f.signature(),
+        assert!(
+            matches!(f.signature(),
                          Signature {
                              type_signature: TypeSignature::OneOf(sigs),
                              volatility: Volatility::Immutable
-                         } if  sigs.len() == 18));
+                         } if  sigs.len() == 15),
+            "{:?}",
+            f.signature()
+        );
     }
 
     #[test]
@@ -243,36 +191,4 @@ mod tests {
             }
         }
     }
-
-    #[test]
-    fn test_datetime_date_add() {
-        let f = DateAddFunction;
-
-        let dates = vec![Some(123), None, Some(42), None];
-        // Intervals in months
-        let intervals = vec![1, 2, 3, 1];
-        let results = [Some(2678400123), None, Some(7776000042), None];
-
-        let date_vector = DateTimeVector::from(dates.clone());
-        let interval_vector = IntervalYearMonthVector::from_vec(intervals);
-        let args: Vec<VectorRef> = vec![Arc::new(date_vector), Arc::new(interval_vector)];
-        let vector = f.eval(FunctionContext::default(), &args).unwrap();
-
-        assert_eq!(4, vector.len());
-        for (i, _t) in dates.iter().enumerate() {
-            let v = vector.get(i);
-            let result = results.get(i).unwrap();
-
-            if result.is_none() {
-                assert_eq!(Value::Null, v);
-                continue;
-            }
-            match v {
-                Value::DateTime(date) => {
-                    assert_eq!(date.val(), result.unwrap());
-                }
-                _ => unreachable!(),
-            }
-        }
-    }
 }
diff --git a/src/common/function/src/scalars/date/date_sub.rs b/src/common/function/src/scalars/date/date_sub.rs
index b1f87e880a..1e00db5331 100644
--- a/src/common/function/src/scalars/date/date_sub.rs
+++ b/src/common/function/src/scalars/date/date_sub.rs
@@ -14,18 +14,19 @@
 
 use std::fmt;
 
-use common_query::error::{InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu};
+use common_query::error::{ArrowComputeSnafu, IntoVectorSnafu, InvalidFuncArgsSnafu, Result};
 use common_query::prelude::Signature;
-use datatypes::data_type::DataType;
+use datatypes::arrow::compute::kernels::numeric;
 use datatypes::prelude::ConcreteDataType;
-use datatypes::value::ValueRef;
-use datatypes::vectors::VectorRef;
-use snafu::ensure;
+use datatypes::vectors::{Helper, VectorRef};
+use snafu::{ensure, ResultExt};
 
 use crate::function::{Function, FunctionContext};
 use crate::helper;
 
-/// A function subtracts an interval value to Timestamp, Date or DateTime, and return the result.
+/// A function subtracts an interval value to Timestamp, Date, and return the result.
+/// The implementation of datetime type is based on Date64 which is incorrect so this function
+/// doesn't support the datetime type.
 #[derive(Clone, Debug, Default)]
 pub struct DateSubFunction;
 
@@ -44,7 +45,6 @@ impl Function for DateSubFunction {
         helper::one_of_sigs2(
             vec![
                 ConcreteDataType::date_datatype(),
-                ConcreteDataType::datetime_datatype(),
                 ConcreteDataType::timestamp_second_datatype(),
                 ConcreteDataType::timestamp_millisecond_datatype(),
                 ConcreteDataType::timestamp_microsecond_datatype(),
@@ -69,65 +69,14 @@ impl Function for DateSubFunction {
             }
         );
 
-        let left = &columns[0];
-        let right = &columns[1];
+        let left = columns[0].to_arrow_array();
+        let right = columns[1].to_arrow_array();
 
-        let size = left.len();
-        let left_datatype = columns[0].data_type();
-
-        match left_datatype {
-            ConcreteDataType::Timestamp(_) => {
-                let mut result = left_datatype.create_mutable_vector(size);
-                for i in 0..size {
-                    let ts = left.get(i).as_timestamp();
-                    let interval = right.get(i).as_interval();
-
-                    let new_ts = match (ts, interval) {
-                        (Some(ts), Some(interval)) => ts.sub_interval(interval),
-                        _ => ts,
-                    };
-
-                    result.push_value_ref(ValueRef::from(new_ts));
-                }
-
-                Ok(result.to_vector())
-            }
-            ConcreteDataType::Date(_) => {
-                let mut result = left_datatype.create_mutable_vector(size);
-                for i in 0..size {
-                    let date = left.get(i).as_date();
-                    let interval = right.get(i).as_interval();
-                    let new_date = match (date, interval) {
-                        (Some(date), Some(interval)) => date.sub_interval(interval),
-                        _ => date,
-                    };
-
-                    result.push_value_ref(ValueRef::from(new_date));
-                }
-
-                Ok(result.to_vector())
-            }
-            ConcreteDataType::DateTime(_) => {
-                let mut result = left_datatype.create_mutable_vector(size);
-                for i in 0..size {
-                    let datetime = left.get(i).as_datetime();
-                    let interval = right.get(i).as_interval();
-                    let new_datetime = match (datetime, interval) {
-                        (Some(datetime), Some(interval)) => datetime.sub_interval(interval),
-                        _ => datetime,
-                    };
-
-                    result.push_value_ref(ValueRef::from(new_datetime));
-                }
-
-                Ok(result.to_vector())
-            }
-            _ => UnsupportedInputDataTypeSnafu {
-                function: NAME,
-                datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
-            }
-            .fail(),
-        }
+        let result = numeric::sub(&left, &right).context(ArrowComputeSnafu)?;
+        let arrow_type = result.data_type().clone();
+        Helper::try_into_vector(result).context(IntoVectorSnafu {
+            data_type: arrow_type,
+        })
     }
 }
 
@@ -145,8 +94,7 @@ mod tests {
     use datatypes::prelude::ConcreteDataType;
     use datatypes::value::Value;
     use datatypes::vectors::{
-        DateTimeVector, DateVector, IntervalDayTimeVector, IntervalYearMonthVector,
-        TimestampSecondVector,
+        DateVector, IntervalDayTimeVector, IntervalYearMonthVector, TimestampSecondVector,
     };
 
     use super::{DateSubFunction, *};
@@ -174,11 +122,15 @@ mod tests {
             f.return_type(&[ConcreteDataType::datetime_datatype()])
                 .unwrap()
         );
-        assert!(matches!(f.signature(),
+        assert!(
+            matches!(f.signature(),
                          Signature {
                              type_signature: TypeSignature::OneOf(sigs),
                              volatility: Volatility::Immutable
-                         } if  sigs.len() == 18));
+                         } if  sigs.len() == 15),
+            "{:?}",
+            f.signature()
+        );
     }
 
     #[test]
@@ -250,42 +202,4 @@ mod tests {
             }
         }
     }
-
-    #[test]
-    fn test_datetime_date_sub() {
-        let f = DateSubFunction;
-        let millis_per_month = 3600 * 24 * 30 * 1000;
-
-        let dates = vec![
-            Some(123 * millis_per_month),
-            None,
-            Some(42 * millis_per_month),
-            None,
-        ];
-        // Intervals in months
-        let intervals = vec![1, 2, 3, 1];
-        let results = [Some(316137600000), None, Some(100915200000), None];
-
-        let date_vector = DateTimeVector::from(dates.clone());
-        let interval_vector = IntervalYearMonthVector::from_vec(intervals);
-        let args: Vec<VectorRef> = vec![Arc::new(date_vector), Arc::new(interval_vector)];
-        let vector = f.eval(FunctionContext::default(), &args).unwrap();
-
-        assert_eq!(4, vector.len());
-        for (i, _t) in dates.iter().enumerate() {
-            let v = vector.get(i);
-            let result = results.get(i).unwrap();
-
-            if result.is_none() {
-                assert_eq!(Value::Null, v);
-                continue;
-            }
-            match v {
-                Value::DateTime(date) => {
-                    assert_eq!(date.val(), result.unwrap());
-                }
-                _ => unreachable!(),
-            }
-        }
-    }
 }
diff --git a/src/common/grpc/src/select.rs b/src/common/grpc/src/select.rs
index ba13acf3b7..493893f49d 100644
--- a/src/common/grpc/src/select.rs
+++ b/src/common/grpc/src/select.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use api::helper::{convert_i128_to_interval, convert_to_pb_decimal128};
+use api::helper::{convert_month_day_nano_to_pb, convert_to_pb_decimal128};
 use api::v1::column::Values;
 use common_base::BitVec;
 use datatypes::types::{IntervalType, TimeType, TimestampType, WrapperType};
@@ -211,7 +211,7 @@ pub fn values(arrays: &[VectorRef]) -> Result<Values> {
             ConcreteDataType::Interval(IntervalType::MonthDayNano(_)),
             IntervalMonthDayNanoVector,
             interval_month_day_nano_values,
-            |x| { convert_i128_to_interval(x.into_native()) }
+            |x| { convert_month_day_nano_to_pb(x) }
         ),
         (
             ConcreteDataType::Decimal128(_),
diff --git a/src/common/time/src/date.rs b/src/common/time/src/date.rs
index 86759d737d..4274913925 100644
--- a/src/common/time/src/date.rs
+++ b/src/common/time/src/date.rs
@@ -14,13 +14,13 @@
 
 use std::fmt::{Display, Formatter, Write};
 
-use chrono::{Datelike, Days, LocalResult, Months, NaiveDate, NaiveTime, TimeZone};
+use chrono::{Datelike, Days, LocalResult, Months, NaiveDate, NaiveTime, TimeDelta, TimeZone};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use snafu::ResultExt;
 
 use crate::error::{InvalidDateStrSnafu, ParseDateStrSnafu, Result};
-use crate::interval::Interval;
+use crate::interval::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth};
 use crate::timezone::get_timezone;
 use crate::util::datetime_to_utc;
 use crate::Timezone;
@@ -134,29 +134,64 @@ impl Date {
         (self.0 as i64) * 24 * 3600
     }
 
-    /// Adds given Interval to the current date.
-    /// Returns None if the resulting date would be out of range.
-    pub fn add_interval(&self, interval: Interval) -> Option<Date> {
+    // FIXME(yingwen): remove add/sub intervals later
+    /// Adds given [IntervalYearMonth] to the current date.
+    pub fn add_year_month(&self, interval: IntervalYearMonth) -> Option<Date> {
         let naive_date = self.to_chrono_date()?;
 
-        let (months, days, _) = interval.to_month_day_nano();
-
         naive_date
-            .checked_add_months(Months::new(months as u32))?
-            .checked_add_days(Days::new(days as u64))
+            .checked_add_months(Months::new(interval.months as u32))
             .map(Into::into)
     }
 
-    /// Subtracts given Interval to the current date.
-    /// Returns None if the resulting date would be out of range.
-    pub fn sub_interval(&self, interval: Interval) -> Option<Date> {
+    /// Adds given [IntervalDayTime] to the current date.
+    pub fn add_day_time(&self, interval: IntervalDayTime) -> Option<Date> {
         let naive_date = self.to_chrono_date()?;
 
-        let (months, days, _) = interval.to_month_day_nano();
+        naive_date
+            .checked_add_days(Days::new(interval.days as u64))?
+            .checked_add_signed(TimeDelta::milliseconds(interval.milliseconds as i64))
+            .map(Into::into)
+    }
+
+    /// Adds given [IntervalMonthDayNano] to the current date.
+    pub fn add_month_day_nano(&self, interval: IntervalMonthDayNano) -> Option<Date> {
+        let naive_date = self.to_chrono_date()?;
 
         naive_date
-            .checked_sub_months(Months::new(months as u32))?
-            .checked_sub_days(Days::new(days as u64))
+            .checked_add_months(Months::new(interval.months as u32))?
+            .checked_add_days(Days::new(interval.days as u64))?
+            .checked_add_signed(TimeDelta::nanoseconds(interval.nanoseconds))
+            .map(Into::into)
+    }
+
+    /// Subtracts given [IntervalYearMonth] to the current date.
+    pub fn sub_year_month(&self, interval: IntervalYearMonth) -> Option<Date> {
+        let naive_date = self.to_chrono_date()?;
+
+        naive_date
+            .checked_sub_months(Months::new(interval.months as u32))
+            .map(Into::into)
+    }
+
+    /// Subtracts given [IntervalDayTime] to the current date.
+    pub fn sub_day_time(&self, interval: IntervalDayTime) -> Option<Date> {
+        let naive_date = self.to_chrono_date()?;
+
+        naive_date
+            .checked_sub_days(Days::new(interval.days as u64))?
+            .checked_sub_signed(TimeDelta::milliseconds(interval.milliseconds as i64))
+            .map(Into::into)
+    }
+
+    /// Subtracts given [IntervalMonthDayNano] to the current date.
+    pub fn sub_month_day_nano(&self, interval: IntervalMonthDayNano) -> Option<Date> {
+        let naive_date = self.to_chrono_date()?;
+
+        naive_date
+            .checked_sub_months(Months::new(interval.months as u32))?
+            .checked_sub_days(Days::new(interval.days as u64))?
+            .checked_sub_signed(TimeDelta::nanoseconds(interval.nanoseconds))
             .map(Into::into)
     }
 
@@ -246,12 +281,12 @@ mod tests {
     fn test_add_sub_interval() {
         let date = Date::new(1000);
 
-        let interval = Interval::from_year_month(3);
+        let interval = IntervalYearMonth::new(3);
 
-        let new_date = date.add_interval(interval).unwrap();
+        let new_date = date.add_year_month(interval).unwrap();
         assert_eq!(new_date.val(), 1091);
 
-        assert_eq!(date, new_date.sub_interval(interval).unwrap());
+        assert_eq!(date, new_date.sub_year_month(interval).unwrap());
     }
 
     #[test]
diff --git a/src/common/time/src/datetime.rs b/src/common/time/src/datetime.rs
index 4a60470aeb..abeae4908a 100644
--- a/src/common/time/src/datetime.rs
+++ b/src/common/time/src/datetime.rs
@@ -13,16 +13,18 @@
 // limitations under the License.
 
 use std::fmt::{Display, Formatter, Write};
-use std::time::Duration;
 
-use chrono::{Days, LocalResult, Months, NaiveDateTime, TimeZone as ChronoTimeZone, Utc};
+use chrono::{
+    Days, LocalResult, Months, NaiveDateTime, TimeDelta, TimeZone as ChronoTimeZone, Utc,
+};
 use serde::{Deserialize, Serialize};
 use snafu::ResultExt;
 
 use crate::error::{InvalidDateStrSnafu, Result};
+use crate::interval::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth};
 use crate::timezone::{get_timezone, Timezone};
 use crate::util::{datetime_to_utc, format_utc_datetime};
-use crate::{Date, Interval};
+use crate::Date;
 
 const DATETIME_FORMAT: &str = "%F %H:%M:%S%.f";
 const DATETIME_FORMAT_WITH_TZ: &str = "%F %H:%M:%S%.f%z";
@@ -160,32 +162,66 @@ impl DateTime {
             None => Utc.from_utc_datetime(&v).naive_local(),
         })
     }
-    /// Adds given Interval to the current datetime.
-    /// Returns None if the resulting datetime would be out of range.
-    pub fn add_interval(&self, interval: Interval) -> Option<Self> {
+
+    // FIXME(yingwen): remove add/sub intervals later
+    /// Adds given [IntervalYearMonth] to the current datetime.
+    pub fn add_year_month(&self, interval: IntervalYearMonth) -> Option<Self> {
         let naive_datetime = self.to_chrono_datetime()?;
-        let (months, days, nsecs) = interval.to_month_day_nano();
 
-        let naive_datetime = naive_datetime
-            .checked_add_months(Months::new(months as u32))?
-            .checked_add_days(Days::new(days as u64))?
-            + Duration::from_nanos(nsecs as u64);
-
-        Some(naive_datetime.into())
+        naive_datetime
+            .checked_add_months(Months::new(interval.months as u32))
+            .map(Into::into)
     }
 
-    /// Subtracts given Interval to the current datetime.
-    /// Returns None if the resulting datetime would be out of range.
-    pub fn sub_interval(&self, interval: Interval) -> Option<Self> {
+    /// Adds given [IntervalDayTime] to the current datetime.
+    pub fn add_day_time(&self, interval: IntervalDayTime) -> Option<Self> {
         let naive_datetime = self.to_chrono_datetime()?;
-        let (months, days, nsecs) = interval.to_month_day_nano();
 
-        let naive_datetime = naive_datetime
-            .checked_sub_months(Months::new(months as u32))?
-            .checked_sub_days(Days::new(days as u64))?
-            - Duration::from_nanos(nsecs as u64);
+        naive_datetime
+            .checked_add_days(Days::new(interval.days as u64))?
+            .checked_add_signed(TimeDelta::milliseconds(interval.milliseconds as i64))
+            .map(Into::into)
+    }
 
-        Some(naive_datetime.into())
+    /// Adds given [IntervalMonthDayNano] to the current datetime.
+    pub fn add_month_day_nano(&self, interval: IntervalMonthDayNano) -> Option<Self> {
+        let naive_datetime = self.to_chrono_datetime()?;
+
+        naive_datetime
+            .checked_add_months(Months::new(interval.months as u32))?
+            .checked_add_days(Days::new(interval.days as u64))?
+            .checked_add_signed(TimeDelta::nanoseconds(interval.nanoseconds))
+            .map(Into::into)
+    }
+
+    /// Subtracts given [IntervalYearMonth] to the current datetime.
+    pub fn sub_year_month(&self, interval: IntervalYearMonth) -> Option<Self> {
+        let naive_datetime = self.to_chrono_datetime()?;
+
+        naive_datetime
+            .checked_sub_months(Months::new(interval.months as u32))
+            .map(Into::into)
+    }
+
+    /// Subtracts given [IntervalDayTime] to the current datetime.
+    pub fn sub_day_time(&self, interval: IntervalDayTime) -> Option<Self> {
+        let naive_datetime = self.to_chrono_datetime()?;
+
+        naive_datetime
+            .checked_sub_days(Days::new(interval.days as u64))?
+            .checked_sub_signed(TimeDelta::milliseconds(interval.milliseconds as i64))
+            .map(Into::into)
+    }
+
+    /// Subtracts given [IntervalMonthDayNano] to the current datetime.
+    pub fn sub_month_day_nano(&self, interval: IntervalMonthDayNano) -> Option<Self> {
+        let naive_datetime = self.to_chrono_datetime()?;
+
+        naive_datetime
+            .checked_sub_months(Months::new(interval.months as u32))?
+            .checked_sub_days(Days::new(interval.days as u64))?
+            .checked_sub_signed(TimeDelta::nanoseconds(interval.nanoseconds))
+            .map(Into::into)
     }
 
     /// Convert to [common_time::date].
@@ -231,12 +267,12 @@ mod tests {
     fn test_add_sub_interval() {
         let datetime = DateTime::new(1000);
 
-        let interval = Interval::from_day_time(1, 200);
+        let interval = IntervalDayTime::new(1, 200);
 
-        let new_datetime = datetime.add_interval(interval).unwrap();
+        let new_datetime = datetime.add_day_time(interval).unwrap();
         assert_eq!(new_datetime.val(), 1000 + 3600 * 24 * 1000 + 200);
 
-        assert_eq!(datetime, new_datetime.sub_interval(interval).unwrap());
+        assert_eq!(datetime, new_datetime.sub_day_time(interval).unwrap());
     }
 
     #[test]
diff --git a/src/common/time/src/interval.rs b/src/common/time/src/interval.rs
index cd57028d29..0ca40f7d79 100644
--- a/src/common/time/src/interval.rs
+++ b/src/common/time/src/interval.rs
@@ -12,18 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::cmp::Ordering;
-use std::default::Default;
-use std::fmt::{self, Display, Formatter, Write};
-use std::hash::{Hash, Hasher};
+use std::hash::Hash;
 
 use arrow::datatypes::IntervalUnit as ArrowIntervalUnit;
 use serde::{Deserialize, Serialize};
-use serde_json::Value;
-use snafu::ResultExt;
-
-use crate::duration::Duration;
-use crate::error::{Result, TimestampOverflowSnafu};
 
 #[derive(
     Debug, Default, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize,
@@ -61,268 +53,269 @@ impl From<ArrowIntervalUnit> for IntervalUnit {
     }
 }
 
-/// Interval Type represents a period of time.
-/// It is composed of months, days and nanoseconds.
-/// 3 kinds of interval are supported: year-month, day-time and
-/// month-day-nano, which will be stored in the following format.
-/// Interval data format:
-/// | months | days   | nsecs      |
-/// | 4bytes | 4bytes | 8bytes     |
-#[derive(Debug, Clone, Default, Copy, Serialize, Deserialize)]
-pub struct Interval {
-    months: i32,
-    days: i32,
-    nsecs: i64,
-    unit: IntervalUnit,
+// The `Value` type requires Serialize, Deserialize.
+#[derive(
+    Debug, Default, Copy, Clone, Eq, PartialEq, Hash, Ord, PartialOrd, Serialize, Deserialize,
+)]
+#[repr(C)]
+pub struct IntervalYearMonth {
+    /// Number of months
+    pub months: i32,
 }
 
-// Nanosecond convert to other time unit
-pub const NANOS_PER_SEC: i64 = 1_000_000_000;
-pub const NANOS_PER_MILLI: i64 = 1_000_000;
-pub const NANOS_PER_MICRO: i64 = 1_000;
-pub const NANOS_PER_HOUR: i64 = 60 * 60 * NANOS_PER_SEC;
-pub const NANOS_PER_DAY: i64 = 24 * NANOS_PER_HOUR;
-pub const NANOS_PER_MONTH: i64 = 30 * NANOS_PER_DAY;
-
-pub const DAYS_PER_MONTH: i64 = 30;
-
-impl Interval {
-    /// Creates a new interval from months, days and nanoseconds.
-    /// Precision is nanosecond.
-    pub fn from_month_day_nano(months: i32, days: i32, nsecs: i64) -> Self {
-        Interval {
-            months,
-            days,
-            nsecs,
-            unit: IntervalUnit::MonthDayNano,
-        }
-    }
-
-    /// Creates a new interval from months.
-    pub fn from_year_month(months: i32) -> Self {
-        Interval {
-            months,
-            days: 0,
-            nsecs: 0,
-            unit: IntervalUnit::YearMonth,
-        }
-    }
-
-    /// Creates a new interval from days and milliseconds.
-    pub fn from_day_time(days: i32, millis: i32) -> Self {
-        Interval {
-            months: 0,
-            days,
-            nsecs: (millis as i64) * NANOS_PER_MILLI,
-            unit: IntervalUnit::DayTime,
-        }
-    }
-
-    pub fn to_duration(&self) -> Result<Duration> {
-        Ok(Duration::new_nanosecond(
-            self.to_nanosecond()
-                .try_into()
-                .context(TimestampOverflowSnafu)?,
-        ))
-    }
-
-    /// Return a tuple(months, days, nanoseconds) from the interval.
-    pub fn to_month_day_nano(&self) -> (i32, i32, i64) {
-        (self.months, self.days, self.nsecs)
-    }
-
-    /// Converts the interval to nanoseconds.
-    pub fn to_nanosecond(&self) -> i128 {
-        let days = (self.days as i64) + DAYS_PER_MONTH * (self.months as i64);
-        (self.nsecs as i128) + (NANOS_PER_DAY as i128) * (days as i128)
-    }
-
-    /// Smallest interval value.
-    pub const MIN: Self = Self {
-        months: i32::MIN,
-        days: i32::MIN,
-        nsecs: i64::MIN,
-        unit: IntervalUnit::MonthDayNano,
-    };
-
-    /// Largest interval value.
-    pub const MAX: Self = Self {
-        months: i32::MAX,
-        days: i32::MAX,
-        nsecs: i64::MAX,
-        unit: IntervalUnit::MonthDayNano,
-    };
-
-    /// Returns the justified interval.
-    /// allows you to adjust the interval of 30-day as one month and the interval of 24-hour as one day
-    pub fn justified_interval(&self) -> Self {
-        let mut result = *self;
-        let extra_months_d = self.days as i64 / DAYS_PER_MONTH;
-        let extra_months_nsecs = self.nsecs / NANOS_PER_MONTH;
-        result.days -= (extra_months_d * DAYS_PER_MONTH) as i32;
-        result.nsecs -= extra_months_nsecs * NANOS_PER_MONTH;
-
-        let extra_days = self.nsecs / NANOS_PER_DAY;
-        result.nsecs -= extra_days * NANOS_PER_DAY;
-
-        result.months += extra_months_d as i32 + extra_months_nsecs as i32;
-        result.days += extra_days as i32;
-
-        result
-    }
-
-    /// Convert Interval to nanoseconds,
-    /// to check whether Interval is positive
-    pub fn is_positive(&self) -> bool {
-        self.to_nanosecond() > 0
-    }
-
-    /// is_zero
-    pub fn is_zero(&self) -> bool {
-        self.months == 0 && self.days == 0 && self.nsecs == 0
-    }
-
-    /// get unit
-    pub fn unit(&self) -> IntervalUnit {
-        self.unit
-    }
-
-    /// Multiple Interval by an integer with overflow check.
-    /// Returns justified Interval, or `None` if overflow occurred.
-    pub fn checked_mul_int<I>(&self, rhs: I) -> Option<Self>
-    where
-        I: TryInto<i32>,
-    {
-        let rhs = rhs.try_into().ok()?;
-        let months = self.months.checked_mul(rhs)?;
-        let days = self.days.checked_mul(rhs)?;
-        let nsecs = self.nsecs.checked_mul(rhs as i64)?;
-
-        Some(
-            Self {
-                months,
-                days,
-                nsecs,
-                unit: self.unit,
-            }
-            .justified_interval(),
-        )
-    }
-
-    /// Convert Interval to ISO 8601 string
-    pub fn to_iso8601_string(self) -> String {
-        IntervalFormat::from(self).to_iso8601_string()
-    }
-
-    /// Convert Interval to postgres verbose string
-    pub fn to_postgres_string(self) -> String {
-        IntervalFormat::from(self).to_postgres_string()
-    }
-
-    /// Convert Interval to sql_standard string
-    pub fn to_sql_standard_string(self) -> String {
-        IntervalFormat::from(self).to_sql_standard_string()
-    }
-
-    /// Interval Type and i128 [IntervalUnit::MonthDayNano] Convert
-    /// v consists of months(i32) | days(i32) | nsecs(i64)
-    pub fn from_i128(v: i128) -> Self {
-        Interval {
-            nsecs: v as i64,
-            days: (v >> 64) as i32,
-            months: (v >> 96) as i32,
-            unit: IntervalUnit::MonthDayNano,
-        }
-    }
-
-    /// `Interval` Type and i64 [IntervalUnit::DayTime] Convert
-    /// v consists of days(i32) | milliseconds(i32)
-    pub fn from_i64(v: i64) -> Self {
-        Interval {
-            nsecs: ((v as i32) as i64) * NANOS_PER_MILLI,
-            days: (v >> 32) as i32,
-            months: 0,
-            unit: IntervalUnit::DayTime,
-        }
-    }
-
-    /// `Interval` Type and i32 [IntervalUnit::YearMonth] Convert
-    /// v consists of months(i32)
-    pub fn from_i32(v: i32) -> Self {
-        Interval {
-            nsecs: 0,
-            days: 0,
-            months: v,
-            unit: IntervalUnit::YearMonth,
-        }
-    }
-
-    pub fn to_i128(&self) -> i128 {
-        // 128            96              64                               0
-        // +-------+-------+-------+-------+-------+-------+-------+-------+
-        // |     months    |      days     |          nanoseconds          |
-        // +-------+-------+-------+-------+-------+-------+-------+-------+
-        let months = (self.months as u128 & u32::MAX as u128) << 96;
-        let days = (self.days as u128 & u32::MAX as u128) << 64;
-        let nsecs = self.nsecs as u128 & u64::MAX as u128;
-        (months | days | nsecs) as i128
-    }
-
-    pub fn to_i64(&self) -> i64 {
-        // 64                              32                              0
-        // +-------+-------+-------+-------+-------+-------+-------+-------+
-        // |             days              |         milliseconds          |
-        // +-------+-------+-------+-------+-------+-------+-------+-------+
-        let days = (self.days as u64 & u32::MAX as u64) << 32;
-        let milliseconds = (self.nsecs / NANOS_PER_MILLI) as u64 & u32::MAX as u64;
-        (days | milliseconds) as i64
+impl IntervalYearMonth {
+    pub fn new(months: i32) -> Self {
+        Self { months }
     }
 
     pub fn to_i32(&self) -> i32 {
         self.months
     }
 
+    pub fn from_i32(months: i32) -> Self {
+        Self { months }
+    }
+
     pub fn negative(&self) -> Self {
-        Self {
-            months: -self.months,
-            days: -self.days,
-            nsecs: -self.nsecs,
-            unit: self.unit,
+        Self::new(-self.months)
+    }
+
+    pub fn to_iso8601_string(&self) -> String {
+        IntervalFormat::from(*self).to_iso8601_string()
+    }
+}
+
+impl From<IntervalYearMonth> for IntervalFormat {
+    fn from(interval: IntervalYearMonth) -> Self {
+        IntervalFormat {
+            years: interval.months / 12,
+            months: interval.months % 12,
+            ..Default::default()
         }
     }
 }
 
-impl From<i128> for Interval {
+impl From<i32> for IntervalYearMonth {
+    fn from(v: i32) -> Self {
+        Self::from_i32(v)
+    }
+}
+
+impl From<IntervalYearMonth> for i32 {
+    fn from(v: IntervalYearMonth) -> Self {
+        v.to_i32()
+    }
+}
+
+impl From<IntervalYearMonth> for serde_json::Value {
+    fn from(v: IntervalYearMonth) -> Self {
+        serde_json::Value::from(v.to_i32())
+    }
+}
+
+#[derive(
+    Debug, Default, Copy, Clone, Eq, PartialEq, Hash, Ord, PartialOrd, Serialize, Deserialize,
+)]
+#[repr(C)]
+pub struct IntervalDayTime {
+    /// Number of days
+    pub days: i32,
+    /// Number of milliseconds
+    pub milliseconds: i32,
+}
+
+impl IntervalDayTime {
+    /// The additive identity i.e. `0`.
+    pub const ZERO: Self = Self::new(0, 0);
+
+    /// The multiplicative inverse, i.e. `-1`.
+    pub const MINUS_ONE: Self = Self::new(-1, -1);
+
+    /// The maximum value that can be represented
+    pub const MAX: Self = Self::new(i32::MAX, i32::MAX);
+
+    /// The minimum value that can be represented
+    pub const MIN: Self = Self::new(i32::MIN, i32::MIN);
+
+    pub const fn new(days: i32, milliseconds: i32) -> Self {
+        Self { days, milliseconds }
+    }
+
+    pub fn to_i64(&self) -> i64 {
+        let d = (self.days as u64 & u32::MAX as u64) << 32;
+        let m = self.milliseconds as u64 & u32::MAX as u64;
+        (d | m) as i64
+    }
+
+    pub fn from_i64(value: i64) -> Self {
+        let days = (value >> 32) as i32;
+        let milliseconds = value as i32;
+        Self { days, milliseconds }
+    }
+
+    pub fn negative(&self) -> Self {
+        Self::new(-self.days, -self.milliseconds)
+    }
+
+    pub fn to_iso8601_string(&self) -> String {
+        IntervalFormat::from(*self).to_iso8601_string()
+    }
+
+    pub fn as_millis(&self) -> i64 {
+        self.days as i64 * MS_PER_DAY + self.milliseconds as i64
+    }
+}
+
+impl From<i64> for IntervalDayTime {
+    fn from(v: i64) -> Self {
+        Self::from_i64(v)
+    }
+}
+
+impl From<IntervalDayTime> for i64 {
+    fn from(v: IntervalDayTime) -> Self {
+        v.to_i64()
+    }
+}
+
+impl From<IntervalDayTime> for serde_json::Value {
+    fn from(v: IntervalDayTime) -> Self {
+        serde_json::Value::from(v.to_i64())
+    }
+}
+
+// Millisecond convert to other time unit
+pub const MS_PER_SEC: i64 = 1_000;
+pub const MS_PER_MINUTE: i64 = 60 * MS_PER_SEC;
+pub const MS_PER_HOUR: i64 = 60 * MS_PER_MINUTE;
+pub const MS_PER_DAY: i64 = 24 * MS_PER_HOUR;
+pub const NANOS_PER_MILLI: i64 = 1_000_000;
+
+impl From<IntervalDayTime> for IntervalFormat {
+    fn from(interval: IntervalDayTime) -> Self {
+        IntervalFormat {
+            days: interval.days,
+            hours: interval.milliseconds as i64 / MS_PER_HOUR,
+            minutes: (interval.milliseconds as i64 % MS_PER_HOUR) / MS_PER_MINUTE,
+            seconds: (interval.milliseconds as i64 % MS_PER_MINUTE) / MS_PER_SEC,
+            microseconds: (interval.milliseconds as i64 % MS_PER_SEC) * MS_PER_SEC,
+            ..Default::default()
+        }
+    }
+}
+
+#[derive(
+    Debug, Default, Copy, Clone, Eq, PartialEq, Hash, Ord, PartialOrd, Serialize, Deserialize,
+)]
+#[repr(C)]
+pub struct IntervalMonthDayNano {
+    /// Number of months
+    pub months: i32,
+    /// Number of days
+    pub days: i32,
+    /// Number of nanoseconds
+    pub nanoseconds: i64,
+}
+
+impl IntervalMonthDayNano {
+    /// The additive identity i.e. `0`.
+    pub const ZERO: Self = Self::new(0, 0, 0);
+
+    /// The multiplicative inverse, i.e. `-1`.
+    pub const MINUS_ONE: Self = Self::new(-1, -1, -1);
+
+    /// The maximum value that can be represented
+    pub const MAX: Self = Self::new(i32::MAX, i32::MAX, i64::MAX);
+
+    /// The minimum value that can be represented
+    pub const MIN: Self = Self::new(i32::MIN, i32::MIN, i64::MIN);
+
+    pub const fn new(months: i32, days: i32, nanoseconds: i64) -> Self {
+        Self {
+            months,
+            days,
+            nanoseconds,
+        }
+    }
+
+    pub fn to_i128(&self) -> i128 {
+        let m = (self.months as u128 & u32::MAX as u128) << 96;
+        let d = (self.days as u128 & u32::MAX as u128) << 64;
+        let n = self.nanoseconds as u128 & u64::MAX as u128;
+        (m | d | n) as i128
+    }
+
+    pub fn from_i128(value: i128) -> Self {
+        let months = (value >> 96) as i32;
+        let days = (value >> 64) as i32;
+        let nanoseconds = value as i64;
+        Self {
+            months,
+            days,
+            nanoseconds,
+        }
+    }
+
+    pub fn negative(&self) -> Self {
+        Self::new(-self.months, -self.days, -self.nanoseconds)
+    }
+
+    pub fn to_iso8601_string(&self) -> String {
+        IntervalFormat::from(*self).to_iso8601_string()
+    }
+}
+
+impl From<i128> for IntervalMonthDayNano {
     fn from(v: i128) -> Self {
         Self::from_i128(v)
     }
 }
 
-impl From<Interval> for i128 {
-    fn from(v: Interval) -> Self {
+impl From<IntervalMonthDayNano> for i128 {
+    fn from(v: IntervalMonthDayNano) -> Self {
         v.to_i128()
     }
 }
 
-impl From<Interval> for serde_json::Value {
-    fn from(v: Interval) -> Self {
-        Value::String(v.to_string())
+impl From<IntervalMonthDayNano> for serde_json::Value {
+    fn from(v: IntervalMonthDayNano) -> Self {
+        serde_json::Value::from(v.to_i128().to_string())
     }
 }
 
-impl Display for Interval {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        let mut s = String::new();
-        if self.months != 0 {
-            write!(s, "{} months ", self.months)?;
+// Nanosecond convert to other time unit
+pub const NS_PER_SEC: i64 = 1_000_000_000;
+pub const NS_PER_MINUTE: i64 = 60 * NS_PER_SEC;
+pub const NS_PER_HOUR: i64 = 60 * NS_PER_MINUTE;
+pub const NS_PER_DAY: i64 = 24 * NS_PER_HOUR;
+
+impl From<IntervalMonthDayNano> for IntervalFormat {
+    fn from(interval: IntervalMonthDayNano) -> Self {
+        IntervalFormat {
+            years: interval.months / 12,
+            months: interval.months % 12,
+            days: interval.days,
+            hours: interval.nanoseconds / NS_PER_HOUR,
+            minutes: (interval.nanoseconds % NS_PER_HOUR) / NS_PER_MINUTE,
+            seconds: (interval.nanoseconds % NS_PER_MINUTE) / NS_PER_SEC,
+            microseconds: (interval.nanoseconds % NS_PER_SEC) / 1_000,
         }
-        if self.days != 0 {
-            write!(s, "{} days ", self.days)?;
-        }
-        if self.nsecs != 0 {
-            write!(s, "{} nsecs", self.nsecs)?;
-        }
-        write!(f, "{}", s.trim())
+    }
+}
+
+pub fn interval_year_month_to_month_day_nano(interval: IntervalYearMonth) -> IntervalMonthDayNano {
+    IntervalMonthDayNano {
+        months: interval.months,
+        days: 0,
+        nanoseconds: 0,
+    }
+}
+
+pub fn interval_day_time_to_month_day_nano(interval: IntervalDayTime) -> IntervalMonthDayNano {
+    IntervalMonthDayNano {
+        months: 0,
+        days: interval.days,
+        nanoseconds: interval.milliseconds as i64 * NANOS_PER_MILLI,
     }
 }
 
@@ -339,31 +332,6 @@ pub struct IntervalFormat {
     pub microseconds: i64,
 }
 
-impl From<Interval> for IntervalFormat {
-    fn from(val: Interval) -> IntervalFormat {
-        let months = val.months;
-        let days = val.days;
-        let microseconds = val.nsecs / NANOS_PER_MICRO;
-        let years = (months - (months % 12)) / 12;
-        let months = months - years * 12;
-        let hours = (microseconds - (microseconds % 3_600_000_000)) / 3_600_000_000;
-        let microseconds = microseconds - hours * 3_600_000_000;
-        let minutes = (microseconds - (microseconds % 60_000_000)) / 60_000_000;
-        let microseconds = microseconds - minutes * 60_000_000;
-        let seconds = (microseconds - (microseconds % 1_000_000)) / 1_000_000;
-        let microseconds = microseconds - seconds * 1_000_000;
-        IntervalFormat {
-            years,
-            months,
-            days,
-            hours,
-            minutes,
-            seconds,
-            microseconds,
-        }
-    }
-}
-
 impl IntervalFormat {
     /// All the field in the interval is 0
     pub fn is_zero(&self) -> bool {
@@ -540,117 +508,37 @@ fn get_time_part(
     interval
 }
 
-/// IntervalCompare is used to compare two intervals
-/// It makes interval into nanoseconds style.
-#[derive(PartialEq, Eq, Hash, PartialOrd, Ord)]
-struct IntervalCompare(i128);
-
-impl From<Interval> for IntervalCompare {
-    fn from(interval: Interval) -> Self {
-        Self(interval.to_nanosecond())
-    }
-}
-
-impl Ord for Interval {
-    fn cmp(&self, other: &Self) -> Ordering {
-        IntervalCompare::from(*self).cmp(&IntervalCompare::from(*other))
-    }
-}
-
-impl PartialOrd for Interval {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Eq for Interval {}
-
-impl PartialEq for Interval {
-    fn eq(&self, other: &Self) -> bool {
-        self.cmp(other).is_eq()
-    }
-}
-
-impl Hash for Interval {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        IntervalCompare::from(*self).hash(state)
-    }
-}
-
 #[cfg(test)]
 mod tests {
-    use std::collections::HashMap;
-
     use super::*;
-    use crate::timestamp::TimeUnit;
 
     #[test]
     fn test_from_year_month() {
-        let interval = Interval::from_year_month(1);
+        let interval = IntervalYearMonth::new(1);
         assert_eq!(interval.months, 1);
     }
 
     #[test]
     fn test_from_date_time() {
-        let interval = Interval::from_day_time(1, 2);
+        let interval = IntervalDayTime::new(1, 2);
         assert_eq!(interval.days, 1);
-        assert_eq!(interval.nsecs, 2_000_000);
+        assert_eq!(interval.milliseconds, 2);
     }
 
     #[test]
-    fn test_to_duration() {
-        let interval = Interval::from_day_time(1, 2);
-
-        let duration = interval.to_duration().unwrap();
-        assert_eq!(86400002000000, duration.value());
-        assert_eq!(TimeUnit::Nanosecond, duration.unit());
-
-        let interval = Interval::from_year_month(12);
-
-        let duration = interval.to_duration().unwrap();
-        assert_eq!(31104000000000000, duration.value());
-        assert_eq!(TimeUnit::Nanosecond, duration.unit());
-    }
-
-    #[test]
-    fn test_interval_is_positive() {
-        let interval = Interval::from_year_month(1);
-        assert!(interval.is_positive());
-        let interval = Interval::from_year_month(-1);
-        assert!(!interval.is_positive());
-
-        let interval = Interval::from_day_time(1, i32::MIN);
-        assert!(!interval.is_positive());
-    }
-
-    #[test]
-    fn test_to_nanosecond() {
-        let interval = Interval::from_year_month(1);
-        assert_eq!(interval.to_nanosecond(), 2592000000000000);
-        let interval = Interval::from_day_time(1, 2);
-        assert_eq!(interval.to_nanosecond(), 86400002000000);
-
-        let max_interval = Interval::from_month_day_nano(i32::MAX, i32::MAX, i64::MAX);
-        assert_eq!(max_interval.to_nanosecond(), 5751829423496836854775807);
-
-        let min_interval = Interval::from_month_day_nano(i32::MIN, i32::MIN, i64::MIN);
-        assert_eq!(min_interval.to_nanosecond(), -5751829426175236854775808);
-    }
-
-    #[test]
-    fn test_interval_is_zero() {
-        let interval = Interval::from_month_day_nano(1, 1, 1);
-        assert!(!interval.is_zero());
-        let interval = Interval::from_month_day_nano(0, 0, 0);
-        assert!(interval.is_zero());
+    fn test_from_month_day_nano() {
+        let interval = IntervalMonthDayNano::new(1, 2, 3);
+        assert_eq!(interval.months, 1);
+        assert_eq!(interval.days, 2);
+        assert_eq!(interval.nanoseconds, 3);
     }
 
     #[test]
     fn test_interval_i128_convert() {
         let test_interval_eq = |month, day, nano| {
-            let interval = Interval::from_month_day_nano(month, day, nano);
+            let interval = IntervalMonthDayNano::new(month, day, nano);
             let interval_i128 = interval.to_i128();
-            let interval2 = Interval::from_i128(interval_i128);
+            let interval2 = IntervalMonthDayNano::from_i128(interval_i128);
             assert_eq!(interval, interval2);
         };
 
@@ -666,11 +554,26 @@ mod tests {
         test_interval_eq(i32::MAX, i32::MIN, i64::MIN);
         test_interval_eq(i32::MIN, i32::MAX, i64::MIN);
         test_interval_eq(i32::MIN, i32::MIN, i64::MIN);
+
+        let interval = IntervalMonthDayNano::from_i128(1);
+        assert_eq!(interval, IntervalMonthDayNano::new(0, 0, 1));
+        assert_eq!(1, IntervalMonthDayNano::new(0, 0, 1).to_i128());
+    }
+
+    #[test]
+    fn test_interval_i64_convert() {
+        let interval = IntervalDayTime::from_i64(1);
+        assert_eq!(interval, IntervalDayTime::new(0, 1));
+        assert_eq!(1, IntervalDayTime::new(0, 1).to_i64());
     }
 
     #[test]
     fn test_convert_interval_format() {
-        let interval = Interval::from_month_day_nano(14, 160, 1000000);
+        let interval = IntervalMonthDayNano {
+            months: 14,
+            days: 160,
+            nanoseconds: 1000000,
+        };
         let interval_format = IntervalFormat::from(interval);
         assert_eq!(interval_format.years, 1);
         assert_eq!(interval_format.months, 2);
@@ -681,94 +584,34 @@ mod tests {
         assert_eq!(interval_format.microseconds, 1000);
     }
 
-    #[test]
-    fn test_interval_hash() {
-        let interval = Interval::from_month_day_nano(1, 31, 1);
-        let interval2 = Interval::from_month_day_nano(2, 1, 1);
-        let mut map = HashMap::new();
-        map.insert(interval, 1);
-        assert_eq!(map.get(&interval2), Some(&1));
-    }
-
-    #[test]
-    fn test_interval_mul_int() {
-        let interval = Interval::from_month_day_nano(1, 1, 1);
-        let interval2 = interval.checked_mul_int(2).unwrap();
-        assert_eq!(interval2.months, 2);
-        assert_eq!(interval2.days, 2);
-        assert_eq!(interval2.nsecs, 2);
-
-        // test justified interval
-        let interval = Interval::from_month_day_nano(1, 31, 1);
-        let interval2 = interval.checked_mul_int(2).unwrap();
-        assert_eq!(interval2.months, 4);
-        assert_eq!(interval2.days, 2);
-        assert_eq!(interval2.nsecs, 2);
-
-        // test overflow situation
-        let interval = Interval::from_month_day_nano(i32::MAX, 1, 1);
-        let interval2 = interval.checked_mul_int(2);
-        assert!(interval2.is_none());
-    }
-
-    #[test]
-    fn test_display() {
-        let interval = Interval::from_month_day_nano(1, 1, 1);
-        assert_eq!(interval.to_string(), "1 months 1 days 1 nsecs");
-
-        let interval = Interval::from_month_day_nano(14, 31, 10000000000);
-        assert_eq!(interval.to_string(), "14 months 31 days 10000000000 nsecs");
-    }
-
-    #[test]
-    fn test_interval_justified() {
-        let interval = Interval::from_month_day_nano(1, 131, 1).justified_interval();
-        let interval2 = Interval::from_month_day_nano(5, 11, 1);
-        assert_eq!(interval, interval2);
-
-        let interval = Interval::from_month_day_nano(1, 1, NANOS_PER_MONTH + 2 * NANOS_PER_DAY)
-            .justified_interval();
-        let interval2 = Interval::from_month_day_nano(2, 3, 0);
-        assert_eq!(interval, interval2);
-    }
-
-    #[test]
-    fn test_serde_json() {
-        let interval = Interval::from_month_day_nano(1, 1, 1);
-        let json = serde_json::to_string(&interval).unwrap();
-        assert_eq!(
-            json,
-            "{\"months\":1,\"days\":1,\"nsecs\":1,\"unit\":\"MonthDayNano\"}"
-        );
-        let interval2: Interval = serde_json::from_str(&json).unwrap();
-        assert_eq!(interval, interval2);
-    }
-
     #[test]
     fn test_to_iso8601_string() {
         // Test interval zero
-        let interval = Interval::from_month_day_nano(0, 0, 0);
+        let interval = IntervalMonthDayNano::new(0, 0, 0);
         assert_eq!(interval.to_iso8601_string(), "PT0S");
 
-        let interval = Interval::from_month_day_nano(1, 1, 1);
+        let interval = IntervalMonthDayNano::new(1, 1, 1);
         assert_eq!(interval.to_iso8601_string(), "P0Y1M1DT0H0M0S");
 
-        let interval = Interval::from_month_day_nano(14, 31, 10000000000);
+        let interval = IntervalMonthDayNano::new(14, 31, 10000000000);
         assert_eq!(interval.to_iso8601_string(), "P1Y2M31DT0H0M10S");
 
-        let interval = Interval::from_month_day_nano(14, 31, 23210200000000);
+        let interval = IntervalMonthDayNano::new(14, 31, 23210200000000);
         assert_eq!(interval.to_iso8601_string(), "P1Y2M31DT6H26M50.2S");
     }
 
     #[test]
     fn test_to_postgres_string() {
         // Test interval zero
-        let interval = Interval::from_month_day_nano(0, 0, 0);
-        assert_eq!(interval.to_postgres_string(), "00:00:00");
-
-        let interval = Interval::from_month_day_nano(23, 100, 23210200000000);
+        let interval = IntervalMonthDayNano::new(0, 0, 0);
         assert_eq!(
-            interval.to_postgres_string(),
+            IntervalFormat::from(interval).to_postgres_string(),
+            "00:00:00"
+        );
+
+        let interval = IntervalMonthDayNano::new(23, 100, 23210200000000);
+        assert_eq!(
+            IntervalFormat::from(interval).to_postgres_string(),
             "1 year 11 mons 100 days 06:26:50.200000"
         );
     }
@@ -776,18 +619,21 @@ mod tests {
     #[test]
     fn test_to_sql_standard_string() {
         // Test zero interval
-        let interval = Interval::from_month_day_nano(0, 0, 0);
-        assert_eq!(interval.to_sql_standard_string(), "0");
+        let interval = IntervalMonthDayNano::new(0, 0, 0);
+        assert_eq!(IntervalFormat::from(interval).to_sql_standard_string(), "0");
 
-        let interval = Interval::from_month_day_nano(23, 100, 23210200000000);
+        let interval = IntervalMonthDayNano::new(23, 100, 23210200000000);
         assert_eq!(
-            interval.to_sql_standard_string(),
+            IntervalFormat::from(interval).to_sql_standard_string(),
             "+1-11 +100 +6:26:50.200000"
         );
 
         // Test interval without year, month, day
-        let interval = Interval::from_month_day_nano(0, 0, 23210200000000);
-        assert_eq!(interval.to_sql_standard_string(), "6:26:50.200000");
+        let interval = IntervalMonthDayNano::new(0, 0, 23210200000000);
+        assert_eq!(
+            IntervalFormat::from(interval).to_sql_standard_string(),
+            "6:26:50.200000"
+        );
     }
 
     #[test]
diff --git a/src/common/time/src/lib.rs b/src/common/time/src/lib.rs
index 770057394c..fa025bf661 100644
--- a/src/common/time/src/lib.rs
+++ b/src/common/time/src/lib.rs
@@ -27,7 +27,7 @@ pub mod util;
 pub use date::Date;
 pub use datetime::DateTime;
 pub use duration::Duration;
-pub use interval::Interval;
+pub use interval::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth};
 pub use range::RangeMillis;
 pub use timestamp::Timestamp;
 pub use timestamp_millis::TimestampMillis;
diff --git a/src/common/time/src/timestamp.rs b/src/common/time/src/timestamp.rs
index 503c44cf99..258e9c1a96 100644
--- a/src/common/time/src/timestamp.rs
+++ b/src/common/time/src/timestamp.rs
@@ -20,16 +20,17 @@ use std::time::Duration;
 
 use arrow::datatypes::TimeUnit as ArrowTimeUnit;
 use chrono::{
-    DateTime, Days, LocalResult, Months, NaiveDate, NaiveDateTime, NaiveTime,
+    DateTime, Days, LocalResult, Months, NaiveDate, NaiveDateTime, NaiveTime, TimeDelta,
     TimeZone as ChronoTimeZone, Utc,
 };
 use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt};
 
+use crate::error;
 use crate::error::{ArithmeticOverflowSnafu, ParseTimestampSnafu, Result, TimestampOverflowSnafu};
+use crate::interval::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth};
 use crate::timezone::{get_timezone, Timezone};
 use crate::util::{datetime_to_utc, div_ceil};
-use crate::{error, Interval};
 
 /// Timestamp represents the value of units(seconds/milliseconds/microseconds/nanoseconds) elapsed
 /// since UNIX epoch. The valid value range of [Timestamp] depends on it's unit (all in UTC timezone):
@@ -140,40 +141,77 @@ impl Timestamp {
         })
     }
 
-    /// Adds given Interval to the current timestamp.
-    /// Returns None if the resulting timestamp would be out of range.
-    pub fn add_interval(&self, interval: Interval) -> Option<Timestamp> {
+    // FIXME(yingwen): remove add/sub intervals later
+    /// Adds given [IntervalYearMonth] to the current timestamp.
+    pub fn add_year_month(&self, interval: IntervalYearMonth) -> Option<Timestamp> {
         let naive_datetime = self.to_chrono_datetime()?;
-        let (months, days, nsecs) = interval.to_month_day_nano();
 
-        let naive_datetime = naive_datetime
-            .checked_add_months(Months::new(months as u32))?
-            .checked_add_days(Days::new(days as u64))?
-            + Duration::from_nanos(nsecs as u64);
+        let naive_datetime =
+            naive_datetime.checked_add_months(Months::new(interval.months as u32))?;
 
-        match Timestamp::from_chrono_datetime(naive_datetime) {
-            // Have to convert the new timestamp by the current unit.
-            Some(ts) => ts.convert_to(self.unit),
-            None => None,
-        }
+        // Have to convert the new timestamp by the current unit.
+        Timestamp::from_chrono_datetime(naive_datetime).and_then(|ts| ts.convert_to(self.unit))
     }
 
-    /// Subtracts given Interval to the current timestamp.
-    /// Returns None if the resulting timestamp would be out of range.
-    pub fn sub_interval(&self, interval: Interval) -> Option<Timestamp> {
+    /// Adds given [IntervalDayTime] to the current timestamp.
+    pub fn add_day_time(&self, interval: IntervalDayTime) -> Option<Timestamp> {
         let naive_datetime = self.to_chrono_datetime()?;
-        let (months, days, nsecs) = interval.to_month_day_nano();
 
         let naive_datetime = naive_datetime
-            .checked_sub_months(Months::new(months as u32))?
-            .checked_sub_days(Days::new(days as u64))?
-            - Duration::from_nanos(nsecs as u64);
+            .checked_add_days(Days::new(interval.days as u64))?
+            .checked_add_signed(TimeDelta::milliseconds(interval.milliseconds as i64))?;
 
-        match Timestamp::from_chrono_datetime(naive_datetime) {
-            // Have to convert the new timestamp by the current unit.
-            Some(ts) => ts.convert_to(self.unit),
-            None => None,
-        }
+        // Have to convert the new timestamp by the current unit.
+        Timestamp::from_chrono_datetime(naive_datetime).and_then(|ts| ts.convert_to(self.unit))
+    }
+
+    /// Adds given [IntervalMonthDayNano] to the current timestamp.
+    pub fn add_month_day_nano(&self, interval: IntervalMonthDayNano) -> Option<Timestamp> {
+        let naive_datetime = self.to_chrono_datetime()?;
+
+        let naive_datetime = naive_datetime
+            .checked_add_months(Months::new(interval.months as u32))?
+            .checked_add_days(Days::new(interval.days as u64))?
+            .checked_add_signed(TimeDelta::nanoseconds(interval.nanoseconds))?;
+
+        // Have to convert the new timestamp by the current unit.
+        Timestamp::from_chrono_datetime(naive_datetime).and_then(|ts| ts.convert_to(self.unit))
+    }
+
+    /// Subtracts given [IntervalYearMonth] to the current timestamp.
+    pub fn sub_year_month(&self, interval: IntervalYearMonth) -> Option<Timestamp> {
+        let naive_datetime = self.to_chrono_datetime()?;
+
+        let naive_datetime =
+            naive_datetime.checked_sub_months(Months::new(interval.months as u32))?;
+
+        // Have to convert the new timestamp by the current unit.
+        Timestamp::from_chrono_datetime(naive_datetime).and_then(|ts| ts.convert_to(self.unit))
+    }
+
+    /// Subtracts given [IntervalDayTime] to the current timestamp.
+    pub fn sub_day_time(&self, interval: IntervalDayTime) -> Option<Timestamp> {
+        let naive_datetime = self.to_chrono_datetime()?;
+
+        let naive_datetime = naive_datetime
+            .checked_sub_days(Days::new(interval.days as u64))?
+            .checked_sub_signed(TimeDelta::milliseconds(interval.milliseconds as i64))?;
+
+        // Have to convert the new timestamp by the current unit.
+        Timestamp::from_chrono_datetime(naive_datetime).and_then(|ts| ts.convert_to(self.unit))
+    }
+
+    /// Subtracts given [IntervalMonthDayNano] to the current timestamp.
+    pub fn sub_month_day_nano(&self, interval: IntervalMonthDayNano) -> Option<Timestamp> {
+        let naive_datetime = self.to_chrono_datetime()?;
+
+        let naive_datetime = naive_datetime
+            .checked_sub_months(Months::new(interval.months as u32))?
+            .checked_sub_days(Days::new(interval.days as u64))?
+            .checked_sub_signed(TimeDelta::nanoseconds(interval.nanoseconds))?;
+
+        // Have to convert the new timestamp by the current unit.
+        Timestamp::from_chrono_datetime(naive_datetime).and_then(|ts| ts.convert_to(self.unit))
     }
 
     /// Subtracts current timestamp with another timestamp, yielding a duration.
@@ -688,13 +726,13 @@ mod tests {
     fn test_add_sub_interval() {
         let ts = Timestamp::new(1000, TimeUnit::Millisecond);
 
-        let interval = Interval::from_day_time(1, 200);
+        let interval = IntervalDayTime::new(1, 200);
 
-        let new_ts = ts.add_interval(interval).unwrap();
+        let new_ts = ts.add_day_time(interval).unwrap();
         assert_eq!(new_ts.unit(), TimeUnit::Millisecond);
         assert_eq!(new_ts.value(), 1000 + 3600 * 24 * 1000 + 200);
 
-        assert_eq!(ts, new_ts.sub_interval(interval).unwrap());
+        assert_eq!(ts, new_ts.sub_day_time(interval).unwrap());
     }
 
     #[test]
diff --git a/src/datatypes/src/interval.rs b/src/datatypes/src/interval.rs
index c0969abc44..b724de8022 100644
--- a/src/datatypes/src/interval.rs
+++ b/src/datatypes/src/interval.rs
@@ -12,11 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common_time::interval::Interval;
+use common_time::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth};
 use paste::paste;
-use serde::{Deserialize, Serialize};
 
-use crate::prelude::{Scalar, Value, ValueRef};
+use crate::prelude::Scalar;
 use crate::scalars::ScalarRef;
 use crate::types::{
     IntervalDayTimeType, IntervalMonthDayNanoType, IntervalYearMonthType, WrapperType,
@@ -26,39 +25,6 @@ use crate::vectors::{IntervalDayTimeVector, IntervalMonthDayNanoVector, Interval
 macro_rules! define_interval_with_unit {
     ($unit: ident, $native_ty: ty) => {
         paste! {
-            #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
-            pub struct [<Interval $unit>](pub Interval);
-
-            impl [<Interval $unit>] {
-                pub fn new(val: $native_ty) -> Self {
-                    Self(Interval:: [<from_ $native_ty>](val))
-                }
-            }
-
-            impl Default for [<Interval $unit>] {
-                fn default() -> Self {
-                    Self::new(0)
-                }
-            }
-
-            impl From<[<Interval $unit>]> for Value {
-                fn from(t: [<Interval $unit>]) -> Value {
-                    Value::Interval(t.0)
-                }
-            }
-
-            impl From<[<Interval $unit>]> for serde_json::Value {
-                fn from(t: [<Interval $unit>]) -> Self {
-                    t.0.into()
-                }
-            }
-
-            impl From<[<Interval $unit>]> for ValueRef<'static> {
-                fn from(t: [<Interval $unit>]) -> Self {
-                    ValueRef::Interval(t.0)
-                }
-            }
-
             impl Scalar for [<Interval $unit>] {
                 type VectorType = [<Interval $unit Vector>];
                 type RefType<'a> = [<Interval $unit>];
@@ -87,41 +53,11 @@ macro_rules! define_interval_with_unit {
                 type Native = $native_ty;
 
                 fn from_native(value: Self::Native) -> Self {
-                    Self::new(value)
+                    Self::[<from_ $native_ty>](value)
                 }
 
                 fn into_native(self) -> Self::Native {
-                    self.0.[<to_ $native_ty>]()
-                }
-            }
-
-            impl From<$native_ty> for [<Interval $unit>] {
-                fn from(val: $native_ty) -> Self {
-                    [<Interval $unit>]::from_native(val as $native_ty)
-                }
-            }
-
-            impl From<[<Interval $unit>]> for $native_ty {
-                fn from(val: [<Interval $unit>]) -> Self {
-                    val.0.[<to_ $native_ty>]()
-                }
-            }
-
-            impl TryFrom<Value> for Option<[<Interval $unit>]> {
-                type Error = $crate::error::Error;
-
-                #[inline]
-                fn try_from(from: Value) -> std::result::Result<Self, Self::Error> {
-                    match from {
-                        Value::Interval(v) if v.unit() == common_time::interval::IntervalUnit::$unit => {
-                            Ok(Some([<Interval $unit>](v)))
-                        },
-                        Value::Null => Ok(None),
-                        _ => $crate::error::TryFromValueSnafu {
-                            reason: format!("{:?} is not a {}", from, stringify!([<Interval $unit>])),
-                        }
-                        .fail(),
-                    }
+                    self.[<to_ $native_ty>]()
                 }
             }
         }
@@ -138,17 +74,17 @@ mod tests {
 
     #[test]
     fn test_interval_scalar() {
-        let interval = IntervalYearMonth::new(1000);
+        let interval = IntervalYearMonth::from(1000);
         assert_eq!(interval, interval.as_scalar_ref());
         assert_eq!(interval, interval.to_owned_scalar());
         assert_eq!(1000, interval.into_native());
 
-        let interval = IntervalDayTime::new(1000);
+        let interval = IntervalDayTime::from(1000);
         assert_eq!(interval, interval.as_scalar_ref());
         assert_eq!(interval, interval.to_owned_scalar());
         assert_eq!(1000, interval.into_native());
 
-        let interval = IntervalMonthDayNano::new(1000);
+        let interval = IntervalMonthDayNano::from(1000);
         assert_eq!(interval, interval.as_scalar_ref());
         assert_eq!(interval, interval.to_owned_scalar());
         assert_eq!(1000, interval.into_native());
@@ -156,15 +92,15 @@ mod tests {
 
     #[test]
     fn test_interval_convert_to_native_type() {
-        let interval = IntervalMonthDayNano::new(1000);
+        let interval = IntervalMonthDayNano::from(1000);
         let native_value: i128 = interval.into();
         assert_eq!(native_value, 1000);
 
-        let interval = IntervalDayTime::new(1000);
+        let interval = IntervalDayTime::from(1000);
         let native_interval: i64 = interval.into();
         assert_eq!(native_interval, 1000);
 
-        let interval = IntervalYearMonth::new(1000);
+        let interval = IntervalYearMonth::from(1000);
         let native_interval: i32 = interval.into();
         assert_eq!(native_interval, 1000);
     }
diff --git a/src/datatypes/src/types/interval_type.rs b/src/datatypes/src/types/interval_type.rs
index 7ee7964982..77f1b47d6d 100644
--- a/src/datatypes/src/types/interval_type.rs
+++ b/src/datatypes/src/types/interval_type.rs
@@ -17,8 +17,9 @@ use arrow::datatypes::{
     IntervalMonthDayNanoType as ArrowIntervalMonthDayNanoType, IntervalUnit as ArrowIntervalUnit,
     IntervalYearMonthType as ArrowIntervalYearMonthType,
 };
-use common_time::interval::IntervalUnit;
-use common_time::Interval;
+use common_time::interval::{
+    IntervalDayTime, IntervalMonthDayNano, IntervalUnit, IntervalYearMonth,
+};
 use enum_dispatch::enum_dispatch;
 use paste::paste;
 use serde::{Deserialize, Serialize};
@@ -26,7 +27,6 @@ use snafu::OptionExt;
 
 use crate::data_type::ConcreteDataType;
 use crate::error;
-use crate::interval::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth};
 use crate::prelude::{
     DataType, LogicalTypeId, MutableVector, ScalarVectorBuilder, Value, ValueRef, Vector,
 };
@@ -75,7 +75,7 @@ macro_rules! impl_data_type_for_interval {
                 }
 
                 fn default_value(&self) -> Value {
-                    Value::Interval(Interval::from_i128(0))
+                    Value::[<Interval $unit>]([<Interval $unit>]::default())
                 }
 
                 fn as_arrow_type(&self) -> ArrowDataType {
@@ -124,7 +124,7 @@ macro_rules! impl_data_type_for_interval {
                 fn cast_value_ref(value: ValueRef) -> crate::Result<Option<Self::Wrapper>> {
                     match value {
                         ValueRef::Null => Ok(None),
-                        ValueRef::Interval(t) => Ok(Some([<Interval $unit>](t))),
+                        ValueRef::[<Interval $unit>](t) => Ok(Some(t)),
                         other => error::CastTypeSnafu {
                             msg: format!("Failed to cast value {:?} to {}", other, stringify!([<Interval $unit>])),
                         }
diff --git a/src/datatypes/src/types/primitive_type.rs b/src/datatypes/src/types/primitive_type.rs
index cae71976fd..86972bd5ee 100644
--- a/src/datatypes/src/types/primitive_type.rs
+++ b/src/datatypes/src/types/primitive_type.rs
@@ -16,7 +16,6 @@ use std::cmp::Ordering;
 use std::fmt;
 
 use arrow::datatypes::{ArrowNativeType, ArrowPrimitiveType, DataType as ArrowDataType};
-use common_time::interval::IntervalUnit;
 use common_time::{Date, DateTime};
 use serde::{Deserialize, Serialize};
 use snafu::OptionExt;
@@ -30,6 +29,7 @@ use crate::types::{DateTimeType, DateType};
 use crate::value::{Value, ValueRef};
 use crate::vectors::{MutableVector, PrimitiveVector, PrimitiveVectorBuilder, Vector};
 
+// TODO(yingwen): Can we remove `Into<serde_json::Value>`?
 /// Represents the wrapper type that wraps a native type using the `newtype pattern`,
 /// such as [Date](`common_time::Date`) is a wrapper type for the underlying native
 /// type `i32`.
@@ -364,11 +364,7 @@ impl DataType for Int64Type {
             Value::DateTime(v) => Some(Value::Int64(v.val())),
             Value::Timestamp(v) => Some(Value::Int64(v.value())),
             Value::Time(v) => Some(Value::Int64(v.value())),
-            Value::Interval(v) => match v.unit() {
-                IntervalUnit::DayTime => Some(Value::Int64(v.to_i64())),
-                IntervalUnit::YearMonth => None,
-                IntervalUnit::MonthDayNano => None,
-            },
+            // We don't allow casting interval type to int.
             _ => None,
         }
     }
@@ -410,11 +406,7 @@ impl DataType for Int32Type {
             Value::Float64(v) => num::cast::cast(v).map(Value::Int32),
             Value::String(v) => v.as_utf8().parse::<i32>().map(Value::Int32).ok(),
             Value::Date(v) => Some(Value::Int32(v.val())),
-            Value::Interval(v) => match v.unit() {
-                IntervalUnit::YearMonth => Some(Value::Int32(v.to_i32())),
-                IntervalUnit::DayTime => None,
-                IntervalUnit::MonthDayNano => None,
-            },
+            // We don't allow casting interval type to int.
             _ => None,
         }
     }
diff --git a/src/datatypes/src/types/string_type.rs b/src/datatypes/src/types/string_type.rs
index 38045a600a..06a5e7c7f6 100644
--- a/src/datatypes/src/types/string_type.rs
+++ b/src/datatypes/src/types/string_type.rs
@@ -78,7 +78,15 @@ impl DataType for StringType {
             Value::DateTime(v) => Some(Value::String(StringBytes::from(v.to_string()))),
             Value::Timestamp(v) => Some(Value::String(StringBytes::from(v.to_iso8601_string()))),
             Value::Time(v) => Some(Value::String(StringBytes::from(v.to_iso8601_string()))),
-            Value::Interval(v) => Some(Value::String(StringBytes::from(v.to_iso8601_string()))),
+            Value::IntervalYearMonth(v) => {
+                Some(Value::String(StringBytes::from(v.to_iso8601_string())))
+            }
+            Value::IntervalDayTime(v) => {
+                Some(Value::String(StringBytes::from(v.to_iso8601_string())))
+            }
+            Value::IntervalMonthDayNano(v) => {
+                Some(Value::String(StringBytes::from(v.to_iso8601_string())))
+            }
             Value::Duration(v) => Some(Value::String(StringBytes::from(v.to_string()))),
             Value::Decimal128(v) => Some(Value::String(StringBytes::from(v.to_string()))),
 
diff --git a/src/datatypes/src/value.rs b/src/datatypes/src/value.rs
index a8e59da513..b973a3156b 100644
--- a/src/datatypes/src/value.rs
+++ b/src/datatypes/src/value.rs
@@ -28,7 +28,7 @@ use common_time::datetime::DateTime;
 use common_time::interval::IntervalUnit;
 use common_time::time::Time;
 use common_time::timestamp::{TimeUnit, Timestamp};
-use common_time::{Duration, Interval, Timezone};
+use common_time::{Duration, IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth, Timezone};
 use datafusion_common::ScalarValue;
 use greptime_proto::v1::value::ValueData;
 pub use ordered_float::OrderedFloat;
@@ -78,7 +78,10 @@ pub enum Value {
     Timestamp(Timestamp),
     Time(Time),
     Duration(Duration),
-    Interval(Interval),
+    // Interval types:
+    IntervalYearMonth(IntervalYearMonth),
+    IntervalDayTime(IntervalDayTime),
+    IntervalMonthDayNano(IntervalMonthDayNano),
 
     List(ListValue),
 }
@@ -111,7 +114,15 @@ impl Display for Value {
             Value::DateTime(v) => write!(f, "{v}"),
             Value::Timestamp(v) => write!(f, "{}", v.to_iso8601_string()),
             Value::Time(t) => write!(f, "{}", t.to_iso8601_string()),
-            Value::Interval(v) => write!(f, "{}", v.to_iso8601_string()),
+            Value::IntervalYearMonth(v) => {
+                write!(f, "{}", v.to_iso8601_string())
+            }
+            Value::IntervalDayTime(v) => {
+                write!(f, "{}", v.to_iso8601_string())
+            }
+            Value::IntervalMonthDayNano(v) => {
+                write!(f, "{}", v.to_iso8601_string())
+            }
             Value::Duration(d) => write!(f, "{d}"),
             Value::List(v) => {
                 let items = v
@@ -153,7 +164,15 @@ macro_rules! define_data_type_func {
                 $struct::DateTime(_) => ConcreteDataType::datetime_datatype(),
                 $struct::Time(t) => ConcreteDataType::time_datatype(*t.unit()),
                 $struct::Timestamp(v) => ConcreteDataType::timestamp_datatype(v.unit()),
-                $struct::Interval(v) => ConcreteDataType::interval_datatype(v.unit()),
+                $struct::IntervalYearMonth(_) => {
+                    ConcreteDataType::interval_datatype(IntervalUnit::YearMonth)
+                }
+                $struct::IntervalDayTime(_) => {
+                    ConcreteDataType::interval_datatype(IntervalUnit::DayTime)
+                }
+                $struct::IntervalMonthDayNano(_) => {
+                    ConcreteDataType::interval_datatype(IntervalUnit::MonthDayNano)
+                }
                 $struct::List(list) => ConcreteDataType::list_datatype(list.datatype().clone()),
                 $struct::Duration(d) => ConcreteDataType::duration_datatype(d.unit()),
                 $struct::Decimal128(d) => {
@@ -206,7 +225,9 @@ impl Value {
             Value::List(v) => ValueRef::List(ListValueRef::Ref { val: v }),
             Value::Timestamp(v) => ValueRef::Timestamp(*v),
             Value::Time(v) => ValueRef::Time(*v),
-            Value::Interval(v) => ValueRef::Interval(*v),
+            Value::IntervalYearMonth(v) => ValueRef::IntervalYearMonth(*v),
+            Value::IntervalDayTime(v) => ValueRef::IntervalDayTime(*v),
+            Value::IntervalMonthDayNano(v) => ValueRef::IntervalMonthDayNano(*v),
             Value::Duration(v) => ValueRef::Duration(*v),
             Value::Decimal128(v) => ValueRef::Decimal128(*v),
         }
@@ -220,14 +241,6 @@ impl Value {
         }
     }
 
-    /// Cast Value to Interval. Return None if value is not a valid interval data type.
-    pub fn as_interval(&self) -> Option<Interval> {
-        match self {
-            Value::Interval(i) => Some(*i),
-            _ => None,
-        }
-    }
-
     /// Cast Value to utf8 String. Return None if value is not a valid string data type.
     pub fn as_string(&self) -> Option<String> {
         match self {
@@ -255,12 +268,35 @@ impl Value {
     /// Cast Value to [Time]. Return None if value is not a valid time data type.
     pub fn as_time(&self) -> Option<Time> {
         match self {
-            Value::Int64(v) => Some(Time::new_millisecond(*v)),
             Value::Time(t) => Some(*t),
             _ => None,
         }
     }
 
+    /// Cast Value to [IntervalYearMonth]. Return None if value is not a valid interval year month data type.
+    pub fn as_interval_year_month(&self) -> Option<IntervalYearMonth> {
+        match self {
+            Value::IntervalYearMonth(v) => Some(*v),
+            _ => None,
+        }
+    }
+
+    /// Cast Value to [IntervalDayTime]. Return None if value is not a valid interval day time data type.
+    pub fn as_interval_day_time(&self) -> Option<IntervalDayTime> {
+        match self {
+            Value::IntervalDayTime(v) => Some(*v),
+            _ => None,
+        }
+    }
+
+    /// Cast Value to [IntervalMonthDayNano]. Return None if value is not a valid interval month day nano data type.
+    pub fn as_interval_month_day_nano(&self) -> Option<IntervalMonthDayNano> {
+        match self {
+            Value::IntervalMonthDayNano(v) => Some(*v),
+            _ => None,
+        }
+    }
+
     /// Cast Value to u64. Return None if value is not a valid uint64 data type.
     pub fn as_u64(&self) -> Option<u64> {
         match self {
@@ -321,11 +357,9 @@ impl Value {
                 TimeUnit::Microsecond => LogicalTypeId::TimeMicrosecond,
                 TimeUnit::Nanosecond => LogicalTypeId::TimeNanosecond,
             },
-            Value::Interval(v) => match v.unit() {
-                IntervalUnit::YearMonth => LogicalTypeId::IntervalYearMonth,
-                IntervalUnit::DayTime => LogicalTypeId::IntervalDayTime,
-                IntervalUnit::MonthDayNano => LogicalTypeId::IntervalMonthDayNano,
-            },
+            Value::IntervalYearMonth(_) => LogicalTypeId::IntervalYearMonth,
+            Value::IntervalDayTime(_) => LogicalTypeId::IntervalDayTime,
+            Value::IntervalMonthDayNano(_) => LogicalTypeId::IntervalMonthDayNano,
             Value::Duration(d) => match d.unit() {
                 TimeUnit::Second => LogicalTypeId::DurationSecond,
                 TimeUnit::Millisecond => LogicalTypeId::DurationMillisecond,
@@ -375,11 +409,9 @@ impl Value {
             }
             Value::Timestamp(t) => timestamp_to_scalar_value(t.unit(), Some(t.value())),
             Value::Time(t) => time_to_scalar_value(*t.unit(), Some(t.value()))?,
-            Value::Interval(v) => match v.unit() {
-                IntervalUnit::YearMonth => ScalarValue::IntervalYearMonth(Some(v.to_i32())),
-                IntervalUnit::DayTime => ScalarValue::IntervalDayTime(Some(v.to_i64())),
-                IntervalUnit::MonthDayNano => ScalarValue::IntervalMonthDayNano(Some(v.to_i128())),
-            },
+            Value::IntervalYearMonth(v) => ScalarValue::IntervalYearMonth(Some(v.to_i32())),
+            Value::IntervalDayTime(v) => ScalarValue::IntervalDayTime(Some(v.to_i64())),
+            Value::IntervalMonthDayNano(v) => ScalarValue::IntervalMonthDayNano(Some(v.to_i128())),
             Value::Duration(d) => duration_to_scalar_value(d.unit(), Some(d.value())),
             Value::Decimal128(d) => {
                 let (v, p, s) = d.to_scalar_value();
@@ -434,7 +466,9 @@ impl Value {
             Value::Timestamp(x) => Some(Value::Timestamp(x.negative())),
             Value::Time(x) => Some(Value::Time(x.negative())),
             Value::Duration(x) => Some(Value::Duration(x.negative())),
-            Value::Interval(x) => Some(Value::Interval(x.negative())),
+            Value::IntervalYearMonth(x) => Some(Value::IntervalYearMonth(x.negative())),
+            Value::IntervalDayTime(x) => Some(Value::IntervalDayTime(x.negative())),
+            Value::IntervalMonthDayNano(x) => Some(Value::IntervalMonthDayNano(x.negative())),
 
             Value::Binary(_) | Value::String(_) | Value::Boolean(_) | Value::List(_) => None,
         }
@@ -571,16 +605,6 @@ pub fn scalar_value_to_timestamp(
     }
 }
 
-/// Convert [ScalarValue] to [Interval].
-pub fn scalar_value_to_interval(scalar: &ScalarValue) -> Option<Interval> {
-    match scalar {
-        ScalarValue::IntervalYearMonth(v) => v.map(Interval::from_i32),
-        ScalarValue::IntervalDayTime(v) => v.map(Interval::from_i64),
-        ScalarValue::IntervalMonthDayNano(v) => v.map(Interval::from_i128),
-        _ => None,
-    }
-}
-
 macro_rules! impl_ord_for_value_like {
     ($Type: ident, $left: ident, $right: ident) => {
         if $left.is_null() && !$right.is_null() {
@@ -607,7 +631,9 @@ macro_rules! impl_ord_for_value_like {
                 ($Type::DateTime(v1), $Type::DateTime(v2)) => v1.cmp(v2),
                 ($Type::Timestamp(v1), $Type::Timestamp(v2)) => v1.cmp(v2),
                 ($Type::Time(v1), $Type::Time(v2)) => v1.cmp(v2),
-                ($Type::Interval(v1), $Type::Interval(v2)) => v1.cmp(v2),
+                ($Type::IntervalYearMonth(v1), $Type::IntervalYearMonth(v2)) => v1.cmp(v2),
+                ($Type::IntervalDayTime(v1), $Type::IntervalDayTime(v2)) => v1.cmp(v2),
+                ($Type::IntervalMonthDayNano(v1), $Type::IntervalMonthDayNano(v2)) => v1.cmp(v2),
                 ($Type::Duration(v1), $Type::Duration(v2)) => v1.cmp(v2),
                 ($Type::List(v1), $Type::List(v2)) => v1.cmp(v2),
                 _ => panic!(
@@ -685,7 +711,9 @@ impl_try_from_value!(Date, Date);
 impl_try_from_value!(Time, Time);
 impl_try_from_value!(DateTime, DateTime);
 impl_try_from_value!(Timestamp, Timestamp);
-impl_try_from_value!(Interval, Interval);
+impl_try_from_value!(IntervalYearMonth, IntervalYearMonth);
+impl_try_from_value!(IntervalDayTime, IntervalDayTime);
+impl_try_from_value!(IntervalMonthDayNano, IntervalMonthDayNano);
 impl_try_from_value!(Duration, Duration);
 impl_try_from_value!(Decimal128, Decimal128);
 
@@ -727,7 +755,9 @@ impl_value_from!(Date, Date);
 impl_value_from!(Time, Time);
 impl_value_from!(DateTime, DateTime);
 impl_value_from!(Timestamp, Timestamp);
-impl_value_from!(Interval, Interval);
+impl_value_from!(IntervalYearMonth, IntervalYearMonth);
+impl_value_from!(IntervalDayTime, IntervalDayTime);
+impl_value_from!(IntervalMonthDayNano, IntervalMonthDayNano);
 impl_value_from!(Duration, Duration);
 impl_value_from!(String, String);
 impl_value_from!(Decimal128, Decimal128);
@@ -774,7 +804,9 @@ impl TryFrom<Value> for serde_json::Value {
             Value::List(v) => serde_json::to_value(v)?,
             Value::Timestamp(v) => serde_json::to_value(v.value())?,
             Value::Time(v) => serde_json::to_value(v.value())?,
-            Value::Interval(v) => serde_json::to_value(v.to_i128())?,
+            Value::IntervalYearMonth(v) => serde_json::to_value(v.to_i32())?,
+            Value::IntervalDayTime(v) => serde_json::to_value(v.to_i64())?,
+            Value::IntervalMonthDayNano(v) => serde_json::to_value(v.to_i128())?,
             Value::Duration(v) => serde_json::to_value(v.value())?,
             Value::Decimal128(v) => serde_json::to_value(v.to_string())?,
         };
@@ -926,13 +958,13 @@ impl TryFrom<ScalarValue> for Value {
                 .unwrap_or(Value::Null),
 
             ScalarValue::IntervalYearMonth(t) => t
-                .map(|x| Value::Interval(Interval::from_i32(x)))
+                .map(|x| Value::IntervalYearMonth(IntervalYearMonth::from_i32(x)))
                 .unwrap_or(Value::Null),
             ScalarValue::IntervalDayTime(t) => t
-                .map(|x| Value::Interval(Interval::from_i64(x)))
+                .map(|x| Value::IntervalDayTime(IntervalDayTime::from_i64(x)))
                 .unwrap_or(Value::Null),
             ScalarValue::IntervalMonthDayNano(t) => t
-                .map(|x| Value::Interval(Interval::from_i128(x)))
+                .map(|x| Value::IntervalMonthDayNano(IntervalMonthDayNano::from_i128(x)))
                 .unwrap_or(Value::Null),
             ScalarValue::DurationSecond(d) => d
                 .map(|x| Value::Duration(Duration::new(x, TimeUnit::Second)))
@@ -987,7 +1019,9 @@ impl From<ValueRef<'_>> for Value {
             ValueRef::DateTime(v) => Value::DateTime(v),
             ValueRef::Timestamp(v) => Value::Timestamp(v),
             ValueRef::Time(v) => Value::Time(v),
-            ValueRef::Interval(v) => Value::Interval(v),
+            ValueRef::IntervalYearMonth(v) => Value::IntervalYearMonth(v),
+            ValueRef::IntervalDayTime(v) => Value::IntervalDayTime(v),
+            ValueRef::IntervalMonthDayNano(v) => Value::IntervalMonthDayNano(v),
             ValueRef::Duration(v) => Value::Duration(v),
             ValueRef::List(v) => v.to_value(),
             ValueRef::Decimal128(v) => Value::Decimal128(v),
@@ -1026,7 +1060,10 @@ pub enum ValueRef<'a> {
     Timestamp(Timestamp),
     Time(Time),
     Duration(Duration),
-    Interval(Interval),
+    // Interval types:
+    IntervalYearMonth(IntervalYearMonth),
+    IntervalDayTime(IntervalDayTime),
+    IntervalMonthDayNano(IntervalMonthDayNano),
 
     // Compound types:
     List(ListValueRef<'a>),
@@ -1150,9 +1187,19 @@ impl<'a> ValueRef<'a> {
         impl_as_for_value_ref!(self, Duration)
     }
 
-    /// Cast itself to [Interval].
-    pub fn as_interval(&self) -> Result<Option<Interval>> {
-        impl_as_for_value_ref!(self, Interval)
+    /// Cast itself to [IntervalYearMonth].
+    pub fn as_interval_year_month(&self) -> Result<Option<IntervalYearMonth>> {
+        impl_as_for_value_ref!(self, IntervalYearMonth)
+    }
+
+    /// Cast itself to [IntervalDayTime].
+    pub fn as_interval_day_time(&self) -> Result<Option<IntervalDayTime>> {
+        impl_as_for_value_ref!(self, IntervalDayTime)
+    }
+
+    /// Cast itself to [IntervalMonthDayNano].
+    pub fn as_interval_month_day_nano(&self) -> Result<Option<IntervalMonthDayNano>> {
+        impl_as_for_value_ref!(self, IntervalMonthDayNano)
     }
 
     /// Cast itself to [ListValueRef].
@@ -1212,7 +1259,9 @@ impl_value_ref_from!(Date, Date);
 impl_value_ref_from!(DateTime, DateTime);
 impl_value_ref_from!(Timestamp, Timestamp);
 impl_value_ref_from!(Time, Time);
-impl_value_ref_from!(Interval, Interval);
+impl_value_ref_from!(IntervalYearMonth, IntervalYearMonth);
+impl_value_ref_from!(IntervalDayTime, IntervalDayTime);
+impl_value_ref_from!(IntervalMonthDayNano, IntervalMonthDayNano);
 impl_value_ref_from!(Duration, Duration);
 impl_value_ref_from!(Decimal128, Decimal128);
 
@@ -1261,7 +1310,9 @@ impl<'a> TryFrom<ValueRef<'a>> for serde_json::Value {
             ValueRef::List(v) => serde_json::to_value(v)?,
             ValueRef::Timestamp(v) => serde_json::to_value(v.value())?,
             ValueRef::Time(v) => serde_json::to_value(v.value())?,
-            ValueRef::Interval(v) => serde_json::to_value(v.to_i128())?,
+            ValueRef::IntervalYearMonth(v) => serde_json::Value::from(v),
+            ValueRef::IntervalDayTime(v) => serde_json::Value::from(v),
+            ValueRef::IntervalMonthDayNano(v) => serde_json::Value::from(v),
             ValueRef::Duration(v) => serde_json::to_value(v.value())?,
             ValueRef::Decimal128(v) => serde_json::to_value(v.to_string())?,
         };
@@ -1359,7 +1410,9 @@ impl<'a> ValueRef<'a> {
             ValueRef::Timestamp(_) => 16,
             ValueRef::Time(_) => 16,
             ValueRef::Duration(_) => 16,
-            ValueRef::Interval(_) => 24,
+            ValueRef::IntervalYearMonth(_) => 4,
+            ValueRef::IntervalDayTime(_) => 8,
+            ValueRef::IntervalMonthDayNano(_) => 16,
             ValueRef::Decimal128(_) => 32,
             ValueRef::List(v) => match v {
                 ListValueRef::Indexed { vector, .. } => vector.memory_size() / vector.len(),
@@ -1428,7 +1481,9 @@ pub fn column_data_to_json(data: ValueData) -> JsonValue {
 mod tests {
     use arrow::datatypes::DataType as ArrowDataType;
     use common_time::timezone::set_default_timezone;
-    use greptime_proto::v1::{Decimal128 as ProtoDecimal128, IntervalMonthDayNano};
+    use greptime_proto::v1::{
+        Decimal128 as ProtoDecimal128, IntervalMonthDayNano as ProtoIntervalMonthDayNano,
+    };
     use num_traits::Float;
 
     use super::*;
@@ -1525,11 +1580,13 @@ mod tests {
             JsonValue::String("interval year [12]".to_string())
         );
         assert_eq!(
-            column_data_to_json(ValueData::IntervalMonthDayNanoValue(IntervalMonthDayNano {
-                months: 1,
-                days: 2,
-                nanoseconds: 3,
-            })),
+            column_data_to_json(ValueData::IntervalMonthDayNanoValue(
+                ProtoIntervalMonthDayNano {
+                    months: 1,
+                    days: 2,
+                    nanoseconds: 3,
+                }
+            )),
             JsonValue::String("interval month [1][2][3]".to_string())
         );
         assert_eq!(
@@ -1740,12 +1797,10 @@ mod tests {
             ScalarValue::IntervalMonthDayNano(None).try_into().unwrap()
         );
         assert_eq!(
-            Value::Interval(Interval::from_month_day_nano(1, 1, 1)),
-            ScalarValue::IntervalMonthDayNano(Some(
-                Interval::from_month_day_nano(1, 1, 1).to_i128()
-            ))
-            .try_into()
-            .unwrap()
+            Value::IntervalMonthDayNano(IntervalMonthDayNano::new(1, 1, 1)),
+            ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNano::new(1, 1, 1).to_i128()))
+                .try_into()
+                .unwrap()
         );
 
         assert_eq!(
@@ -1975,9 +2030,17 @@ mod tests {
             &ConcreteDataType::time_nanosecond_datatype(),
             &Value::Time(Time::new_nanosecond(1)),
         );
+        check_type_and_value(
+            &ConcreteDataType::interval_year_month_datatype(),
+            &Value::IntervalYearMonth(IntervalYearMonth::new(1)),
+        );
+        check_type_and_value(
+            &ConcreteDataType::interval_day_time_datatype(),
+            &Value::IntervalDayTime(IntervalDayTime::new(1, 2)),
+        );
         check_type_and_value(
             &ConcreteDataType::interval_month_day_nano_datatype(),
-            &Value::Interval(Interval::from_month_day_nano(1, 2, 3)),
+            &Value::IntervalMonthDayNano(IntervalMonthDayNano::new(1, 2, 3)),
         );
         check_type_and_value(
             &ConcreteDataType::duration_second_datatype(),
@@ -2160,7 +2223,9 @@ mod tests {
         check_as_value_ref!(Float64, OrderedF64::from(16.0));
         check_as_value_ref!(Timestamp, Timestamp::new_millisecond(1));
         check_as_value_ref!(Time, Time::new_millisecond(1));
-        check_as_value_ref!(Interval, Interval::from_month_day_nano(1, 2, 3));
+        check_as_value_ref!(IntervalYearMonth, IntervalYearMonth::new(1));
+        check_as_value_ref!(IntervalDayTime, IntervalDayTime::new(1, 2));
+        check_as_value_ref!(IntervalMonthDayNano, IntervalMonthDayNano::new(1, 2, 3));
         check_as_value_ref!(Duration, Duration::new_millisecond(1));
 
         assert_eq!(
@@ -2672,9 +2737,11 @@ mod tests {
         check_value_ref_size_eq(&ValueRef::DateTime(DateTime::new(1)), 8);
         check_value_ref_size_eq(&ValueRef::Timestamp(Timestamp::new_millisecond(1)), 16);
         check_value_ref_size_eq(&ValueRef::Time(Time::new_millisecond(1)), 16);
+        check_value_ref_size_eq(&ValueRef::IntervalYearMonth(IntervalYearMonth::new(1)), 4);
+        check_value_ref_size_eq(&ValueRef::IntervalDayTime(IntervalDayTime::new(1, 2)), 8);
         check_value_ref_size_eq(
-            &ValueRef::Interval(Interval::from_month_day_nano(1, 2, 3)),
-            24,
+            &ValueRef::IntervalMonthDayNano(IntervalMonthDayNano::new(1, 2, 3)),
+            16,
         );
         check_value_ref_size_eq(&ValueRef::Duration(Duration::new_millisecond(1)), 16);
         check_value_ref_size_eq(
diff --git a/src/datatypes/src/vectors/helper.rs b/src/datatypes/src/vectors/helper.rs
index 892d6d2942..ab176d126f 100644
--- a/src/datatypes/src/vectors/helper.rs
+++ b/src/datatypes/src/vectors/helper.rs
@@ -421,7 +421,7 @@ mod tests {
     use common_decimal::Decimal128;
     use common_time::time::Time;
     use common_time::timestamp::TimeUnit;
-    use common_time::{Date, DateTime, Duration, Interval};
+    use common_time::{Date, DateTime, Duration, IntervalMonthDayNano};
 
     use super::*;
     use crate::value::Value;
@@ -689,7 +689,10 @@ mod tests {
         );
         assert_eq!(3, vector.len());
         for i in 0..vector.len() {
-            assert_eq!(Value::Interval(Interval::from_i128(2000)), vector.get(i));
+            assert_eq!(
+                Value::IntervalMonthDayNano(IntervalMonthDayNano::from_i128(2000)),
+                vector.get(i)
+            );
         }
     }
 
diff --git a/src/flow/src/compute/render/reduce.rs b/src/flow/src/compute/render/reduce.rs
index 8de6b7a17a..8cff13781b 100644
--- a/src/flow/src/compute/render/reduce.rs
+++ b/src/flow/src/compute/render/reduce.rs
@@ -560,7 +560,7 @@ fn reduce_batch_subgraph(
                     .get_mut(i)
                     .context(InternalSnafu{
                         reason: format!(
-                            "Output builder should have the same length as the row, expected at most {} but got {}", 
+                            "Output builder should have the same length as the row, expected at most {} but got {}",
                             column_cnt - 1,
                             i
                         )
@@ -1162,7 +1162,9 @@ fn from_val_to_slice_idx(
 #[cfg(test)]
 mod test {
 
-    use common_time::{DateTime, Interval, Timestamp};
+    use std::time::Duration;
+
+    use common_time::Timestamp;
     use datatypes::data_type::{ConcreteDataType, ConcreteDataType as CDT};
     use hydroflow::scheduled::graph::Hydroflow;
 
@@ -1214,8 +1216,8 @@ mod test {
         let expected = TypedPlan {
             schema: RelationType::new(vec![
                 ColumnType::new(CDT::uint64_datatype(), true), // sum(number)
-                ColumnType::new(CDT::datetime_datatype(), false), // window start
-                ColumnType::new(CDT::datetime_datatype(), false), // window end
+                ColumnType::new(CDT::timestamp_millisecond_datatype(), false), // window start
+                ColumnType::new(CDT::timestamp_millisecond_datatype(), false), // window end
             ])
             .into_unnamed(),
             // TODO(discord9): mfp indirectly ref to key columns
@@ -1232,7 +1234,10 @@ mod test {
                             .with_types(
                                 RelationType::new(vec![
                                     ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                                    ColumnType::new(
+                                        ConcreteDataType::timestamp_millisecond_datatype(),
+                                        false,
+                                    ),
                                 ])
                                 .into_unnamed(),
                             ),
@@ -1242,22 +1247,18 @@ mod test {
                                 .map(vec![
                                     ScalarExpr::Column(1).call_unary(
                                         UnaryFunc::TumbleWindowFloor {
-                                            window_size: Interval::from_month_day_nano(
-                                                0,
-                                                0,
-                                                1_000_000_000,
-                                            ),
-                                            start_time: Some(DateTime::new(1625097600000)),
+                                            window_size: Duration::from_nanos(1_000_000_000),
+                                            start_time: Some(Timestamp::new_millisecond(
+                                                1625097600000,
+                                            )),
                                         },
                                     ),
                                     ScalarExpr::Column(1).call_unary(
                                         UnaryFunc::TumbleWindowCeiling {
-                                            window_size: Interval::from_month_day_nano(
-                                                0,
-                                                0,
-                                                1_000_000_000,
-                                            ),
-                                            start_time: Some(DateTime::new(1625097600000)),
+                                            window_size: Duration::from_nanos(1_000_000_000),
+                                            start_time: Some(Timestamp::new_millisecond(
+                                                1625097600000,
+                                            )),
                                         },
                                     ),
                                 ])
@@ -1278,9 +1279,9 @@ mod test {
                     }
                     .with_types(
                         RelationType::new(vec![
-                            ColumnType::new(CDT::datetime_datatype(), false), // window start
-                            ColumnType::new(CDT::datetime_datatype(), false), // window end
-                            ColumnType::new(CDT::uint64_datatype(), true),    //sum(number)
+                            ColumnType::new(CDT::timestamp_millisecond_datatype(), false), // window start
+                            ColumnType::new(CDT::timestamp_millisecond_datatype(), false), // window end
+                            ColumnType::new(CDT::uint64_datatype(), true), //sum(number)
                         ])
                         .with_key(vec![1])
                         .with_time_index(Some(0))
diff --git a/src/flow/src/expr/df_func.rs b/src/flow/src/expr/df_func.rs
index 620615c9eb..a19017e7f3 100644
--- a/src/flow/src/expr/df_func.rs
+++ b/src/flow/src/expr/df_func.rs
@@ -171,9 +171,13 @@ impl DfScalarFunction {
     }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct RawDfScalarFn {
-    /// The raw bytes encoded datafusion scalar function
+    /// The raw bytes encoded datafusion scalar function,
+    /// due to substrait have too many layers of nested struct and `ScalarFunction` 's derive is different
+    /// for simplicity's sake
+    /// so we store bytes instead of `ScalarFunction` here
+    /// but in unit test we will still compare decoded struct(using `f_decoded` field in Debug impl)
     pub(crate) f: bytes::BytesMut,
     /// The input schema of the function
     pub(crate) input_schema: RelationDesc,
@@ -181,6 +185,17 @@ pub struct RawDfScalarFn {
     pub(crate) extensions: FunctionExtensions,
 }
 
+impl std::fmt::Debug for RawDfScalarFn {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RawDfScalarFn")
+            .field("f", &self.f)
+            .field("f_decoded", &ScalarFunction::decode(&mut self.f.as_ref()))
+            .field("df_schema", &self.input_schema)
+            .field("extensions", &self.extensions)
+            .finish()
+    }
+}
+
 impl RawDfScalarFn {
     pub fn from_proto(
         f: &substrait::substrait_proto_df::proto::expression::ScalarFunction,
diff --git a/src/flow/src/expr/func.rs b/src/flow/src/expr/func.rs
index 36a1029725..4f0a944b16 100644
--- a/src/flow/src/expr/func.rs
+++ b/src/flow/src/expr/func.rs
@@ -16,19 +16,18 @@
 
 use std::collections::HashMap;
 use std::sync::{Arc, OnceLock};
+use std::time::Duration;
 
 use arrow::array::{ArrayRef, BooleanArray};
 use common_error::ext::BoxedError;
 use common_time::timestamp::TimeUnit;
-use common_time::{DateTime, Timestamp};
+use common_time::Timestamp;
 use datafusion_expr::Operator;
 use datatypes::data_type::ConcreteDataType;
 use datatypes::prelude::DataType;
 use datatypes::types::cast;
 use datatypes::value::Value;
-use datatypes::vectors::{
-    BooleanVector, DateTimeVector, Helper, TimestampMillisecondVector, VectorRef,
-};
+use datatypes::vectors::{BooleanVector, Helper, TimestampMillisecondVector, VectorRef};
 use serde::{Deserialize, Serialize};
 use smallvec::smallvec;
 use snafu::{ensure, OptionExt, ResultExt};
@@ -52,8 +51,8 @@ pub enum UnmaterializableFunc {
     CurrentSchema,
     TumbleWindow {
         ts: Box<TypedExpr>,
-        window_size: common_time::Interval,
-        start_time: Option<DateTime>,
+        window_size: Duration,
+        start_time: Option<Timestamp>,
     },
 }
 
@@ -63,7 +62,8 @@ impl UnmaterializableFunc {
         match self {
             Self::Now => Signature {
                 input: smallvec![],
-                output: ConcreteDataType::datetime_datatype(),
+                // TODO(yingwen): Maybe return timestamp.
+                output: ConcreteDataType::timestamp_millisecond_datatype(),
                 generic_fn: GenericFn::Now,
             },
             Self::CurrentSchema => Signature {
@@ -110,12 +110,12 @@ pub enum UnaryFunc {
     StepTimestamp,
     Cast(ConcreteDataType),
     TumbleWindowFloor {
-        window_size: common_time::Interval,
-        start_time: Option<DateTime>,
+        window_size: Duration,
+        start_time: Option<Timestamp>,
     },
     TumbleWindowCeiling {
-        window_size: common_time::Interval,
-        start_time: Option<DateTime>,
+        window_size: Duration,
+        start_time: Option<Timestamp>,
     },
 }
 
@@ -139,8 +139,8 @@ impl UnaryFunc {
                 },
             },
             Self::StepTimestamp => Signature {
-                input: smallvec![ConcreteDataType::datetime_datatype()],
-                output: ConcreteDataType::datetime_datatype(),
+                input: smallvec![ConcreteDataType::timestamp_millisecond_datatype()],
+                output: ConcreteDataType::timestamp_millisecond_datatype(),
                 generic_fn: GenericFn::StepTimestamp,
             },
             Self::Cast(to) => Signature {
@@ -238,19 +238,19 @@ impl UnaryFunc {
                 }
             }
             Self::StepTimestamp => {
-                let datetime_array = get_datetime_array(&arg_col)?;
-                let date_array_ref = datetime_array
+                let timestamp_array = get_timestamp_array(&arg_col)?;
+                let timestamp_array_ref = timestamp_array
                     .as_any()
-                    .downcast_ref::<arrow::array::Date64Array>()
+                    .downcast_ref::<arrow::array::TimestampMillisecondArray>()
                     .context({
                         TypeMismatchSnafu {
                             expected: ConcreteDataType::boolean_datatype(),
-                            actual: ConcreteDataType::from_arrow_type(datetime_array.data_type()),
+                            actual: ConcreteDataType::from_arrow_type(timestamp_array.data_type()),
                         }
                     })?;
 
-                let ret = arrow::compute::unary(date_array_ref, |arr| arr + 1);
-                let ret = DateTimeVector::from(ret);
+                let ret = arrow::compute::unary(timestamp_array_ref, |arr| arr + 1);
+                let ret = TimestampMillisecondVector::from(ret);
                 Ok(Arc::new(ret))
             }
             Self::Cast(to) => {
@@ -266,19 +266,19 @@ impl UnaryFunc {
                 window_size,
                 start_time,
             } => {
-                let datetime_array = get_datetime_array(&arg_col)?;
-                let date_array_ref = datetime_array
+                let timestamp_array = get_timestamp_array(&arg_col)?;
+                let date_array_ref = timestamp_array
                     .as_any()
-                    .downcast_ref::<arrow::array::Date64Array>()
+                    .downcast_ref::<arrow::array::TimestampMillisecondArray>()
                     .context({
                         TypeMismatchSnafu {
                             expected: ConcreteDataType::boolean_datatype(),
-                            actual: ConcreteDataType::from_arrow_type(datetime_array.data_type()),
+                            actual: ConcreteDataType::from_arrow_type(timestamp_array.data_type()),
                         }
                     })?;
 
-                let start_time = start_time.map(|t| t.val());
-                let window_size = (window_size.to_nanosecond() / 1_000_000) as repr::Duration; // nanosecond to millisecond
+                let start_time = start_time.map(|t| t.value());
+                let window_size = window_size.as_millis() as repr::Duration;
 
                 let ret = arrow::compute::unary(date_array_ref, |ts| {
                     get_window_start(ts, window_size, start_time)
@@ -291,19 +291,19 @@ impl UnaryFunc {
                 window_size,
                 start_time,
             } => {
-                let datetime_array = get_datetime_array(&arg_col)?;
-                let date_array_ref = datetime_array
+                let timestamp_array = get_timestamp_array(&arg_col)?;
+                let date_array_ref = timestamp_array
                     .as_any()
-                    .downcast_ref::<arrow::array::Date64Array>()
+                    .downcast_ref::<arrow::array::TimestampMillisecondArray>()
                     .context({
                         TypeMismatchSnafu {
                             expected: ConcreteDataType::boolean_datatype(),
-                            actual: ConcreteDataType::from_arrow_type(datetime_array.data_type()),
+                            actual: ConcreteDataType::from_arrow_type(timestamp_array.data_type()),
                         }
                     })?;
 
-                let start_time = start_time.map(|t| t.val());
-                let window_size = (window_size.to_nanosecond() / 1_000_000) as repr::Duration; // nanosecond to millisecond
+                let start_time = start_time.map(|t| t.value());
+                let window_size = window_size.as_millis() as repr::Duration;
 
                 let ret = arrow::compute::unary(date_array_ref, |ts| {
                     get_window_start(ts, window_size, start_time) + window_size
@@ -330,19 +330,20 @@ impl UnaryFunc {
                     })?;
                     if let Some(window_size) = window_size_untyped.as_string() {
                         // cast as interval
-                        cast(
+                        let interval = cast(
                             Value::from(window_size),
-                            &ConcreteDataType::interval_month_day_nano_datatype(),
+                            &ConcreteDataType::interval_day_time_datatype(),
                         )
                         .map_err(BoxedError::new)
                         .context(ExternalSnafu)?
-                        .as_interval()
+                        .as_interval_day_time()
                         .context(UnexpectedSnafu {
                             reason: "Expect window size arg to be interval after successful cast"
                                 .to_string(),
-                        })?
-                    } else if let Some(interval) = window_size_untyped.as_interval() {
-                        interval
+                        })?;
+                        Duration::from_millis(interval.as_millis() as u64)
+                    } else if let Some(interval) = window_size_untyped.as_interval_day_time() {
+                        Duration::from_millis(interval.as_millis() as u64)
                     } else {
                         InvalidQuerySnafu {
                                 reason: format!(
@@ -357,16 +358,19 @@ impl UnaryFunc {
                 let start_time = match args.get(2) {
                     Some(start_time) => {
                         if let Some(value) = start_time.expr.as_literal() {
-                            // cast as DateTime
-                            let ret = cast(value, &ConcreteDataType::datetime_datatype())
-                                .map_err(BoxedError::new)
-                                .context(ExternalSnafu)?
-                                .as_datetime()
-                                .context(UnexpectedSnafu {
-                                    reason:
-                                        "Expect start time arg to be datetime after successful cast"
-                                            .to_string(),
-                                })?;
+                            // cast as timestamp
+                            let ret = cast(
+                                value,
+                                &ConcreteDataType::timestamp_millisecond_datatype(),
+                            )
+                            .map_err(BoxedError::new)
+                            .context(ExternalSnafu)?
+                            .as_timestamp()
+                            .context(UnexpectedSnafu {
+                                reason:
+                                    "Expect start time arg to be timestamp after successful cast"
+                                        .to_string(),
+                            })?;
                             Some(ret)
                         } else {
                             UnexpectedSnafu {
@@ -446,15 +450,15 @@ impl UnaryFunc {
             }
             Self::StepTimestamp => {
                 let ty = arg.data_type();
-                if let Value::DateTime(datetime) = arg {
-                    let datetime = DateTime::from(datetime.val() + 1);
-                    Ok(Value::from(datetime))
+                if let Value::Timestamp(timestamp) = arg {
+                    let timestamp = Timestamp::new_millisecond(timestamp.value() + 1);
+                    Ok(Value::from(timestamp))
                 } else if let Ok(v) = value_to_internal_ts(arg) {
-                    let datetime = DateTime::from(v + 1);
-                    Ok(Value::from(datetime))
+                    let timestamp = Timestamp::new_millisecond(v + 1);
+                    Ok(Value::from(timestamp))
                 } else {
                     TypeMismatchSnafu {
-                        expected: ConcreteDataType::datetime_datatype(),
+                        expected: ConcreteDataType::timestamp_millisecond_datatype(),
                         actual: ty,
                     }
                     .fail()?
@@ -474,8 +478,8 @@ impl UnaryFunc {
                 start_time,
             } => {
                 let ts = get_ts_as_millisecond(arg)?;
-                let start_time = start_time.map(|t| t.val());
-                let window_size = (window_size.to_nanosecond() / 1_000_000) as repr::Duration; // nanosecond to millisecond
+                let start_time = start_time.map(|t| t.value());
+                let window_size = window_size.as_millis() as repr::Duration;
                 let window_start = get_window_start(ts, window_size, start_time);
 
                 let ret = Timestamp::new_millisecond(window_start);
@@ -486,8 +490,8 @@ impl UnaryFunc {
                 start_time,
             } => {
                 let ts = get_ts_as_millisecond(arg)?;
-                let start_time = start_time.map(|t| t.val());
-                let window_size = (window_size.to_nanosecond() / 1_000_000) as repr::Duration; // nanosecond to millisecond
+                let start_time = start_time.map(|t| t.value());
+                let window_size = window_size.as_millis() as repr::Duration;
                 let window_start = get_window_start(ts, window_size, start_time);
 
                 let window_end = window_start + window_size;
@@ -498,21 +502,22 @@ impl UnaryFunc {
     }
 }
 
-fn get_datetime_array(vector: &VectorRef) -> Result<arrow::array::ArrayRef, EvalError> {
+fn get_timestamp_array(vector: &VectorRef) -> Result<arrow::array::ArrayRef, EvalError> {
     let arrow_array = vector.to_arrow_array();
-    let datetime_array =
-        if *arrow_array.data_type() == ConcreteDataType::datetime_datatype().as_arrow_type() {
-            arrow_array
-        } else {
-            arrow::compute::cast(
-                &arrow_array,
-                &ConcreteDataType::datetime_datatype().as_arrow_type(),
-            )
-            .context(ArrowSnafu {
-                context: "Trying to cast to datetime in StepTimestamp",
-            })?
-        };
-    Ok(datetime_array)
+    let timestamp_array = if *arrow_array.data_type()
+        == ConcreteDataType::timestamp_millisecond_datatype().as_arrow_type()
+    {
+        arrow_array
+    } else {
+        arrow::compute::cast(
+            &arrow_array,
+            &ConcreteDataType::timestamp_millisecond_datatype().as_arrow_type(),
+        )
+        .context(ArrowSnafu {
+            context: "Trying to cast to timestamp in StepTimestamp",
+        })?
+    };
+    Ok(timestamp_array)
 }
 
 fn get_window_start(
@@ -1284,7 +1289,6 @@ where
 mod test {
     use std::sync::Arc;
 
-    use common_time::Interval;
     use datatypes::vectors::Vector;
     use pretty_assertions::assert_eq;
 
@@ -1292,18 +1296,18 @@ mod test {
 
     #[test]
     fn test_tumble_batch() {
-        let datetime_vector = DateTimeVector::from_vec(vec![1, 2, 10, 13, 14, 20, 25]);
+        let timestamp_vector = TimestampMillisecondVector::from_vec(vec![1, 2, 10, 13, 14, 20, 25]);
         let tumble_start = UnaryFunc::TumbleWindowFloor {
-            window_size: Interval::from_day_time(0, 10),
+            window_size: Duration::from_millis(10),
             start_time: None,
         };
         let tumble_end = UnaryFunc::TumbleWindowCeiling {
-            window_size: Interval::from_day_time(0, 10),
+            window_size: Duration::from_millis(10),
             start_time: None,
         };
 
-        let len = datetime_vector.len();
-        let batch = Batch::try_new(vec![Arc::new(datetime_vector)], len).unwrap();
+        let len = timestamp_vector.len();
+        let batch = Batch::try_new(vec![Arc::new(timestamp_vector)], len).unwrap();
         let arg = ScalarExpr::Column(0);
 
         let start = tumble_start.eval_batch(&batch, &arg).unwrap();
@@ -1459,4 +1463,17 @@ mod test {
             Err(Error::InvalidQuery { .. })
         );
     }
+
+    #[test]
+    fn test_cast_int() {
+        let interval = cast(
+            Value::from("1 second"),
+            &ConcreteDataType::interval_day_time_datatype(),
+        )
+        .unwrap();
+        assert_eq!(
+            interval,
+            Value::from(common_time::IntervalDayTime::new(0, 1000))
+        );
+    }
 }
diff --git a/src/flow/src/repr.rs b/src/flow/src/repr.rs
index acec71ebcf..1c257c8f3c 100644
--- a/src/flow/src/repr.rs
+++ b/src/flow/src/repr.rs
@@ -61,7 +61,7 @@ pub const BATCH_SIZE: usize = 32 * 16384;
 /// Convert a value that is or can be converted to Datetime to internal timestamp
 ///
 /// support types are: `Date`, `DateTime`, `TimeStamp`, `i64`
-pub fn value_to_internal_ts(value: Value) -> Result<Timestamp, EvalError> {
+pub fn value_to_internal_ts(value: Value) -> Result<i64, EvalError> {
     let is_supported_time_type = |arg: &Value| {
         let ty = arg.data_type();
         matches!(
@@ -76,14 +76,14 @@ pub fn value_to_internal_ts(value: Value) -> Result<Timestamp, EvalError> {
         Value::Int64(ts) => Ok(ts),
         arg if is_supported_time_type(&arg) => {
             let arg_ty = arg.data_type();
-            let res = cast(arg, &ConcreteDataType::datetime_datatype()).context({
+            let res = cast(arg, &ConcreteDataType::timestamp_millisecond_datatype()).context({
                 CastValueSnafu {
                     from: arg_ty,
-                    to: ConcreteDataType::datetime_datatype(),
+                    to: ConcreteDataType::timestamp_millisecond_datatype(),
                 }
             })?;
-            if let Value::DateTime(ts) = res {
-                Ok(ts.val())
+            if let Value::Timestamp(ts) = res {
+                Ok(ts.value())
             } else {
                 unreachable!()
             }
diff --git a/src/flow/src/transform.rs b/src/flow/src/transform.rs
index 6d4de97417..e33b023560 100644
--- a/src/flow/src/transform.rs
+++ b/src/flow/src/transform.rs
@@ -156,10 +156,10 @@ mod test {
 
     use catalog::RegisterTableRequest;
     use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, NUMBERS_TABLE_ID};
-    use common_time::DateTime;
     use datatypes::prelude::*;
     use datatypes::schema::Schema;
-    use datatypes::vectors::VectorRef;
+    use datatypes::timestamp::TimestampMillisecond;
+    use datatypes::vectors::{TimestampMillisecondVectorBuilder, VectorRef};
     use itertools::Itertools;
     use prost::Message;
     use query::parser::QueryLanguageParser;
@@ -202,7 +202,7 @@ mod test {
             ];
             let schema = RelationType::new(vec![
                 ColumnType::new(CDT::uint32_datatype(), false),
-                ColumnType::new(CDT::datetime_datatype(), false),
+                ColumnType::new(CDT::timestamp_millisecond_datatype(), false),
             ]);
             schemas.insert(
                 gid,
@@ -232,7 +232,11 @@ mod test {
 
         let schema = vec![
             datatypes::schema::ColumnSchema::new("number", CDT::uint32_datatype(), false),
-            datatypes::schema::ColumnSchema::new("ts", CDT::datetime_datatype(), false),
+            datatypes::schema::ColumnSchema::new(
+                "ts",
+                CDT::timestamp_millisecond_datatype(),
+                false,
+            ),
         ];
         let mut columns = vec![];
         let numbers = (1..=10).collect_vec();
@@ -240,7 +244,11 @@ mod test {
         columns.push(column);
 
         let ts = (1..=10).collect_vec();
-        let column: VectorRef = Arc::new(<DateTime as Scalar>::VectorType::from_vec(ts));
+        let mut builder = TimestampMillisecondVectorBuilder::with_capacity(10);
+        ts.into_iter()
+            .map(|v| builder.push(Some(TimestampMillisecond::new(v))))
+            .count();
+        let column: VectorRef = builder.to_vector_cloned();
         columns.push(column);
 
         let schema = Arc::new(Schema::new(schema));
diff --git a/src/flow/src/transform/aggr.rs b/src/flow/src/transform/aggr.rs
index 3eec5242cc..2d0a6f66da 100644
--- a/src/flow/src/transform/aggr.rs
+++ b/src/flow/src/transform/aggr.rs
@@ -345,9 +345,10 @@ impl TypedPlan {
 #[cfg(test)]
 mod test {
     use std::collections::BTreeMap;
+    use std::time::Duration;
 
     use bytes::BytesMut;
-    use common_time::{DateTime, Interval};
+    use common_time::{IntervalMonthDayNano, Timestamp};
     use datatypes::prelude::ConcreteDataType;
     use datatypes::value::Value;
     use pretty_assertions::assert_eq;
@@ -398,7 +399,10 @@ mod test {
                             .with_types(
                                 RelationType::new(vec![
                                     ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                                    ColumnType::new(
+                                        ConcreteDataType::timestamp_millisecond_datatype(),
+                                        false,
+                                    ),
                                 ])
                                 .into_named(vec![
                                     Some("number".to_string()),
@@ -413,22 +417,18 @@ mod test {
                                 .map(vec![
                                     ScalarExpr::Column(1).call_unary(
                                         UnaryFunc::TumbleWindowFloor {
-                                            window_size: Interval::from_month_day_nano(
-                                                0,
-                                                0,
-                                                1_000_000_000,
-                                            ),
-                                            start_time: Some(DateTime::new(1625097600000)),
+                                            window_size: Duration::from_nanos(1_000_000_000),
+                                            start_time: Some(Timestamp::new_millisecond(
+                                                1625097600000,
+                                            )),
                                         },
                                     ),
                                     ScalarExpr::Column(1).call_unary(
                                         UnaryFunc::TumbleWindowCeiling {
-                                            window_size: Interval::from_month_day_nano(
-                                                0,
-                                                0,
-                                                1_000_000_000,
-                                            ),
-                                            start_time: Some(DateTime::new(1625097600000)),
+                                            window_size: Duration::from_nanos(1_000_000_000),
+                                            start_time: Some(Timestamp::new_millisecond(
+                                                1625097600000,
+                                            )),
                                         },
                                     ),
                                 ])
@@ -539,7 +539,10 @@ mod test {
                             .with_types(
                                 RelationType::new(vec![
                                     ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                                    ColumnType::new(
+                                        ConcreteDataType::timestamp_millisecond_datatype(),
+                                        false,
+                                    ),
                                 ])
                                 .into_named(vec![
                                     Some("number".to_string()),
@@ -554,22 +557,18 @@ mod test {
                                 .map(vec![
                                     ScalarExpr::Column(1).call_unary(
                                         UnaryFunc::TumbleWindowFloor {
-                                            window_size: Interval::from_month_day_nano(
-                                                0,
-                                                0,
-                                                1_000_000_000,
-                                            ),
-                                            start_time: Some(DateTime::new(1625097600000)),
+                                            window_size: Duration::from_nanos(1_000_000_000),
+                                            start_time: Some(Timestamp::new_millisecond(
+                                                1625097600000,
+                                            )),
                                         },
                                     ),
                                     ScalarExpr::Column(1).call_unary(
                                         UnaryFunc::TumbleWindowCeiling {
-                                            window_size: Interval::from_month_day_nano(
-                                                0,
-                                                0,
-                                                1_000_000_000,
-                                            ),
-                                            start_time: Some(DateTime::new(1625097600000)),
+                                            window_size: Duration::from_nanos(1_000_000_000),
+                                            start_time: Some(Timestamp::new_millisecond(
+                                                1625097600000,
+                                            )),
                                         },
                                     ),
                                 ])
@@ -686,7 +685,10 @@ mod test {
                             .with_types(
                                 RelationType::new(vec![
                                     ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                                    ColumnType::new(
+                                        ConcreteDataType::timestamp_millisecond_datatype(),
+                                        false,
+                                    ),
                                 ])
                                 .into_named(vec![
                                     Some("number".to_string()),
@@ -701,21 +703,13 @@ mod test {
                                 .map(vec![
                                     ScalarExpr::Column(1).call_unary(
                                         UnaryFunc::TumbleWindowFloor {
-                                            window_size: Interval::from_month_day_nano(
-                                                0,
-                                                0,
-                                                3_600_000_000_000,
-                                            ),
+                                            window_size: Duration::from_nanos(3_600_000_000_000),
                                             start_time: None,
                                         },
                                     ),
                                     ScalarExpr::Column(1).call_unary(
                                         UnaryFunc::TumbleWindowCeiling {
-                                            window_size: Interval::from_month_day_nano(
-                                                0,
-                                                0,
-                                                3_600_000_000_000,
-                                            ),
+                                            window_size: Duration::from_nanos(3_600_000_000_000),
                                             start_time: None,
                                         },
                                     ),
@@ -833,7 +827,10 @@ mod test {
                             .with_types(
                                 RelationType::new(vec![
                                     ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                                    ColumnType::new(
+                                        ConcreteDataType::timestamp_millisecond_datatype(),
+                                        false,
+                                    ),
                                 ])
                                 .into_named(vec![
                                     Some("number".to_string()),
@@ -848,21 +845,13 @@ mod test {
                                 .map(vec![
                                     ScalarExpr::Column(1).call_unary(
                                         UnaryFunc::TumbleWindowFloor {
-                                            window_size: Interval::from_month_day_nano(
-                                                0,
-                                                0,
-                                                3_600_000_000_000,
-                                            ),
+                                            window_size: Duration::from_nanos(3_600_000_000_000),
                                             start_time: None,
                                         },
                                     ),
                                     ScalarExpr::Column(1).call_unary(
                                         UnaryFunc::TumbleWindowCeiling {
-                                            window_size: Interval::from_month_day_nano(
-                                                0,
-                                                0,
-                                                3_600_000_000_000,
-                                            ),
+                                            window_size: Duration::from_nanos(3_600_000_000_000),
                                             start_time: None,
                                         },
                                     ),
@@ -948,7 +937,10 @@ mod test {
                             .with_types(
                                 RelationType::new(vec![
                                     ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                                    ColumnType::new(
+                                        ConcreteDataType::timestamp_millisecond_datatype(),
+                                        false,
+                                    ),
                                 ])
                                 .into_named(vec![
                                     Some("number".to_string()),
@@ -963,22 +955,18 @@ mod test {
                                 .map(vec![
                                     ScalarExpr::Column(1).call_unary(
                                         UnaryFunc::TumbleWindowFloor {
-                                            window_size: Interval::from_month_day_nano(
-                                                0,
-                                                0,
-                                                3_600_000_000_000,
-                                            ),
-                                            start_time: Some(DateTime::new(1625097600000)),
+                                            window_size: Duration::from_nanos(3_600_000_000_000),
+                                            start_time: Some(Timestamp::new_millisecond(
+                                                1625097600000,
+                                            )),
                                         },
                                     ),
                                     ScalarExpr::Column(1).call_unary(
                                         UnaryFunc::TumbleWindowCeiling {
-                                            window_size: Interval::from_month_day_nano(
-                                                0,
-                                                0,
-                                                3_600_000_000_000,
-                                            ),
-                                            start_time: Some(DateTime::new(1625097600000)),
+                                            window_size: Duration::from_nanos(3_600_000_000_000),
+                                            start_time: Some(Timestamp::new_millisecond(
+                                                1625097600000,
+                                            )),
                                         },
                                     ),
                                 ])
@@ -1512,7 +1500,7 @@ mod test {
                             .with_types(
                                 RelationType::new(vec![
                                     ColumnType::new(ConcreteDataType::uint32_datatype(), false),
-                                    ColumnType::new(ConcreteDataType::datetime_datatype(), false),
+                                    ColumnType::new(ConcreteDataType::timestamp_millisecond_datatype(), false),
                                 ])
                                 .into_named(vec![
                                     Some("number".to_string()),
@@ -1536,7 +1524,7 @@ mod test {
                                                 true,
                                             ),ColumnType::new(
                                                 ConcreteDataType::timestamp_millisecond_datatype(),
-                                                true,
+                                                false,
                                             )])
                                             .into_unnamed(),
                                             extensions: FunctionExtensions {
@@ -1554,10 +1542,10 @@ mod test {
                                     .unwrap(),
                                     exprs: vec![
                                         ScalarExpr::Literal(
-                                            Value::Interval(Interval::from_month_day_nano(0, 0, 30000000000)),
+                                            Value::IntervalMonthDayNano(IntervalMonthDayNano::new(0, 0, 30000000000)),
                                             CDT::interval_month_day_nano_datatype()
                                         ),
-                                        ScalarExpr::Column(1).cast(CDT::timestamp_millisecond_datatype())
+                                        ScalarExpr::Column(1)
                                         ],
                                 }])
                                 .unwrap()
diff --git a/src/flow/src/transform/literal.rs b/src/flow/src/transform/literal.rs
index f9dd8b9535..ffb62ff14d 100644
--- a/src/flow/src/transform/literal.rs
+++ b/src/flow/src/transform/literal.rs
@@ -178,14 +178,14 @@ pub(crate) fn from_substrait_literal(lit: &Literal) -> Result<(Value, CDT), Erro
             let (days, seconds, microseconds) =
                 (interval.days, interval.seconds, interval.microseconds);
             let millis = microseconds / 1000 + seconds * 1000;
-            let value_interval = common_time::Interval::from_day_time(days, millis);
+            let value_interval = common_time::IntervalDayTime::new(days, millis);
             (
-                Value::Interval(value_interval),
+                Value::IntervalDayTime(value_interval),
                 CDT::interval_day_time_datatype(),
             )
         }
         Some(LiteralType::IntervalYearToMonth(interval)) => (
-            Value::Interval(common_time::Interval::from_year_month(
+            Value::IntervalYearMonth(common_time::IntervalYearMonth::new(
                 interval.years * 12 + interval.months,
             )),
             CDT::interval_year_month_datatype(),
@@ -239,9 +239,9 @@ fn from_substrait_user_defined_type(user_defined: &UserDefined) -> Result<(Value
                     }
                 );
                 let i: i32 = from_bytes(&val.value)?;
-                let value_interval = common_time::Interval::from_year_month(i);
+                let value_interval = common_time::IntervalYearMonth::new(i);
                 (
-                    Value::Interval(value_interval),
+                    Value::IntervalYearMonth(value_interval),
                     CDT::interval_year_month_datatype(),
                 )
             }
@@ -255,12 +255,12 @@ fn from_substrait_user_defined_type(user_defined: &UserDefined) -> Result<(Value
                         )
                     }
                 );
+                // TODO(yingwen): Datafusion may change the representation of the interval type.
                 let i: i128 = from_bytes(&val.value)?;
                 let (months, days, nsecs) = ((i >> 96) as i32, (i >> 64) as i32, i as i64);
-                let value_interval =
-                    common_time::Interval::from_month_day_nano(months, days, nsecs);
+                let value_interval = common_time::IntervalMonthDayNano::new(months, days, nsecs);
                 (
-                    Value::Interval(value_interval),
+                    Value::IntervalMonthDayNano(value_interval),
                     CDT::interval_month_day_nano_datatype(),
                 )
             }
@@ -274,11 +274,12 @@ fn from_substrait_user_defined_type(user_defined: &UserDefined) -> Result<(Value
                         )
                     }
                 );
+                // TODO(yingwen): Datafusion may change the representation of the interval type.
                 let i: i64 = from_bytes(&val.value)?;
                 let (days, millis) = ((i >> 32) as i32, i as i32);
-                let value_interval = common_time::Interval::from_day_time(days, millis);
+                let value_interval = common_time::IntervalDayTime::new(days, millis);
                 (
-                    Value::Interval(value_interval),
+                    Value::IntervalDayTime(value_interval),
                     CDT::interval_day_time_datatype(),
                 )
             }
diff --git a/src/mito2/src/row_converter.rs b/src/mito2/src/row_converter.rs
index a7f6f1644e..38e6b68a65 100644
--- a/src/mito2/src/row_converter.rs
+++ b/src/mito2/src/row_converter.rs
@@ -16,9 +16,10 @@ use bytes::Buf;
 use common_base::bytes::Bytes;
 use common_decimal::Decimal128;
 use common_time::time::Time;
-use common_time::{Date, Duration, Interval};
+use common_time::{Date, Duration, IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth};
 use datatypes::data_type::ConcreteDataType;
 use datatypes::prelude::Value;
+use datatypes::types::IntervalType;
 use datatypes::value::ValueRef;
 use memcomparable::{Deserializer, Serializer};
 use paste::paste;
@@ -117,6 +118,24 @@ impl SortField {
                             .serialize($serializer)
                             .context(SerializeFieldSnafu)?;
                     }
+                    ConcreteDataType::Interval(IntervalType::YearMonth(_)) => {
+                        let interval = value.as_interval_year_month().context(FieldTypeMismatchSnafu)?;
+                        interval.map(|i| i.to_i32())
+                            .serialize($serializer)
+                            .context(SerializeFieldSnafu)?;
+                    }
+                    ConcreteDataType::Interval(IntervalType::DayTime(_)) => {
+                        let interval = value.as_interval_day_time().context(FieldTypeMismatchSnafu)?;
+                        interval.map(|i| i.to_i64())
+                            .serialize($serializer)
+                            .context(SerializeFieldSnafu)?;
+                    }
+                    ConcreteDataType::Interval(IntervalType::MonthDayNano(_)) => {
+                        let interval = value.as_interval_month_day_nano().context(FieldTypeMismatchSnafu)?;
+                        interval.map(|i| i.to_i128())
+                            .serialize($serializer)
+                            .context(SerializeFieldSnafu)?;
+                    }
                     ConcreteDataType::List(_) |
                     ConcreteDataType::Dictionary(_) |
                     ConcreteDataType::Null(_) => {
@@ -144,7 +163,6 @@ impl SortField {
             Date, date,
             DateTime, datetime,
             Time, time,
-            Interval, interval,
             Duration, duration,
             Decimal128, decimal128,
             Json, binary
@@ -181,6 +199,24 @@ impl SortField {
                             .map(|t|ty.create_timestamp(t));
                         Ok(Value::from(timestamp))
                     }
+                    ConcreteDataType::Interval(IntervalType::YearMonth(_)) => {
+                        let interval = Option::<i32>::deserialize(deserializer)
+                            .context(error::DeserializeFieldSnafu)?
+                            .map(IntervalYearMonth::from_i32);
+                        Ok(Value::from(interval))
+                    }
+                    ConcreteDataType::Interval(IntervalType::DayTime(_)) => {
+                        let interval = Option::<i64>::deserialize(deserializer)
+                            .context(error::DeserializeFieldSnafu)?
+                            .map(IntervalDayTime::from_i64);
+                        Ok(Value::from(interval))
+                    }
+                    ConcreteDataType::Interval(IntervalType::MonthDayNano(_)) => {
+                        let interval = Option::<i128>::deserialize(deserializer)
+                            .context(error::DeserializeFieldSnafu)?
+                            .map(IntervalMonthDayNano::from_i128);
+                        Ok(Value::from(interval))
+                    }
                     ConcreteDataType::List(l) => NotSupportedFieldSnafu {
                         data_type: ConcreteDataType::List(l.clone()),
                     }
@@ -212,7 +248,6 @@ impl SortField {
             Date, Date,
             Time, Time,
             DateTime, DateTime,
-            Interval, Interval,
             Duration, Duration,
             Decimal128, Decimal128
         )
@@ -263,7 +298,9 @@ impl SortField {
             ConcreteDataType::Timestamp(_) => 9, // We treat timestamp as Option<i64>
             ConcreteDataType::Time(_) => 10,     // i64 and 1 byte time unit
             ConcreteDataType::Duration(_) => 10,
-            ConcreteDataType::Interval(_) => 18,
+            ConcreteDataType::Interval(IntervalType::YearMonth(_)) => 5,
+            ConcreteDataType::Interval(IntervalType::DayTime(_)) => 9,
+            ConcreteDataType::Interval(IntervalType::MonthDayNano(_)) => 17,
             ConcreteDataType::Decimal128(_) => 19,
             ConcreteDataType::Null(_)
             | ConcreteDataType::List(_)
@@ -387,7 +424,9 @@ impl RowCodec for McmpRowCodec {
 #[cfg(test)]
 mod tests {
     use common_base::bytes::StringBytes;
-    use common_time::{DateTime, Timestamp};
+    use common_time::{
+        DateTime, IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth, Timestamp,
+    };
     use datatypes::value::Value;
 
     use super::*;
@@ -563,6 +602,8 @@ mod tests {
                 ConcreteDataType::timestamp_millisecond_datatype(),
                 ConcreteDataType::time_millisecond_datatype(),
                 ConcreteDataType::duration_millisecond_datatype(),
+                ConcreteDataType::interval_year_month_datatype(),
+                ConcreteDataType::interval_day_time_datatype(),
                 ConcreteDataType::interval_month_day_nano_datatype(),
                 ConcreteDataType::decimal128_default_datatype(),
             ],
@@ -585,7 +626,9 @@ mod tests {
                 Value::Timestamp(Timestamp::new_millisecond(12)),
                 Value::Time(Time::new_millisecond(13)),
                 Value::Duration(Duration::new_millisecond(14)),
-                Value::Interval(Interval::from_month_day_nano(1, 1, 15)),
+                Value::IntervalYearMonth(IntervalYearMonth::new(1)),
+                Value::IntervalDayTime(IntervalDayTime::new(1, 15)),
+                Value::IntervalMonthDayNano(IntervalMonthDayNano::new(1, 1, 15)),
                 Value::Decimal128(Decimal128::from(16)),
             ],
         );
diff --git a/src/query/src/range_select/plan_rewrite.rs b/src/query/src/range_select/plan_rewrite.rs
index 43b0928539..f7e0a3dd0d 100644
--- a/src/query/src/range_select/plan_rewrite.rs
+++ b/src/query/src/range_select/plan_rewrite.rs
@@ -20,9 +20,9 @@ use arrow_schema::DataType;
 use async_recursion::async_recursion;
 use catalog::table_source::DfTableSourceProvider;
 use chrono::Utc;
-use common_time::interval::NANOS_PER_MILLI;
+use common_time::interval::{MS_PER_DAY, NANOS_PER_MILLI};
 use common_time::timestamp::TimeUnit;
-use common_time::{Interval, Timestamp, Timezone};
+use common_time::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth, Timestamp, Timezone};
 use datafusion::datasource::DefaultTableSource;
 use datafusion::prelude::Column;
 use datafusion::scalar::ScalarValue;
@@ -145,8 +145,6 @@ fn evaluate_expr_to_millisecond(args: &[Expr], i: usize, interval_only: bool) ->
     }
     let execution_props = ExecutionProps::new().with_query_execution_start_time(Utc::now());
     let info = SimplifyContext::new(&execution_props).with_schema(Arc::new(DFSchema::empty()));
-    let interval_to_ms =
-        |interval: Interval| -> i64 { (interval.to_nanosecond() / NANOS_PER_MILLI as i128) as i64 };
     let simplify_expr = ExprSimplifier::new(info).simplify(expr.clone())?;
     match simplify_expr {
         Expr::Literal(ScalarValue::TimestampNanosecond(ts_nanos, _))
@@ -161,15 +159,37 @@ fn evaluate_expr_to_millisecond(args: &[Expr], i: usize, interval_only: bool) ->
         | Expr::Literal(ScalarValue::DurationMillisecond(ts_millis)) => ts_millis,
         Expr::Literal(ScalarValue::TimestampSecond(ts_secs, _))
         | Expr::Literal(ScalarValue::DurationSecond(ts_secs)) => ts_secs.map(|v| v * 1_000),
-        Expr::Literal(ScalarValue::IntervalYearMonth(interval)) => {
-            interval.map(|v| interval_to_ms(Interval::from_i32(v)))
-        }
-        Expr::Literal(ScalarValue::IntervalDayTime(interval)) => {
-            interval.map(|v| interval_to_ms(Interval::from_i64(v)))
-        }
-        Expr::Literal(ScalarValue::IntervalMonthDayNano(interval)) => {
-            interval.map(|v| interval_to_ms(Interval::from_i128(v)))
-        }
+        // We don't support interval with months as days in a month is unclear.
+        Expr::Literal(ScalarValue::IntervalYearMonth(interval)) => interval
+            .map(|v| {
+                let interval = IntervalYearMonth::from_i32(v);
+                if interval.months != 0 {
+                    return Err(DataFusionError::Plan(format!(
+                        "Year or month interval is not allowed in range query: {}",
+                        expr.display_name().unwrap_or_default()
+                    )));
+                }
+
+                Ok(0)
+            })
+            .transpose()?,
+        Expr::Literal(ScalarValue::IntervalDayTime(interval)) => interval.map(|v| {
+            let interval = IntervalDayTime::from_i64(v);
+            interval.as_millis()
+        }),
+        Expr::Literal(ScalarValue::IntervalMonthDayNano(interval)) => interval
+            .map(|v| {
+                let interval = IntervalMonthDayNano::from_i128(v);
+                if interval.months != 0 {
+                    return Err(DataFusionError::Plan(format!(
+                        "Year or month interval is not allowed in range query: {}",
+                        expr.display_name().unwrap_or_default()
+                    )));
+                }
+
+                Ok(interval.days as i64 * MS_PER_DAY + interval.nanoseconds / NANOS_PER_MILLI)
+            })
+            .transpose()?,
         _ => None,
     }
     .ok_or_else(|| {
@@ -547,6 +567,7 @@ mod test {
     use catalog::memory::MemoryCatalogManager;
     use catalog::RegisterTableRequest;
     use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+    use common_time::IntervalYearMonth;
     use datafusion_expr::{BinaryExpr, Operator};
     use datatypes::prelude::ConcreteDataType;
     use datatypes::schema::{ColumnSchema, Schema};
@@ -788,35 +809,29 @@ mod test {
 
     #[test]
     fn test_parse_duration_expr() {
-        let interval_to_ms = |interval: Interval| -> u128 {
-            (interval.to_nanosecond() / NANOS_PER_MILLI as i128) as u128
-        };
         // test IntervalYearMonth
-        let interval = Interval::from_year_month(10);
+        let interval = IntervalYearMonth::new(10);
         let args = vec![Expr::Literal(ScalarValue::IntervalYearMonth(Some(
             interval.to_i32(),
         )))];
-        assert_eq!(
-            parse_duration_expr(&args, 0).unwrap().as_millis(),
-            interval_to_ms(interval)
-        );
+        assert!(parse_duration_expr(&args, 0).is_err(),);
         // test IntervalDayTime
-        let interval = Interval::from_day_time(10, 10);
+        let interval = IntervalDayTime::new(10, 10);
         let args = vec![Expr::Literal(ScalarValue::IntervalDayTime(Some(
             interval.to_i64(),
         )))];
         assert_eq!(
-            parse_duration_expr(&args, 0).unwrap().as_millis(),
-            interval_to_ms(interval)
+            parse_duration_expr(&args, 0).unwrap().as_millis() as i64,
+            interval.as_millis()
         );
         // test IntervalMonthDayNano
-        let interval = Interval::from_month_day_nano(10, 10, 10);
+        let interval = IntervalMonthDayNano::new(0, 10, 10);
         let args = vec![Expr::Literal(ScalarValue::IntervalMonthDayNano(Some(
             interval.to_i128(),
         )))];
         assert_eq!(
-            parse_duration_expr(&args, 0).unwrap().as_millis(),
-            interval_to_ms(interval)
+            parse_duration_expr(&args, 0).unwrap().as_millis() as i64,
+            interval.days as i64 * MS_PER_DAY + interval.nanoseconds / NANOS_PER_MILLI,
         );
         // test Duration
         let args = vec![Expr::Literal(ScalarValue::Utf8(Some("1y4w".into())))];
@@ -828,25 +843,25 @@ mod test {
         assert!(parse_duration_expr(&args, 10).is_err());
         // test evaluate expr
         let args = vec![Expr::BinaryExpr(BinaryExpr {
-            left: Box::new(Expr::Literal(ScalarValue::IntervalYearMonth(Some(
-                Interval::from_year_month(10).to_i32(),
+            left: Box::new(Expr::Literal(ScalarValue::IntervalDayTime(Some(
+                IntervalDayTime::new(0, 10).to_i64(),
             )))),
             op: Operator::Plus,
-            right: Box::new(Expr::Literal(ScalarValue::IntervalYearMonth(Some(
-                Interval::from_year_month(10).to_i32(),
+            right: Box::new(Expr::Literal(ScalarValue::IntervalDayTime(Some(
+                IntervalDayTime::new(0, 10).to_i64(),
             )))),
         })];
         assert_eq!(
-            parse_duration_expr(&args, 0).unwrap().as_millis(),
-            interval_to_ms(Interval::from_year_month(20))
+            parse_duration_expr(&args, 0).unwrap(),
+            Duration::from_millis(20)
         );
         let args = vec![Expr::BinaryExpr(BinaryExpr {
-            left: Box::new(Expr::Literal(ScalarValue::IntervalYearMonth(Some(
-                Interval::from_year_month(10).to_i32(),
+            left: Box::new(Expr::Literal(ScalarValue::IntervalDayTime(Some(
+                IntervalDayTime::new(0, 10).to_i64(),
             )))),
             op: Operator::Minus,
-            right: Box::new(Expr::Literal(ScalarValue::IntervalYearMonth(Some(
-                Interval::from_year_month(10).to_i32(),
+            right: Box::new(Expr::Literal(ScalarValue::IntervalDayTime(Some(
+                IntervalDayTime::new(0, 10).to_i64(),
             )))),
         })];
         // test zero interval error
@@ -854,7 +869,7 @@ mod test {
         // test must all be interval
         let args = vec![Expr::BinaryExpr(BinaryExpr {
             left: Box::new(Expr::Literal(ScalarValue::IntervalYearMonth(Some(
-                Interval::from_year_month(10).to_i32(),
+                IntervalYearMonth::new(10).to_i32(),
             )))),
             op: Operator::Minus,
             right: Box::new(Expr::Literal(ScalarValue::Time64Microsecond(Some(0)))),
@@ -907,19 +922,15 @@ mod test {
         );
         // test evaluate expr
         let args = vec![Expr::BinaryExpr(BinaryExpr {
-            left: Box::new(Expr::Literal(ScalarValue::IntervalYearMonth(Some(
-                Interval::from_year_month(10).to_i32(),
+            left: Box::new(Expr::Literal(ScalarValue::IntervalDayTime(Some(
+                IntervalDayTime::new(0, 10).to_i64(),
             )))),
             op: Operator::Plus,
-            right: Box::new(Expr::Literal(ScalarValue::IntervalYearMonth(Some(
-                Interval::from_year_month(10).to_i32(),
+            right: Box::new(Expr::Literal(ScalarValue::IntervalDayTime(Some(
+                IntervalDayTime::new(0, 10).to_i64(),
             )))),
         })];
-        assert_eq!(
-            parse_align_to(&args, 0, None).unwrap(),
-            // 20 month
-            20 * 30 * 24 * 60 * 60 * 1000
-        );
+        assert_eq!(parse_align_to(&args, 0, None).unwrap(), 20);
     }
 
     #[test]
@@ -927,18 +938,18 @@ mod test {
         let expr = Expr::BinaryExpr(BinaryExpr {
             left: Box::new(Expr::Literal(ScalarValue::DurationMillisecond(Some(20)))),
             op: Operator::Minus,
-            right: Box::new(Expr::Literal(ScalarValue::IntervalYearMonth(Some(
-                Interval::from_year_month(10).to_i32(),
+            right: Box::new(Expr::Literal(ScalarValue::IntervalDayTime(Some(
+                IntervalDayTime::new(10, 0).to_i64(),
             )))),
         });
         assert!(!interval_only_in_expr(&expr));
         let expr = Expr::BinaryExpr(BinaryExpr {
-            left: Box::new(Expr::Literal(ScalarValue::IntervalYearMonth(Some(
-                Interval::from_year_month(10).to_i32(),
+            left: Box::new(Expr::Literal(ScalarValue::IntervalDayTime(Some(
+                IntervalDayTime::new(10, 0).to_i64(),
             )))),
             op: Operator::Minus,
-            right: Box::new(Expr::Literal(ScalarValue::IntervalYearMonth(Some(
-                Interval::from_year_month(10).to_i32(),
+            right: Box::new(Expr::Literal(ScalarValue::IntervalDayTime(Some(
+                IntervalDayTime::new(10, 0).to_i64(),
             )))),
         });
         assert!(interval_only_in_expr(&expr));
diff --git a/src/servers/src/mysql/writer.rs b/src/servers/src/mysql/writer.rs
index 8a5814d8fe..6c7cff7494 100644
--- a/src/servers/src/mysql/writer.rs
+++ b/src/servers/src/mysql/writer.rs
@@ -226,7 +226,11 @@ impl<'a, W: AsyncWrite + Unpin> MysqlResultWriter<'a, W> {
                     Value::Timestamp(v) => row_writer.write_col(
                         v.to_chrono_datetime_with_timezone(Some(&query_context.timezone())),
                     )?,
-                    Value::Interval(v) => row_writer.write_col(v.to_iso8601_string())?,
+                    Value::IntervalYearMonth(v) => row_writer.write_col(v.to_iso8601_string())?,
+                    Value::IntervalDayTime(v) => row_writer.write_col(v.to_iso8601_string())?,
+                    Value::IntervalMonthDayNano(v) => {
+                        row_writer.write_col(v.to_iso8601_string())?
+                    }
                     Value::Duration(v) => row_writer.write_col(v.to_std_duration())?,
                     Value::List(_) => {
                         return Err(Error::Internal {
diff --git a/src/servers/src/postgres/types.rs b/src/servers/src/postgres/types.rs
index a5d1d392ac..c268b893d8 100644
--- a/src/servers/src/postgres/types.rs
+++ b/src/servers/src/postgres/types.rs
@@ -21,12 +21,12 @@ use std::collections::HashMap;
 use std::ops::Deref;
 
 use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
-use common_time::Interval;
+use common_time::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth};
 use datafusion_common::ScalarValue;
 use datafusion_expr::LogicalPlan;
 use datatypes::prelude::{ConcreteDataType, Value};
 use datatypes::schema::Schema;
-use datatypes::types::TimestampType;
+use datatypes::types::{IntervalType, TimestampType};
 use datatypes::value::ListValue;
 use pgwire::api::portal::{Format, Portal};
 use pgwire::api::results::{DataRowEncoder, FieldInfo};
@@ -325,7 +325,9 @@ fn encode_array(
                 .iter()
                 .map(|v| match v {
                     Value::Null => Ok(None),
-                    Value::Interval(v) => Ok(Some(PgInterval::from(*v))),
+                    Value::IntervalYearMonth(v) => Ok(Some(PgInterval::from(*v))),
+                    Value::IntervalDayTime(v) => Ok(Some(PgInterval::from(*v))),
+                    Value::IntervalMonthDayNano(v) => Ok(Some(PgInterval::from(*v))),
                     _ => Err(PgWireError::ApiError(Box::new(Error::Internal {
                         err_msg: format!("Invalid list item type, find {v:?}, expected interval",),
                     }))),
@@ -443,7 +445,9 @@ pub(super) fn encode_value(
                 })))
             }
         }
-        Value::Interval(v) => builder.encode_field(&PgInterval::from(*v)),
+        Value::IntervalYearMonth(v) => builder.encode_field(&PgInterval::from(*v)),
+        Value::IntervalDayTime(v) => builder.encode_field(&PgInterval::from(*v)),
+        Value::IntervalMonthDayNano(v) => builder.encode_field(&PgInterval::from(*v)),
         Value::Decimal128(v) => builder.encode_field(&v.to_string()),
         Value::List(values) => encode_array(query_ctx, values, builder),
         Value::Duration(_) => Err(PgWireError::ApiError(Box::new(Error::Internal {
@@ -875,9 +879,51 @@ pub(super) fn parameters_to_scalar_values(
                 let data = portal.parameter::<PgInterval>(idx, &client_type)?;
                 if let Some(server_type) = &server_type {
                     match server_type {
-                        ConcreteDataType::Interval(_) => ScalarValue::IntervalMonthDayNano(
-                            data.map(|i| Interval::from(i).to_i128()),
-                        ),
+                        ConcreteDataType::Interval(IntervalType::YearMonth(_)) => {
+                            ScalarValue::IntervalYearMonth(
+                                data.map(|i| {
+                                    if i.days != 0 || i.microseconds != 0 {
+                                        Err(invalid_parameter_error(
+                                            "invalid_parameter_type",
+                                            Some(format!(
+                                                "Expected: {}, found: {}",
+                                                server_type, client_type
+                                            )),
+                                        ))
+                                    } else {
+                                        Ok(IntervalYearMonth::new(i.months).to_i32())
+                                    }
+                                })
+                                .transpose()?,
+                            )
+                        }
+                        ConcreteDataType::Interval(IntervalType::DayTime(_)) => {
+                            ScalarValue::IntervalDayTime(
+                                data.map(|i| {
+                                    if i.months != 0 || i.microseconds % 1000 != 0 {
+                                        Err(invalid_parameter_error(
+                                            "invalid_parameter_type",
+                                            Some(format!(
+                                                "Expected: {}, found: {}",
+                                                server_type, client_type
+                                            )),
+                                        ))
+                                    } else {
+                                        Ok(IntervalDayTime::new(
+                                            i.days,
+                                            (i.microseconds / 1000) as i32,
+                                        )
+                                        .to_i64())
+                                    }
+                                })
+                                .transpose()?,
+                            )
+                        }
+                        ConcreteDataType::Interval(IntervalType::MonthDayNano(_)) => {
+                            ScalarValue::IntervalMonthDayNano(
+                                data.map(|i| IntervalMonthDayNano::from(i).to_i128()),
+                            )
+                        }
                         _ => {
                             return Err(invalid_parameter_error(
                                 "invalid_parameter_type",
@@ -886,7 +932,9 @@ pub(super) fn parameters_to_scalar_values(
                         }
                     }
                 } else {
-                    ScalarValue::IntervalMonthDayNano(data.map(|i| Interval::from(i).to_i128()))
+                    ScalarValue::IntervalMonthDayNano(
+                        data.map(|i| IntervalMonthDayNano::from(i).to_i128()),
+                    )
                 }
             }
             &Type::BYTEA => {
@@ -1150,7 +1198,21 @@ mod test {
                 FieldFormat::Text,
             ),
             FieldInfo::new(
-                "intervals".into(),
+                "interval_year_month".into(),
+                None,
+                None,
+                Type::INTERVAL,
+                FieldFormat::Text,
+            ),
+            FieldInfo::new(
+                "interval_day_time".into(),
+                None,
+                None,
+                Type::INTERVAL,
+                FieldFormat::Text,
+            ),
+            FieldInfo::new(
+                "interval_month_day_nano".into(),
                 None,
                 None,
                 Type::INTERVAL,
@@ -1214,6 +1276,8 @@ mod test {
             ConcreteDataType::datetime_datatype(),
             ConcreteDataType::timestamp_datatype(TimeUnit::Second),
             ConcreteDataType::interval_datatype(IntervalUnit::YearMonth),
+            ConcreteDataType::interval_datatype(IntervalUnit::DayTime),
+            ConcreteDataType::interval_datatype(IntervalUnit::MonthDayNano),
             ConcreteDataType::list_datatype(ConcreteDataType::int64_datatype()),
             ConcreteDataType::list_datatype(ConcreteDataType::float64_datatype()),
             ConcreteDataType::list_datatype(ConcreteDataType::string_datatype()),
@@ -1246,7 +1310,9 @@ mod test {
             Value::Time(1001i64.into()),
             Value::DateTime(1000001i64.into()),
             Value::Timestamp(1000001i64.into()),
-            Value::Interval(1000001i128.into()),
+            Value::IntervalYearMonth(IntervalYearMonth::new(1)),
+            Value::IntervalDayTime(IntervalDayTime::new(1, 10)),
+            Value::IntervalMonthDayNano(IntervalMonthDayNano::new(1, 1, 10)),
             Value::List(ListValue::new(
                 vec![Value::Int64(1i64)],
                 ConcreteDataType::int64_datatype(),
diff --git a/src/servers/src/postgres/types/interval.rs b/src/servers/src/postgres/types/interval.rs
index 9bc643916a..c15833f86c 100644
--- a/src/servers/src/postgres/types/interval.rs
+++ b/src/servers/src/postgres/types/interval.rs
@@ -15,31 +15,51 @@
 use std::fmt::Display;
 
 use bytes::{Buf, BufMut};
-use common_time::Interval;
+use common_time::interval::IntervalFormat;
+use common_time::{IntervalDayTime, IntervalMonthDayNano, IntervalYearMonth};
 use pgwire::types::ToSqlText;
 use postgres_types::{to_sql_checked, FromSql, IsNull, ToSql, Type};
 
 #[derive(Debug, Clone, Copy, Default)]
 pub struct PgInterval {
-    months: i32,
-    days: i32,
-    microseconds: i64,
+    pub(crate) months: i32,
+    pub(crate) days: i32,
+    pub(crate) microseconds: i64,
 }
 
-impl From<Interval> for PgInterval {
-    fn from(interval: Interval) -> Self {
-        let (months, days, nanos) = interval.to_month_day_nano();
+impl From<IntervalYearMonth> for PgInterval {
+    fn from(interval: IntervalYearMonth) -> Self {
         Self {
-            months,
-            days,
-            microseconds: nanos / 1000,
+            months: interval.months,
+            days: 0,
+            microseconds: 0,
         }
     }
 }
 
-impl From<PgInterval> for Interval {
+impl From<IntervalDayTime> for PgInterval {
+    fn from(interval: IntervalDayTime) -> Self {
+        Self {
+            months: 0,
+            days: interval.days,
+            microseconds: interval.milliseconds as i64 * 1000,
+        }
+    }
+}
+
+impl From<IntervalMonthDayNano> for PgInterval {
+    fn from(interval: IntervalMonthDayNano) -> Self {
+        Self {
+            months: interval.months,
+            days: interval.days,
+            microseconds: interval.nanoseconds / 1000,
+        }
+    }
+}
+
+impl From<PgInterval> for IntervalMonthDayNano {
     fn from(interval: PgInterval) -> Self {
-        Interval::from_month_day_nano(
+        IntervalMonthDayNano::new(
             interval.months,
             interval.days,
             // Maybe overflow, but most scenarios ok.
@@ -56,7 +76,11 @@ impl From<PgInterval> for Interval {
 
 impl Display for PgInterval {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", Interval::from(*self).to_postgres_string())
+        write!(
+            f,
+            "{}",
+            IntervalFormat::from(IntervalMonthDayNano::from(*self)).to_postgres_string()
+        )
     }
 }
 
diff --git a/src/sql/src/statements.rs b/src/sql/src/statements.rs
index 30af7ae517..e62565d323 100644
--- a/src/sql/src/statements.rs
+++ b/src/sql/src/statements.rs
@@ -322,7 +322,9 @@ pub fn sql_value_to_value(
             | Value::Timestamp(_)
             | Value::Time(_)
             | Value::Duration(_)
-            | Value::Interval(_) => match unary_op {
+            | Value::IntervalYearMonth(_)
+            | Value::IntervalDayTime(_)
+            | Value::IntervalMonthDayNano(_) => match unary_op {
                 UnaryOperator::Plus => {}
                 UnaryOperator::Minus => {
                     value = value
diff --git a/tests/cases/standalone/common/range/interval.result b/tests/cases/standalone/common/range/interval.result
index 10da6cf90d..b67310ab85 100644
--- a/tests/cases/standalone/common/range/interval.result
+++ b/tests/cases/standalone/common/range/interval.result
@@ -20,25 +20,18 @@ Affected Rows: 8
 
 SELECT ts, host, min(val) RANGE (INTERVAL '1 year') FROM host ALIGN (INTERVAL '1 year') ORDER BY host, ts;
 
-+---------------------+-------+----------------------------------------------------------------------------+
-| ts                  | host  | MIN(host.val) RANGE IntervalMonthDayNano("950737950171172051122527404032") |
-+---------------------+-------+----------------------------------------------------------------------------+
-| 1970-01-01T00:00:00 | host1 | 0                                                                          |
-| 1970-12-27T00:00:00 | host1 | 2                                                                          |
-| 1970-01-01T00:00:00 | host2 | 4                                                                          |
-| 1970-12-27T00:00:00 | host2 | 6                                                                          |
-+---------------------+-------+----------------------------------------------------------------------------+
+Error: 3000(PlanQuery), DataFusion error: Error during planning: Year or month interval is not allowed in range query: IntervalMonthDayNano("950737950171172051122527404032")
 
-SELECT ts, host, min(val) RANGE (INTERVAL '1' year) FROM host ALIGN (INTERVAL '1' year) ORDER BY host, ts;
+SELECT ts, host, min(val) RANGE (INTERVAL '1' day) FROM host ALIGN (INTERVAL '1' day) ORDER BY host, ts;
 
-+---------------------+-------+----------------------------------------------------------------------------+
-| ts                  | host  | MIN(host.val) RANGE IntervalMonthDayNano("950737950171172051122527404032") |
-+---------------------+-------+----------------------------------------------------------------------------+
-| 1970-01-01T00:00:00 | host1 | 0                                                                          |
-| 1970-12-27T00:00:00 | host1 | 2                                                                          |
-| 1970-01-01T00:00:00 | host2 | 4                                                                          |
-| 1970-12-27T00:00:00 | host2 | 6                                                                          |
-+---------------------+-------+----------------------------------------------------------------------------+
++---------------------+-------+------------------------------------------------------------------+
+| ts                  | host  | MIN(host.val) RANGE IntervalMonthDayNano("18446744073709551616") |
++---------------------+-------+------------------------------------------------------------------+
+| 1970-01-01T00:00:00 | host1 | 0                                                                |
+| 1971-01-02T00:00:00 | host1 | 2                                                                |
+| 1970-01-01T00:00:00 | host2 | 4                                                                |
+| 1971-01-02T00:00:00 | host2 | 6                                                                |
++---------------------+-------+------------------------------------------------------------------+
 
 DROP TABLE host;
 
diff --git a/tests/cases/standalone/common/range/interval.sql b/tests/cases/standalone/common/range/interval.sql
index 3e17ecae58..3c5bae3ed1 100644
--- a/tests/cases/standalone/common/range/interval.sql
+++ b/tests/cases/standalone/common/range/interval.sql
@@ -16,6 +16,6 @@ INSERT INTO TABLE host VALUES
 
 SELECT ts, host, min(val) RANGE (INTERVAL '1 year') FROM host ALIGN (INTERVAL '1 year') ORDER BY host, ts;
 
-SELECT ts, host, min(val) RANGE (INTERVAL '1' year) FROM host ALIGN (INTERVAL '1' year) ORDER BY host, ts;
+SELECT ts, host, min(val) RANGE (INTERVAL '1' day) FROM host ALIGN (INTERVAL '1' day) ORDER BY host, ts;
 
 DROP TABLE host;

From a889ea88ca595c184c14a3c65fe64027d9ed23a6 Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Mon, 14 Oct 2024 11:18:59 +0800
Subject: [PATCH 073/128] fix: case sensitive for __field__ matcher (#4822)

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 src/query/src/promql/planner.rs               |  3 ++-
 .../cases/standalone/common/tql/basic.result  | 27 +++++++++++++++++++
 tests/cases/standalone/common/tql/basic.sql   | 10 +++++++
 3 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs
index ad00c900c8..bafd69aa95 100644
--- a/src/query/src/promql/planner.rs
+++ b/src/query/src/promql/planner.rs
@@ -805,10 +805,11 @@ impl PromPlanner {
 
             let exprs = result_set
                 .into_iter()
-                .map(|col| DfExpr::Column(col.into()))
+                .map(|col| DfExpr::Column(Column::new_unqualified(col)))
                 .chain(self.create_tag_column_exprs()?)
                 .chain(Some(self.create_time_index_column_expr()?))
                 .collect::<Vec<_>>();
+
             // reuse this variable for simplicity
             table_scan = LogicalPlanBuilder::from(table_scan)
                 .project(exprs)
diff --git a/tests/cases/standalone/common/tql/basic.result b/tests/cases/standalone/common/tql/basic.result
index f38055ea1a..5c6725dbcd 100644
--- a/tests/cases/standalone/common/tql/basic.result
+++ b/tests/cases/standalone/common/tql/basic.result
@@ -123,3 +123,30 @@ DROP TABLE test;
 
 Affected Rows: 0
 
+CREATE TABLE test (`Field_I` DOUBLE, `Ts_J` TIMESTAMP TIME INDEX, `Tag_K` STRING PRIMARY KEY);
+
+Affected Rows: 0
+
+INSERT INTO test VALUES (1, 1, "a"), (1, 1, "b"), (2, 2, "a");
+
+Affected Rows: 3
+
+TQL EVAL (0, 10, '5s') test{__field__="Field_I"};
+
++---------+-------+---------------------+
+| Field_I | Tag_K | Ts_J                |
++---------+-------+---------------------+
+| 2.0     | a     | 1970-01-01T00:00:05 |
+| 2.0     | a     | 1970-01-01T00:00:10 |
+| 1.0     | b     | 1970-01-01T00:00:05 |
+| 1.0     | b     | 1970-01-01T00:00:10 |
++---------+-------+---------------------+
+
+TQL EVAL (0, 10, '5s') test{__field__="field_i"};
+
+Error: 1004(InvalidArguments), Cannot find column field_i
+
+drop table test;
+
+Affected Rows: 0
+
diff --git a/tests/cases/standalone/common/tql/basic.sql b/tests/cases/standalone/common/tql/basic.sql
index 3cd8da8fcf..85f2948148 100644
--- a/tests/cases/standalone/common/tql/basic.sql
+++ b/tests/cases/standalone/common/tql/basic.sql
@@ -32,3 +32,13 @@ TQL EVAL ('1970-01-01T00:00:00'::timestamp, '1970-01-01T00:00:00'::timestamp + '
 TQL EVAL (now() - now(), now() -  (now() - '10 seconds'::interval), '1s')  test{k="a"};
 
 DROP TABLE test;
+
+CREATE TABLE test (`Field_I` DOUBLE, `Ts_J` TIMESTAMP TIME INDEX, `Tag_K` STRING PRIMARY KEY);
+
+INSERT INTO test VALUES (1, 1, "a"), (1, 1, "b"), (2, 2, "a");
+
+TQL EVAL (0, 10, '5s') test{__field__="Field_I"};
+
+TQL EVAL (0, 10, '5s') test{__field__="field_i"};
+
+drop table test;

From 3b2ce31a19e097ec727e2c59c28de93356800e63 Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Mon, 14 Oct 2024 11:32:47 +0800
Subject: [PATCH 074/128] feat: enable prof features by default (#4815)

* feat: enable prof by default

* docs: don't need to build with features

* feat: add common-pprof as optional dep for pprof feature

* build: remove optional

* feat: use dump_text
---
 Cargo.lock                           | 13 ++++
 Cargo.toml                           |  2 +
 docs/how-to/how-to-profile-cpu.md    |  6 --
 docs/how-to/how-to-profile-memory.md |  6 --
 src/cmd/Cargo.toml                   |  2 +-
 src/common/pprof/Cargo.toml          | 22 +++++++
 src/common/pprof/src/lib.rs          | 99 ++++++++++++++++++++++++++++
 src/common/pprof/src/nix.rs          | 78 ++++++++++++++++++++++
 src/servers/Cargo.toml               | 10 +--
 src/servers/src/error.rs             |  4 +-
 src/servers/src/http/pprof.rs        |  9 +--
 11 files changed, 222 insertions(+), 29 deletions(-)
 create mode 100644 src/common/pprof/Cargo.toml
 create mode 100644 src/common/pprof/src/lib.rs
 create mode 100644 src/common/pprof/src/nix.rs

diff --git a/Cargo.lock b/Cargo.lock
index c2356a4c2e..eee5d4ee5d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2214,6 +2214,18 @@ dependencies = [
 name = "common-plugins"
 version = "0.9.3"
 
+[[package]]
+name = "common-pprof"
+version = "0.9.3"
+dependencies = [
+ "common-error",
+ "common-macro",
+ "pprof",
+ "prost 0.12.6",
+ "snafu 0.8.5",
+ "tokio",
+]
+
 [[package]]
 name = "common-procedure"
 version = "0.9.3"
@@ -10705,6 +10717,7 @@ dependencies = [
  "common-mem-prof",
  "common-meta",
  "common-plugins",
+ "common-pprof",
  "common-query",
  "common-recordbatch",
  "common-runtime",
diff --git a/Cargo.toml b/Cargo.toml
index 63d7ad3ba7..72ad968ca7 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -20,6 +20,7 @@ members = [
     "src/common/mem-prof",
     "src/common/meta",
     "src/common/plugins",
+    "src/common/pprof",
     "src/common/procedure",
     "src/common/procedure-test",
     "src/common/query",
@@ -208,6 +209,7 @@ common-macro = { path = "src/common/macro" }
 common-mem-prof = { path = "src/common/mem-prof" }
 common-meta = { path = "src/common/meta" }
 common-plugins = { path = "src/common/plugins" }
+common-pprof = { path = "src/common/pprof" }
 common-procedure = { path = "src/common/procedure" }
 common-procedure-test = { path = "src/common/procedure-test" }
 common-query = { path = "src/common/query" }
diff --git a/docs/how-to/how-to-profile-cpu.md b/docs/how-to/how-to-profile-cpu.md
index b73c85ea2f..b1c5ded09e 100644
--- a/docs/how-to/how-to-profile-cpu.md
+++ b/docs/how-to/how-to-profile-cpu.md
@@ -1,11 +1,5 @@
 # Profiling CPU
 
-## Build GreptimeDB with `pprof` feature
-
-```bash
-cargo build --features=pprof
-```
-
 ## HTTP API
 Sample at 99 Hertz, for 5 seconds, output report in [protobuf format](https://github.com/google/pprof/blob/master/proto/profile.proto).
 ```bash
diff --git a/docs/how-to/how-to-profile-memory.md b/docs/how-to/how-to-profile-memory.md
index 7211683190..a0fe42df55 100644
--- a/docs/how-to/how-to-profile-memory.md
+++ b/docs/how-to/how-to-profile-memory.md
@@ -18,12 +18,6 @@ sudo apt install libjemalloc-dev
 curl https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl > ./flamegraph.pl
 ```
 
-### Build GreptimeDB with `mem-prof` feature.
-
-```bash
-cargo build --features=mem-prof
-```
-
 ## Profiling
 
 Start GreptimeDB instance with environment variables:
diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml
index b57d221187..501f40a047 100644
--- a/src/cmd/Cargo.toml
+++ b/src/cmd/Cargo.toml
@@ -10,7 +10,7 @@ name = "greptime"
 path = "src/bin/greptime.rs"
 
 [features]
-default = ["python"]
+default = ["python", "servers/pprof", "servers/mem-prof"]
 tokio-console = ["common-telemetry/tokio-console"]
 python = ["frontend/python"]
 
diff --git a/src/common/pprof/Cargo.toml b/src/common/pprof/Cargo.toml
new file mode 100644
index 0000000000..1657244d21
--- /dev/null
+++ b/src/common/pprof/Cargo.toml
@@ -0,0 +1,22 @@
+[package]
+name = "common-pprof"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+common-error.workspace = true
+common-macro.workspace = true
+prost.workspace = true
+snafu.workspace = true
+tokio.workspace = true
+
+[target.'cfg(unix)'.dependencies]
+pprof = { version = "0.13", features = [
+    "flamegraph",
+    "prost-codec",
+    "protobuf",
+] }
+
+[lints]
+workspace = true
diff --git a/src/common/pprof/src/lib.rs b/src/common/pprof/src/lib.rs
new file mode 100644
index 0000000000..71bcda6559
--- /dev/null
+++ b/src/common/pprof/src/lib.rs
@@ -0,0 +1,99 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#[cfg(unix)]
+pub mod nix;
+
+pub mod error {
+    use std::any::Any;
+
+    use common_error::ext::ErrorExt;
+    use common_error::status_code::StatusCode;
+    use common_macro::stack_trace_debug;
+    use snafu::{Location, Snafu};
+
+    #[derive(Snafu)]
+    #[stack_trace_debug]
+    #[snafu(visibility(pub(crate)))]
+    pub enum Error {
+        #[cfg(unix)]
+        #[snafu(display("Pprof error"))]
+        Pprof {
+            #[snafu(source)]
+            error: pprof::Error,
+            #[snafu(implicit)]
+            location: Location,
+        },
+
+        #[snafu(display("Pprof is unsupported on this platform"))]
+        Unsupported {
+            #[snafu(implicit)]
+            location: Location,
+        },
+    }
+
+    pub type Result<T> = std::result::Result<T, Error>;
+
+    impl ErrorExt for Error {
+        fn status_code(&self) -> StatusCode {
+            match self {
+                #[cfg(unix)]
+                Error::Pprof { .. } => StatusCode::Unexpected,
+                Error::Unsupported { .. } => StatusCode::Unsupported,
+            }
+        }
+
+        fn as_any(&self) -> &dyn Any {
+            self
+        }
+    }
+}
+
+#[cfg(not(unix))]
+pub mod dummy {
+    use std::time::Duration;
+
+    use crate::error::{Result, UnsupportedSnafu};
+
+    /// Dummpy CPU profiler utility.
+    #[derive(Debug)]
+    pub struct Profiling {}
+
+    impl Profiling {
+        /// Creates a new profiler.
+        pub fn new(_duration: Duration, _frequency: i32) -> Profiling {
+            Profiling {}
+        }
+
+        /// Profiles and returns a generated text.
+        pub async fn dump_text(&self) -> Result<String> {
+            UnsupportedSnafu {}.fail()
+        }
+
+        /// Profiles and returns a generated flamegraph.
+        pub async fn dump_flamegraph(&self) -> Result<Vec<u8>> {
+            UnsupportedSnafu {}.fail()
+        }
+
+        /// Profiles and returns a generated proto.
+        pub async fn dump_proto(&self) -> Result<Vec<u8>> {
+            UnsupportedSnafu {}.fail()
+        }
+    }
+}
+
+#[cfg(not(unix))]
+pub use dummy::Profiling;
+#[cfg(unix)]
+pub use nix::Profiling;
diff --git a/src/common/pprof/src/nix.rs b/src/common/pprof/src/nix.rs
new file mode 100644
index 0000000000..bd76f8fb31
--- /dev/null
+++ b/src/common/pprof/src/nix.rs
@@ -0,0 +1,78 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Duration;
+
+use pprof::protos::Message;
+use snafu::ResultExt;
+
+use crate::error::{PprofSnafu, Result};
+
+/// CPU profiler utility.
+// Inspired by https://github.com/datafuselabs/databend/blob/67f445e83cd4eceda98f6c1c114858929d564029/src/common/base/src/base/profiling.rs
+#[derive(Debug)]
+pub struct Profiling {
+    /// Sample duration.
+    duration: Duration,
+    /// Sample frequency.
+    frequency: i32,
+}
+
+impl Profiling {
+    /// Creates a new profiler.
+    pub fn new(duration: Duration, frequency: i32) -> Profiling {
+        Profiling {
+            duration,
+            frequency,
+        }
+    }
+
+    /// Profiles and returns a generated pprof report.
+    pub async fn report(&self) -> Result<pprof::Report> {
+        let guard = pprof::ProfilerGuardBuilder::default()
+            .frequency(self.frequency)
+            .blocklist(&["libc", "libgcc", "pthread", "vdso"])
+            .build()
+            .context(PprofSnafu)?;
+        tokio::time::sleep(self.duration).await;
+        guard.report().build().context(PprofSnafu)
+    }
+
+    /// Profiles and returns a generated text.
+    pub async fn dump_text(&self) -> Result<String> {
+        let report = self.report().await?;
+        let text = format!("{report:?}");
+        Ok(text)
+    }
+
+    /// Profiles and returns a generated flamegraph.
+    pub async fn dump_flamegraph(&self) -> Result<Vec<u8>> {
+        let mut body: Vec<u8> = Vec::new();
+
+        let report = self.report().await?;
+        report.flamegraph(&mut body).context(PprofSnafu)?;
+
+        Ok(body)
+    }
+
+    /// Profiles and returns a generated proto.
+    pub async fn dump_proto(&self) -> Result<Vec<u8>> {
+        let report = self.report().await?;
+        // Generate google’s pprof format report.
+        let profile = report.pprof().context(PprofSnafu)?;
+        let body = profile.encode_to_vec();
+
+        Ok(body)
+    }
+}
diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml
index 725ff497a4..df02a44855 100644
--- a/src/servers/Cargo.toml
+++ b/src/servers/Cargo.toml
@@ -7,7 +7,7 @@ license.workspace = true
 [features]
 dashboard = []
 mem-prof = ["dep:common-mem-prof"]
-pprof = ["dep:pprof"]
+pprof = ["dep:common-pprof"]
 testing = []
 
 [lints]
@@ -37,6 +37,7 @@ common-macro.workspace = true
 common-mem-prof = { workspace = true, optional = true }
 common-meta.workspace = true
 common-plugins.workspace = true
+common-pprof = { workspace = true, optional = true }
 common-query.workspace = true
 common-recordbatch.workspace = true
 common-runtime.workspace = true
@@ -75,11 +76,6 @@ pgwire = { version = "0.25.0", default-features = false, features = ["server-api
 pin-project = "1.0"
 pipeline.workspace = true
 postgres-types = { version = "0.2", features = ["with-chrono-0_4", "with-serde_json-1"] }
-pprof = { version = "0.13", features = [
-    "flamegraph",
-    "prost-codec",
-    "protobuf",
-], optional = true }
 prometheus.workspace = true
 promql-parser.workspace = true
 prost.workspace = true
@@ -136,7 +132,7 @@ tokio-postgres = "0.7"
 tokio-postgres-rustls = "0.12"
 tokio-test = "0.4"
 
-[target.'cfg(not(windows))'.dev-dependencies]
+[target.'cfg(unix)'.dev-dependencies]
 pprof = { version = "0.13", features = ["criterion", "flamegraph"] }
 
 [target.'cfg(windows)'.dependencies]
diff --git a/src/servers/src/error.rs b/src/servers/src/error.rs
index 0fde3b527c..a796b89521 100644
--- a/src/servers/src/error.rs
+++ b/src/servers/src/error.rs
@@ -424,9 +424,7 @@ pub enum Error {
 
     #[cfg(feature = "pprof")]
     #[snafu(display("Failed to dump pprof data"))]
-    DumpPprof {
-        source: crate::http::pprof::nix::Error,
-    },
+    DumpPprof { source: common_pprof::error::Error },
 
     #[cfg(not(windows))]
     #[snafu(display("Failed to update jemalloc metrics"))]
diff --git a/src/servers/src/http/pprof.rs b/src/servers/src/http/pprof.rs
index 12479444db..c00d160aaf 100644
--- a/src/servers/src/http/pprof.rs
+++ b/src/servers/src/http/pprof.rs
@@ -12,9 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(feature = "pprof")]
-pub(crate) mod nix;
-
 #[cfg(feature = "pprof")]
 pub mod handler {
     use std::num::NonZeroI32;
@@ -23,13 +20,13 @@ pub mod handler {
     use axum::extract::Query;
     use axum::http::StatusCode;
     use axum::response::IntoResponse;
+    use common_pprof::Profiling;
     use common_telemetry::info;
     use schemars::JsonSchema;
     use serde::{Deserialize, Serialize};
     use snafu::ResultExt;
 
     use crate::error::{DumpPprofSnafu, Result};
-    use crate::http::pprof::nix::Profiling;
 
     /// Output format.
     #[derive(Debug, Serialize, Deserialize, JsonSchema)]
@@ -70,8 +67,8 @@ pub mod handler {
         let body = match req.output {
             Output::Proto => profiling.dump_proto().await.context(DumpPprofSnafu)?,
             Output::Text => {
-                let report = profiling.report().await.context(DumpPprofSnafu)?;
-                format!("{:?}", report).into_bytes()
+                let report = profiling.dump_text().await.context(DumpPprofSnafu)?;
+                report.into_bytes()
             }
             Output::Flamegraph => profiling.dump_flamegraph().await.context(DumpPprofSnafu)?,
         };

From 06e565d25a8232ee693d6d793951fe72df5de5f7 Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Mon, 14 Oct 2024 16:44:13 +0800
Subject: [PATCH 075/128] feat: cache logical region's metadata (#4827)

* feat: cache logical region's metadata

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* feat: implement logical region locking for metadata operations

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix: correct typo in comment for MetadataRegion struct

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 src/metric-engine/src/engine/alter.rs         | 32 ++++++---
 src/metric-engine/src/engine/open.rs          | 29 +++++---
 src/metric-engine/src/engine/read.rs          | 10 ++-
 .../src/engine/region_metadata.rs             | 61 +++++++++++++++--
 src/metric-engine/src/engine/state.rs         | 30 ++++++++
 src/metric-engine/src/metadata_region.rs      | 68 +++++++++++++++++--
 6 files changed, 198 insertions(+), 32 deletions(-)

diff --git a/src/metric-engine/src/engine/alter.rs b/src/metric-engine/src/engine/alter.rs
index c99942dec6..76066ab97a 100644
--- a/src/metric-engine/src/engine/alter.rs
+++ b/src/metric-engine/src/engine/alter.rs
@@ -64,15 +64,19 @@ impl MetricEngineInner {
     /// Return the physical region id behind this logical region
     async fn alter_logical_region(
         &self,
-        region_id: RegionId,
+        logical_region_id: RegionId,
         request: RegionAlterRequest,
     ) -> Result<RegionId> {
         let physical_region_id = {
             let state = &self.state.read().unwrap();
-            state.get_physical_region_id(region_id).with_context(|| {
-                error!("Trying to alter an nonexistent region {region_id}");
-                LogicalRegionNotFoundSnafu { region_id }
-            })?
+            state
+                .get_physical_region_id(logical_region_id)
+                .with_context(|| {
+                    error!("Trying to alter an nonexistent region {logical_region_id}");
+                    LogicalRegionNotFoundSnafu {
+                        region_id: logical_region_id,
+                    }
+                })?
         };
 
         // only handle adding column
@@ -80,6 +84,12 @@ impl MetricEngineInner {
             return Ok(physical_region_id);
         };
 
+        // lock metadata region for this logical region id
+        let _write_guard = self
+            .metadata_region
+            .write_lock_logical_region(logical_region_id)
+            .await;
+
         let metadata_region_id = to_metadata_region_id(physical_region_id);
         let mut columns_to_add = vec![];
         for col in &columns {
@@ -87,7 +97,7 @@ impl MetricEngineInner {
                 .metadata_region
                 .column_semantic_type(
                     metadata_region_id,
-                    region_id,
+                    logical_region_id,
                     &col.column_metadata.column_schema.name,
                 )
                 .await?
@@ -102,7 +112,7 @@ impl MetricEngineInner {
         self.add_columns_to_physical_data_region(
             data_region_id,
             metadata_region_id,
-            region_id,
+            logical_region_id,
             columns_to_add,
         )
         .await?;
@@ -110,10 +120,16 @@ impl MetricEngineInner {
         // register columns to logical region
         for col in columns {
             self.metadata_region
-                .add_column(metadata_region_id, region_id, &col.column_metadata)
+                .add_column(metadata_region_id, logical_region_id, &col.column_metadata)
                 .await?;
         }
 
+        // invalid logical column cache
+        self.state
+            .write()
+            .unwrap()
+            .invalid_logical_column_cache(logical_region_id);
+
         Ok(physical_region_id)
     }
 
diff --git a/src/metric-engine/src/engine/open.rs b/src/metric-engine/src/engine/open.rs
index 815ffc1143..97b049e01d 100644
--- a/src/metric-engine/src/engine/open.rs
+++ b/src/metric-engine/src/engine/open.rs
@@ -134,17 +134,26 @@ impl MetricEngineInner {
             .await?;
         let logical_region_num = logical_regions.len();
 
-        let mut state = self.state.write().unwrap();
-        // recover physical column names
-        let physical_column_names = physical_columns
-            .into_iter()
-            .map(|col| col.column_schema.name)
-            .collect();
-        state.add_physical_region(physical_region_id, physical_column_names);
-        // recover logical regions
-        for logical_region_id in logical_regions {
-            state.add_logical_region(physical_region_id, logical_region_id);
+        {
+            let mut state = self.state.write().unwrap();
+            // recover physical column names
+            let physical_column_names = physical_columns
+                .into_iter()
+                .map(|col| col.column_schema.name)
+                .collect();
+            state.add_physical_region(physical_region_id, physical_column_names);
+            // recover logical regions
+            for logical_region_id in &logical_regions {
+                state.add_logical_region(physical_region_id, *logical_region_id);
+            }
         }
+
+        for logical_region_id in logical_regions {
+            self.metadata_region
+                .open_logical_region(logical_region_id)
+                .await;
+        }
+
         LOGICAL_REGION_COUNT.add(logical_region_num as i64);
 
         Ok(())
diff --git a/src/metric-engine/src/engine/read.rs b/src/metric-engine/src/engine/read.rs
index af01e1bb9e..2ba4c213c6 100644
--- a/src/metric-engine/src/engine/read.rs
+++ b/src/metric-engine/src/engine/read.rs
@@ -169,11 +169,11 @@ impl MetricEngineInner {
     ) -> Result<Vec<usize>> {
         // project on logical columns
         let all_logical_columns = self
-            .load_logical_columns(physical_region_id, logical_region_id)
+            .load_logical_column_names(physical_region_id, logical_region_id)
             .await?;
         let projected_logical_names = origin_projection
             .iter()
-            .map(|i| all_logical_columns[*i].column_schema.name.clone())
+            .map(|i| all_logical_columns[*i].clone())
             .collect::<Vec<_>>();
 
         // generate physical projection
@@ -200,10 +200,8 @@ impl MetricEngineInner {
         logical_region_id: RegionId,
     ) -> Result<Vec<usize>> {
         let logical_columns = self
-            .load_logical_columns(physical_region_id, logical_region_id)
-            .await?
-            .into_iter()
-            .map(|col| col.column_schema.name);
+            .load_logical_column_names(physical_region_id, logical_region_id)
+            .await?;
         let mut projection = Vec::with_capacity(logical_columns.len());
         let data_region_id = utils::to_data_region_id(physical_region_id);
         let physical_metadata = self
diff --git a/src/metric-engine/src/engine/region_metadata.rs b/src/metric-engine/src/engine/region_metadata.rs
index 1fa33ec39d..171480c589 100644
--- a/src/metric-engine/src/engine/region_metadata.rs
+++ b/src/metric-engine/src/engine/region_metadata.rs
@@ -23,13 +23,29 @@ use crate::error::Result;
 impl MetricEngineInner {
     /// Load column metadata of a logical region.
     ///
-    /// The return value is ordered on [ColumnId].
+    /// The return value is ordered on column name.
     pub async fn load_logical_columns(
         &self,
         physical_region_id: RegionId,
         logical_region_id: RegionId,
     ) -> Result<Vec<ColumnMetadata>> {
-        // load logical and physical columns, and intersect them to get logical column metadata
+        // First try to load from state cache
+        if let Some(columns) = self
+            .state
+            .read()
+            .unwrap()
+            .logical_columns()
+            .get(&logical_region_id)
+        {
+            return Ok(columns.clone());
+        }
+
+        // Else load from metadata region and update the cache.
+        let _read_guard = self
+            .metadata_region
+            .read_lock_logical_region(logical_region_id)
+            .await;
+        // Load logical and physical columns, and intersect them to get logical column metadata.
         let mut logical_column_metadata = self
             .metadata_region
             .logical_columns(physical_region_id, logical_region_id)
@@ -37,11 +53,48 @@ impl MetricEngineInner {
             .into_iter()
             .map(|(_, column_metadata)| column_metadata)
             .collect::<Vec<_>>();
-
-        // sort columns on column id to ensure the order
+        // Sort columns on column name to ensure the order
         logical_column_metadata
             .sort_unstable_by(|c1, c2| c1.column_schema.name.cmp(&c2.column_schema.name));
+        // Update cache
+        self.state
+            .write()
+            .unwrap()
+            .add_logical_columns(logical_region_id, logical_column_metadata.clone());
 
         Ok(logical_column_metadata)
     }
+
+    /// Load logical column names of a logical region.
+    ///
+    /// The return value is ordered on column name alphabetically.
+    pub async fn load_logical_column_names(
+        &self,
+        physical_region_id: RegionId,
+        logical_region_id: RegionId,
+    ) -> Result<Vec<String>> {
+        // First try to load from state cache
+        if let Some(columns) = self
+            .state
+            .read()
+            .unwrap()
+            .logical_columns()
+            .get(&logical_region_id)
+        {
+            return Ok(columns
+                .iter()
+                .map(|c| c.column_schema.name.clone())
+                .collect());
+        }
+
+        // Else load from metadata region
+        let columns = self
+            .load_logical_columns(physical_region_id, logical_region_id)
+            .await?
+            .into_iter()
+            .map(|c| c.column_schema.name)
+            .collect::<Vec<_>>();
+
+        Ok(columns)
+    }
 }
diff --git a/src/metric-engine/src/engine/state.rs b/src/metric-engine/src/engine/state.rs
index 5214ae1fbf..24ab5a31bf 100644
--- a/src/metric-engine/src/engine/state.rs
+++ b/src/metric-engine/src/engine/state.rs
@@ -17,6 +17,7 @@
 use std::collections::{HashMap, HashSet};
 
 use snafu::OptionExt;
+use store_api::metadata::ColumnMetadata;
 use store_api::storage::RegionId;
 
 use crate::error::{PhysicalRegionNotFoundSnafu, Result};
@@ -35,6 +36,10 @@ pub(crate) struct MetricEngineState {
     /// Cache for the columns of physical regions.
     /// The region id in key is the data region id.
     physical_columns: HashMap<RegionId, HashSet<String>>,
+    /// Cache for the column metadata of logical regions.
+    /// The column order is the same with the order in the metadata, which is
+    /// alphabetically ordered on column name.
+    logical_columns: HashMap<RegionId, Vec<ColumnMetadata>>,
 }
 
 impl MetricEngineState {
@@ -80,6 +85,21 @@ impl MetricEngineState {
             .insert(logical_region_id, physical_region_id);
     }
 
+    /// Add and reorder logical columns.
+    ///
+    /// Caller should make sure:
+    /// 1. there is no duplicate columns
+    /// 2. the column order is the same with the order in the metadata, which is
+    ///    alphabetically ordered on column name.
+    pub fn add_logical_columns(
+        &mut self,
+        logical_region_id: RegionId,
+        new_columns: impl IntoIterator<Item = ColumnMetadata>,
+    ) {
+        let columns = self.logical_columns.entry(logical_region_id).or_default();
+        columns.extend(new_columns);
+    }
+
     pub fn get_physical_region_id(&self, logical_region_id: RegionId) -> Option<RegionId> {
         self.logical_regions.get(&logical_region_id).copied()
     }
@@ -88,6 +108,10 @@ impl MetricEngineState {
         &self.physical_columns
     }
 
+    pub fn logical_columns(&self) -> &HashMap<RegionId, Vec<ColumnMetadata>> {
+        &self.logical_columns
+    }
+
     pub fn physical_regions(&self) -> &HashMap<RegionId, HashSet<RegionId>> {
         &self.physical_regions
     }
@@ -129,9 +153,15 @@ impl MetricEngineState {
             .unwrap() // Safety: physical_region_id is got from physical_regions
             .remove(&logical_region_id);
 
+        self.logical_columns.remove(&logical_region_id);
+
         Ok(())
     }
 
+    pub fn invalid_logical_column_cache(&mut self, logical_region_id: RegionId) {
+        self.logical_columns.remove(&logical_region_id);
+    }
+
     pub fn is_logical_region_exist(&self, logical_region_id: RegionId) -> bool {
         self.logical_regions().contains_key(&logical_region_id)
     }
diff --git a/src/metric-engine/src/metadata_region.rs b/src/metric-engine/src/metadata_region.rs
index 8e892e7ba7..b02fa3de51 100644
--- a/src/metric-engine/src/metadata_region.rs
+++ b/src/metric-engine/src/metadata_region.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::collections::HashMap;
+use std::sync::Arc;
 
 use api::v1::value::ValueData;
 use api::v1::{ColumnDataType, ColumnSchema, Row, Rows, SemanticType, Value};
@@ -21,7 +22,7 @@ use base64::Engine;
 use common_recordbatch::util::collect;
 use datafusion::prelude::{col, lit};
 use mito2::engine::MitoEngine;
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
 use store_api::metadata::ColumnMetadata;
 use store_api::metric_engine_consts::{
     METADATA_SCHEMA_KEY_COLUMN_INDEX, METADATA_SCHEMA_KEY_COLUMN_NAME,
@@ -31,11 +32,12 @@ use store_api::metric_engine_consts::{
 use store_api::region_engine::RegionEngine;
 use store_api::region_request::{RegionDeleteRequest, RegionPutRequest};
 use store_api::storage::{RegionId, ScanRequest};
+use tokio::sync::{OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock};
 
 use crate::error::{
     CollectRecordBatchStreamSnafu, DecodeColumnValueSnafu, DeserializeColumnMetadataSnafu,
-    MitoReadOperationSnafu, MitoWriteOperationSnafu, ParseRegionIdSnafu, RegionAlreadyExistsSnafu,
-    Result,
+    LogicalRegionNotFoundSnafu, MitoReadOperationSnafu, MitoWriteOperationSnafu,
+    ParseRegionIdSnafu, RegionAlreadyExistsSnafu, Result,
 };
 use crate::utils;
 
@@ -56,11 +58,19 @@ const COLUMN_PREFIX: &str = "__column_";
 /// itself.
 pub struct MetadataRegion {
     mito: MitoEngine,
+    /// Logical lock for operations that need to be serialized. Like update & read region columns.
+    ///
+    /// Region entry will be registered on creating and opening logical region, and deregistered on
+    /// removing logical region.
+    logical_region_lock: RwLock<HashMap<RegionId, Arc<RwLock<()>>>>,
 }
 
 impl MetadataRegion {
     pub fn new(mito: MitoEngine) -> Self {
-        Self { mito }
+        Self {
+            mito,
+            logical_region_lock: RwLock::new(HashMap::new()),
+        }
     }
 
     /// Add a new table key to metadata.
@@ -85,10 +95,21 @@ impl MetadataRegion {
             }
             .fail()
         } else {
+            self.logical_region_lock
+                .write()
+                .await
+                .insert(logical_region_id, Arc::new(RwLock::new(())));
             Ok(())
         }
     }
 
+    pub async fn open_logical_region(&self, logical_region_id: RegionId) {
+        self.logical_region_lock
+            .write()
+            .await
+            .insert(logical_region_id, Arc::new(RwLock::new(())));
+    }
+
     /// Add a new column key to metadata.
     ///
     /// This method won't check if the column already exists. But
@@ -111,6 +132,40 @@ impl MetadataRegion {
         .await
     }
 
+    /// Retrieve a read lock guard of given logical region id.
+    pub async fn read_lock_logical_region(
+        &self,
+        logical_region_id: RegionId,
+    ) -> Result<OwnedRwLockReadGuard<()>> {
+        let lock = self
+            .logical_region_lock
+            .read()
+            .await
+            .get(&logical_region_id)
+            .context(LogicalRegionNotFoundSnafu {
+                region_id: logical_region_id,
+            })?
+            .clone();
+        Ok(RwLock::read_owned(lock).await)
+    }
+
+    /// Retrieve a write lock guard of given logical region id.
+    pub async fn write_lock_logical_region(
+        &self,
+        logical_region_id: RegionId,
+    ) -> Result<OwnedRwLockWriteGuard<()>> {
+        let lock = self
+            .logical_region_lock
+            .read()
+            .await
+            .get(&logical_region_id)
+            .context(LogicalRegionNotFoundSnafu {
+                region_id: logical_region_id,
+            })?
+            .clone();
+        Ok(RwLock::write_owned(lock).await)
+    }
+
     /// Remove a registered logical region from metadata.
     ///
     /// This method doesn't check if the previous key exists.
@@ -136,6 +191,11 @@ impl MetadataRegion {
         column_keys.push(region_key);
         self.delete(region_id, &column_keys).await?;
 
+        self.logical_region_lock
+            .write()
+            .await
+            .remove(&logical_region_id);
+
         Ok(())
     }
 

From b61a388d046dc752d313a58a48e6f8352eca4597 Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Mon, 14 Oct 2024 20:46:07 +0800
Subject: [PATCH 076/128] refactor: replace info logs with debug logs in region
 server (#4829)

* refactor: replace info logs with debug logs in region server

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix: update error handling for closing and opening nonexistent regions

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 src/datanode/src/region_server.rs     | 8 +++++---
 src/metric-engine/src/engine.rs       | 4 ++--
 src/metric-engine/src/engine/close.rs | 6 ++++--
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs
index 0bac53e4d6..b6e1eb0404 100644
--- a/src/datanode/src/region_server.rs
+++ b/src/datanode/src/region_server.rs
@@ -32,7 +32,7 @@ use common_recordbatch::SendableRecordBatchStream;
 use common_runtime::Runtime;
 use common_telemetry::tracing::{self, info_span};
 use common_telemetry::tracing_context::{FutureExt, TracingContext};
-use common_telemetry::{error, info, warn};
+use common_telemetry::{debug, error, info, warn};
 use dashmap::DashMap;
 use datafusion::datasource::{provider_as_source, TableProvider};
 use datafusion::error::Result as DfResult;
@@ -893,7 +893,7 @@ impl RegionServerInner {
         for region in logical_regions {
             self.region_map
                 .insert(region, RegionEngineWithStatus::Ready(engine.clone()));
-            info!("Logical region {} is registered!", region);
+            debug!("Logical region {} is registered!", region);
         }
         Ok(())
     }
@@ -935,17 +935,19 @@ impl RegionServerInner {
             .iter()
             .map(|x| (*x.key(), x.value().clone()))
             .collect::<Vec<_>>();
+        let num_regions = regions.len();
 
         for (region_id, engine) in regions {
             let closed = engine
                 .handle_request(region_id, RegionRequest::Close(RegionCloseRequest {}))
                 .await;
             match closed {
-                Ok(_) => info!("Region {region_id} is closed"),
+                Ok(_) => debug!("Region {region_id} is closed"),
                 Err(e) => warn!("Failed to close region {region_id}, err: {e}"),
             }
         }
         self.region_map.clear();
+        info!("closed {num_regions} regions");
 
         let engines = self.engines.write().unwrap().drain().collect::<Vec<_>>();
         for (engine_name, engine) in engines {
diff --git a/src/metric-engine/src/engine.rs b/src/metric-engine/src/engine.rs
index 42948aa6cd..ab789db661 100644
--- a/src/metric-engine/src/engine.rs
+++ b/src/metric-engine/src/engine.rs
@@ -338,7 +338,7 @@ mod test {
             .await
             .unwrap();
 
-        // close nonexistent region
+        // close nonexistent region won't report error
         let nonexistent_region_id = RegionId::new(12313, 12);
         engine
             .handle_request(
@@ -346,7 +346,7 @@ mod test {
                 RegionRequest::Close(RegionCloseRequest {}),
             )
             .await
-            .unwrap_err();
+            .unwrap();
 
         // open nonexistent region won't report error
         let invalid_open_request = RegionOpenRequest {
diff --git a/src/metric-engine/src/engine/close.rs b/src/metric-engine/src/engine/close.rs
index ad89fce36a..677f488b73 100644
--- a/src/metric-engine/src/engine/close.rs
+++ b/src/metric-engine/src/engine/close.rs
@@ -14,13 +14,14 @@
 
 //! Close a metric region
 
+use common_telemetry::debug;
 use snafu::ResultExt;
 use store_api::region_engine::RegionEngine;
 use store_api::region_request::{AffectedRows, RegionCloseRequest, RegionRequest};
 use store_api::storage::RegionId;
 
 use super::MetricEngineInner;
-use crate::error::{CloseMitoRegionSnafu, LogicalRegionNotFoundSnafu, Result};
+use crate::error::{CloseMitoRegionSnafu, Result};
 use crate::metrics::PHYSICAL_REGION_COUNT;
 use crate::utils;
 
@@ -54,7 +55,8 @@ impl MetricEngineInner {
         {
             Ok(0)
         } else {
-            Err(LogicalRegionNotFoundSnafu { region_id }.build())
+            debug!("Closing a non-existent logical region {}", region_id);
+            Ok(0)
         }
     }
 

From b5233e500bce55d80887edcb7fe9398d2c627b9e Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Tue, 15 Oct 2024 11:35:31 +0800
Subject: [PATCH 077/128] feat: defer `HeartbeatHandlerGroup` construction  and
 enhance `LeadershipChangeNotifier` (#4826)

* feat: enhance `HeartbeatHandlerGroup`

* chore: apply suggestions from CR

* chore: minor refactoring

* chore: code styling

* chore: apply suggestions from CR
---
 src/cmd/src/metasrv.rs                        |  4 +++
 src/cmd/src/standalone.rs                     |  2 +-
 src/common/meta/src/leadership_notifier.rs    |  9 ++++++-
 src/meta-srv/src/bootstrap.rs                 |  4 +++
 src/meta-srv/src/handler.rs                   | 10 ++++---
 src/meta-srv/src/handler/failure_handler.rs   |  1 +
 .../src/handler/region_lease_handler.rs       |  1 +
 src/meta-srv/src/metasrv.rs                   | 27 ++++++++++++++-----
 src/meta-srv/src/metasrv/builder.rs           | 25 ++++++++---------
 src/meta-srv/src/mocks.rs                     |  2 +-
 src/meta-srv/src/region/supervisor.rs         |  1 +
 src/meta-srv/src/service/heartbeat.rs         |  9 ++++++-
 12 files changed, 68 insertions(+), 27 deletions(-)

diff --git a/src/cmd/src/metasrv.rs b/src/cmd/src/metasrv.rs
index 2389770f52..757bb27341 100644
--- a/src/cmd/src/metasrv.rs
+++ b/src/cmd/src/metasrv.rs
@@ -48,6 +48,10 @@ impl Instance {
             _guard: guard,
         }
     }
+
+    pub fn mut_inner(&mut self) -> &mut MetasrvInstance {
+        &mut self.instance
+    }
 }
 
 #[async_trait]
diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
index d4a0d823c9..8e1185e389 100644
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -653,7 +653,7 @@ impl StartCommand {
     }
 }
 
-struct StandaloneInformationExtension {
+pub struct StandaloneInformationExtension {
     region_server: RegionServer,
     procedure_manager: ProcedureManagerRef,
     start_time_ms: u64,
diff --git a/src/common/meta/src/leadership_notifier.rs b/src/common/meta/src/leadership_notifier.rs
index 4ba65d1adc..0fed50d388 100644
--- a/src/common/meta/src/leadership_notifier.rs
+++ b/src/common/meta/src/leadership_notifier.rs
@@ -15,7 +15,7 @@
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use common_telemetry::error;
+use common_telemetry::{error, info};
 
 use crate::error::Result;
 
@@ -45,6 +45,13 @@ pub struct LeadershipChangeNotifier {
     listeners: Vec<Arc<dyn LeadershipChangeListener>>,
 }
 
+impl LeadershipChangeNotifierCustomizer for LeadershipChangeNotifier {
+    fn customize(&self, notifier: &mut LeadershipChangeNotifier) {
+        info!("Customizing leadership change notifier");
+        notifier.listeners.extend(self.listeners.clone());
+    }
+}
+
 impl LeadershipChangeNotifier {
     /// Adds a listener to the notifier.
     pub fn add_listener(&mut self, listener: Arc<dyn LeadershipChangeListener>) {
diff --git a/src/meta-srv/src/bootstrap.rs b/src/meta-srv/src/bootstrap.rs
index d10b22c610..79773498cf 100644
--- a/src/meta-srv/src/bootstrap.rs
+++ b/src/meta-srv/src/bootstrap.rs
@@ -148,6 +148,10 @@ impl MetasrvInstance {
     pub fn plugins(&self) -> Plugins {
         self.plugins.clone()
     }
+
+    pub fn mut_inner(&mut self) -> &mut Metasrv {
+        &mut self.metasrv
+    }
 }
 
 pub async fn bootstrap_metasrv_with_router(
diff --git a/src/meta-srv/src/handler.rs b/src/meta-srv/src/handler.rs
index b45a1ac7e3..ce75ff5ff1 100644
--- a/src/meta-srv/src/handler.rs
+++ b/src/meta-srv/src/handler.rs
@@ -209,15 +209,16 @@ impl Pushers {
     }
 }
 
+#[derive(Clone)]
 struct NameCachedHandler {
     name: &'static str,
-    handler: Box<dyn HeartbeatHandler>,
+    handler: Arc<dyn HeartbeatHandler>,
 }
 
 impl NameCachedHandler {
     fn new(handler: impl HeartbeatHandler + 'static) -> Self {
         let name = handler.name();
-        let handler = Box::new(handler);
+        let handler = Arc::new(handler);
         Self { name, handler }
     }
 }
@@ -225,7 +226,7 @@ impl NameCachedHandler {
 pub type HeartbeatHandlerGroupRef = Arc<HeartbeatHandlerGroup>;
 
 /// The group of heartbeat handlers.
-#[derive(Default)]
+#[derive(Default, Clone)]
 pub struct HeartbeatHandlerGroup {
     handlers: Vec<NameCachedHandler>,
     pushers: Pushers,
@@ -442,6 +443,7 @@ impl Mailbox for HeartbeatMailbox {
 }
 
 /// The builder to build the group of heartbeat handlers.
+#[derive(Clone)]
 pub struct HeartbeatHandlerGroupBuilder {
     /// The handler to handle region failure.
     region_failure_handler: Option<RegionFailureHandler>,
@@ -540,7 +542,7 @@ impl HeartbeatHandlerGroupBuilder {
         }
 
         Ok(HeartbeatHandlerGroup {
-            handlers: self.handlers.into_iter().collect(),
+            handlers: self.handlers,
             pushers: self.pushers,
         })
     }
diff --git a/src/meta-srv/src/handler/failure_handler.rs b/src/meta-srv/src/handler/failure_handler.rs
index 02f423c4b4..8908881d17 100644
--- a/src/meta-srv/src/handler/failure_handler.rs
+++ b/src/meta-srv/src/handler/failure_handler.rs
@@ -21,6 +21,7 @@ use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
 use crate::metasrv::Context;
 use crate::region::supervisor::{DatanodeHeartbeat, HeartbeatAcceptor, RegionSupervisor};
 
+#[derive(Clone)]
 pub struct RegionFailureHandler {
     heartbeat_acceptor: HeartbeatAcceptor,
 }
diff --git a/src/meta-srv/src/handler/region_lease_handler.rs b/src/meta-srv/src/handler/region_lease_handler.rs
index de491da371..40a554e453 100644
--- a/src/meta-srv/src/handler/region_lease_handler.rs
+++ b/src/meta-srv/src/handler/region_lease_handler.rs
@@ -26,6 +26,7 @@ use crate::metasrv::Context;
 use crate::region::lease_keeper::{RegionLeaseKeeperRef, RenewRegionLeasesResponse};
 use crate::region::RegionLeaseKeeper;
 
+#[derive(Clone)]
 pub struct RegionLeaseHandler {
     region_lease_seconds: u64,
     region_lease_keeper: RegionLeaseKeeperRef,
diff --git a/src/meta-srv/src/metasrv.rs b/src/meta-srv/src/metasrv.rs
index c43eec60d4..71d15be2a0 100644
--- a/src/meta-srv/src/metasrv.rs
+++ b/src/meta-srv/src/metasrv.rs
@@ -44,18 +44,18 @@ use common_wal::config::MetasrvWalConfig;
 use serde::{Deserialize, Serialize};
 use servers::export_metrics::ExportMetricsOption;
 use servers::http::HttpOptions;
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
 use table::metadata::TableId;
 use tokio::sync::broadcast::error::RecvError;
 
 use crate::cluster::MetaPeerClientRef;
 use crate::election::{Election, LeaderChangeMessage};
 use crate::error::{
-    InitMetadataSnafu, KvBackendSnafu, Result, StartProcedureManagerSnafu, StartTelemetryTaskSnafu,
-    StopProcedureManagerSnafu,
+    self, InitMetadataSnafu, KvBackendSnafu, Result, StartProcedureManagerSnafu,
+    StartTelemetryTaskSnafu, StopProcedureManagerSnafu,
 };
 use crate::failure_detector::PhiAccrualFailureDetectorOptions;
-use crate::handler::HeartbeatHandlerGroupRef;
+use crate::handler::{HeartbeatHandlerGroupBuilder, HeartbeatHandlerGroupRef};
 use crate::lease::lookup_datanode_peer;
 use crate::procedure::region_migration::manager::RegionMigrationManagerRef;
 use crate::procedure::ProcedureManagerListenerAdapter;
@@ -353,7 +353,8 @@ pub struct Metasrv {
     selector: SelectorRef,
     // The flow selector is used to select a target flownode.
     flow_selector: SelectorRef,
-    handler_group: HeartbeatHandlerGroupRef,
+    handler_group: Option<HeartbeatHandlerGroupRef>,
+    handler_group_builder: Option<HeartbeatHandlerGroupBuilder>,
     election: Option<ElectionRef>,
     procedure_manager: ProcedureManagerRef,
     mailbox: MailboxRef,
@@ -370,7 +371,15 @@ pub struct Metasrv {
 }
 
 impl Metasrv {
-    pub async fn try_start(&self) -> Result<()> {
+    pub async fn try_start(&mut self) -> Result<()> {
+        let builder = self
+            .handler_group_builder
+            .take()
+            .context(error::UnexpectedSnafu {
+                violated: "expected heartbeat handler group builder",
+            })?;
+        self.handler_group = Some(Arc::new(builder.build()?));
+
         if self
             .started
             .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed)
@@ -558,10 +567,14 @@ impl Metasrv {
         &self.flow_selector
     }
 
-    pub fn handler_group(&self) -> &HeartbeatHandlerGroupRef {
+    pub fn handler_group(&self) -> &Option<HeartbeatHandlerGroupRef> {
         &self.handler_group
     }
 
+    pub fn handler_group_builder(&mut self) -> &mut Option<HeartbeatHandlerGroupBuilder> {
+        &mut self.handler_group_builder
+    }
+
     pub fn election(&self) -> Option<&ElectionRef> {
         self.election.as_ref()
     }
diff --git a/src/meta-srv/src/metasrv/builder.rs b/src/meta-srv/src/metasrv/builder.rs
index db01b3ec9d..32129b8538 100644
--- a/src/meta-srv/src/metasrv/builder.rs
+++ b/src/meta-srv/src/metasrv/builder.rs
@@ -48,9 +48,7 @@ use crate::flow_meta_alloc::FlowPeerAllocator;
 use crate::greptimedb_telemetry::get_greptimedb_telemetry_task;
 use crate::handler::failure_handler::RegionFailureHandler;
 use crate::handler::region_lease_handler::RegionLeaseHandler;
-use crate::handler::{
-    HeartbeatHandlerGroup, HeartbeatHandlerGroupBuilder, HeartbeatMailbox, Pushers,
-};
+use crate::handler::{HeartbeatHandlerGroupBuilder, HeartbeatMailbox, Pushers};
 use crate::lease::MetaPeerLookupService;
 use crate::metasrv::{
     ElectionRef, Metasrv, MetasrvInfo, MetasrvOptions, SelectorContext, SelectorRef, TABLE_ID_SEQ,
@@ -74,7 +72,7 @@ pub struct MetasrvBuilder {
     kv_backend: Option<KvBackendRef>,
     in_memory: Option<ResettableKvBackendRef>,
     selector: Option<SelectorRef>,
-    handler_group: Option<HeartbeatHandlerGroup>,
+    handler_group_builder: Option<HeartbeatHandlerGroupBuilder>,
     election: Option<ElectionRef>,
     meta_peer_client: Option<MetaPeerClientRef>,
     node_manager: Option<NodeManagerRef>,
@@ -88,7 +86,7 @@ impl MetasrvBuilder {
             kv_backend: None,
             in_memory: None,
             selector: None,
-            handler_group: None,
+            handler_group_builder: None,
             meta_peer_client: None,
             election: None,
             options: None,
@@ -118,8 +116,11 @@ impl MetasrvBuilder {
         self
     }
 
-    pub fn heartbeat_handler(mut self, handler_group: HeartbeatHandlerGroup) -> Self {
-        self.handler_group = Some(handler_group);
+    pub fn heartbeat_handler(
+        mut self,
+        handler_group_builder: HeartbeatHandlerGroupBuilder,
+    ) -> Self {
+        self.handler_group_builder = Some(handler_group_builder);
         self
     }
 
@@ -161,7 +162,7 @@ impl MetasrvBuilder {
             kv_backend,
             in_memory,
             selector,
-            handler_group,
+            handler_group_builder,
             node_manager,
             plugins,
             table_metadata_allocator,
@@ -338,8 +339,8 @@ impl MetasrvBuilder {
             .context(error::InitDdlManagerSnafu)?,
         );
 
-        let handler_group = match handler_group {
-            Some(handler_group) => handler_group,
+        let handler_group_builder = match handler_group_builder {
+            Some(handler_group_builder) => handler_group_builder,
             None => {
                 let region_lease_handler = RegionLeaseHandler::new(
                     distributed_time_constants::REGION_LEASE_SECS,
@@ -352,7 +353,6 @@ impl MetasrvBuilder {
                     .with_region_failure_handler(region_failover_handler)
                     .with_region_lease_handler(Some(region_lease_handler))
                     .add_default_handlers()
-                    .build()?
             }
         };
 
@@ -371,7 +371,8 @@ impl MetasrvBuilder {
             selector,
             // TODO(jeremy): We do not allow configuring the flow selector.
             flow_selector: Arc::new(RoundRobinSelector::new(SelectTarget::Flownode)),
-            handler_group: Arc::new(handler_group),
+            handler_group: None,
+            handler_group_builder: Some(handler_group_builder),
             election,
             procedure_manager,
             mailbox,
diff --git a/src/meta-srv/src/mocks.rs b/src/meta-srv/src/mocks.rs
index 014d524927..4a88701e1a 100644
--- a/src/meta-srv/src/mocks.rs
+++ b/src/meta-srv/src/mocks.rs
@@ -74,7 +74,7 @@ pub async fn mock(
         None => builder,
     };
 
-    let metasrv = builder.build().await.unwrap();
+    let mut metasrv = builder.build().await.unwrap();
     metasrv.try_start().await.unwrap();
 
     let (client, server) = tokio::io::duplex(1024);
diff --git a/src/meta-srv/src/region/supervisor.rs b/src/meta-srv/src/region/supervisor.rs
index 32da666f2b..c6db67c761 100644
--- a/src/meta-srv/src/region/supervisor.rs
+++ b/src/meta-srv/src/region/supervisor.rs
@@ -258,6 +258,7 @@ impl RegionFailureDetectorController for RegionFailureDetectorControl {
 }
 
 /// [`HeartbeatAcceptor`] forwards heartbeats to [`RegionSupervisor`].
+#[derive(Clone)]
 pub(crate) struct HeartbeatAcceptor {
     sender: Sender<Event>,
 }
diff --git a/src/meta-srv/src/service/heartbeat.rs b/src/meta-srv/src/service/heartbeat.rs
index 569d6a8089..2470129be9 100644
--- a/src/meta-srv/src/service/heartbeat.rs
+++ b/src/meta-srv/src/service/heartbeat.rs
@@ -23,6 +23,7 @@ use api::v1::meta::{
 use common_telemetry::{debug, error, info, warn};
 use futures::StreamExt;
 use once_cell::sync::OnceCell;
+use snafu::OptionExt;
 use tokio::sync::mpsc;
 use tokio::sync::mpsc::Sender;
 use tokio_stream::wrappers::ReceiverStream;
@@ -45,7 +46,13 @@ impl heartbeat_server::Heartbeat for Metasrv {
     ) -> GrpcResult<Self::HeartbeatStream> {
         let mut in_stream = req.into_inner();
         let (tx, rx) = mpsc::channel(128);
-        let handler_group = self.handler_group().clone();
+        let handler_group = self
+            .handler_group()
+            .clone()
+            .context(error::UnexpectedSnafu {
+                violated: "expected heartbeat handlers",
+            })?;
+
         let ctx = self.new_ctx();
         let _handle = common_runtime::spawn_global(async move {
             let mut pusher_key = None;

From bb8b54b5d3228d6bd4d48523191b8307a12dfc71 Mon Sep 17 00:00:00 2001
From: Ning Sun <sunng@protonmail.com>
Date: Mon, 14 Oct 2024 23:47:29 -0700
Subject: [PATCH 078/128] feat: add some s2 geo functions (#4823)

* feat: add first batch of s2 functions

* refactor: update reusable code from main

* test: add sqlness tests for s2

* feat: add tostring function for s2

* Update src/common/function/src/scalars/geo/s2.rs

Co-authored-by: Lei, HUANG <6406592+v0y4g3r@users.noreply.github.com>

* Apply suggestions from code review

* one more change

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
Co-authored-by: Ruihang Xia <waynestxia@gmail.com>
Co-authored-by: Lei, HUANG <6406592+v0y4g3r@users.noreply.github.com>
---
 Cargo.lock                                    |  52 +++-
 src/common/function/Cargo.toml                |   3 +-
 src/common/function/src/scalars/geo.rs        |  13 +-
 .../function/src/scalars/geo/encoding.rs      |   2 +-
 src/common/function/src/scalars/geo/h3.rs     |  28 +-
 .../function/src/scalars/geo/helpers.rs       |  26 +-
 src/common/function/src/scalars/geo/s2.rs     | 275 ++++++++++++++++++
 .../standalone/common/function/geo.result     |  15 +
 .../cases/standalone/common/function/geo.sql  |   9 +
 9 files changed, 383 insertions(+), 40 deletions(-)
 create mode 100644 src/common/function/src/scalars/geo/s2.rs

diff --git a/Cargo.lock b/Cargo.lock
index eee5d4ee5d..b096335763 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -230,6 +230,15 @@ dependencies = [
  "tonic-build",
 ]
 
+[[package]]
+name = "approx"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f2a05fd1bd10b2527e20a2cd32d8873d115b8b39fe219ee25f42a8aca6ba278"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "approx"
 version = "0.5.1"
@@ -985,6 +994,7 @@ dependencies = [
  "num-bigint",
  "num-integer",
  "num-traits",
+ "serde",
 ]
 
 [[package]]
@@ -1548,6 +1558,16 @@ dependencies = [
  "vob",
 ]
 
+[[package]]
+name = "cgmath"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a98d30140e3296250832bbaaff83b27dcd6fa3cc70fb6f1f3e5c9c0023b5317"
+dependencies = [
+ "approx 0.4.0",
+ "num-traits",
+]
+
 [[package]]
 name = "chrono"
 version = "0.4.38"
@@ -2054,6 +2074,7 @@ dependencies = [
  "once_cell",
  "paste",
  "ron",
+ "s2",
  "serde",
  "serde_json",
  "session",
@@ -3971,6 +3992,15 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28a80e3145d8ad11ba0995949bbcf48b9df2be62772b3d351ef017dff6ecb853"
 
+[[package]]
+name = "float_extras"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b22b70f8649ea2315955f1a36d964b0e4da482dfaa5f0d04df0d1fb7c338ab7a"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "flow"
 version = "0.9.3"
@@ -4401,7 +4431,7 @@ version = "0.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ff16065e5720f376fbced200a5ae0f47ace85fd70b7e54269790281353b6d61"
 dependencies = [
- "approx",
+ "approx 0.5.1",
  "num-traits",
  "serde",
 ]
@@ -6883,7 +6913,7 @@ version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d506eb7e08d6329505faa8a3a00a5dcc6de9f76e0c77e4b75763ae3c770831ff"
 dependencies = [
- "approx",
+ "approx 0.5.1",
  "matrixmultiply",
  "nalgebra-macros",
  "num-complex",
@@ -10274,6 +10304,20 @@ version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
 
+[[package]]
+name = "s2"
+version = "0.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc7fbc04bb52c40b5f48c9bb2d2961375301916e0c25d9f373750654d588cd5c"
+dependencies = [
+ "bigdecimal 0.3.1",
+ "cgmath",
+ "float_extras",
+ "lazy_static",
+ "libm",
+ "serde",
+]
+
 [[package]]
 name = "safe-proc-macro2"
 version = "1.0.67"
@@ -10912,7 +10956,7 @@ version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f0b7840f121a46d63066ee7a99fc81dcabbc6105e437cae43528cea199b5a05f"
 dependencies = [
- "approx",
+ "approx 0.5.1",
  "num-complex",
  "num-traits",
  "paste",
@@ -11383,7 +11427,7 @@ version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b35a062dbadac17a42e0fc64c27f419b25d6fae98572eb43c8814c9e873d7721"
 dependencies = [
- "approx",
+ "approx 0.5.1",
  "lazy_static",
  "nalgebra",
  "num-traits",
diff --git a/src/common/function/Cargo.toml b/src/common/function/Cargo.toml
index e9a3dd6b55..6b23762d90 100644
--- a/src/common/function/Cargo.toml
+++ b/src/common/function/Cargo.toml
@@ -9,7 +9,7 @@ workspace = true
 
 [features]
 default = ["geo"]
-geo = ["geohash", "h3o"]
+geo = ["geohash", "h3o", "s2"]
 
 [dependencies]
 api.workspace = true
@@ -35,6 +35,7 @@ num = "0.4"
 num-traits = "0.2"
 once_cell.workspace = true
 paste = "1.0"
+s2 = { version = "0.0.12", optional = true }
 serde.workspace = true
 serde_json.workspace = true
 session.workspace = true
diff --git a/src/common/function/src/scalars/geo.rs b/src/common/function/src/scalars/geo.rs
index e47a1de9f2..866acddc4d 100644
--- a/src/common/function/src/scalars/geo.rs
+++ b/src/common/function/src/scalars/geo.rs
@@ -17,8 +17,7 @@ pub(crate) mod encoding;
 mod geohash;
 mod h3;
 mod helpers;
-
-use geohash::{GeohashFunction, GeohashNeighboursFunction};
+mod s2;
 
 use crate::function_registry::FunctionRegistry;
 
@@ -27,8 +26,8 @@ pub(crate) struct GeoFunctions;
 impl GeoFunctions {
     pub fn register(registry: &FunctionRegistry) {
         // geohash
-        registry.register(Arc::new(GeohashFunction));
-        registry.register(Arc::new(GeohashNeighboursFunction));
+        registry.register(Arc::new(geohash::GeohashFunction));
+        registry.register(Arc::new(geohash::GeohashNeighboursFunction));
 
         // h3 index
         registry.register(Arc::new(h3::H3LatLngToCell));
@@ -55,5 +54,11 @@ impl GeoFunctions {
         registry.register(Arc::new(h3::H3GridDiskDistances));
         registry.register(Arc::new(h3::H3GridDistance));
         registry.register(Arc::new(h3::H3GridPathCells));
+
+        // s2
+        registry.register(Arc::new(s2::S2LatLngToCell));
+        registry.register(Arc::new(s2::S2CellLevel));
+        registry.register(Arc::new(s2::S2CellToToken));
+        registry.register(Arc::new(s2::S2CellParent));
     }
 }
diff --git a/src/common/function/src/scalars/geo/encoding.rs b/src/common/function/src/scalars/geo/encoding.rs
index 0600120598..145cd24bff 100644
--- a/src/common/function/src/scalars/geo/encoding.rs
+++ b/src/common/function/src/scalars/geo/encoding.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
 use common_error::ext::{BoxedError, PlainError};
 use common_error::status_code::StatusCode;
 use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
-use common_query::error::{self, InvalidFuncArgsSnafu, InvalidInputStateSnafu, Result};
+use common_query::error::{self, InvalidInputStateSnafu, Result};
 use common_query::logical_plan::accumulator::AggrFuncTypeStore;
 use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
 use common_query::prelude::AccumulatorCreatorFunction;
diff --git a/src/common/function/src/scalars/geo/h3.rs b/src/common/function/src/scalars/geo/h3.rs
index 00c567d8d7..7f98c3147a 100644
--- a/src/common/function/src/scalars/geo/h3.rs
+++ b/src/common/function/src/scalars/geo/h3.rs
@@ -16,7 +16,7 @@ use std::str::FromStr;
 
 use common_error::ext::{BoxedError, PlainError};
 use common_error::status_code::StatusCode;
-use common_query::error::{self, InvalidFuncArgsSnafu, Result};
+use common_query::error::{self, Result};
 use common_query::prelude::{Signature, TypeSignature};
 use datafusion::logical_expr::Volatility;
 use datatypes::prelude::ConcreteDataType;
@@ -29,9 +29,9 @@ use datatypes::vectors::{
 use derive_more::Display;
 use h3o::{CellIndex, LatLng, Resolution};
 use once_cell::sync::Lazy;
-use snafu::{ensure, ResultExt};
+use snafu::ResultExt;
 
-use super::helpers::{ensure_columns_len, ensure_columns_n};
+use super::helpers::{ensure_and_coerce, ensure_columns_len, ensure_columns_n};
 use crate::function::{Function, FunctionContext};
 
 static CELL_TYPES: Lazy<Vec<ConcreteDataType>> = Lazy::new(|| {
@@ -382,15 +382,7 @@ impl Function for H3CellResolution {
     }
 
     fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
-        ensure!(
-            columns.len() == 1,
-            InvalidFuncArgsSnafu {
-                err_msg: format!(
-                    "The length of the args is not correct, expect 1, provided : {}",
-                    columns.len()
-                ),
-            }
-        );
+        ensure_columns_n!(columns, 1);
 
         let cell_vec = &columns[0];
         let size = cell_vec.len();
@@ -982,18 +974,6 @@ fn value_to_resolution(v: Value) -> Result<Resolution> {
         .context(error::ExecuteSnafu)
 }
 
-macro_rules! ensure_and_coerce {
-    ($compare:expr, $coerce:expr) => {{
-        ensure!(
-            $compare,
-            InvalidFuncArgsSnafu {
-                err_msg: "Argument was outside of acceptable range "
-            }
-        );
-        Ok($coerce)
-    }};
-}
-
 fn value_to_position(v: Value) -> Result<u64> {
     match v {
         Value::Int8(v) => ensure_and_coerce!(v >= 0, v as u64),
diff --git a/src/common/function/src/scalars/geo/helpers.rs b/src/common/function/src/scalars/geo/helpers.rs
index f07eaefb15..22d47f54e4 100644
--- a/src/common/function/src/scalars/geo/helpers.rs
+++ b/src/common/function/src/scalars/geo/helpers.rs
@@ -14,15 +14,15 @@
 
 macro_rules! ensure_columns_len {
     ($columns:ident) => {
-        ensure!(
+        snafu::ensure!(
             $columns.windows(2).all(|c| c[0].len() == c[1].len()),
-            InvalidFuncArgsSnafu {
+            common_query::error::InvalidFuncArgsSnafu {
                 err_msg: "The length of input columns are in different size"
             }
         )
     };
     ($column_a:ident, $column_b:ident, $($column_n:ident),*) => {
-        ensure!(
+        snafu::ensure!(
             {
                 let mut result = $column_a.len() == $column_b.len();
                 $(
@@ -30,7 +30,7 @@ macro_rules! ensure_columns_len {
                 )*
                 result
             }
-            InvalidFuncArgsSnafu {
+            common_query::error::InvalidFuncArgsSnafu {
                 err_msg: "The length of input columns are in different size"
             }
         )
@@ -41,9 +41,9 @@ pub(super) use ensure_columns_len;
 
 macro_rules! ensure_columns_n {
     ($columns:ident, $n:literal) => {
-        ensure!(
+        snafu::ensure!(
             $columns.len() == $n,
-            InvalidFuncArgsSnafu {
+            common_query::error::InvalidFuncArgsSnafu {
                 err_msg: format!(
                     "The length of arguments is not correct, expect {}, provided : {}",
                     stringify!($n),
@@ -59,3 +59,17 @@ macro_rules! ensure_columns_n {
 }
 
 pub(super) use ensure_columns_n;
+
+macro_rules! ensure_and_coerce {
+    ($compare:expr, $coerce:expr) => {{
+        snafu::ensure!(
+            $compare,
+            common_query::error::InvalidFuncArgsSnafu {
+                err_msg: "Argument was outside of acceptable range "
+            }
+        );
+        Ok($coerce)
+    }};
+}
+
+pub(super) use ensure_and_coerce;
diff --git a/src/common/function/src/scalars/geo/s2.rs b/src/common/function/src/scalars/geo/s2.rs
new file mode 100644
index 0000000000..6e40dc300f
--- /dev/null
+++ b/src/common/function/src/scalars/geo/s2.rs
@@ -0,0 +1,275 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_query::error::{InvalidFuncArgsSnafu, Result};
+use common_query::prelude::{Signature, TypeSignature};
+use datafusion::logical_expr::Volatility;
+use datatypes::prelude::ConcreteDataType;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::value::Value;
+use datatypes::vectors::{MutableVector, StringVectorBuilder, UInt64VectorBuilder, VectorRef};
+use derive_more::Display;
+use once_cell::sync::Lazy;
+use s2::cellid::{CellID, MAX_LEVEL};
+use s2::latlng::LatLng;
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+use crate::scalars::geo::helpers::{ensure_and_coerce, ensure_columns_len, ensure_columns_n};
+
+static CELL_TYPES: Lazy<Vec<ConcreteDataType>> = Lazy::new(|| {
+    vec![
+        ConcreteDataType::int64_datatype(),
+        ConcreteDataType::uint64_datatype(),
+    ]
+});
+
+static COORDINATE_TYPES: Lazy<Vec<ConcreteDataType>> = Lazy::new(|| {
+    vec![
+        ConcreteDataType::float32_datatype(),
+        ConcreteDataType::float64_datatype(),
+    ]
+});
+
+static LEVEL_TYPES: Lazy<Vec<ConcreteDataType>> = Lazy::new(|| {
+    vec![
+        ConcreteDataType::int8_datatype(),
+        ConcreteDataType::int16_datatype(),
+        ConcreteDataType::int32_datatype(),
+        ConcreteDataType::int64_datatype(),
+        ConcreteDataType::uint8_datatype(),
+        ConcreteDataType::uint16_datatype(),
+        ConcreteDataType::uint32_datatype(),
+        ConcreteDataType::uint64_datatype(),
+    ]
+});
+
+/// Function that returns [s2] encoding cellid for a given geospatial coordinate.
+///
+/// [s2]: http://s2geometry.io
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct S2LatLngToCell;
+
+impl Function for S2LatLngToCell {
+    fn name(&self) -> &str {
+        "s2_latlng_to_cell"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::uint64_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        let mut signatures = Vec::with_capacity(COORDINATE_TYPES.len());
+        for coord_type in COORDINATE_TYPES.as_slice() {
+            signatures.push(TypeSignature::Exact(vec![
+                // latitude
+                coord_type.clone(),
+                // longitude
+                coord_type.clone(),
+            ]));
+        }
+        Signature::one_of(signatures, Volatility::Stable)
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure_columns_n!(columns, 2);
+
+        let lat_vec = &columns[0];
+        let lon_vec = &columns[1];
+
+        let size = lat_vec.len();
+        let mut results = UInt64VectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let lat = lat_vec.get(i).as_f64_lossy();
+            let lon = lon_vec.get(i).as_f64_lossy();
+
+            let result = match (lat, lon) {
+                (Some(lat), Some(lon)) => {
+                    let coord = LatLng::from_degrees(lat, lon);
+                    ensure!(
+                        coord.is_valid(),
+                        InvalidFuncArgsSnafu {
+                            err_msg: "The input coordinates are invalid",
+                        }
+                    );
+                    let cellid = CellID::from(coord);
+                    let encoded: u64 = cellid.0;
+                    Some(encoded)
+                }
+                _ => None,
+            };
+
+            results.push(result);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Return the level of current s2 cell
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct S2CellLevel;
+
+impl Function for S2CellLevel {
+    fn name(&self) -> &str {
+        "s2_cell_level"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::uint64_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure_columns_n!(columns, 1);
+
+        let cell_vec = &columns[0];
+        let size = cell_vec.len();
+        let mut results = UInt64VectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i));
+            let res = cell.map(|cell| cell.level());
+
+            results.push(res);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Return the string presentation of the cell
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct S2CellToToken;
+
+impl Function for S2CellToToken {
+    fn name(&self) -> &str {
+        "s2_cell_to_token"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::string_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure_columns_n!(columns, 1);
+
+        let cell_vec = &columns[0];
+        let size = cell_vec.len();
+        let mut results = StringVectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i));
+            let res = cell.map(|cell| cell.to_token());
+
+            results.push(res.as_deref());
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+/// Return parent at given level of current s2 cell
+#[derive(Clone, Debug, Default, Display)]
+#[display("{}", self.name())]
+pub struct S2CellParent;
+
+impl Function for S2CellParent {
+    fn name(&self) -> &str {
+        "s2_cell_parent"
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::uint64_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        signature_of_cell_and_level()
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure_columns_n!(columns, 2);
+
+        let cell_vec = &columns[0];
+        let level_vec = &columns[1];
+        let size = cell_vec.len();
+        let mut results = UInt64VectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let cell = cell_from_value(cell_vec.get(i));
+            let level = value_to_level(level_vec.get(i))?;
+            let result = cell.map(|cell| cell.parent(level).0);
+
+            results.push(result);
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+fn signature_of_cell() -> Signature {
+    let mut signatures = Vec::with_capacity(CELL_TYPES.len());
+    for cell_type in CELL_TYPES.as_slice() {
+        signatures.push(TypeSignature::Exact(vec![cell_type.clone()]));
+    }
+
+    Signature::one_of(signatures, Volatility::Stable)
+}
+
+fn signature_of_cell_and_level() -> Signature {
+    let mut signatures = Vec::with_capacity(CELL_TYPES.len() * LEVEL_TYPES.len());
+    for cell_type in CELL_TYPES.as_slice() {
+        for level_type in LEVEL_TYPES.as_slice() {
+            signatures.push(TypeSignature::Exact(vec![
+                cell_type.clone(),
+                level_type.clone(),
+            ]));
+        }
+    }
+    Signature::one_of(signatures, Volatility::Stable)
+}
+
+fn cell_from_value(v: Value) -> Option<CellID> {
+    match v {
+        Value::Int64(v) => Some(CellID(v as u64)),
+        Value::UInt64(v) => Some(CellID(v)),
+        _ => None,
+    }
+}
+
+fn value_to_level(v: Value) -> Result<u64> {
+    match v {
+        Value::Int8(v) => ensure_and_coerce!(v >= 0 && v <= MAX_LEVEL as i8, v as u64),
+        Value::Int16(v) => ensure_and_coerce!(v >= 0 && v <= MAX_LEVEL as i16, v as u64),
+        Value::Int32(v) => ensure_and_coerce!(v >= 0 && v <= MAX_LEVEL as i32, v as u64),
+        Value::Int64(v) => ensure_and_coerce!(v >= 0 && v <= MAX_LEVEL as i64, v as u64),
+        Value::UInt8(v) => ensure_and_coerce!(v <= MAX_LEVEL as u8, v as u64),
+        Value::UInt16(v) => ensure_and_coerce!(v <= MAX_LEVEL as u16, v as u64),
+        Value::UInt32(v) => ensure_and_coerce!(v <= MAX_LEVEL as u32, v as u64),
+        Value::UInt64(v) => ensure_and_coerce!(v <= MAX_LEVEL, v),
+        _ => unreachable!(),
+    }
+}
diff --git a/tests/cases/standalone/common/function/geo.result b/tests/cases/standalone/common/function/geo.result
index 75caeb886b..8c9460c738 100644
--- a/tests/cases/standalone/common/function/geo.result
+++ b/tests/cases/standalone/common/function/geo.result
@@ -259,6 +259,21 @@ SELECT geohash_neighbours(37.76938, -122.3889, 11);
 | [9q8yygxnefv, 9q8yygxnefu, 9q8yygxnefs, 9q8yygxnefk, 9q8yygxnefm, 9q8yygxnefq, 9q8yygxnefw, 9q8yygxnefy] |
 +----------------------------------------------------------------------------------------------------------+
 
+WITH cell_cte AS (
+  SELECT s2_latlng_to_cell(37.76938, -122.3889) AS cell
+)
+SELECT cell,
+       s2_cell_to_token(cell),
+       s2_cell_level(cell),
+       s2_cell_parent(cell, 3)
+FROM cell_cte;
+
++---------------------+---------------------------------+------------------------------+----------------------------------------+
+| cell                | s2_cell_to_token(cell_cte.cell) | s2_cell_level(cell_cte.cell) | s2_cell_parent(cell_cte.cell,Int64(3)) |
++---------------------+---------------------------------+------------------------------+----------------------------------------+
+| 9263763445276221387 | 808f7fc59ef01fcb                | 30                           | 9277415232383221760                    |
++---------------------+---------------------------------+------------------------------+----------------------------------------+
+
 SELECT json_encode_path(37.76938, -122.3889, 1728083375::TimestampSecond);
 
 +----------------------------------------------------------------------------------------------------------------------+
diff --git a/tests/cases/standalone/common/function/geo.sql b/tests/cases/standalone/common/function/geo.sql
index af2a16517d..205d45ddd5 100644
--- a/tests/cases/standalone/common/function/geo.sql
+++ b/tests/cases/standalone/common/function/geo.sql
@@ -83,6 +83,15 @@ SELECT geohash(37.76938, -122.3889, 11::UInt64);
 
 SELECT geohash_neighbours(37.76938, -122.3889, 11);
 
+WITH cell_cte AS (
+  SELECT s2_latlng_to_cell(37.76938, -122.3889) AS cell
+)
+SELECT cell,
+       s2_cell_to_token(cell),
+       s2_cell_level(cell),
+       s2_cell_parent(cell, 3)
+FROM cell_cte;
+
 SELECT json_encode_path(37.76938, -122.3889, 1728083375::TimestampSecond);
 
 SELECT json_encode_path(lat, lon, ts)

From 972c2441af26ad68fd31f5e0a92b1646b1c3704f Mon Sep 17 00:00:00 2001
From: zyy17 <zyylsxm@gmail.com>
Date: Tue, 15 Oct 2024 16:50:37 +0800
Subject: [PATCH 079/128] chore: bump promql-parser to v0.4.1 and use
 `to_string()` for EvalStmt (#4832)

chore: bump promql-parser to v0.4.1 and use to_string() for EvalStmt
---
 Cargo.lock             | 5 +++--
 Cargo.toml             | 2 +-
 src/query/src/stats.rs | 3 +--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index b096335763..00748b4a5a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8612,11 +8612,12 @@ dependencies = [
 
 [[package]]
 name = "promql-parser"
-version = "0.4.0"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "007a331efb31f6ddb644590ef22359c9469784931162aad92599e34bcfa66583"
+checksum = "0c1ad4a4cfa84ec4aa5831c82e57af0a3faf3f0af83bee13fa1390b2d0a32dc9"
 dependencies = [
  "cfgrammar",
+ "chrono",
  "lazy_static",
  "lrlex",
  "lrpar",
diff --git a/Cargo.toml b/Cargo.toml
index 72ad968ca7..704909be9f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -143,7 +143,7 @@ parquet = { version = "51.0.0", default-features = false, features = ["arrow", "
 paste = "1.0"
 pin-project = "1.0"
 prometheus = { version = "0.13.3", features = ["process"] }
-promql-parser = { version = "0.4" }
+promql-parser = { version = "0.4.1" }
 prost = "0.12"
 raft-engine = { version = "0.4.1", default-features = false }
 rand = "0.8"
diff --git a/src/query/src/stats.rs b/src/query/src/stats.rs
index 896271897d..085642fbc1 100644
--- a/src/query/src/stats.rs
+++ b/src/query/src/stats.rs
@@ -70,8 +70,7 @@ impl SlowQueryTimer {
                 slow!(
                     cost = elapsed.as_millis() as u64,
                     threshold = threshold.as_millis() as u64,
-                    // TODO(zyy17): It's better to implement Display for EvalStmt for pretty print.
-                    promql = format!("{:?}", stmt)
+                    promql = stmt.to_string()
                 );
             }
         }

From 3197b8b53562f52b1955bad3571c9d3762593915 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Tue, 15 Oct 2024 17:48:13 +0800
Subject: [PATCH 080/128] feat: introduce default customizers (#4831)

* feat: introduce `DefaultHeartbeatHandlerGroupBuilderCustomizer` and `DefaultLeadershipChangeNotifierCustomizer`

* chore: code styling
---
 src/cmd/src/metasrv.rs                        |  24 +++-
 src/common/base/src/plugins.rs                |  12 ++
 src/common/meta/src/leadership_notifier.rs    |  26 +++-
 src/meta-srv/src/bootstrap.rs                 |   4 +-
 src/meta-srv/src/handler.rs                   | 128 ++++++++++++++++--
 .../handler/filter_inactive_region_stats.rs   |   2 +
 6 files changed, 178 insertions(+), 18 deletions(-)

diff --git a/src/cmd/src/metasrv.rs b/src/cmd/src/metasrv.rs
index 757bb27341..664066132e 100644
--- a/src/cmd/src/metasrv.rs
+++ b/src/cmd/src/metasrv.rs
@@ -49,8 +49,8 @@ impl Instance {
         }
     }
 
-    pub fn mut_inner(&mut self) -> &mut MetasrvInstance {
-        &mut self.instance
+    pub fn get_inner(&self) -> &MetasrvInstance {
+        &self.instance
     }
 }
 
@@ -90,6 +90,14 @@ impl Command {
     pub fn load_options(&self, global_options: &GlobalOptions) -> Result<MetasrvOptions> {
         self.subcmd.load_options(global_options)
     }
+
+    pub fn config_file(&self) -> &Option<String> {
+        self.subcmd.config_file()
+    }
+
+    pub fn env_prefix(&self) -> &String {
+        self.subcmd.env_prefix()
+    }
 }
 
 #[derive(Parser)]
@@ -109,6 +117,18 @@ impl SubCommand {
             SubCommand::Start(cmd) => cmd.load_options(global_options),
         }
     }
+
+    fn config_file(&self) -> &Option<String> {
+        match self {
+            SubCommand::Start(cmd) => &cmd.config_file,
+        }
+    }
+
+    fn env_prefix(&self) -> &String {
+        match self {
+            SubCommand::Start(cmd) => &cmd.env_prefix,
+        }
+    }
 }
 
 #[derive(Debug, Default, Parser)]
diff --git a/src/common/base/src/plugins.rs b/src/common/base/src/plugins.rs
index 98b40cd059..c392422b64 100644
--- a/src/common/base/src/plugins.rs
+++ b/src/common/base/src/plugins.rs
@@ -38,6 +38,18 @@ impl Plugins {
         self.read().get::<T>().cloned()
     }
 
+    pub fn get_or_insert<T, F>(&self, f: F) -> T
+    where
+        T: 'static + Send + Sync + Clone,
+        F: FnOnce() -> T,
+    {
+        let mut binding = self.write();
+        if !binding.contains::<T>() {
+            binding.insert(f());
+        }
+        binding.get::<T>().cloned().unwrap()
+    }
+
     pub fn map_mut<T: 'static + Send + Sync, F, R>(&self, mapper: F) -> R
     where
         F: FnOnce(Option<&mut T>) -> R,
diff --git a/src/common/meta/src/leadership_notifier.rs b/src/common/meta/src/leadership_notifier.rs
index 0fed50d388..393bf3cb82 100644
--- a/src/common/meta/src/leadership_notifier.rs
+++ b/src/common/meta/src/leadership_notifier.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 
 use async_trait::async_trait;
 use common_telemetry::{error, info};
@@ -24,6 +24,8 @@ pub type LeadershipChangeNotifierCustomizerRef = Arc<dyn LeadershipChangeNotifie
 /// A trait for customizing the leadership change notifier.
 pub trait LeadershipChangeNotifierCustomizer: Send + Sync {
     fn customize(&self, notifier: &mut LeadershipChangeNotifier);
+
+    fn add_listener(&self, listener: Arc<dyn LeadershipChangeListener>);
 }
 
 /// A trait for handling leadership change events in a distributed system.
@@ -45,10 +47,28 @@ pub struct LeadershipChangeNotifier {
     listeners: Vec<Arc<dyn LeadershipChangeListener>>,
 }
 
-impl LeadershipChangeNotifierCustomizer for LeadershipChangeNotifier {
+#[derive(Default)]
+pub struct DefaultLeadershipChangeNotifierCustomizer {
+    listeners: Mutex<Vec<Arc<dyn LeadershipChangeListener>>>,
+}
+
+impl DefaultLeadershipChangeNotifierCustomizer {
+    pub fn new() -> Self {
+        Self {
+            listeners: Mutex::new(Vec::new()),
+        }
+    }
+}
+
+impl LeadershipChangeNotifierCustomizer for DefaultLeadershipChangeNotifierCustomizer {
     fn customize(&self, notifier: &mut LeadershipChangeNotifier) {
         info!("Customizing leadership change notifier");
-        notifier.listeners.extend(self.listeners.clone());
+        let listeners = self.listeners.lock().unwrap().clone();
+        notifier.listeners.extend(listeners);
+    }
+
+    fn add_listener(&self, listener: Arc<dyn LeadershipChangeListener>) {
+        self.listeners.lock().unwrap().push(listener);
     }
 }
 
diff --git a/src/meta-srv/src/bootstrap.rs b/src/meta-srv/src/bootstrap.rs
index 79773498cf..8323a7f8df 100644
--- a/src/meta-srv/src/bootstrap.rs
+++ b/src/meta-srv/src/bootstrap.rs
@@ -149,8 +149,8 @@ impl MetasrvInstance {
         self.plugins.clone()
     }
 
-    pub fn mut_inner(&mut self) -> &mut Metasrv {
-        &mut self.metasrv
+    pub fn get_inner(&self) -> &Metasrv {
+        &self.metasrv
     }
 }
 
diff --git a/src/meta-srv/src/handler.rs b/src/meta-srv/src/handler.rs
index ce75ff5ff1..5499cb259b 100644
--- a/src/meta-srv/src/handler.rs
+++ b/src/meta-srv/src/handler.rs
@@ -14,7 +14,7 @@
 
 use std::collections::{BTreeMap, HashSet};
 use std::ops::Range;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant};
 
 use api::v1::meta::mailbox_message::Payload;
@@ -210,7 +210,7 @@ impl Pushers {
 }
 
 #[derive(Clone)]
-struct NameCachedHandler {
+pub struct NameCachedHandler {
     name: &'static str,
     handler: Arc<dyn HeartbeatHandler>,
 }
@@ -547,15 +547,27 @@ impl HeartbeatHandlerGroupBuilder {
         })
     }
 
+    fn add_handler_after_inner(&mut self, target: &str, handler: NameCachedHandler) -> Result<()> {
+        if let Some(pos) = self.handlers.iter().position(|x| x.name == target) {
+            self.handlers.insert(pos + 1, handler);
+            return Ok(());
+        }
+
+        error::HandlerNotFoundSnafu { name: target }.fail()
+    }
+
     /// Adds the handler after the specified handler.
     pub fn add_handler_after(
         &mut self,
         target: &'static str,
         handler: impl HeartbeatHandler + 'static,
     ) -> Result<()> {
+        self.add_handler_after_inner(target, NameCachedHandler::new(handler))
+    }
+
+    fn add_handler_before_inner(&mut self, target: &str, handler: NameCachedHandler) -> Result<()> {
         if let Some(pos) = self.handlers.iter().position(|x| x.name == target) {
-            self.handlers
-                .insert(pos + 1, NameCachedHandler::new(handler));
+            self.handlers.insert(pos, handler);
             return Ok(());
         }
 
@@ -568,8 +580,12 @@ impl HeartbeatHandlerGroupBuilder {
         target: &'static str,
         handler: impl HeartbeatHandler + 'static,
     ) -> Result<()> {
+        self.add_handler_before_inner(target, NameCachedHandler::new(handler))
+    }
+
+    fn replace_handler_inner(&mut self, target: &str, handler: NameCachedHandler) -> Result<()> {
         if let Some(pos) = self.handlers.iter().position(|x| x.name == target) {
-            self.handlers.insert(pos, NameCachedHandler::new(handler));
+            self.handlers[pos] = handler;
             return Ok(());
         }
 
@@ -582,25 +598,115 @@ impl HeartbeatHandlerGroupBuilder {
         target: &'static str,
         handler: impl HeartbeatHandler + 'static,
     ) -> Result<()> {
-        if let Some(pos) = self.handlers.iter().position(|x| x.name == target) {
-            self.handlers[pos] = NameCachedHandler::new(handler);
-            return Ok(());
-        }
+        self.replace_handler_inner(target, NameCachedHandler::new(handler))
+    }
 
-        error::HandlerNotFoundSnafu { name: target }.fail()
+    fn add_handler_last_inner(&mut self, handler: NameCachedHandler) {
+        self.handlers.push(handler);
     }
 
     fn add_handler_last(&mut self, handler: impl HeartbeatHandler + 'static) {
-        self.handlers.push(NameCachedHandler::new(handler));
+        self.add_handler_last_inner(NameCachedHandler::new(handler));
     }
 }
 
 pub type HeartbeatHandlerGroupBuilderCustomizerRef =
     Arc<dyn HeartbeatHandlerGroupBuilderCustomizer>;
 
+pub enum CustomizeHeartbeatGroupAction {
+    AddHandlerAfter {
+        target: String,
+        handler: NameCachedHandler,
+    },
+    AddHandlerBefore {
+        target: String,
+        handler: NameCachedHandler,
+    },
+    ReplaceHandler {
+        target: String,
+        handler: NameCachedHandler,
+    },
+    AddHandlerLast {
+        handler: NameCachedHandler,
+    },
+}
+
+impl CustomizeHeartbeatGroupAction {
+    pub fn new_add_handler_after(
+        target: &'static str,
+        handler: impl HeartbeatHandler + 'static,
+    ) -> Self {
+        Self::AddHandlerAfter {
+            target: target.to_string(),
+            handler: NameCachedHandler::new(handler),
+        }
+    }
+
+    pub fn new_add_handler_before(
+        target: &'static str,
+        handler: impl HeartbeatHandler + 'static,
+    ) -> Self {
+        Self::AddHandlerBefore {
+            target: target.to_string(),
+            handler: NameCachedHandler::new(handler),
+        }
+    }
+
+    pub fn new_replace_handler(
+        target: &'static str,
+        handler: impl HeartbeatHandler + 'static,
+    ) -> Self {
+        Self::ReplaceHandler {
+            target: target.to_string(),
+            handler: NameCachedHandler::new(handler),
+        }
+    }
+
+    pub fn new_add_handler_last(handler: impl HeartbeatHandler + 'static) -> Self {
+        Self::AddHandlerLast {
+            handler: NameCachedHandler::new(handler),
+        }
+    }
+}
+
 /// The customizer of the [`HeartbeatHandlerGroupBuilder`].
 pub trait HeartbeatHandlerGroupBuilderCustomizer: Send + Sync {
     fn customize(&self, builder: &mut HeartbeatHandlerGroupBuilder) -> Result<()>;
+
+    fn add_action(&self, action: CustomizeHeartbeatGroupAction);
+}
+
+#[derive(Default)]
+pub struct DefaultHeartbeatHandlerGroupBuilderCustomizer {
+    actions: Mutex<Vec<CustomizeHeartbeatGroupAction>>,
+}
+
+impl HeartbeatHandlerGroupBuilderCustomizer for DefaultHeartbeatHandlerGroupBuilderCustomizer {
+    fn customize(&self, builder: &mut HeartbeatHandlerGroupBuilder) -> Result<()> {
+        info!("Customizing the heartbeat handler group builder");
+        let mut actions = self.actions.lock().unwrap();
+        for action in actions.drain(..) {
+            match action {
+                CustomizeHeartbeatGroupAction::AddHandlerAfter { target, handler } => {
+                    builder.add_handler_after_inner(&target, handler)?;
+                }
+                CustomizeHeartbeatGroupAction::AddHandlerBefore { target, handler } => {
+                    builder.add_handler_before_inner(&target, handler)?;
+                }
+                CustomizeHeartbeatGroupAction::ReplaceHandler { target, handler } => {
+                    builder.replace_handler_inner(&target, handler)?;
+                }
+                CustomizeHeartbeatGroupAction::AddHandlerLast { handler } => {
+                    builder.add_handler_last_inner(handler);
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn add_action(&self, action: CustomizeHeartbeatGroupAction) {
+        self.actions.lock().unwrap().push(action);
+    }
 }
 
 #[cfg(test)]
diff --git a/src/meta-srv/src/handler/filter_inactive_region_stats.rs b/src/meta-srv/src/handler/filter_inactive_region_stats.rs
index fc1518f8b6..8504db7d5b 100644
--- a/src/meta-srv/src/handler/filter_inactive_region_stats.rs
+++ b/src/meta-srv/src/handler/filter_inactive_region_stats.rs
@@ -20,6 +20,8 @@ use crate::error::Result;
 use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
 use crate::metasrv::Context;
 
+pub const NAME: &str = "FilterInactiveRegionStatsHandler";
+
 pub struct FilterInactiveRegionStatsHandler;
 
 #[async_trait]

From 16b8cdc3d5599c51e04e639c0bc2450b9d55fff5 Mon Sep 17 00:00:00 2001
From: discord9 <55937128+discord9@users.noreply.github.com>
Date: Tue, 15 Oct 2024 18:48:03 +0800
Subject: [PATCH 081/128] chore: bump version v0.9.4 (#4833)

---
 Cargo.lock | 138 ++++++++++++++++++++++++++---------------------------
 Cargo.toml |   2 +-
 2 files changed, 70 insertions(+), 70 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 00748b4a5a..c3736b2d9f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -214,7 +214,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c"
 
 [[package]]
 name = "api"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "common-base",
  "common-decimal",
@@ -775,7 +775,7 @@ dependencies = [
 
 [[package]]
 name = "auth"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "async-trait",
@@ -1385,7 +1385,7 @@ dependencies = [
 
 [[package]]
 name = "cache"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "catalog",
  "common-error",
@@ -1393,7 +1393,7 @@ dependencies = [
  "common-meta",
  "moka",
  "snafu 0.8.5",
- "substrait 0.9.3",
+ "substrait 0.9.4",
 ]
 
 [[package]]
@@ -1420,7 +1420,7 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
 [[package]]
 name = "catalog"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "arrow",
@@ -1759,7 +1759,7 @@ checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
 
 [[package]]
 name = "client"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "arc-swap",
@@ -1789,7 +1789,7 @@ dependencies = [
  "serde_json",
  "snafu 0.8.5",
  "substrait 0.37.3",
- "substrait 0.9.3",
+ "substrait 0.9.4",
  "tokio",
  "tokio-stream",
  "tonic 0.11.0",
@@ -1819,7 +1819,7 @@ dependencies = [
 
 [[package]]
 name = "cmd"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "async-trait",
  "auth",
@@ -1876,7 +1876,7 @@ dependencies = [
  "similar-asserts",
  "snafu 0.8.5",
  "store-api",
- "substrait 0.9.3",
+ "substrait 0.9.4",
  "table",
  "temp-env",
  "tempfile",
@@ -1922,7 +1922,7 @@ checksum = "55b672471b4e9f9e95499ea597ff64941a309b2cdbffcc46f2cc5e2d971fd335"
 
 [[package]]
 name = "common-base"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "anymap2",
  "async-trait",
@@ -1940,7 +1940,7 @@ dependencies = [
 
 [[package]]
 name = "common-catalog"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "chrono",
  "common-error",
@@ -1951,7 +1951,7 @@ dependencies = [
 
 [[package]]
 name = "common-config"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "common-base",
  "common-error",
@@ -1974,7 +1974,7 @@ dependencies = [
 
 [[package]]
 name = "common-datasource"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "arrow",
  "arrow-schema",
@@ -2011,7 +2011,7 @@ dependencies = [
 
 [[package]]
 name = "common-decimal"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "bigdecimal 0.4.5",
  "common-error",
@@ -2024,7 +2024,7 @@ dependencies = [
 
 [[package]]
 name = "common-error"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "snafu 0.8.5",
  "strum 0.25.0",
@@ -2033,7 +2033,7 @@ dependencies = [
 
 [[package]]
 name = "common-frontend"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "async-trait",
@@ -2048,7 +2048,7 @@ dependencies = [
 
 [[package]]
 name = "common-function"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "arc-swap",
@@ -2088,7 +2088,7 @@ dependencies = [
 
 [[package]]
 name = "common-greptimedb-telemetry"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "async-trait",
  "common-runtime",
@@ -2105,7 +2105,7 @@ dependencies = [
 
 [[package]]
 name = "common-grpc"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "arrow-flight",
@@ -2131,7 +2131,7 @@ dependencies = [
 
 [[package]]
 name = "common-grpc-expr"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "common-base",
@@ -2149,7 +2149,7 @@ dependencies = [
 
 [[package]]
 name = "common-macro"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "arc-swap",
  "common-query",
@@ -2163,7 +2163,7 @@ dependencies = [
 
 [[package]]
 name = "common-mem-prof"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "common-error",
  "common-macro",
@@ -2176,7 +2176,7 @@ dependencies = [
 
 [[package]]
 name = "common-meta"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "anymap2",
  "api",
@@ -2233,11 +2233,11 @@ dependencies = [
 
 [[package]]
 name = "common-plugins"
-version = "0.9.3"
+version = "0.9.4"
 
 [[package]]
 name = "common-pprof"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "common-error",
  "common-macro",
@@ -2249,7 +2249,7 @@ dependencies = [
 
 [[package]]
 name = "common-procedure"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -2276,7 +2276,7 @@ dependencies = [
 
 [[package]]
 name = "common-procedure-test"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "async-trait",
  "common-procedure",
@@ -2284,7 +2284,7 @@ dependencies = [
 
 [[package]]
 name = "common-query"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "async-trait",
@@ -2310,7 +2310,7 @@ dependencies = [
 
 [[package]]
 name = "common-recordbatch"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "arc-swap",
  "common-error",
@@ -2329,7 +2329,7 @@ dependencies = [
 
 [[package]]
 name = "common-runtime"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "async-trait",
  "common-error",
@@ -2351,7 +2351,7 @@ dependencies = [
 
 [[package]]
 name = "common-telemetry"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "atty",
  "backtrace",
@@ -2379,7 +2379,7 @@ dependencies = [
 
 [[package]]
 name = "common-test-util"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "client",
  "common-query",
@@ -2391,7 +2391,7 @@ dependencies = [
 
 [[package]]
 name = "common-time"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "arrow",
  "chrono",
@@ -2407,7 +2407,7 @@ dependencies = [
 
 [[package]]
 name = "common-version"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "build-data",
  "const_format",
@@ -2418,7 +2418,7 @@ dependencies = [
 
 [[package]]
 name = "common-wal"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "common-base",
  "common-error",
@@ -3227,7 +3227,7 @@ dependencies = [
 
 [[package]]
 name = "datanode"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "arrow-flight",
@@ -3277,7 +3277,7 @@ dependencies = [
  "session",
  "snafu 0.8.5",
  "store-api",
- "substrait 0.9.3",
+ "substrait 0.9.4",
  "table",
  "tokio",
  "toml 0.8.19",
@@ -3286,7 +3286,7 @@ dependencies = [
 
 [[package]]
 name = "datatypes"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -3892,7 +3892,7 @@ dependencies = [
 
 [[package]]
 name = "file-engine"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "async-trait",
@@ -4003,7 +4003,7 @@ dependencies = [
 
 [[package]]
 name = "flow"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "arrow",
@@ -4060,7 +4060,7 @@ dependencies = [
  "snafu 0.8.5",
  "store-api",
  "strum 0.25.0",
- "substrait 0.9.3",
+ "substrait 0.9.4",
  "table",
  "tokio",
  "tonic 0.11.0",
@@ -4122,7 +4122,7 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa"
 
 [[package]]
 name = "frontend"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "arc-swap",
@@ -5170,7 +5170,7 @@ dependencies = [
 
 [[package]]
 name = "index"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "async-trait",
  "asynchronous-codec",
@@ -6001,7 +6001,7 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
 
 [[package]]
 name = "log-store"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -6321,7 +6321,7 @@ dependencies = [
 
 [[package]]
 name = "meta-client"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "async-trait",
@@ -6347,7 +6347,7 @@ dependencies = [
 
 [[package]]
 name = "meta-srv"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "async-trait",
@@ -6425,7 +6425,7 @@ dependencies = [
 
 [[package]]
 name = "metric-engine"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "aquamarine",
@@ -6528,7 +6528,7 @@ dependencies = [
 
 [[package]]
 name = "mito2"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "aquamarine",
@@ -7264,7 +7264,7 @@ dependencies = [
 
 [[package]]
 name = "object-store"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "anyhow",
  "bytes",
@@ -7554,7 +7554,7 @@ dependencies = [
 
 [[package]]
 name = "operator"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "async-trait",
@@ -7599,7 +7599,7 @@ dependencies = [
  "sql",
  "sqlparser 0.45.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=54a267ac89c09b11c0c88934690530807185d3e7)",
  "store-api",
- "substrait 0.9.3",
+ "substrait 0.9.4",
  "table",
  "tokio",
  "tokio-util",
@@ -7849,7 +7849,7 @@ dependencies = [
 
 [[package]]
 name = "partition"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "async-trait",
@@ -8150,7 +8150,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "pipeline"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "ahash 0.8.11",
  "api",
@@ -8312,7 +8312,7 @@ dependencies = [
 
 [[package]]
 name = "plugins"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "auth",
  "common-base",
@@ -8586,7 +8586,7 @@ dependencies = [
 
 [[package]]
 name = "promql"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "ahash 0.8.11",
  "async-trait",
@@ -8822,7 +8822,7 @@ dependencies = [
 
 [[package]]
 name = "puffin"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "async-compression 0.4.13",
  "async-trait",
@@ -8944,7 +8944,7 @@ dependencies = [
 
 [[package]]
 name = "query"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "ahash 0.8.11",
  "api",
@@ -9007,7 +9007,7 @@ dependencies = [
  "stats-cli",
  "store-api",
  "streaming-stats",
- "substrait 0.9.3",
+ "substrait 0.9.4",
  "table",
  "tokio",
  "tokio-stream",
@@ -10441,7 +10441,7 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
 [[package]]
 name = "script"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "arc-swap",
@@ -10735,7 +10735,7 @@ dependencies = [
 
 [[package]]
 name = "servers"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "aide",
  "api",
@@ -10845,7 +10845,7 @@ dependencies = [
 
 [[package]]
 name = "session"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "arc-swap",
@@ -11166,7 +11166,7 @@ dependencies = [
 
 [[package]]
 name = "sql"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "chrono",
@@ -11227,7 +11227,7 @@ dependencies = [
 
 [[package]]
 name = "sqlness-runner"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "async-trait",
  "clap 4.5.19",
@@ -11447,7 +11447,7 @@ dependencies = [
 
 [[package]]
 name = "store-api"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "aquamarine",
@@ -11616,7 +11616,7 @@ dependencies = [
 
 [[package]]
 name = "substrait"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "async-trait",
  "bytes",
@@ -11815,7 +11815,7 @@ dependencies = [
 
 [[package]]
 name = "table"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "async-trait",
@@ -12081,7 +12081,7 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
 
 [[package]]
 name = "tests-fuzz"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "arbitrary",
  "async-trait",
@@ -12123,7 +12123,7 @@ dependencies = [
 
 [[package]]
 name = "tests-integration"
-version = "0.9.3"
+version = "0.9.4"
 dependencies = [
  "api",
  "arrow-flight",
@@ -12185,7 +12185,7 @@ dependencies = [
  "sql",
  "sqlx",
  "store-api",
- "substrait 0.9.3",
+ "substrait 0.9.4",
  "table",
  "tempfile",
  "time",
diff --git a/Cargo.toml b/Cargo.toml
index 704909be9f..133db91819 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -65,7 +65,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.9.3"
+version = "0.9.4"
 edition = "2021"
 license = "Apache-2.0"
 

From 59ec90299b4edb975aad844d6864b40dcd50e3be Mon Sep 17 00:00:00 2001
From: jeremyhi <jiachun_feng@proton.me>
Date: Tue, 15 Oct 2024 19:36:48 +0800
Subject: [PATCH 082/128] refactor: metasrv cannot be cloned (#4834)

* refactor: metasrv cannot be cloned

* chore: remove MetasrvInstance's clone
---
 src/meta-srv/src/bootstrap.rs                 | 16 ++++-----
 src/meta-srv/src/handler.rs                   |  1 -
 src/meta-srv/src/handler/failure_handler.rs   |  1 -
 .../src/handler/region_lease_handler.rs       |  1 -
 src/meta-srv/src/metasrv.rs                   | 35 +++++++++----------
 src/meta-srv/src/metasrv/builder.rs           |  6 ++--
 src/meta-srv/src/mocks.rs                     | 11 +++---
 src/meta-srv/src/service/admin.rs             |  2 +-
 src/meta-srv/src/service/heartbeat.rs         |  9 ++---
 tests-integration/src/cluster.rs              |  2 +-
 10 files changed, 38 insertions(+), 46 deletions(-)

diff --git a/src/meta-srv/src/bootstrap.rs b/src/meta-srv/src/bootstrap.rs
index 8323a7f8df..d5f369a348 100644
--- a/src/meta-srv/src/bootstrap.rs
+++ b/src/meta-srv/src/bootstrap.rs
@@ -56,9 +56,8 @@ use crate::selector::SelectorType;
 use crate::service::admin;
 use crate::{error, Result};
 
-#[derive(Clone)]
 pub struct MetasrvInstance {
-    metasrv: Metasrv,
+    metasrv: Arc<Metasrv>,
 
     httpsrv: Arc<HttpServer>,
 
@@ -83,8 +82,9 @@ impl MetasrvInstance {
                 .with_greptime_config_options(opts.to_toml().context(TomlFormatSnafu)?)
                 .build(),
         );
+        let metasrv = Arc::new(metasrv);
         // put metasrv into plugins for later use
-        plugins.insert::<Arc<Metasrv>>(Arc::new(metasrv.clone()));
+        plugins.insert::<Arc<Metasrv>>(metasrv.clone());
         let export_metrics_task = ExportMetricsTask::try_new(&opts.export_metrics, Some(&plugins))
             .context(InitExportMetricsTaskSnafu)?;
         Ok(MetasrvInstance {
@@ -178,13 +178,13 @@ pub async fn bootstrap_metasrv_with_router(
     Ok(())
 }
 
-pub fn router(metasrv: Metasrv) -> Router {
+pub fn router(metasrv: Arc<Metasrv>) -> Router {
     tonic::transport::Server::builder()
         .accept_http1(true) // for admin services
-        .add_service(HeartbeatServer::new(metasrv.clone()))
-        .add_service(StoreServer::new(metasrv.clone()))
-        .add_service(ClusterServer::new(metasrv.clone()))
-        .add_service(ProcedureServiceServer::new(metasrv.clone()))
+        .add_service(HeartbeatServer::from_arc(metasrv.clone()))
+        .add_service(StoreServer::from_arc(metasrv.clone()))
+        .add_service(ClusterServer::from_arc(metasrv.clone()))
+        .add_service(ProcedureServiceServer::from_arc(metasrv.clone()))
         .add_service(admin::make_admin_service(metasrv))
 }
 
diff --git a/src/meta-srv/src/handler.rs b/src/meta-srv/src/handler.rs
index 5499cb259b..15eff85a88 100644
--- a/src/meta-srv/src/handler.rs
+++ b/src/meta-srv/src/handler.rs
@@ -443,7 +443,6 @@ impl Mailbox for HeartbeatMailbox {
 }
 
 /// The builder to build the group of heartbeat handlers.
-#[derive(Clone)]
 pub struct HeartbeatHandlerGroupBuilder {
     /// The handler to handle region failure.
     region_failure_handler: Option<RegionFailureHandler>,
diff --git a/src/meta-srv/src/handler/failure_handler.rs b/src/meta-srv/src/handler/failure_handler.rs
index 8908881d17..02f423c4b4 100644
--- a/src/meta-srv/src/handler/failure_handler.rs
+++ b/src/meta-srv/src/handler/failure_handler.rs
@@ -21,7 +21,6 @@ use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
 use crate::metasrv::Context;
 use crate::region::supervisor::{DatanodeHeartbeat, HeartbeatAcceptor, RegionSupervisor};
 
-#[derive(Clone)]
 pub struct RegionFailureHandler {
     heartbeat_acceptor: HeartbeatAcceptor,
 }
diff --git a/src/meta-srv/src/handler/region_lease_handler.rs b/src/meta-srv/src/handler/region_lease_handler.rs
index 40a554e453..de491da371 100644
--- a/src/meta-srv/src/handler/region_lease_handler.rs
+++ b/src/meta-srv/src/handler/region_lease_handler.rs
@@ -26,7 +26,6 @@ use crate::metasrv::Context;
 use crate::region::lease_keeper::{RegionLeaseKeeperRef, RenewRegionLeasesResponse};
 use crate::region::RegionLeaseKeeper;
 
-#[derive(Clone)]
 pub struct RegionLeaseHandler {
     region_lease_seconds: u64,
     region_lease_keeper: RegionLeaseKeeperRef,
diff --git a/src/meta-srv/src/metasrv.rs b/src/meta-srv/src/metasrv.rs
index 71d15be2a0..5c53a1abbb 100644
--- a/src/meta-srv/src/metasrv.rs
+++ b/src/meta-srv/src/metasrv.rs
@@ -16,7 +16,7 @@ pub mod builder;
 
 use std::fmt::Display;
 use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
+use std::sync::{Arc, Mutex, RwLock};
 use std::time::Duration;
 
 use clap::ValueEnum;
@@ -337,7 +337,6 @@ impl MetaStateHandler {
     }
 }
 
-#[derive(Clone)]
 pub struct Metasrv {
     state: StateRef,
     started: Arc<AtomicBool>,
@@ -353,8 +352,8 @@ pub struct Metasrv {
     selector: SelectorRef,
     // The flow selector is used to select a target flownode.
     flow_selector: SelectorRef,
-    handler_group: Option<HeartbeatHandlerGroupRef>,
-    handler_group_builder: Option<HeartbeatHandlerGroupBuilder>,
+    handler_group: RwLock<Option<HeartbeatHandlerGroupRef>>,
+    handler_group_builder: Mutex<Option<HeartbeatHandlerGroupBuilder>>,
     election: Option<ElectionRef>,
     procedure_manager: ProcedureManagerRef,
     mailbox: MailboxRef,
@@ -371,15 +370,7 @@ pub struct Metasrv {
 }
 
 impl Metasrv {
-    pub async fn try_start(&mut self) -> Result<()> {
-        let builder = self
-            .handler_group_builder
-            .take()
-            .context(error::UnexpectedSnafu {
-                violated: "expected heartbeat handler group builder",
-            })?;
-        self.handler_group = Some(Arc::new(builder.build()?));
-
+    pub async fn try_start(&self) -> Result<()> {
         if self
             .started
             .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed)
@@ -389,6 +380,16 @@ impl Metasrv {
             return Ok(());
         }
 
+        let handler_group_builder =
+            self.handler_group_builder
+                .lock()
+                .unwrap()
+                .take()
+                .context(error::UnexpectedSnafu {
+                    violated: "expected heartbeat handler group builder",
+                })?;
+        *self.handler_group.write().unwrap() = Some(Arc::new(handler_group_builder.build()?));
+
         // Creates default schema if not exists
         self.table_metadata_manager
             .init()
@@ -567,12 +568,8 @@ impl Metasrv {
         &self.flow_selector
     }
 
-    pub fn handler_group(&self) -> &Option<HeartbeatHandlerGroupRef> {
-        &self.handler_group
-    }
-
-    pub fn handler_group_builder(&mut self) -> &mut Option<HeartbeatHandlerGroupBuilder> {
-        &mut self.handler_group_builder
+    pub fn handler_group(&self) -> Option<HeartbeatHandlerGroupRef> {
+        self.handler_group.read().unwrap().clone()
     }
 
     pub fn election(&self) -> Option<&ElectionRef> {
diff --git a/src/meta-srv/src/metasrv/builder.rs b/src/meta-srv/src/metasrv/builder.rs
index 32129b8538..31c17bb8f2 100644
--- a/src/meta-srv/src/metasrv/builder.rs
+++ b/src/meta-srv/src/metasrv/builder.rs
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use std::sync::atomic::AtomicBool;
-use std::sync::{Arc, RwLock};
+use std::sync::{Arc, Mutex, RwLock};
 use std::time::Duration;
 
 use client::client_manager::NodeClients;
@@ -371,8 +371,8 @@ impl MetasrvBuilder {
             selector,
             // TODO(jeremy): We do not allow configuring the flow selector.
             flow_selector: Arc::new(RoundRobinSelector::new(SelectTarget::Flownode)),
-            handler_group: None,
-            handler_group_builder: Some(handler_group_builder),
+            handler_group: RwLock::new(None),
+            handler_group_builder: Mutex::new(Some(handler_group_builder)),
             election,
             procedure_manager,
             mailbox,
diff --git a/src/meta-srv/src/mocks.rs b/src/meta-srv/src/mocks.rs
index 4a88701e1a..318fd16c87 100644
--- a/src/meta-srv/src/mocks.rs
+++ b/src/meta-srv/src/mocks.rs
@@ -33,7 +33,7 @@ use crate::metasrv::{Metasrv, MetasrvOptions, SelectorRef};
 pub struct MockInfo {
     pub server_addr: String,
     pub channel_manager: ChannelManager,
-    pub metasrv: Metasrv,
+    pub metasrv: Arc<Metasrv>,
 }
 
 pub async fn mock_with_memstore() -> MockInfo {
@@ -74,16 +74,17 @@ pub async fn mock(
         None => builder,
     };
 
-    let mut metasrv = builder.build().await.unwrap();
+    let metasrv = builder.build().await.unwrap();
     metasrv.try_start().await.unwrap();
 
     let (client, server) = tokio::io::duplex(1024);
+    let metasrv = Arc::new(metasrv);
     let service = metasrv.clone();
     let _handle = tokio::spawn(async move {
         tonic::transport::Server::builder()
-            .add_service(HeartbeatServer::new(service.clone()))
-            .add_service(StoreServer::new(service.clone()))
-            .add_service(ProcedureServiceServer::new(service.clone()))
+            .add_service(HeartbeatServer::from_arc(service.clone()))
+            .add_service(StoreServer::from_arc(service.clone()))
+            .add_service(ProcedureServiceServer::from_arc(service.clone()))
             .serve_with_incoming(futures::stream::iter(vec![Ok::<_, std::io::Error>(server)]))
             .await
     });
diff --git a/src/meta-srv/src/service/admin.rs b/src/meta-srv/src/service/admin.rs
index f60461fa7c..fca8cf3295 100644
--- a/src/meta-srv/src/service/admin.rs
+++ b/src/meta-srv/src/service/admin.rs
@@ -30,7 +30,7 @@ use tonic::server::NamedService;
 
 use crate::metasrv::Metasrv;
 
-pub fn make_admin_service(metasrv: Metasrv) -> Admin {
+pub fn make_admin_service(metasrv: Arc<Metasrv>) -> Admin {
     let router = Router::new().route("/health", health::HealthHandler);
 
     let router = router.route(
diff --git a/src/meta-srv/src/service/heartbeat.rs b/src/meta-srv/src/service/heartbeat.rs
index 2470129be9..9569d0e431 100644
--- a/src/meta-srv/src/service/heartbeat.rs
+++ b/src/meta-srv/src/service/heartbeat.rs
@@ -46,12 +46,9 @@ impl heartbeat_server::Heartbeat for Metasrv {
     ) -> GrpcResult<Self::HeartbeatStream> {
         let mut in_stream = req.into_inner();
         let (tx, rx) = mpsc::channel(128);
-        let handler_group = self
-            .handler_group()
-            .clone()
-            .context(error::UnexpectedSnafu {
-                violated: "expected heartbeat handlers",
-            })?;
+        let handler_group = self.handler_group().context(error::UnexpectedSnafu {
+            violated: "expected heartbeat handlers",
+        })?;
 
         let ctx = self.new_ctx();
         let _handle = common_runtime::spawn_global(async move {
diff --git a/tests-integration/src/cluster.rs b/tests-integration/src/cluster.rs
index 44e5946769..769dad4c1e 100644
--- a/tests-integration/src/cluster.rs
+++ b/tests-integration/src/cluster.rs
@@ -72,7 +72,7 @@ pub struct GreptimeDbCluster {
 
     pub datanode_instances: HashMap<DatanodeId, Datanode>,
     pub kv_backend: KvBackendRef,
-    pub metasrv: Metasrv,
+    pub metasrv: Arc<Metasrv>,
     pub frontend: Arc<FeInstance>,
 }
 

From 4622412dfec496ccc723fd0960ec61bdf428a79f Mon Sep 17 00:00:00 2001
From: localhost <xpaomian@gmail.com>
Date: Wed, 16 Oct 2024 12:36:08 +0800
Subject: [PATCH 083/128] feat: add API to write OpenTelemetry logs to
 GreptimeDB (#4755)

* chore: otlp logs api

* feat: add API to write OpenTelemetry logs to GreptimeDB

* chore: fix test data schema error

* chore: modify the underlying data structure of the pipeline value map type from hashmap to btremap to keep key order

* chore: fix by pr comment

* chore: resolve conflicts and add some test

* chore: remove useless error

* chore: change otlp header name

* chore: fmt code

* chore: fix integration test for otlp log write api

* chore: fix by pr comment

* chore: set otlp body with fulltext default
---
 Cargo.lock                                    |   2 +
 Cargo.toml                                    |   1 +
 src/datatypes/src/value.rs                    |  78 +--
 src/frontend/src/instance/otlp.rs             |  31 +-
 src/frontend/src/metrics.rs                   |  12 +
 src/pipeline/src/etl.rs                       |  36 ++
 src/pipeline/src/etl/error.rs                 |  16 +-
 src/pipeline/src/etl/processor/cmcd.rs        |   7 +-
 src/pipeline/src/etl/processor/regex.rs       |   6 +-
 .../transform/transformer/greptime/coerce.rs  | 130 +++--
 src/pipeline/src/etl/value.rs                 |  93 +++-
 src/pipeline/src/etl/value/map.rs             |  34 +-
 src/pipeline/src/lib.rs                       |   2 +-
 src/pipeline/tests/pipeline.rs                |  52 ++
 src/servers/Cargo.toml                        |   1 +
 src/servers/src/error.rs                      |  10 +-
 src/servers/src/http.rs                       |  10 +-
 src/servers/src/http/header.rs                |   3 +
 src/servers/src/http/otlp.rs                  | 179 ++++++-
 src/servers/src/metrics.rs                    |   7 +
 src/servers/src/otlp.rs                       |   1 +
 src/servers/src/otlp/logs.rs                  | 506 ++++++++++++++++++
 src/servers/src/query_handler.rs              |  13 +-
 tests-integration/tests/http.rs               |  58 +-
 24 files changed, 1158 insertions(+), 130 deletions(-)
 create mode 100644 src/servers/src/otlp/logs.rs

diff --git a/Cargo.lock b/Cargo.lock
index c3736b2d9f..bf76687e72 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7549,6 +7549,7 @@ dependencies = [
  "ordered-float 4.3.0",
  "percent-encoding",
  "rand",
+ "serde_json",
  "thiserror",
 ]
 
@@ -10737,6 +10738,7 @@ dependencies = [
 name = "servers"
 version = "0.9.4"
 dependencies = [
+ "ahash 0.8.11",
  "aide",
  "api",
  "arrow",
diff --git a/Cargo.toml b/Cargo.toml
index 133db91819..484cdcf55d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -138,6 +138,7 @@ opentelemetry-proto = { version = "0.5", features = [
     "metrics",
     "trace",
     "with-serde",
+    "logs",
 ] }
 parquet = { version = "51.0.0", default-features = false, features = ["arrow", "async", "object_store"] }
 paste = "1.0"
diff --git a/src/datatypes/src/value.rs b/src/datatypes/src/value.rs
index b973a3156b..76b59a8db7 100644
--- a/src/datatypes/src/value.rs
+++ b/src/datatypes/src/value.rs
@@ -38,6 +38,7 @@ use snafu::{ensure, ResultExt};
 
 use crate::error::{self, ConvertArrowArrayToScalarsSnafu, Error, Result, TryFromValueSnafu};
 use crate::prelude::*;
+use crate::schema::ColumnSchema;
 use crate::type_id::LogicalTypeId;
 use crate::types::{IntervalType, ListType};
 use crate::vectors::ListVector;
@@ -1286,39 +1287,52 @@ impl<'a> From<Option<ListValueRef<'a>>> for ValueRef<'a> {
     }
 }
 
-impl<'a> TryFrom<ValueRef<'a>> for serde_json::Value {
-    type Error = serde_json::Error;
+/// transform a [ValueRef] to a [serde_json::Value].
+/// The json type will be handled specially
+pub fn transform_value_ref_to_json_value<'a>(
+    value: ValueRef<'a>,
+    schema: &'a ColumnSchema,
+) -> serde_json::Result<serde_json::Value> {
+    let json_value = match value {
+        ValueRef::Null => serde_json::Value::Null,
+        ValueRef::Boolean(v) => serde_json::Value::Bool(v),
+        ValueRef::UInt8(v) => serde_json::Value::from(v),
+        ValueRef::UInt16(v) => serde_json::Value::from(v),
+        ValueRef::UInt32(v) => serde_json::Value::from(v),
+        ValueRef::UInt64(v) => serde_json::Value::from(v),
+        ValueRef::Int8(v) => serde_json::Value::from(v),
+        ValueRef::Int16(v) => serde_json::Value::from(v),
+        ValueRef::Int32(v) => serde_json::Value::from(v),
+        ValueRef::Int64(v) => serde_json::Value::from(v),
+        ValueRef::Float32(v) => serde_json::Value::from(v.0),
+        ValueRef::Float64(v) => serde_json::Value::from(v.0),
+        ValueRef::String(bytes) => serde_json::Value::String(bytes.to_string()),
+        ValueRef::Binary(bytes) => {
+            if let ConcreteDataType::Json(_) = schema.data_type {
+                match jsonb::from_slice(bytes) {
+                    Ok(json) => json.into(),
+                    Err(e) => {
+                        error!(e; "Failed to parse jsonb");
+                        serde_json::Value::Null
+                    }
+                }
+            } else {
+                serde_json::to_value(bytes)?
+            }
+        }
+        ValueRef::Date(v) => serde_json::Value::Number(v.val().into()),
+        ValueRef::DateTime(v) => serde_json::Value::Number(v.val().into()),
+        ValueRef::List(v) => serde_json::to_value(v)?,
+        ValueRef::Timestamp(v) => serde_json::to_value(v.value())?,
+        ValueRef::Time(v) => serde_json::to_value(v.value())?,
+        ValueRef::IntervalYearMonth(v) => serde_json::Value::from(v),
+        ValueRef::IntervalDayTime(v) => serde_json::Value::from(v),
+        ValueRef::IntervalMonthDayNano(v) => serde_json::Value::from(v),
+        ValueRef::Duration(v) => serde_json::to_value(v.value())?,
+        ValueRef::Decimal128(v) => serde_json::to_value(v.to_string())?,
+    };
 
-    fn try_from(value: ValueRef<'a>) -> serde_json::Result<serde_json::Value> {
-        let json_value = match value {
-            ValueRef::Null => serde_json::Value::Null,
-            ValueRef::Boolean(v) => serde_json::Value::Bool(v),
-            ValueRef::UInt8(v) => serde_json::Value::from(v),
-            ValueRef::UInt16(v) => serde_json::Value::from(v),
-            ValueRef::UInt32(v) => serde_json::Value::from(v),
-            ValueRef::UInt64(v) => serde_json::Value::from(v),
-            ValueRef::Int8(v) => serde_json::Value::from(v),
-            ValueRef::Int16(v) => serde_json::Value::from(v),
-            ValueRef::Int32(v) => serde_json::Value::from(v),
-            ValueRef::Int64(v) => serde_json::Value::from(v),
-            ValueRef::Float32(v) => serde_json::Value::from(v.0),
-            ValueRef::Float64(v) => serde_json::Value::from(v.0),
-            ValueRef::String(bytes) => serde_json::Value::String(bytes.to_string()),
-            ValueRef::Binary(bytes) => serde_json::to_value(bytes)?,
-            ValueRef::Date(v) => serde_json::Value::Number(v.val().into()),
-            ValueRef::DateTime(v) => serde_json::Value::Number(v.val().into()),
-            ValueRef::List(v) => serde_json::to_value(v)?,
-            ValueRef::Timestamp(v) => serde_json::to_value(v.value())?,
-            ValueRef::Time(v) => serde_json::to_value(v.value())?,
-            ValueRef::IntervalYearMonth(v) => serde_json::Value::from(v),
-            ValueRef::IntervalDayTime(v) => serde_json::Value::from(v),
-            ValueRef::IntervalMonthDayNano(v) => serde_json::Value::from(v),
-            ValueRef::Duration(v) => serde_json::to_value(v.value())?,
-            ValueRef::Decimal128(v) => serde_json::to_value(v.to_string())?,
-        };
-
-        Ok(json_value)
-    }
+    Ok(json_value)
 }
 
 /// Reference to a [ListValue].
diff --git a/src/frontend/src/instance/otlp.rs b/src/frontend/src/instance/otlp.rs
index 09335af080..7ed1609715 100644
--- a/src/frontend/src/instance/otlp.rs
+++ b/src/frontend/src/instance/otlp.rs
@@ -17,8 +17,10 @@ use auth::{PermissionChecker, PermissionCheckerRef, PermissionReq};
 use client::Output;
 use common_error::ext::BoxedError;
 use common_telemetry::tracing;
+use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
 use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest;
 use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
+use pipeline::PipelineWay;
 use servers::error::{self, AuthSnafu, Result as ServerResult};
 use servers::interceptor::{OpenTelemetryProtocolInterceptor, OpenTelemetryProtocolInterceptorRef};
 use servers::otlp;
@@ -28,7 +30,7 @@ use session::context::QueryContextRef;
 use snafu::ResultExt;
 
 use crate::instance::Instance;
-use crate::metrics::{OTLP_METRICS_ROWS, OTLP_TRACES_ROWS};
+use crate::metrics::{OTLP_LOGS_ROWS, OTLP_METRICS_ROWS, OTLP_TRACES_ROWS};
 
 #[async_trait]
 impl OpenTelemetryProtocolHandler for Instance {
@@ -92,4 +94,31 @@ impl OpenTelemetryProtocolHandler for Instance {
             .map_err(BoxedError::new)
             .context(error::ExecuteGrpcQuerySnafu)
     }
+
+    #[tracing::instrument(skip_all)]
+    async fn logs(
+        &self,
+        request: ExportLogsServiceRequest,
+        pipeline: PipelineWay,
+        table_name: String,
+        ctx: QueryContextRef,
+    ) -> ServerResult<Output> {
+        self.plugins
+            .get::<PermissionCheckerRef>()
+            .as_ref()
+            .check_permission(ctx.current_user(), PermissionReq::Otlp)
+            .context(AuthSnafu)?;
+
+        let interceptor_ref = self
+            .plugins
+            .get::<OpenTelemetryProtocolInterceptorRef<servers::error::Error>>();
+        interceptor_ref.pre_execute(ctx.clone())?;
+        let (requests, rows) = otlp::logs::to_grpc_insert_requests(request, pipeline, table_name)?;
+
+        self.handle_log_inserts(requests, ctx)
+            .await
+            .inspect(|_| OTLP_LOGS_ROWS.inc_by(rows as u64))
+            .map_err(BoxedError::new)
+            .context(error::ExecuteGrpcQuerySnafu)
+    }
 }
diff --git a/src/frontend/src/metrics.rs b/src/frontend/src/metrics.rs
index 33580c550e..a8bf4eb76e 100644
--- a/src/frontend/src/metrics.rs
+++ b/src/frontend/src/metrics.rs
@@ -41,16 +41,28 @@ lazy_static! {
         .with_label_values(&["insert"]);
     pub static ref EXECUTE_SCRIPT_ELAPSED: Histogram = HANDLE_SCRIPT_ELAPSED
         .with_label_values(&["execute"]);
+
+    /// The number of OpenTelemetry metrics send by frontend node.
     pub static ref OTLP_METRICS_ROWS: IntCounter = register_int_counter!(
         "greptime_frontend_otlp_metrics_rows",
         "frontend otlp metrics rows"
     )
     .unwrap();
+
+    /// The number of OpenTelemetry traces send by frontend node.
     pub static ref OTLP_TRACES_ROWS: IntCounter = register_int_counter!(
         "greptime_frontend_otlp_traces_rows",
         "frontend otlp traces rows"
     )
     .unwrap();
+
+    /// The number of OpenTelemetry logs send by frontend node.
+    pub static ref OTLP_LOGS_ROWS: IntCounter = register_int_counter!(
+        "greptime_frontend_otlp_logs_rows",
+        "frontend otlp logs rows"
+    )
+    .unwrap();
+
     /// The number of heartbeats send by frontend node.
     pub static ref HEARTBEAT_SENT_COUNT: IntCounter = register_int_counter!(
         "greptime_frontend_heartbeat_send_count",
diff --git a/src/pipeline/src/etl.rs b/src/pipeline/src/etl.rs
index f29032e4f8..bec06cab8b 100644
--- a/src/pipeline/src/etl.rs
+++ b/src/pipeline/src/etl.rs
@@ -210,6 +210,37 @@ where
         self.transformer.transform_mut(val)
     }
 
+    pub fn prepare_pipeline_value(&self, val: Value, result: &mut [Value]) -> Result<()> {
+        match val {
+            Value::Map(map) => {
+                let mut search_from = 0;
+                // because of the key in the json map is ordered
+                for (payload_key, payload_value) in map.values.into_iter() {
+                    if search_from >= self.required_keys.len() {
+                        break;
+                    }
+
+                    // because of map key is ordered, required_keys is ordered too
+                    if let Some(pos) = self.required_keys[search_from..]
+                        .iter()
+                        .position(|k| k == &payload_key)
+                    {
+                        result[search_from + pos] = payload_value;
+                        // next search from is always after the current key
+                        search_from += pos;
+                    }
+                }
+            }
+            Value::String(_) => {
+                result[0] = val;
+            }
+            _ => {
+                return PrepareValueMustBeObjectSnafu.fail();
+            }
+        }
+        Ok(())
+    }
+
     pub fn prepare(&self, val: serde_json::Value, result: &mut [Value]) -> Result<()> {
         match val {
             serde_json::Value::Object(map) => {
@@ -286,6 +317,11 @@ pub(crate) fn find_key_index(intermediate_keys: &[String], key: &str, kind: &str
         .context(IntermediateKeyIndexSnafu { kind, key })
 }
 
+pub enum PipelineWay {
+    Identity,
+    Custom(std::sync::Arc<Pipeline<crate::GreptimeTransformer>>),
+}
+
 #[cfg(test)]
 mod tests {
 
diff --git a/src/pipeline/src/etl/error.rs b/src/pipeline/src/etl/error.rs
index 08654c2c82..9fe0af7cef 100644
--- a/src/pipeline/src/etl/error.rs
+++ b/src/pipeline/src/etl/error.rs
@@ -438,18 +438,26 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("failed to coerce complex value, not supported"))]
-    CoerceComplexType {
+    #[snafu(display("Can not coerce json type to {ty}"))]
+    CoerceJsonTypeTo {
+        ty: String,
         #[snafu(implicit)]
         location: Location,
     },
-    #[snafu(display("failed to coerce value: {msg}"))]
+    #[snafu(display(
+        "Can not coerce {ty} to json type. we only consider object and array to be json types."
+    ))]
+    CoerceTypeToJson {
+        ty: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+    #[snafu(display("Failed to coerce value: {msg}"))]
     CoerceIncompatibleTypes {
         msg: String,
         #[snafu(implicit)]
         location: Location,
     },
-
     #[snafu(display(
         "Invalid resolution: '{resolution}'. Available resolutions: {valid_resolution}"
     ))]
diff --git a/src/pipeline/src/etl/processor/cmcd.rs b/src/pipeline/src/etl/processor/cmcd.rs
index f43186b94a..06cfeb7c69 100644
--- a/src/pipeline/src/etl/processor/cmcd.rs
+++ b/src/pipeline/src/etl/processor/cmcd.rs
@@ -402,7 +402,8 @@ impl Processor for CmcdProcessor {
 
 #[cfg(test)]
 mod tests {
-    use ahash::HashMap;
+    use std::collections::BTreeMap;
+
     use urlencoding::decode;
 
     use super::{CmcdProcessorBuilder, CMCD_KEYS};
@@ -563,14 +564,14 @@ mod tests {
             let values = vec
                 .into_iter()
                 .map(|(k, v)| (k.to_string(), v))
-                .collect::<HashMap<String, Value>>();
+                .collect::<BTreeMap<String, Value>>();
             let expected = Map { values };
 
             let actual = processor.parse(0, &decoded).unwrap();
             let actual = actual
                 .into_iter()
                 .map(|(index, value)| (intermediate_keys[index].clone(), value))
-                .collect::<HashMap<String, Value>>();
+                .collect::<BTreeMap<String, Value>>();
             let actual = Map { values: actual };
             assert_eq!(actual, expected);
         }
diff --git a/src/pipeline/src/etl/processor/regex.rs b/src/pipeline/src/etl/processor/regex.rs
index a74c19140c..de25195f99 100644
--- a/src/pipeline/src/etl/processor/regex.rs
+++ b/src/pipeline/src/etl/processor/regex.rs
@@ -383,6 +383,8 @@ impl Processor for RegexProcessor {
 }
 #[cfg(test)]
 mod tests {
+    use std::collections::BTreeMap;
+
     use ahash::{HashMap, HashMapExt};
     use itertools::Itertools;
 
@@ -475,14 +477,14 @@ ignore_missing: false"#;
             .map(|k| k.to_string())
             .collect_vec();
             let processor = builder.build(&intermediate_keys).unwrap();
-            let mut result = HashMap::new();
+            let mut result = BTreeMap::new();
             for (index, pattern) in processor.patterns.iter().enumerate() {
                 let r = processor
                     .process(&breadcrumbs_str, pattern, (0, index))
                     .unwrap()
                     .into_iter()
                     .map(|(k, v)| (intermediate_keys[k].clone(), v))
-                    .collect::<HashMap<_, _>>();
+                    .collect::<BTreeMap<_, _>>();
                 result.extend(r);
             }
             let map = Map { values: result };
diff --git a/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs b/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs
index 8d07b34d46..a0a96b3562 100644
--- a/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs
+++ b/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs
@@ -12,16 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use api::v1::column_data_type_extension::TypeExt;
 use api::v1::column_def::options_from_fulltext;
-use api::v1::ColumnOptions;
+use api::v1::{ColumnDataTypeExtension, ColumnOptions, JsonTypeExtension};
 use datatypes::schema::FulltextOptions;
 use greptime_proto::v1::value::ValueData;
 use greptime_proto::v1::{ColumnDataType, ColumnSchema, SemanticType};
 use snafu::ResultExt;
 
 use crate::etl::error::{
-    CoerceComplexTypeSnafu, CoerceIncompatibleTypesSnafu, CoerceStringToTypeSnafu,
-    CoerceUnsupportedEpochTypeSnafu, CoerceUnsupportedNullTypeSnafu,
+    CoerceIncompatibleTypesSnafu, CoerceJsonTypeToSnafu, CoerceStringToTypeSnafu,
+    CoerceTypeToJsonSnafu, CoerceUnsupportedEpochTypeSnafu, CoerceUnsupportedNullTypeSnafu,
     CoerceUnsupportedNullTypeToSnafu, ColumnOptionsSnafu, Error, Result,
 };
 use crate::etl::transform::index::Index;
@@ -62,7 +63,10 @@ impl TryFrom<Value> for ValueData {
             }
             Value::Timestamp(Timestamp::Second(s)) => Ok(ValueData::TimestampSecondValue(s)),
 
-            Value::Array(_) | Value::Map(_) => CoerceComplexTypeSnafu.fail(),
+            Value::Array(_) | Value::Map(_) => {
+                let data: jsonb::Value = value.into();
+                Ok(ValueData::BinaryValue(data.to_vec()))
+            }
         }
     }
 }
@@ -74,15 +78,15 @@ pub(crate) fn coerce_columns(transform: &Transform) -> Result<Vec<ColumnSchema>>
     for field in transform.real_fields.iter() {
         let column_name = field.output_name().to_string();
 
-        let datatype = coerce_type(transform)? as i32;
+        let (datatype, datatype_extension) = coerce_type(transform)?;
 
         let semantic_type = coerce_semantic_type(transform) as i32;
 
         let column = ColumnSchema {
             column_name,
-            datatype,
+            datatype: datatype as i32,
             semantic_type,
-            datatype_extension: None,
+            datatype_extension,
             options: coerce_options(transform)?,
         };
         columns.push(column);
@@ -111,30 +115,41 @@ fn coerce_options(transform: &Transform) -> Result<Option<ColumnOptions>> {
     }
 }
 
-fn coerce_type(transform: &Transform) -> Result<ColumnDataType> {
+fn coerce_type(transform: &Transform) -> Result<(ColumnDataType, Option<ColumnDataTypeExtension>)> {
     match transform.type_ {
-        Value::Int8(_) => Ok(ColumnDataType::Int8),
-        Value::Int16(_) => Ok(ColumnDataType::Int16),
-        Value::Int32(_) => Ok(ColumnDataType::Int32),
-        Value::Int64(_) => Ok(ColumnDataType::Int64),
+        Value::Int8(_) => Ok((ColumnDataType::Int8, None)),
+        Value::Int16(_) => Ok((ColumnDataType::Int16, None)),
+        Value::Int32(_) => Ok((ColumnDataType::Int32, None)),
+        Value::Int64(_) => Ok((ColumnDataType::Int64, None)),
 
-        Value::Uint8(_) => Ok(ColumnDataType::Uint8),
-        Value::Uint16(_) => Ok(ColumnDataType::Uint16),
-        Value::Uint32(_) => Ok(ColumnDataType::Uint32),
-        Value::Uint64(_) => Ok(ColumnDataType::Uint64),
+        Value::Uint8(_) => Ok((ColumnDataType::Uint8, None)),
+        Value::Uint16(_) => Ok((ColumnDataType::Uint16, None)),
+        Value::Uint32(_) => Ok((ColumnDataType::Uint32, None)),
+        Value::Uint64(_) => Ok((ColumnDataType::Uint64, None)),
 
-        Value::Float32(_) => Ok(ColumnDataType::Float32),
-        Value::Float64(_) => Ok(ColumnDataType::Float64),
+        Value::Float32(_) => Ok((ColumnDataType::Float32, None)),
+        Value::Float64(_) => Ok((ColumnDataType::Float64, None)),
 
-        Value::Boolean(_) => Ok(ColumnDataType::Boolean),
-        Value::String(_) => Ok(ColumnDataType::String),
+        Value::Boolean(_) => Ok((ColumnDataType::Boolean, None)),
+        Value::String(_) => Ok((ColumnDataType::String, None)),
 
-        Value::Timestamp(Timestamp::Nanosecond(_)) => Ok(ColumnDataType::TimestampNanosecond),
-        Value::Timestamp(Timestamp::Microsecond(_)) => Ok(ColumnDataType::TimestampMicrosecond),
-        Value::Timestamp(Timestamp::Millisecond(_)) => Ok(ColumnDataType::TimestampMillisecond),
-        Value::Timestamp(Timestamp::Second(_)) => Ok(ColumnDataType::TimestampSecond),
+        Value::Timestamp(Timestamp::Nanosecond(_)) => {
+            Ok((ColumnDataType::TimestampNanosecond, None))
+        }
+        Value::Timestamp(Timestamp::Microsecond(_)) => {
+            Ok((ColumnDataType::TimestampMicrosecond, None))
+        }
+        Value::Timestamp(Timestamp::Millisecond(_)) => {
+            Ok((ColumnDataType::TimestampMillisecond, None))
+        }
+        Value::Timestamp(Timestamp::Second(_)) => Ok((ColumnDataType::TimestampSecond, None)),
 
-        Value::Array(_) | Value::Map(_) => CoerceComplexTypeSnafu.fail(),
+        Value::Array(_) | Value::Map(_) => Ok((
+            ColumnDataType::Binary,
+            Some(ColumnDataTypeExtension {
+                type_ext: Some(TypeExt::JsonType(JsonTypeExtension::JsonBinary.into())),
+            }),
+        )),
 
         Value::Null => CoerceUnsupportedNullTypeToSnafu {
             ty: transform.type_.to_str_type(),
@@ -191,12 +206,12 @@ pub(crate) fn coerce_value(val: &Value, transform: &Transform) -> Result<Option<
                 ))),
             },
             _ => CoerceIncompatibleTypesSnafu {
-                msg: "Timestamp can only be coerced to another timestamp",
+                msg: "Timestamp can only be coerced to another type",
             }
             .fail(),
         },
 
-        Value::Array(_) | Value::Map(_) => CoerceComplexTypeSnafu.fail(),
+        Value::Array(_) | Value::Map(_) => coerce_json_value(val, transform),
     }
 }
 
@@ -228,7 +243,12 @@ fn coerce_bool_value(b: bool, transform: &Transform) -> Result<Option<ValueData>
             }
         },
 
-        Value::Array(_) | Value::Map(_) => return CoerceComplexTypeSnafu.fail(),
+        Value::Array(_) | Value::Map(_) => {
+            return CoerceJsonTypeToSnafu {
+                ty: transform.type_.to_str_type(),
+            }
+            .fail()
+        }
 
         Value::Null => return Ok(None),
     };
@@ -264,7 +284,12 @@ fn coerce_i64_value(n: i64, transform: &Transform) -> Result<Option<ValueData>>
             }
         },
 
-        Value::Array(_) | Value::Map(_) => return CoerceComplexTypeSnafu.fail(),
+        Value::Array(_) | Value::Map(_) => {
+            return CoerceJsonTypeToSnafu {
+                ty: transform.type_.to_str_type(),
+            }
+            .fail()
+        }
 
         Value::Null => return Ok(None),
     };
@@ -300,7 +325,12 @@ fn coerce_u64_value(n: u64, transform: &Transform) -> Result<Option<ValueData>>
             }
         },
 
-        Value::Array(_) | Value::Map(_) => return CoerceComplexTypeSnafu.fail(),
+        Value::Array(_) | Value::Map(_) => {
+            return CoerceJsonTypeToSnafu {
+                ty: transform.type_.to_str_type(),
+            }
+            .fail()
+        }
 
         Value::Null => return Ok(None),
     };
@@ -336,7 +366,12 @@ fn coerce_f64_value(n: f64, transform: &Transform) -> Result<Option<ValueData>>
             }
         },
 
-        Value::Array(_) | Value::Map(_) => return CoerceComplexTypeSnafu.fail(),
+        Value::Array(_) | Value::Map(_) => {
+            return CoerceJsonTypeToSnafu {
+                ty: transform.type_.to_str_type(),
+            }
+            .fail()
+        }
 
         Value::Null => return Ok(None),
     };
@@ -411,12 +446,43 @@ fn coerce_string_value(s: &String, transform: &Transform) -> Result<Option<Value
             None => CoerceUnsupportedEpochTypeSnafu { ty: "String" }.fail(),
         },
 
-        Value::Array(_) | Value::Map(_) => CoerceComplexTypeSnafu.fail(),
+        Value::Array(_) | Value::Map(_) => {
+            return CoerceJsonTypeToSnafu {
+                ty: transform.type_.to_str_type(),
+            }
+            .fail()
+        }
 
         Value::Null => Ok(None),
     }
 }
 
+fn coerce_json_value(v: &Value, transform: &Transform) -> Result<Option<ValueData>> {
+    match &transform.type_ {
+        Value::Array(_) | Value::Map(_) => (),
+        t => {
+            return CoerceTypeToJsonSnafu {
+                ty: t.to_str_type(),
+            }
+            .fail();
+        }
+    }
+    match v {
+        Value::Map(_) => {
+            let data: jsonb::Value = v.into();
+            Ok(Some(ValueData::BinaryValue(data.to_vec())))
+        }
+        Value::Array(_) => {
+            let data: jsonb::Value = v.into();
+            Ok(Some(ValueData::BinaryValue(data.to_vec())))
+        }
+        _ => CoerceTypeToJsonSnafu {
+            ty: v.to_str_type(),
+        }
+        .fail(),
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/src/pipeline/src/etl/value.rs b/src/pipeline/src/etl/value.rs
index 3adde2514b..5316000256 100644
--- a/src/pipeline/src/etl/value.rs
+++ b/src/pipeline/src/etl/value.rs
@@ -16,8 +16,10 @@ pub mod array;
 pub mod map;
 pub mod time;
 
-use ahash::{HashMap, HashMapExt};
+use std::collections::BTreeMap;
+
 pub use array::Array;
+use jsonb::{Number as JsonbNumber, Object as JsonbObject, Value as JsonbValue};
 pub use map::Map;
 use snafu::{OptionExt, ResultExt};
 pub use time::Timestamp;
@@ -57,6 +59,7 @@ pub enum Value {
 
     Timestamp(Timestamp),
 
+    /// We only consider object and array to be json types.
     Array(Array),
     Map(Map),
 }
@@ -110,8 +113,9 @@ impl Value {
                 _ => Ok(Value::Timestamp(Timestamp::Nanosecond(0))),
             },
 
-            "array" => Ok(Value::Array(Array::default())),
-            "map" => Ok(Value::Map(Map::default())),
+            // We only consider object and array to be json types. and use Map to represent json
+            // TODO(qtang): Needs to be defined with better semantics
+            "json" => Ok(Value::Map(Map::default())),
 
             _ => ValueParseTypeSnafu { t }.fail(),
         }
@@ -221,8 +225,7 @@ impl Value {
 
             Value::Timestamp(_) => "epoch",
 
-            Value::Array(_) => "array",
-            Value::Map(_) => "map",
+            Value::Array(_) | Value::Map(_) => "json",
 
             Value::Null => "null",
         }
@@ -287,7 +290,7 @@ impl TryFrom<serde_json::Value> for Value {
                 Ok(Value::Array(Array { values }))
             }
             serde_json::Value::Object(v) => {
-                let mut values = HashMap::with_capacity(v.len());
+                let mut values = BTreeMap::new();
                 for (k, v) in v {
                     values.insert(k, Value::try_from(v)?);
                 }
@@ -318,7 +321,7 @@ impl TryFrom<&yaml_rust::Yaml> for Value {
                 Ok(Value::Array(Array { values }))
             }
             yaml_rust::Yaml::Hash(v) => {
-                let mut values = HashMap::new();
+                let mut values = BTreeMap::new();
                 for (k, v) in v {
                     let key = k
                         .as_str()
@@ -331,3 +334,79 @@ impl TryFrom<&yaml_rust::Yaml> for Value {
         }
     }
 }
+
+impl<'a> From<&Value> for JsonbValue<'a> {
+    fn from(value: &Value) -> Self {
+        match value {
+            Value::Null => JsonbValue::Null,
+            Value::Boolean(v) => JsonbValue::Bool(*v),
+
+            Value::Int8(v) => JsonbValue::Number(JsonbNumber::Int64(*v as i64)),
+            Value::Int16(v) => JsonbValue::Number(JsonbNumber::Int64(*v as i64)),
+            Value::Int32(v) => JsonbValue::Number(JsonbNumber::Int64(*v as i64)),
+            Value::Int64(v) => JsonbValue::Number(JsonbNumber::Int64(*v)),
+
+            Value::Uint8(v) => JsonbValue::Number(JsonbNumber::UInt64(*v as u64)),
+            Value::Uint16(v) => JsonbValue::Number(JsonbNumber::UInt64(*v as u64)),
+            Value::Uint32(v) => JsonbValue::Number(JsonbNumber::UInt64(*v as u64)),
+            Value::Uint64(v) => JsonbValue::Number(JsonbNumber::UInt64(*v)),
+            Value::Float32(v) => JsonbValue::Number(JsonbNumber::Float64(*v as f64)),
+            Value::Float64(v) => JsonbValue::Number(JsonbNumber::Float64(*v)),
+            Value::String(v) => JsonbValue::String(v.clone().into()),
+            Value::Timestamp(v) => JsonbValue::String(v.to_string().into()),
+            Value::Array(arr) => {
+                let mut vals: Vec<JsonbValue> = Vec::with_capacity(arr.len());
+                for val in arr.iter() {
+                    vals.push(val.into());
+                }
+                JsonbValue::Array(vals)
+            }
+            Value::Map(obj) => {
+                let mut map = JsonbObject::new();
+                for (k, v) in obj.iter() {
+                    let val: JsonbValue = v.into();
+                    map.insert(k.to_string(), val);
+                }
+                JsonbValue::Object(map)
+            }
+        }
+    }
+}
+
+impl<'a> From<Value> for JsonbValue<'a> {
+    fn from(value: Value) -> Self {
+        match value {
+            Value::Null => JsonbValue::Null,
+            Value::Boolean(v) => JsonbValue::Bool(v),
+
+            Value::Int8(v) => JsonbValue::Number(JsonbNumber::Int64(v as i64)),
+            Value::Int16(v) => JsonbValue::Number(JsonbNumber::Int64(v as i64)),
+            Value::Int32(v) => JsonbValue::Number(JsonbNumber::Int64(v as i64)),
+            Value::Int64(v) => JsonbValue::Number(JsonbNumber::Int64(v)),
+
+            Value::Uint8(v) => JsonbValue::Number(JsonbNumber::UInt64(v as u64)),
+            Value::Uint16(v) => JsonbValue::Number(JsonbNumber::UInt64(v as u64)),
+            Value::Uint32(v) => JsonbValue::Number(JsonbNumber::UInt64(v as u64)),
+            Value::Uint64(v) => JsonbValue::Number(JsonbNumber::UInt64(v)),
+            Value::Float32(v) => JsonbValue::Number(JsonbNumber::Float64(v as f64)),
+            Value::Float64(v) => JsonbValue::Number(JsonbNumber::Float64(v)),
+            Value::String(v) => JsonbValue::String(v.into()),
+            Value::Timestamp(v) => JsonbValue::String(v.to_string().into()),
+            Value::Array(arr) => {
+                let mut vals: Vec<JsonbValue> = Vec::with_capacity(arr.len());
+                for val in arr.into_iter() {
+                    vals.push(val.into());
+                }
+                JsonbValue::Array(vals)
+            }
+            Value::Map(obj) => {
+                let mut map = JsonbObject::new();
+                for (k, v) in obj.into_iter() {
+                    let val: JsonbValue = v.into();
+                    map.insert(k, val);
+                }
+                JsonbValue::Object(map)
+            }
+        }
+    }
+}
diff --git a/src/pipeline/src/etl/value/map.rs b/src/pipeline/src/etl/value/map.rs
index b8b81da756..004a617b0f 100644
--- a/src/pipeline/src/etl/value/map.rs
+++ b/src/pipeline/src/etl/value/map.rs
@@ -12,21 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use ahash::{HashMap, HashMapExt};
+use std::collections::BTreeMap;
+
+use ahash::HashMap;
 
 use crate::etl::value::Value;
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Default)]
 pub struct Map {
-    pub values: HashMap<String, Value>,
-}
-
-impl Default for Map {
-    fn default() -> Self {
-        Self {
-            values: HashMap::with_capacity(30),
-        }
-    }
+    pub values: BTreeMap<String, Value>,
 }
 
 impl Map {
@@ -47,12 +41,16 @@ impl Map {
 
 impl From<HashMap<String, Value>> for Map {
     fn from(values: HashMap<String, Value>) -> Self {
-        Map { values }
+        let mut map = Map::default();
+        for (k, v) in values.into_iter() {
+            map.insert(k, v);
+        }
+        map
     }
 }
 
 impl std::ops::Deref for Map {
-    type Target = HashMap<String, Value>;
+    type Target = BTreeMap<String, Value>;
 
     fn deref(&self) -> &Self::Target {
         &self.values
@@ -65,6 +63,16 @@ impl std::ops::DerefMut for Map {
     }
 }
 
+impl std::iter::IntoIterator for Map {
+    type Item = (String, Value);
+
+    type IntoIter = std::collections::btree_map::IntoIter<String, Value>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.values.into_iter()
+    }
+}
+
 impl std::fmt::Display for Map {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         let values = self
diff --git a/src/pipeline/src/lib.rs b/src/pipeline/src/lib.rs
index 36ef3023f6..1043006b73 100644
--- a/src/pipeline/src/lib.rs
+++ b/src/pipeline/src/lib.rs
@@ -21,7 +21,7 @@ pub use etl::processor::Processor;
 pub use etl::transform::transformer::identity_pipeline;
 pub use etl::transform::{GreptimeTransformer, Transformer};
 pub use etl::value::{Array, Map, Value};
-pub use etl::{parse, Content, Pipeline};
+pub use etl::{error as etl_error, parse, Content, Pipeline, PipelineWay};
 pub use manager::{
     error, pipeline_operator, table, util, PipelineInfo, PipelineRef, PipelineTableRef,
     PipelineVersion,
diff --git a/src/pipeline/tests/pipeline.rs b/src/pipeline/tests/pipeline.rs
index d5712eaedd..c5b8bfc201 100644
--- a/src/pipeline/tests/pipeline.rs
+++ b/src/pipeline/tests/pipeline.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use api::v1::value::ValueData;
 use api::v1::Rows;
 use common_telemetry::tracing::info;
 use greptime_proto::v1::value::ValueData::{
@@ -466,6 +467,57 @@ transform:
     }
 }
 
+#[test]
+fn test_json_type() {
+    let input_value_str = r#"
+{
+    "product_object": {"hello":"world"},
+    "product_array": ["hello", "world"]
+}
+"#;
+    let input_value = serde_json::from_str::<serde_json::Value>(input_value_str).unwrap();
+
+    let pipeline_yaml = r#"
+processors:
+
+transform:
+    - fields:
+        - product_object
+        - product_array
+      type: json
+"#;
+
+    let yaml_content = Content::Yaml(pipeline_yaml.into());
+    let pipeline: Pipeline<GreptimeTransformer> = parse(&yaml_content).unwrap();
+
+    let mut status = pipeline.init_intermediate_state();
+    pipeline.prepare(input_value, &mut status).unwrap();
+    let row = pipeline.exec_mut(&mut status).unwrap();
+    let r = row
+        .values
+        .into_iter()
+        .map(|v| v.value_data.unwrap())
+        .collect::<Vec<_>>();
+
+    let product_object = r[0].clone();
+    let product_array = r[1].clone();
+    match product_object {
+        ValueData::BinaryValue(data) => {
+            let jsonb = jsonb::from_slice(&data).unwrap().to_string();
+            assert_eq!(r#"{"hello":"world"}"#, jsonb);
+        }
+        _ => panic!("unexpected value"),
+    }
+
+    match product_array {
+        ValueData::BinaryValue(data) => {
+            let jsonb = jsonb::from_slice(&data).unwrap().to_string();
+            assert_eq!(r#"["hello","world"]"#, jsonb);
+        }
+        _ => panic!("unexpected value"),
+    }
+}
+
 #[test]
 fn test_simple_data() {
     let input_value_str = r#"
diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml
index df02a44855..354bdf642f 100644
--- a/src/servers/Cargo.toml
+++ b/src/servers/Cargo.toml
@@ -14,6 +14,7 @@ testing = []
 workspace = true
 
 [dependencies]
+ahash = "0.8"
 aide = { version = "0.9", features = ["axum"] }
 api.workspace = true
 arrow.workspace = true
diff --git a/src/servers/src/error.rs b/src/servers/src/error.rs
index a796b89521..9d7b2f9f8a 100644
--- a/src/servers/src/error.rs
+++ b/src/servers/src/error.rs
@@ -531,6 +531,13 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("OpenTelemetry log error"))]
+    OpenTelemetryLog {
+        source: pipeline::etl_error::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -595,7 +602,8 @@ impl ErrorExt for Error {
             | MysqlValueConversion { .. }
             | ParseJson { .. }
             | UnsupportedContentType { .. }
-            | TimestampOverflow { .. } => StatusCode::InvalidArguments,
+            | TimestampOverflow { .. }
+            | OpenTelemetryLog { .. } => StatusCode::InvalidArguments,
 
             Catalog { source, .. } => source.status_code(),
             RowWriter { source, .. } => source.status_code(),
diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs
index 67309b7244..b0c4843a1b 100644
--- a/src/servers/src/http.rs
+++ b/src/servers/src/http.rs
@@ -36,6 +36,7 @@ use common_time::timestamp::TimeUnit;
 use common_time::Timestamp;
 use datatypes::data_type::DataType;
 use datatypes::schema::SchemaRef;
+use datatypes::value::transform_value_ref_to_json_value;
 use event::{LogState, LogValidatorRef};
 use futures::FutureExt;
 use schemars::JsonSchema;
@@ -241,14 +242,18 @@ impl HttpRecordsOutput {
         } else {
             let num_rows = recordbatches.iter().map(|r| r.num_rows()).sum::<usize>();
             let mut rows = Vec::with_capacity(num_rows);
+            let schemas = schema.column_schemas();
             let num_cols = schema.column_schemas().len();
             rows.resize_with(num_rows, || Vec::with_capacity(num_cols));
 
             let mut finished_row_cursor = 0;
             for recordbatch in recordbatches {
-                for col in recordbatch.columns() {
+                for (col_idx, col) in recordbatch.columns().iter().enumerate() {
+                    // safety here: schemas length is equal to the number of columns in the recordbatch
+                    let schema = &schemas[col_idx];
                     for row_idx in 0..recordbatch.num_rows() {
-                        let value = Value::try_from(col.get_ref(row_idx)).context(ToJsonSnafu)?;
+                        let value = transform_value_ref_to_json_value(col.get_ref(row_idx), schema)
+                            .context(ToJsonSnafu)?;
                         rows[row_idx + finished_row_cursor].push(value);
                     }
                 }
@@ -882,6 +887,7 @@ impl HttpServer {
         Router::new()
             .route("/v1/metrics", routing::post(otlp::metrics))
             .route("/v1/traces", routing::post(otlp::traces))
+            .route("/v1/logs", routing::post(otlp::logs))
             .layer(
                 ServiceBuilder::new()
                     .layer(HandleErrorLayer::new(handle_error))
diff --git a/src/servers/src/http/header.rs b/src/servers/src/http/header.rs
index f9ff1485ef..2d621db9e7 100644
--- a/src/servers/src/http/header.rs
+++ b/src/servers/src/http/header.rs
@@ -44,6 +44,9 @@ pub mod constants {
     pub const GREPTIME_DB_HEADER_NAME: &str = "x-greptime-db-name";
     pub const GREPTIME_TIMEZONE_HEADER_NAME: &str = "x-greptime-timezone";
     pub const GREPTIME_DB_HEADER_ERROR_CODE: &str = common_error::GREPTIME_DB_HEADER_ERROR_CODE;
+    pub const GREPTIME_LOG_PIPELINE_NAME_HEADER_NAME: &str = "x-greptime-log-pipeline-name";
+    pub const GREPTIME_LOG_PIPELINE_VERSION_HEADER_NAME: &str = "x-greptime-log-pipeline-version";
+    pub const GREPTIME_LOG_TABLE_NAME_HEADER_NAME: &str = "x-greptime-log-table-name";
 }
 
 pub static GREPTIME_DB_HEADER_FORMAT: HeaderName =
diff --git a/src/servers/src/http/otlp.rs b/src/servers/src/http/otlp.rs
index 3efdaeec96..ddd318acc7 100644
--- a/src/servers/src/http/otlp.rs
+++ b/src/servers/src/http/otlp.rs
@@ -12,26 +12,39 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use core::str;
+use std::result::Result as StdResult;
 use std::sync::Arc;
 
-use axum::extract::State;
-use axum::http::header;
+use axum::extract::{FromRequestParts, State};
+use axum::http::header::HeaderValue;
+use axum::http::request::Parts;
+use axum::http::{header, StatusCode};
 use axum::response::IntoResponse;
-use axum::Extension;
+use axum::{async_trait, Extension};
 use bytes::Bytes;
 use common_telemetry::tracing;
+use opentelemetry_proto::tonic::collector::logs::v1::{
+    ExportLogsServiceRequest, ExportLogsServiceResponse,
+};
 use opentelemetry_proto::tonic::collector::metrics::v1::{
     ExportMetricsServiceRequest, ExportMetricsServiceResponse,
 };
 use opentelemetry_proto::tonic::collector::trace::v1::{
     ExportTraceServiceRequest, ExportTraceServiceResponse,
 };
+use pipeline::util::to_pipeline_version;
+use pipeline::PipelineWay;
 use prost::Message;
 use session::context::{Channel, QueryContext};
 use snafu::prelude::*;
 
 use super::header::{write_cost_header_map, CONTENT_TYPE_PROTOBUF};
 use crate::error::{self, Result};
+use crate::http::header::constants::{
+    GREPTIME_LOG_PIPELINE_NAME_HEADER_NAME, GREPTIME_LOG_PIPELINE_VERSION_HEADER_NAME,
+    GREPTIME_LOG_TABLE_NAME_HEADER_NAME,
+};
 use crate::query_handler::OpenTelemetryProtocolHandlerRef;
 
 #[axum_macros::debug_handler]
@@ -39,8 +52,9 @@ use crate::query_handler::OpenTelemetryProtocolHandlerRef;
 pub async fn metrics(
     State(handler): State<OpenTelemetryProtocolHandlerRef>,
     Extension(mut query_ctx): Extension<QueryContext>,
+
     bytes: Bytes,
-) -> Result<OtlpMetricsResponse> {
+) -> Result<OtlpResponse<ExportMetricsServiceResponse>> {
     let db = query_ctx.get_db_string();
     query_ctx.set_channel(Channel::Otlp);
     let query_ctx = Arc::new(query_ctx);
@@ -53,7 +67,7 @@ pub async fn metrics(
     handler
         .metrics(request, query_ctx)
         .await
-        .map(|o| OtlpMetricsResponse {
+        .map(|o| OtlpResponse {
             resp_body: ExportMetricsServiceResponse {
                 partial_success: None,
             },
@@ -61,27 +75,13 @@ pub async fn metrics(
         })
 }
 
-pub struct OtlpMetricsResponse {
-    resp_body: ExportMetricsServiceResponse,
-    write_cost: usize,
-}
-
-impl IntoResponse for OtlpMetricsResponse {
-    fn into_response(self) -> axum::response::Response {
-        let mut header_map = write_cost_header_map(self.write_cost);
-        header_map.insert(header::CONTENT_TYPE, CONTENT_TYPE_PROTOBUF.clone());
-
-        (header_map, self.resp_body.encode_to_vec()).into_response()
-    }
-}
-
 #[axum_macros::debug_handler]
 #[tracing::instrument(skip_all, fields(protocol = "otlp", request_type = "traces"))]
 pub async fn traces(
     State(handler): State<OpenTelemetryProtocolHandlerRef>,
     Extension(mut query_ctx): Extension<QueryContext>,
     bytes: Bytes,
-) -> Result<OtlpTracesResponse> {
+) -> Result<OtlpResponse<ExportTraceServiceResponse>> {
     let db = query_ctx.get_db_string();
     query_ctx.set_channel(Channel::Otlp);
     let query_ctx = Arc::new(query_ctx);
@@ -93,7 +93,7 @@ pub async fn traces(
     handler
         .traces(request, query_ctx)
         .await
-        .map(|o| OtlpTracesResponse {
+        .map(|o| OtlpResponse {
             resp_body: ExportTraceServiceResponse {
                 partial_success: None,
             },
@@ -101,12 +101,143 @@ pub async fn traces(
         })
 }
 
-pub struct OtlpTracesResponse {
-    resp_body: ExportTraceServiceResponse,
+pub struct PipelineInfo {
+    pub pipeline_name: Option<String>,
+    pub pipeline_version: Option<String>,
+}
+
+fn pipeline_header_error(
+    header: &HeaderValue,
+    key: &str,
+) -> StdResult<String, (http::StatusCode, String)> {
+    let header_utf8 = str::from_utf8(header.as_bytes());
+    match header_utf8 {
+        Ok(s) => Ok(s.to_string()),
+        Err(_) => Err((
+            StatusCode::BAD_REQUEST,
+            format!("`{}` header is not valid UTF-8 string type.", key),
+        )),
+    }
+}
+
+#[async_trait]
+impl<S> FromRequestParts<S> for PipelineInfo
+where
+    S: Send + Sync,
+{
+    type Rejection = (StatusCode, String);
+
+    async fn from_request_parts(parts: &mut Parts, _state: &S) -> StdResult<Self, Self::Rejection> {
+        let pipeline_name = parts.headers.get(GREPTIME_LOG_PIPELINE_NAME_HEADER_NAME);
+        let pipeline_version = parts.headers.get(GREPTIME_LOG_PIPELINE_VERSION_HEADER_NAME);
+        match (pipeline_name, pipeline_version) {
+            (Some(name), Some(version)) => Ok(PipelineInfo {
+                pipeline_name: Some(pipeline_header_error(
+                    name,
+                    GREPTIME_LOG_PIPELINE_NAME_HEADER_NAME,
+                )?),
+                pipeline_version: Some(pipeline_header_error(
+                    version,
+                    GREPTIME_LOG_PIPELINE_VERSION_HEADER_NAME,
+                )?),
+            }),
+            (None, _) => Ok(PipelineInfo {
+                pipeline_name: None,
+                pipeline_version: None,
+            }),
+            (Some(name), None) => Ok(PipelineInfo {
+                pipeline_name: Some(pipeline_header_error(
+                    name,
+                    GREPTIME_LOG_PIPELINE_NAME_HEADER_NAME,
+                )?),
+                pipeline_version: None,
+            }),
+        }
+    }
+}
+
+pub struct TableInfo {
+    table_name: String,
+}
+
+#[async_trait]
+impl<S> FromRequestParts<S> for TableInfo
+where
+    S: Send + Sync,
+{
+    type Rejection = (StatusCode, String);
+
+    async fn from_request_parts(parts: &mut Parts, _state: &S) -> StdResult<Self, Self::Rejection> {
+        let table_name = parts.headers.get(GREPTIME_LOG_TABLE_NAME_HEADER_NAME);
+
+        match table_name {
+            Some(name) => Ok(TableInfo {
+                table_name: pipeline_header_error(name, GREPTIME_LOG_TABLE_NAME_HEADER_NAME)?,
+            }),
+            None => Ok(TableInfo {
+                table_name: "opentelemetry_logs".to_string(),
+            }),
+        }
+    }
+}
+
+#[axum_macros::debug_handler]
+#[tracing::instrument(skip_all, fields(protocol = "otlp", request_type = "traces"))]
+pub async fn logs(
+    State(handler): State<OpenTelemetryProtocolHandlerRef>,
+    Extension(mut query_ctx): Extension<QueryContext>,
+    pipeline_info: PipelineInfo,
+    table_info: TableInfo,
+    bytes: Bytes,
+) -> Result<OtlpResponse<ExportLogsServiceResponse>> {
+    let db = query_ctx.get_db_string();
+    query_ctx.set_channel(Channel::Otlp);
+    let query_ctx = Arc::new(query_ctx);
+    let _timer = crate::metrics::METRIC_HTTP_OPENTELEMETRY_LOGS_ELAPSED
+        .with_label_values(&[db.as_str()])
+        .start_timer();
+    let request = ExportLogsServiceRequest::decode(bytes).context(error::DecodeOtlpRequestSnafu)?;
+
+    let pipeline_way;
+    if let Some(pipeline_name) = &pipeline_info.pipeline_name {
+        let pipeline_version =
+            to_pipeline_version(pipeline_info.pipeline_version).map_err(|_| {
+                error::InvalidParameterSnafu {
+                    reason: GREPTIME_LOG_PIPELINE_VERSION_HEADER_NAME,
+                }
+                .build()
+            })?;
+        let pipeline = match handler
+            .get_pipeline(pipeline_name, pipeline_version, query_ctx.clone())
+            .await
+        {
+            Ok(p) => p,
+            Err(e) => {
+                return Err(e);
+            }
+        };
+        pipeline_way = PipelineWay::Custom(pipeline);
+    } else {
+        pipeline_way = PipelineWay::Identity;
+    }
+
+    handler
+        .logs(request, pipeline_way, table_info.table_name, query_ctx)
+        .await
+        .map(|o| OtlpResponse {
+            resp_body: ExportLogsServiceResponse {
+                partial_success: None,
+            },
+            write_cost: o.meta.cost,
+        })
+}
+
+pub struct OtlpResponse<T: Message> {
+    resp_body: T,
     write_cost: usize,
 }
 
-impl IntoResponse for OtlpTracesResponse {
+impl<T: Message> IntoResponse for OtlpResponse<T> {
     fn into_response(self) -> axum::response::Response {
         let mut header_map = write_cost_header_map(self.write_cost);
         header_map.insert(header::CONTENT_TYPE, CONTENT_TYPE_PROTOBUF.clone());
diff --git a/src/servers/src/metrics.rs b/src/servers/src/metrics.rs
index cdf927536f..ead86f3ad8 100644
--- a/src/servers/src/metrics.rs
+++ b/src/servers/src/metrics.rs
@@ -141,6 +141,13 @@ lazy_static! {
             &[METRIC_DB_LABEL]
         )
         .unwrap();
+    pub static ref METRIC_HTTP_OPENTELEMETRY_LOGS_ELAPSED: HistogramVec =
+    register_histogram_vec!(
+        "greptime_servers_http_otlp_logs_elapsed",
+        "servers http otlp logs elapsed",
+        &[METRIC_DB_LABEL]
+    )
+    .unwrap();
     pub static ref METRIC_HTTP_LOGS_INGESTION_COUNTER: IntCounterVec = register_int_counter_vec!(
         "greptime_servers_http_logs_ingestion_counter",
         "servers http logs ingestion counter",
diff --git a/src/servers/src/otlp.rs b/src/servers/src/otlp.rs
index 5b92d12cb6..cc1321ac70 100644
--- a/src/servers/src/otlp.rs
+++ b/src/servers/src/otlp.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub mod logs;
 pub mod metrics;
 pub mod plugin;
 pub mod trace;
diff --git a/src/servers/src/otlp/logs.rs b/src/servers/src/otlp/logs.rs
new file mode 100644
index 0000000000..4c47da29c8
--- /dev/null
+++ b/src/servers/src/otlp/logs.rs
@@ -0,0 +1,506 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::{BTreeMap, HashMap};
+
+use api::v1::column_data_type_extension::TypeExt;
+use api::v1::value::ValueData;
+use api::v1::{
+    ColumnDataType, ColumnDataTypeExtension, ColumnOptions, ColumnSchema, JsonTypeExtension, Row,
+    RowInsertRequest, RowInsertRequests, Rows, SemanticType, Value as GreptimeValue,
+};
+use jsonb::{Number as JsonbNumber, Value as JsonbValue};
+use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
+use opentelemetry_proto::tonic::common::v1::{any_value, AnyValue, InstrumentationScope, KeyValue};
+use opentelemetry_proto::tonic::logs::v1::LogRecord;
+use pipeline::{Array, Map, PipelineWay, Value as PipelineValue};
+use snafu::ResultExt;
+
+use super::trace::attributes::OtlpAnyValue;
+use crate::error::{OpenTelemetryLogSnafu, Result};
+use crate::otlp::trace::span::bytes_to_hex_string;
+
+/// Convert OpenTelemetry metrics to GreptimeDB insert requests
+///
+/// See
+/// <https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/metrics/v1/metrics.proto>
+/// for data structure of OTLP metrics.
+///
+/// Returns `InsertRequests` and total number of rows to ingest
+pub fn to_grpc_insert_requests(
+    request: ExportLogsServiceRequest,
+    pipeline: PipelineWay,
+    table_name: String,
+) -> Result<(RowInsertRequests, usize)> {
+    match pipeline {
+        PipelineWay::Identity => {
+            let rows = parse_export_logs_service_request_to_rows(request);
+            let len = rows.rows.len();
+            let insert_request = RowInsertRequest {
+                rows: Some(rows),
+                table_name,
+            };
+            Ok((
+                RowInsertRequests {
+                    inserts: vec![insert_request],
+                },
+                len,
+            ))
+        }
+        PipelineWay::Custom(p) => {
+            let request = parse_export_logs_service_request(request);
+            let mut result = Vec::new();
+            let mut intermediate_state = p.init_intermediate_state();
+            for v in request {
+                p.prepare_pipeline_value(v, &mut intermediate_state)
+                    .context(OpenTelemetryLogSnafu)?;
+                let r = p
+                    .exec_mut(&mut intermediate_state)
+                    .context(OpenTelemetryLogSnafu)?;
+                result.push(r);
+            }
+            let len = result.len();
+            let rows = Rows {
+                schema: p.schemas().clone(),
+                rows: result,
+            };
+            let insert_request = RowInsertRequest {
+                rows: Some(rows),
+                table_name,
+            };
+            let insert_requests = RowInsertRequests {
+                inserts: vec![insert_request],
+            };
+            Ok((insert_requests, len))
+        }
+    }
+}
+
+fn scope_to_pipeline_value(
+    scope: Option<InstrumentationScope>,
+) -> (PipelineValue, PipelineValue, PipelineValue) {
+    scope
+        .map(|x| {
+            (
+                PipelineValue::Map(Map {
+                    values: key_value_to_map(x.attributes),
+                }),
+                PipelineValue::String(x.version),
+                PipelineValue::String(x.name),
+            )
+        })
+        .unwrap_or((
+            PipelineValue::Null,
+            PipelineValue::Null,
+            PipelineValue::Null,
+        ))
+}
+
+fn scope_to_jsonb(
+    scope: Option<InstrumentationScope>,
+) -> (JsonbValue<'static>, Option<String>, Option<String>) {
+    scope
+        .map(|x| {
+            (
+                key_value_to_jsonb(x.attributes),
+                Some(x.version),
+                Some(x.name),
+            )
+        })
+        .unwrap_or((JsonbValue::Null, None, None))
+}
+
+fn log_to_pipeline_value(
+    log: LogRecord,
+    resource_schema_url: PipelineValue,
+    resource_attr: PipelineValue,
+    scope_schema_url: PipelineValue,
+    scope_name: PipelineValue,
+    scope_version: PipelineValue,
+    scope_attrs: PipelineValue,
+) -> PipelineValue {
+    let log_attrs = PipelineValue::Map(Map {
+        values: key_value_to_map(log.attributes),
+    });
+    let mut map = BTreeMap::new();
+    map.insert(
+        "Timestamp".to_string(),
+        PipelineValue::Uint64(log.time_unix_nano),
+    );
+    map.insert(
+        "ObservedTimestamp".to_string(),
+        PipelineValue::Uint64(log.observed_time_unix_nano),
+    );
+
+    // need to be convert to string
+    map.insert(
+        "TraceId".to_string(),
+        PipelineValue::String(bytes_to_hex_string(&log.trace_id)),
+    );
+    map.insert(
+        "SpanId".to_string(),
+        PipelineValue::String(bytes_to_hex_string(&log.span_id)),
+    );
+    map.insert("TraceFlags".to_string(), PipelineValue::Uint32(log.flags));
+    map.insert(
+        "SeverityText".to_string(),
+        PipelineValue::String(log.severity_text),
+    );
+    map.insert(
+        "SeverityNumber".to_string(),
+        PipelineValue::Int32(log.severity_number),
+    );
+    // need to be convert to string
+    map.insert(
+        "Body".to_string(),
+        log.body
+            .as_ref()
+            .map(|x| PipelineValue::String(log_body_to_string(x)))
+            .unwrap_or(PipelineValue::Null),
+    );
+    map.insert("ResourceSchemaUrl".to_string(), resource_schema_url);
+
+    map.insert("ResourceAttributes".to_string(), resource_attr);
+    map.insert("ScopeSchemaUrl".to_string(), scope_schema_url);
+    map.insert("ScopeName".to_string(), scope_name);
+    map.insert("ScopeVersion".to_string(), scope_version);
+    map.insert("ScopeAttributes".to_string(), scope_attrs);
+    map.insert("LogAttributes".to_string(), log_attrs);
+    PipelineValue::Map(Map { values: map })
+}
+
+fn build_otlp_logs_identity_schema() -> Vec<ColumnSchema> {
+    [
+        (
+            "scope_name",
+            ColumnDataType::String,
+            SemanticType::Tag,
+            None,
+            None,
+        ),
+        (
+            "scope_version",
+            ColumnDataType::String,
+            SemanticType::Field,
+            None,
+            None,
+        ),
+        (
+            "scope_attributes",
+            ColumnDataType::Binary,
+            SemanticType::Field,
+            Some(ColumnDataTypeExtension {
+                type_ext: Some(TypeExt::JsonType(JsonTypeExtension::JsonBinary.into())),
+            }),
+            None,
+        ),
+        (
+            "resource_attributes",
+            ColumnDataType::Binary,
+            SemanticType::Field,
+            Some(ColumnDataTypeExtension {
+                type_ext: Some(TypeExt::JsonType(JsonTypeExtension::JsonBinary.into())),
+            }),
+            None,
+        ),
+        (
+            "log_attributes",
+            ColumnDataType::Binary,
+            SemanticType::Field,
+            Some(ColumnDataTypeExtension {
+                type_ext: Some(TypeExt::JsonType(JsonTypeExtension::JsonBinary.into())),
+            }),
+            None,
+        ),
+        (
+            "timestamp",
+            ColumnDataType::TimestampNanosecond,
+            SemanticType::Timestamp,
+            None,
+            None,
+        ),
+        (
+            "observed_timestamp",
+            ColumnDataType::TimestampNanosecond,
+            SemanticType::Field,
+            None,
+            None,
+        ),
+        (
+            "trace_id",
+            ColumnDataType::String,
+            SemanticType::Tag,
+            None,
+            None,
+        ),
+        (
+            "span_id",
+            ColumnDataType::String,
+            SemanticType::Tag,
+            None,
+            None,
+        ),
+        (
+            "trace_flags",
+            ColumnDataType::Uint32,
+            SemanticType::Field,
+            None,
+            None,
+        ),
+        (
+            "severity_text",
+            ColumnDataType::String,
+            SemanticType::Field,
+            None,
+            None,
+        ),
+        (
+            "severity_number",
+            ColumnDataType::Int32,
+            SemanticType::Field,
+            None,
+            None,
+        ),
+        (
+            "body",
+            ColumnDataType::String,
+            SemanticType::Field,
+            None,
+            Some(ColumnOptions {
+                options: HashMap::from([(
+                    "fulltext".to_string(),
+                    r#"{"enable":true}"#.to_string(),
+                )]),
+            }),
+        ),
+    ]
+    .into_iter()
+    .map(
+        |(field_name, column_type, semantic_type, datatype_extension, options)| ColumnSchema {
+            column_name: field_name.to_string(),
+            datatype: column_type as i32,
+            semantic_type: semantic_type as i32,
+            datatype_extension,
+            options,
+        },
+    )
+    .collect::<Vec<ColumnSchema>>()
+}
+
+fn build_identity_row(
+    log: LogRecord,
+    resource_attr: JsonbValue<'_>,
+    scope_name: Option<String>,
+    scope_version: Option<String>,
+    scope_attrs: JsonbValue<'_>,
+) -> Row {
+    let row = vec![
+        GreptimeValue {
+            value_data: scope_name.map(ValueData::StringValue),
+        },
+        GreptimeValue {
+            value_data: scope_version.map(ValueData::StringValue),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::BinaryValue(scope_attrs.to_vec())),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::BinaryValue(resource_attr.to_vec())),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::BinaryValue(
+                key_value_to_jsonb(log.attributes).to_vec(),
+            )),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::TimestampNanosecondValue(
+                log.time_unix_nano as i64,
+            )),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::TimestampNanosecondValue(
+                log.observed_time_unix_nano as i64,
+            )),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::StringValue(bytes_to_hex_string(&log.trace_id))),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::StringValue(bytes_to_hex_string(&log.span_id))),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::U32Value(log.flags)),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::StringValue(log.severity_text)),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::I32Value(log.severity_number)),
+        },
+        GreptimeValue {
+            value_data: log
+                .body
+                .as_ref()
+                .map(|x| ValueData::StringValue(log_body_to_string(x))),
+        },
+    ];
+
+    Row { values: row }
+}
+
+fn parse_export_logs_service_request_to_rows(request: ExportLogsServiceRequest) -> Rows {
+    let mut result = Vec::new();
+    for r in request.resource_logs {
+        let resource_attr = r
+            .resource
+            .map(|x| key_value_to_jsonb(x.attributes))
+            .unwrap_or(JsonbValue::Null);
+        for scope_logs in r.scope_logs {
+            let (scope_attrs, scope_version, scope_name) = scope_to_jsonb(scope_logs.scope);
+            for log in scope_logs.log_records {
+                let value = build_identity_row(
+                    log,
+                    resource_attr.clone(),
+                    scope_name.clone(),
+                    scope_version.clone(),
+                    scope_attrs.clone(),
+                );
+                result.push(value);
+            }
+        }
+    }
+    Rows {
+        schema: build_otlp_logs_identity_schema(),
+        rows: result,
+    }
+}
+
+/// transform otlp logs request to pipeline value
+/// https://opentelemetry.io/docs/concepts/signals/logs/
+fn parse_export_logs_service_request(request: ExportLogsServiceRequest) -> Vec<PipelineValue> {
+    let mut result = Vec::new();
+    for r in request.resource_logs {
+        let resource_attr = r
+            .resource
+            .map(|x| {
+                PipelineValue::Map(Map {
+                    values: key_value_to_map(x.attributes),
+                })
+            })
+            .unwrap_or(PipelineValue::Null);
+        let resource_schema_url = PipelineValue::String(r.schema_url);
+        for scope_logs in r.scope_logs {
+            let (scope_attrs, scope_version, scope_name) =
+                scope_to_pipeline_value(scope_logs.scope);
+            let scope_schema_url = PipelineValue::String(scope_logs.schema_url);
+            for log in scope_logs.log_records {
+                let value = log_to_pipeline_value(
+                    log,
+                    resource_schema_url.clone(),
+                    resource_attr.clone(),
+                    scope_schema_url.clone(),
+                    scope_name.clone(),
+                    scope_version.clone(),
+                    scope_attrs.clone(),
+                );
+                result.push(value);
+            }
+        }
+    }
+    result
+}
+
+// convert AnyValue to pipeline value
+fn any_value_to_pipeline_value(value: any_value::Value) -> PipelineValue {
+    match value {
+        any_value::Value::StringValue(s) => PipelineValue::String(s),
+        any_value::Value::IntValue(i) => PipelineValue::Int64(i),
+        any_value::Value::DoubleValue(d) => PipelineValue::Float64(d),
+        any_value::Value::BoolValue(b) => PipelineValue::Boolean(b),
+        any_value::Value::ArrayValue(a) => {
+            let values = a
+                .values
+                .into_iter()
+                .map(|v| match v.value {
+                    Some(value) => any_value_to_pipeline_value(value),
+                    None => PipelineValue::Null,
+                })
+                .collect();
+            PipelineValue::Array(Array { values })
+        }
+        any_value::Value::KvlistValue(kv) => {
+            let value = key_value_to_map(kv.values);
+            PipelineValue::Map(Map { values: value })
+        }
+        any_value::Value::BytesValue(b) => PipelineValue::String(bytes_to_hex_string(&b)),
+    }
+}
+
+// convert otlp keyValue vec to map
+fn key_value_to_map(key_values: Vec<KeyValue>) -> BTreeMap<String, PipelineValue> {
+    let mut map = BTreeMap::new();
+    for kv in key_values {
+        let value = match kv.value {
+            Some(value) => match value.value {
+                Some(value) => any_value_to_pipeline_value(value),
+                None => PipelineValue::Null,
+            },
+            None => PipelineValue::Null,
+        };
+        map.insert(kv.key.clone(), value);
+    }
+    map
+}
+
+fn any_value_to_jsonb(value: any_value::Value) -> JsonbValue<'static> {
+    match value {
+        any_value::Value::StringValue(s) => JsonbValue::String(s.into()),
+        any_value::Value::IntValue(i) => JsonbValue::Number(JsonbNumber::Int64(i)),
+        any_value::Value::DoubleValue(d) => JsonbValue::Number(JsonbNumber::Float64(d)),
+        any_value::Value::BoolValue(b) => JsonbValue::Bool(b),
+        any_value::Value::ArrayValue(a) => {
+            let values = a
+                .values
+                .into_iter()
+                .map(|v| match v.value {
+                    Some(value) => any_value_to_jsonb(value),
+                    None => JsonbValue::Null,
+                })
+                .collect();
+            JsonbValue::Array(values)
+        }
+        any_value::Value::KvlistValue(kv) => key_value_to_jsonb(kv.values),
+        any_value::Value::BytesValue(b) => JsonbValue::String(bytes_to_hex_string(&b).into()),
+    }
+}
+
+fn key_value_to_jsonb(key_values: Vec<KeyValue>) -> JsonbValue<'static> {
+    let mut map = BTreeMap::new();
+    for kv in key_values {
+        let value = match kv.value {
+            Some(value) => match value.value {
+                Some(value) => any_value_to_jsonb(value),
+                None => JsonbValue::Null,
+            },
+            None => JsonbValue::Null,
+        };
+        map.insert(kv.key.clone(), value);
+    }
+    JsonbValue::Object(map)
+}
+
+fn log_body_to_string(body: &AnyValue) -> String {
+    let otlp_value = OtlpAnyValue::from(body);
+    otlp_value.to_string()
+}
diff --git a/src/servers/src/query_handler.rs b/src/servers/src/query_handler.rs
index 1fe64e6522..a1ad9997ba 100644
--- a/src/servers/src/query_handler.rs
+++ b/src/servers/src/query_handler.rs
@@ -33,9 +33,10 @@ use api::v1::RowInsertRequests;
 use async_trait::async_trait;
 use common_query::Output;
 use headers::HeaderValue;
+use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
 use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest;
 use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
-use pipeline::{GreptimeTransformer, Pipeline, PipelineInfo, PipelineVersion};
+use pipeline::{GreptimeTransformer, Pipeline, PipelineInfo, PipelineVersion, PipelineWay};
 use serde_json::Value;
 use session::context::QueryContextRef;
 
@@ -105,7 +106,7 @@ pub trait PromStoreProtocolHandler {
 }
 
 #[async_trait]
-pub trait OpenTelemetryProtocolHandler {
+pub trait OpenTelemetryProtocolHandler: LogHandler {
     /// Handling opentelemetry metrics request
     async fn metrics(
         &self,
@@ -119,6 +120,14 @@ pub trait OpenTelemetryProtocolHandler {
         request: ExportTraceServiceRequest,
         ctx: QueryContextRef,
     ) -> Result<Output>;
+
+    async fn logs(
+        &self,
+        request: ExportLogsServiceRequest,
+        pipeline: PipelineWay,
+        table_name: String,
+        ctx: QueryContextRef,
+    ) -> Result<Output>;
 }
 
 /// LogHandler is responsible for handling log related requests.
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index 7a030aad56..8541adc37a 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -17,10 +17,11 @@ use std::io::Write;
 
 use api::prom_store::remote::WriteRequest;
 use auth::user_provider_from_option;
-use axum::http::{HeaderName, StatusCode};
+use axum::http::{HeaderName, HeaderValue, StatusCode};
 use common_error::status_code::StatusCode as ErrorCode;
 use flate2::write::GzEncoder;
 use flate2::Compression;
+use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
 use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest;
 use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
 use opentelemetry_proto::tonic::metrics::v1::ResourceMetrics;
@@ -90,6 +91,7 @@ macro_rules! http_tests {
 
                 test_otlp_metrics,
                 test_otlp_traces,
+                test_otlp_logs,
             );
         )*
     };
@@ -1520,7 +1522,7 @@ pub async fn test_otlp_metrics(store_type: StorageType) {
     let client = TestClient::new(app);
 
     // write metrics data
-    let res = send_req(&client, "/v1/otlp/v1/metrics", body.clone(), false).await;
+    let res = send_req(&client, vec![], "/v1/otlp/v1/metrics", body.clone(), false).await;
     assert_eq!(StatusCode::OK, res.status());
 
     // select metrics data
@@ -1532,7 +1534,7 @@ pub async fn test_otlp_metrics(store_type: StorageType) {
     assert_eq!(res.status(), StatusCode::OK);
 
     // write metrics data with gzip
-    let res = send_req(&client, "/v1/otlp/v1/metrics", body.clone(), true).await;
+    let res = send_req(&client, vec![], "/v1/otlp/v1/metrics", body.clone(), true).await;
     assert_eq!(StatusCode::OK, res.status());
 
     // select metrics data again
@@ -1557,7 +1559,7 @@ pub async fn test_otlp_traces(store_type: StorageType) {
     let client = TestClient::new(app);
 
     // write traces data
-    let res = send_req(&client, "/v1/otlp/v1/traces", body.clone(), false).await;
+    let res = send_req(&client, vec![], "/v1/otlp/v1/traces", body.clone(), false).await;
     assert_eq!(StatusCode::OK, res.status());
 
     // select traces data
@@ -1572,7 +1574,7 @@ pub async fn test_otlp_traces(store_type: StorageType) {
     assert_eq!(res.status(), StatusCode::OK);
 
     // write metrics data with gzip
-    let res = send_req(&client, "/v1/otlp/v1/traces", body.clone(), true).await;
+    let res = send_req(&client, vec![], "/v1/otlp/v1/traces", body.clone(), true).await;
     assert_eq!(StatusCode::OK, res.status());
 
     // select metrics data again
@@ -1581,6 +1583,40 @@ pub async fn test_otlp_traces(store_type: StorageType) {
     guard.remove_all().await;
 }
 
+pub async fn test_otlp_logs(store_type: StorageType) {
+    common_telemetry::init_default_ut_logging();
+    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_otlp_traces").await;
+
+    let content = r#"
+{"resourceLogs":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"schemaUrl":"https://opentelemetry.io/schemas/1.0.0/resourceLogs","scopeLogs":[{"scope":{},"schemaUrl":"https://opentelemetry.io/schemas/1.0.0/scopeLogs","logRecords":[{"flags":1,"timeUnixNano":1581452773000009875,"observedTimeUnixNano":1581452773000009875,"severityNumber":9,"severityText":"Info","body":{"value":{"stringValue":"This is a log message"}},"attributes":[{"key":"app","value":{"stringValue":"server"}},{"key":"instance_num","value":{"intValue":1}}],"droppedAttributesCount":1,"traceId":[48,56,48,52,48,50,48,49,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48],"spanId":[48,49,48,50,48,52,48,56,48,48,48,48,48,48,48,48]},{"flags":1,"timeUnixNano":1581452773000000789,"observedTimeUnixNano":1581452773000000789,"severityNumber":9,"severityText":"Info","body":{"value":{"stringValue":"something happened"}},"attributes":[{"key":"customer","value":{"stringValue":"acme"}},{"key":"env","value":{"stringValue":"dev"}}],"droppedAttributesCount":1,"traceId":[48],"spanId":[48]}]}]}]}
+"#;
+
+    let req: ExportLogsServiceRequest = serde_json::from_str(content).unwrap();
+    let body = req.encode_to_vec();
+
+    // handshake
+    let client = TestClient::new(app);
+
+    // write traces data
+    let res = send_req(
+        &client,
+        vec![(
+            HeaderName::from_static("x-greptime-log-table-name"),
+            HeaderValue::from_static("logs"),
+        )],
+        "/v1/otlp/v1/logs?db=public",
+        body.clone(),
+        false,
+    )
+    .await;
+    assert_eq!(StatusCode::OK, res.status());
+
+    let expected = r#"[["","",{},{"resource-attr":"resource-attr-val-1"},{"customer":"acme","env":"dev"},1581452773000000789,1581452773000000789,"30","30",1,"Info",9,"something happened"],["","",{},{"resource-attr":"resource-attr-val-1"},{"app":"server","instance_num":1},1581452773000009875,1581452773000009875,"3038303430323031303030303030303030303030303030303030303030303030","30313032303430383030303030303030",1,"Info",9,"This is a log message"]]"#;
+    validate_data(&client, "select * from logs;", expected).await;
+
+    guard.remove_all().await;
+}
+
 async fn validate_data(client: &TestClient, sql: &str, expected: &str) {
     let res = client
         .get(format!("/v1/sql?sql={sql}").as_str())
@@ -1593,11 +1629,21 @@ async fn validate_data(client: &TestClient, sql: &str, expected: &str) {
     assert_eq!(v, expected);
 }
 
-async fn send_req(client: &TestClient, path: &str, body: Vec<u8>, with_gzip: bool) -> TestResponse {
+async fn send_req(
+    client: &TestClient,
+    headers: Vec<(HeaderName, HeaderValue)>,
+    path: &str,
+    body: Vec<u8>,
+    with_gzip: bool,
+) -> TestResponse {
     let mut req = client
         .post(path)
         .header("content-type", "application/x-protobuf");
 
+    for (k, v) in headers {
+        req = req.header(k, v);
+    }
+
     let mut len = body.len();
 
     if with_gzip {

From 176f2df5b3e85f18be9ea2988d6d8bfa6b92a5b3 Mon Sep 17 00:00:00 2001
From: Yiran <cuiyiran3@gmail.com>
Date: Wed, 16 Oct 2024 15:43:14 +0800
Subject: [PATCH 084/128] fix: dead links (#4837)

---
 docs/benchmarks/log/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/benchmarks/log/README.md b/docs/benchmarks/log/README.md
index c3984b3ee6..2a884266b4 100644
--- a/docs/benchmarks/log/README.md
+++ b/docs/benchmarks/log/README.md
@@ -48,4 +48,4 @@ Please refer to [SQL query](./query.sql) for GreptimeDB and Clickhouse, and [que
 
 ## Addition
 - You can tune GreptimeDB's configuration to get better performance.
-- You can setup GreptimeDB to use S3 as storage, see [here](https://docs.greptime.com/user-guide/operations/configuration/#storage-options).
+- You can setup GreptimeDB to use S3 as storage, see [here](https://docs.greptime.com/user-guide/deployments/configuration#storage-options).

From c231eee7c123b8341d9d776a087cd8b993576e0e Mon Sep 17 00:00:00 2001
From: Ning Sun <sunng@protonmail.com>
Date: Wed, 16 Oct 2024 00:46:31 -0700
Subject: [PATCH 085/128] fix: respect feature flags for geo function (#4836)

---
 src/common/function/src/scalars/aggregate.rs | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/common/function/src/scalars/aggregate.rs b/src/common/function/src/scalars/aggregate.rs
index f7ce316b6c..08edf43568 100644
--- a/src/common/function/src/scalars/aggregate.rs
+++ b/src/common/function/src/scalars/aggregate.rs
@@ -31,7 +31,6 @@ pub use polyval::PolyvalAccumulatorCreator;
 pub use scipy_stats_norm_cdf::ScipyStatsNormCdfAccumulatorCreator;
 pub use scipy_stats_norm_pdf::ScipyStatsNormPdfAccumulatorCreator;
 
-use super::geo::encoding::JsonPathEncodeFunctionCreator;
 use crate::function_registry::FunctionRegistry;
 
 /// A function creates `AggregateFunctionCreator`.
@@ -93,6 +92,11 @@ impl AggregateFunctions {
         register_aggr_func!("scipystatsnormcdf", 2, ScipyStatsNormCdfAccumulatorCreator);
         register_aggr_func!("scipystatsnormpdf", 2, ScipyStatsNormPdfAccumulatorCreator);
 
-        register_aggr_func!("json_encode_path", 3, JsonPathEncodeFunctionCreator);
+        #[cfg(feature = "geo")]
+        register_aggr_func!(
+            "json_encode_path",
+            3,
+            super::geo::encoding::JsonPathEncodeFunctionCreator
+        );
     }
 }

From 0ce93f0b88fbde8b524f24f5e1434537f105ad19 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Wed, 16 Oct 2024 17:36:57 +0800
Subject: [PATCH 086/128] chore: add more metrics for region migration (#4838)

---
 src/meta-srv/src/metrics.rs                     | 15 ++++++++++++++-
 src/meta-srv/src/procedure/region_migration.rs  | 17 +++++++++++++++++
 .../src/procedure/region_migration/manager.rs   |  8 ++++++++
 3 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/src/meta-srv/src/metrics.rs b/src/meta-srv/src/metrics.rs
index 07d0a4a12d..7a7bdce9a8 100644
--- a/src/meta-srv/src/metrics.rs
+++ b/src/meta-srv/src/metrics.rs
@@ -45,7 +45,20 @@ lazy_static! {
     /// Meta kv cache miss counter.
     pub static ref METRIC_META_KV_CACHE_MISS: IntCounterVec =
         register_int_counter_vec!("greptime_meta_kv_cache_miss", "meta kv cache miss", &["op"]).unwrap();
-    //  Heartbeat received by metasrv.
+    ///  Heartbeat received by metasrv.
     pub static ref METRIC_META_HEARTBEAT_RECV: IntCounterVec =
         register_int_counter_vec!("greptime_meta_heartbeat_recv", "heartbeats received by metasrv", &["pusher_key"]).unwrap();
+    /// The migration execute histogram.
+    pub static ref METRIC_META_REGION_MIGRATION_EXECUTE: HistogramVec =
+        register_histogram_vec!("greptime_meta_region_migration_execute", "meta region migration execute", &["state"]).unwrap();
+    /// The migration error counter.
+    pub static ref METRIC_META_REGION_MIGRATION_ERROR: IntCounterVec =
+        register_int_counter_vec!("greptime_meta_region_migration_error", "meta region migration abort", &["state", "error_type"]).unwrap();
+    /// The migration datanode counter.
+    pub static ref METRIC_META_REGION_MIGRATION_DATANODES: IntCounterVec =
+        register_int_counter_vec!("greptime_meta_region_migration_stat", "meta region migration stat", &["datanode_type", "datanode_id"]).unwrap();
+    /// The migration fail counter.
+    pub static ref METRIC_META_REGION_MIGRATION_FAIL: IntCounter =
+        register_int_counter!("greptime_meta_region_migration_fail", "meta region migration fail").unwrap();
+
 }
diff --git a/src/meta-srv/src/procedure/region_migration.rs b/src/meta-srv/src/procedure/region_migration.rs
index 72b886418c..40df9401cb 100644
--- a/src/meta-srv/src/procedure/region_migration.rs
+++ b/src/meta-srv/src/procedure/region_migration.rs
@@ -54,6 +54,7 @@ use tokio::time::Instant;
 
 use self::migration_start::RegionMigrationStart;
 use crate::error::{self, Result};
+use crate::metrics::{METRIC_META_REGION_MIGRATION_ERROR, METRIC_META_REGION_MIGRATION_EXECUTE};
 use crate::service::mailbox::MailboxRef;
 
 /// It's shared in each step and available even after recovering.
@@ -390,6 +391,12 @@ impl Context {
 #[async_trait::async_trait]
 #[typetag::serde(tag = "region_migration_state")]
 pub(crate) trait State: Sync + Send + Debug {
+    fn name(&self) -> &'static str {
+        let type_name = std::any::type_name::<Self>();
+        // short name
+        type_name.split("::").last().unwrap_or(type_name)
+    }
+
     /// Yields the next [State] and [Status].
     async fn next(&mut self, ctx: &mut Context) -> Result<(Box<dyn State>, Status)>;
 
@@ -478,10 +485,20 @@ impl Procedure for RegionMigrationProcedure {
     async fn execute(&mut self, _ctx: &ProcedureContext) -> ProcedureResult<Status> {
         let state = &mut self.state;
 
+        let name = state.name();
+        let _timer = METRIC_META_REGION_MIGRATION_EXECUTE
+            .with_label_values(&[name])
+            .start_timer();
         let (next, status) = state.next(&mut self.context).await.map_err(|e| {
             if e.is_retryable() {
+                METRIC_META_REGION_MIGRATION_ERROR
+                    .with_label_values(&[name, "retryable"])
+                    .inc();
                 ProcedureError::retry_later(e)
             } else {
+                METRIC_META_REGION_MIGRATION_ERROR
+                    .with_label_values(&[name, "external"])
+                    .inc();
                 ProcedureError::external(e)
             }
         })?;
diff --git a/src/meta-srv/src/procedure/region_migration/manager.rs b/src/meta-srv/src/procedure/region_migration/manager.rs
index 1059c1794a..c61a9a4b6c 100644
--- a/src/meta-srv/src/procedure/region_migration/manager.rs
+++ b/src/meta-srv/src/procedure/region_migration/manager.rs
@@ -30,6 +30,7 @@ use store_api::storage::RegionId;
 use table::table_name::TableName;
 
 use crate::error::{self, Result};
+use crate::metrics::{METRIC_META_REGION_MIGRATION_DATANODES, METRIC_META_REGION_MIGRATION_FAIL};
 use crate::procedure::region_migration::{
     DefaultContextFactory, PersistentContext, RegionMigrationProcedure,
 };
@@ -323,6 +324,12 @@ impl RegionMigrationManager {
             schema_name,
             ..
         } = table_info.table_name();
+        METRIC_META_REGION_MIGRATION_DATANODES
+            .with_label_values(&["src", &task.from_peer.id.to_string()])
+            .inc();
+        METRIC_META_REGION_MIGRATION_DATANODES
+            .with_label_values(&["desc", &task.to_peer.id.to_string()])
+            .inc();
         let RegionMigrationProcedureTask {
             cluster_id,
             region_id,
@@ -358,6 +365,7 @@ impl RegionMigrationManager {
 
             if let Err(e) = watcher::wait(watcher).await {
                 error!(e; "Failed to wait region migration procedure {procedure_id} for {task}");
+                METRIC_META_REGION_MIGRATION_FAIL.inc();
                 return;
             }
 

From 613e07afb41913f380a8090d44f10f74f6886999 Mon Sep 17 00:00:00 2001
From: discord9 <55937128+discord9@users.noreply.github.com>
Date: Wed, 16 Oct 2024 19:50:25 +0800
Subject: [PATCH 087/128] feat: window sort physical plan (#4814)

* WIP

* feat: range split& tests

* WIP: split range

* add sort exprs

* chore: typo

* WIP

* feat: find successive runs

* WIP

* READY FOR REVIEW PART ONE: more tests

* refactor: break into smaller functions

* feat: precompute working range(need testing)

* tests: on working range

* tests: on working range

* feat: support rev working range

* feat(to be tested): core logic of merge sort

* fix: poll results

* fix: find_slice_from_range&test

* chore: remove some unused util func&fields

* chore: typos

* chore: impl exec plan for WindowedSortExec

* test(WIP): window sort stream

* test: window sort stream

* chore: remove unused

* fix: fetch

* fix: WIP intersection remaining

* test: fix and test!

* chore: remove outdated comments

* chore: rename test

* chore: remove dbg line

* chore: sorted runs

* feat: handling unexpected data

* chore: unused

* chore: remove a print in test

* chore: per review

* docs: wrong comment

* chore: more test cases
---
 Cargo.lock                   |    4 +
 src/query/Cargo.toml         |    4 +
 src/query/src/lib.rs         |    1 +
 src/query/src/window_sort.rs | 3221 ++++++++++++++++++++++++++++++++++
 4 files changed, 3230 insertions(+)
 create mode 100644 src/query/src/window_sort.rs

diff --git a/Cargo.lock b/Cargo.lock
index bf76687e72..46effb6a84 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8980,6 +8980,7 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-sql",
  "datatypes",
+ "fastrand",
  "format_num",
  "futures",
  "futures-util",
@@ -8994,12 +8995,15 @@ dependencies = [
  "object-store",
  "once_cell",
  "paste",
+ "pretty_assertions",
  "prometheus",
  "promql",
  "promql-parser",
  "prost 0.12.6",
  "rand",
  "regex",
+ "serde",
+ "serde_json",
  "session",
  "snafu 0.8.5",
  "sql",
diff --git a/src/query/Cargo.toml b/src/query/Cargo.toml
index 4bcda7b0a7..3f983bb9e3 100644
--- a/src/query/Cargo.toml
+++ b/src/query/Cargo.toml
@@ -71,11 +71,15 @@ arrow.workspace = true
 catalog = { workspace = true, features = ["testing"] }
 common-macro.workspace = true
 common-query = { workspace = true, features = ["testing"] }
+fastrand = "2.0"
 format_num = "0.1"
 num = "0.4"
 num-traits = "0.2"
 paste = "1.0"
+pretty_assertions = "1.4.0"
 rand.workspace = true
+serde.workspace = true
+serde_json.workspace = true
 session = { workspace = true, features = ["testing"] }
 statrs = "0.16"
 stats-cli = "3.0"
diff --git a/src/query/src/lib.rs b/src/query/src/lib.rs
index e2843c708e..e4e527a477 100644
--- a/src/query/src/lib.rs
+++ b/src/query/src/lib.rs
@@ -36,6 +36,7 @@ mod range_select;
 pub mod region_query;
 pub mod sql;
 pub mod stats;
+mod window_sort;
 
 #[cfg(test)]
 mod tests;
diff --git a/src/query/src/window_sort.rs b/src/query/src/window_sort.rs
new file mode 100644
index 0000000000..c2cc9b2389
--- /dev/null
+++ b/src/query/src/window_sort.rs
@@ -0,0 +1,3221 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! A physical plan for window sort(Which is sorting multiple sorted ranges according to input `PartitionRange`).
+//!
+
+use std::any::Any;
+use std::collections::{BTreeMap, BTreeSet, VecDeque};
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+use arrow::array::types::{
+    TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
+    TimestampSecondType,
+};
+use arrow::array::{Array, ArrayRef, PrimitiveArray};
+use arrow::compute::SortColumn;
+use arrow_schema::{DataType, SchemaRef, SortOptions};
+use common_error::ext::{BoxedError, PlainError};
+use common_error::status_code::StatusCode;
+use common_recordbatch::{DfRecordBatch, DfSendableRecordBatchStream};
+use common_time::Timestamp;
+use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool};
+use datafusion::execution::{RecordBatchStream, TaskContext};
+use datafusion::physical_plan::memory::MemoryStream;
+use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};
+use datafusion::physical_plan::sorts::streaming_merge::streaming_merge;
+use datafusion::physical_plan::{
+    DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
+};
+use datafusion_common::utils::bisect;
+use datafusion_common::{internal_err, DataFusionError};
+use datafusion_physical_expr::PhysicalSortExpr;
+use datatypes::value::Value;
+use futures::Stream;
+use itertools::Itertools;
+use snafu::ResultExt;
+use store_api::region_engine::PartitionRange;
+
+use crate::error::{QueryExecutionSnafu, Result};
+
+/// A complex stream sort execution plan which accepts a list of `PartitionRange` and
+/// merge sort them whenever possible, and emit the sorted result as soon as possible.
+/// This sorting plan only accept sort by ts and will not sort by other fields.
+///
+/// internally, it call [`streaming_merge`] multiple times to merge multiple sorted "working ranges"
+///
+/// the input stream must be concated in the order of `PartitionRange` in `ranges`(and each `PartitionRange` is sorted within itself), or else the result will be incorrect
+#[derive(Debug, Clone)]
+pub struct WindowedSortExec {
+    /// Physical sort expressions(that is, sort by timestamp)
+    expression: PhysicalSortExpr,
+    /// Optional number of rows to fetch. Stops producing rows after this fetch
+    fetch: Option<usize>,
+    /// The input ranges indicate input stream will be composed of those ranges in given order
+    ranges: Vec<PartitionRange>,
+    /// All available working ranges and their corresponding working set
+    ///
+    /// working ranges promise once input stream get a value out of current range, future values will never be in this range
+    all_avail_working_range: Vec<(TimeRange, BTreeSet<usize>)>,
+    input: Arc<dyn ExecutionPlan>,
+    /// Execution metrics
+    metrics: ExecutionPlanMetricsSet,
+    properties: PlanProperties,
+}
+
+fn check_partition_range_monotonicity(ranges: &[PartitionRange], descending: bool) -> bool {
+    if descending {
+        ranges.windows(2).all(|w| w[0].end >= w[1].end)
+    } else {
+        ranges.windows(2).all(|w| w[0].start <= w[1].start)
+    }
+}
+
+impl WindowedSortExec {
+    pub fn try_new(
+        expression: PhysicalSortExpr,
+        fetch: Option<usize>,
+        ranges: Vec<PartitionRange>,
+        input: Arc<dyn ExecutionPlan>,
+    ) -> Result<Self> {
+        if !check_partition_range_monotonicity(&ranges, expression.options.descending) {
+            let msg = if expression.options.descending {
+                "Input `PartitionRange`s's upper bound is not monotonic non-increase"
+            } else {
+                "Input `PartitionRange`s's lower bound is not monotonic non-decrease"
+            };
+            let plain_error = PlainError::new(msg.to_string(), StatusCode::Unexpected);
+            return Err(BoxedError::new(plain_error)).context(QueryExecutionSnafu {});
+        }
+
+        let properties = PlanProperties::new(
+            input.equivalence_properties().clone(),
+            input.output_partitioning().clone(),
+            input.execution_mode(),
+        );
+
+        let overlap_counts = split_overlapping_ranges(&ranges);
+        let all_avail_working_range =
+            compute_all_working_ranges(&overlap_counts, expression.options.descending);
+        Ok(Self {
+            expression,
+            fetch,
+            ranges,
+            all_avail_working_range,
+            input,
+            metrics: ExecutionPlanMetricsSet::new(),
+            properties,
+        })
+    }
+
+    /// During receiving partial-sorted RecordBatch, we need to update the working set which is the
+    /// `PartitionRange` we think those RecordBatch belongs to. And when we receive something outside of working set, we can merge results before whenever possible.
+    pub fn to_stream(
+        &self,
+        context: Arc<TaskContext>,
+        partition: usize,
+    ) -> datafusion_common::Result<DfSendableRecordBatchStream> {
+        let input_stream: DfSendableRecordBatchStream =
+            self.input.execute(partition, context.clone())?;
+
+        let df_stream = Box::pin(WindowedSortStream::new(context, self, input_stream)) as _;
+
+        Ok(df_stream)
+    }
+}
+
+impl DisplayAs for WindowedSortExec {
+    fn fmt_as(&self, _t: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "WindowedSortExec")
+    }
+}
+
+impl ExecutionPlan for WindowedSortExec {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.input.schema()
+    }
+
+    fn properties(&self) -> &PlanProperties {
+        &self.properties
+    }
+
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+        vec![&self.input]
+    }
+
+    fn with_new_children(
+        self: Arc<Self>,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {
+        let new_input = if let Some(first) = children.first() {
+            first
+        } else {
+            internal_err!("No children found")?
+        };
+        let new = Self::try_new(
+            self.expression.clone(),
+            self.fetch,
+            self.ranges.clone(),
+            new_input.clone(),
+        )?;
+        Ok(Arc::new(new))
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        context: Arc<TaskContext>,
+    ) -> datafusion_common::Result<DfSendableRecordBatchStream> {
+        self.to_stream(context, partition)
+    }
+
+    fn metrics(&self) -> Option<MetricsSet> {
+        Some(self.metrics.clone_inner())
+    }
+}
+
+/// The core logic of merging sort multiple sorted ranges
+///
+/// the flow of data is:
+/// ```md
+/// input --check if sorted--> in_progress --find sorted run--> sorted_input_runs --call merge sort--> merge_stream --> output
+/// ```
+pub struct WindowedSortStream {
+    /// Memory pool for this stream
+    memory_pool: Arc<dyn MemoryPool>,
+    /// currently assembling RecordBatches, will be put to `sort_partition_rbs` when it's done
+    in_progress: Vec<DfRecordBatch>,
+    /// last `Timestamp` of the last input RecordBatch in `in_progress`, use to found partial sorted run's boundary
+    last_value: Option<Timestamp>,
+    /// Current working set of `PartitionRange` sorted RecordBatches
+    sorted_input_runs: Vec<DfSendableRecordBatchStream>,
+    /// Merge-sorted result streams, should be polled to end before start a new merge sort again
+    merge_stream: VecDeque<DfSendableRecordBatchStream>,
+    /// The number of times merge sort has been called
+    merge_count: usize,
+    /// Index into current `working_range` in `all_avail_working_range`
+    working_idx: usize,
+    /// input stream assumed reading in order of `PartitionRange`
+    input: DfSendableRecordBatchStream,
+    /// Whether the input stream is terminated, if it did, we should poll input again
+    is_input_terminated: bool,
+    /// Output Schema, which is the same as input schema, since this is a sort plan
+    schema: SchemaRef,
+    /// Physical sort expressions(that is, sort by timestamp)
+    expression: PhysicalSortExpr,
+    /// Optional number of rows to fetch. Stops producing rows after this fetch
+    fetch: Option<usize>,
+    /// number of rows produced
+    produced: usize,
+    /// Resulting Stream(`merge_stream`)'s batch size, merely a suggestion
+    batch_size: usize,
+    /// All available working ranges and their corresponding working set
+    ///
+    /// working ranges promise once input stream get a value out of current range, future values will never be in this range
+    all_avail_working_range: Vec<(TimeRange, BTreeSet<usize>)>,
+    /// Execution metrics
+    metrics: ExecutionPlanMetricsSet,
+}
+
+impl WindowedSortStream {
+    pub fn new(
+        context: Arc<TaskContext>,
+        exec: &WindowedSortExec,
+        input: DfSendableRecordBatchStream,
+    ) -> Self {
+        Self {
+            memory_pool: context.runtime_env().memory_pool.clone(),
+            in_progress: Vec::new(),
+            last_value: None,
+            sorted_input_runs: Vec::new(),
+            merge_stream: VecDeque::new(),
+            merge_count: 0,
+            working_idx: 0,
+            schema: input.schema(),
+            input,
+            is_input_terminated: false,
+            expression: exec.expression.clone(),
+            fetch: exec.fetch,
+            produced: 0,
+            batch_size: context.session_config().batch_size(),
+            all_avail_working_range: exec.all_avail_working_range.clone(),
+            metrics: exec.metrics.clone(),
+        }
+    }
+}
+
+impl WindowedSortStream {
+    /// Poll the next RecordBatch from the merge-sort's output stream
+    fn poll_result_stream(
+        &mut self,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<datafusion_common::Result<DfRecordBatch>>> {
+        while let Some(merge_stream) = &mut self.merge_stream.front_mut() {
+            match merge_stream.as_mut().poll_next(cx) {
+                Poll::Ready(Some(Ok(batch))) => {
+                    let ret = if let Some(remaining) = self.remaining_fetch() {
+                        if remaining == 0 {
+                            None
+                        } else if remaining < batch.num_rows() {
+                            self.produced += remaining;
+                            Some(Ok(batch.slice(0, remaining)))
+                        } else {
+                            self.produced += batch.num_rows();
+                            Some(Ok(batch))
+                        }
+                    } else {
+                        self.produced += batch.num_rows();
+                        Some(Ok(batch))
+                    };
+                    return Poll::Ready(ret);
+                }
+                Poll::Ready(Some(Err(e))) => {
+                    return Poll::Ready(Some(Err(e)));
+                }
+                Poll::Ready(None) => {
+                    // current merge stream is done, we can start polling the next one
+
+                    self.merge_stream.pop_front();
+                    continue;
+                }
+                Poll::Pending => {
+                    return Poll::Pending;
+                }
+            }
+        }
+        // if no output stream is available
+        Poll::Ready(None)
+    }
+    /// The core logic of merging sort multiple sorted ranges
+    ///
+    /// We try to maximize the number of sorted runs we can merge in one go, while emit the result as soon as possible.
+    pub fn poll_next_inner(
+        mut self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<datafusion_common::Result<DfRecordBatch>>> {
+        // first check and send out the merge result
+        match self.poll_result_stream(cx) {
+            Poll::Ready(None) => {
+                if self.is_input_terminated {
+                    return Poll::Ready(None);
+                }
+            }
+            x => return x,
+        };
+
+        // consume input stream
+        while !self.is_input_terminated {
+            // then we get a new RecordBatch from input stream
+            let new_input_rbs = match self.input.as_mut().poll_next(cx) {
+                Poll::Ready(Some(Ok(batch))) => {
+                    Some(split_batch_to_sorted_run(batch, &self.expression)?)
+                }
+                Poll::Ready(Some(Err(e))) => {
+                    return Poll::Ready(Some(Err(e)));
+                }
+                Poll::Ready(None) => {
+                    // input stream is done, we need to merge sort the remaining working set
+                    self.is_input_terminated = true;
+                    None
+                }
+                Poll::Pending => return Poll::Pending,
+            };
+
+            // The core logic to eargerly merge sort the working set
+            match new_input_rbs {
+                Some(SortedRunSet {
+                    runs_with_batch,
+                    sort_column,
+                }) => {
+                    // compare with last_value to find boundary, then merge runs if needed
+
+                    // iterate over runs_with_batch to merge sort, might create zero or more stream to put to `sort_partition_rbs`
+
+                    let mut last_remaining = None;
+                    let mut run_iter = runs_with_batch.into_iter();
+                    loop {
+                        let (sorted_rb, run_info) = if let Some(r) = last_remaining.take() {
+                            r
+                        } else if let Some(r) = run_iter.next() {
+                            r
+                        } else {
+                            break;
+                        };
+                        if sorted_rb.num_rows() == 0 {
+                            continue;
+                        }
+                        // determine if this batch is in current working range
+                        let cur_range = {
+                            match run_info.get_time_range() {
+                                Some(r) => r,
+                                _ => internal_err!("Found NULL in time index column")?,
+                            }
+                        };
+                        let working_range = if let Some(r) = self.get_working_range() {
+                            r
+                        } else {
+                            internal_err!("No working range found")?
+                        };
+
+                        if sort_column.options.unwrap_or_default().descending {
+                            if cur_range.end > working_range.end {
+                                internal_err!("Current batch have data on the right side of working range, something is very wrong")?;
+                            }
+                        } else if cur_range.start < working_range.start {
+                            internal_err!("Current batch have data on the left side of working range, something is very wrong")?;
+                        }
+
+                        if cur_range.is_subset(&working_range) {
+                            // data still in range, can't merge sort yet
+                            // see if can concat entire sorted rb, merge sort need to wait
+                            self.try_concat_batch(
+                                sorted_rb.clone(),
+                                &run_info,
+                                sort_column.options,
+                            )?;
+                        } else if let Some(intersection) = cur_range.intersection(&working_range) {
+                            // slice rb by intersection and concat it then merge sort
+                            let cur_sort_column =
+                                sort_column.values.slice(run_info.offset, run_info.len);
+                            let (offset, len) = find_slice_from_range(
+                                &SortColumn {
+                                    values: cur_sort_column.clone(),
+                                    options: sort_column.options,
+                                },
+                                &intersection,
+                            )?;
+
+                            if offset != 0 {
+                                internal_err!("Current batch have data on the left side of working range, something is very wrong")?;
+                            }
+
+                            let sliced_rb = sorted_rb.slice(offset, len);
+
+                            // try to concat the sliced input batch to the current `in_progress` run
+                            self.try_concat_batch(sliced_rb, &run_info, sort_column.options)?;
+                            // since no more sorted data in this range will come in now, build stream now
+                            self.build_sorted_stream()?;
+
+                            // since we are crossing working range, we need to merge sort the working set
+                            self.start_new_merge_sort()?;
+
+                            let (r_offset, r_len) =
+                                (offset + len, sorted_rb.num_rows() - offset - len);
+                            if r_len != 0 {
+                                // we have remaining data in this batch, put it back to input queue
+                                let remaining_rb = sorted_rb.slice(r_offset, r_len);
+                                let new_first_val =
+                                    get_timestamp_from_idx(&cur_sort_column, r_offset)?;
+                                let new_run_info = SucRun {
+                                    offset: run_info.offset + r_offset,
+                                    len: r_len,
+                                    first_val: new_first_val,
+                                    last_val: run_info.last_val,
+                                };
+                                last_remaining = Some((remaining_rb, new_run_info));
+                            }
+                            // deal with remaining batch cross working range problem
+                            // i.e: this example require more slice, and we are currently at point A
+                            // |---1---|       |---3---|
+                            // |-------A--2------------|
+                            //  put the remaining batch back to iter and deal it in next loop
+                        } else {
+                            // no overlap, we can merge sort the working set
+
+                            self.build_sorted_stream()?;
+                            self.start_new_merge_sort()?;
+
+                            // always put it back to input queue until some batch is in working range
+                            last_remaining = Some((sorted_rb, run_info));
+                        }
+                    }
+                }
+                None => {
+                    // input stream is done, we need to merge sort the remaining working set
+                    // and emit the result
+                    self.build_sorted_stream()?;
+                    self.start_new_merge_sort()?;
+                }
+            }
+        }
+        // emit the merge result
+        self.poll_result_stream(cx)
+    }
+
+    fn push_batch(&mut self, batch: DfRecordBatch) {
+        self.in_progress.push(batch);
+    }
+
+    /// Try to concat the input batch to the current `in_progress` run
+    ///
+    /// if the input batch is not sorted, build old run to stream and start a new run with new batch
+    fn try_concat_batch(
+        &mut self,
+        batch: DfRecordBatch,
+        run_info: &SucRun<Timestamp>,
+        opt: Option<SortOptions>,
+    ) -> datafusion_common::Result<()> {
+        let is_ok_to_concat =
+            cmp_with_opts(&self.last_value, &run_info.first_val, &opt) <= std::cmp::Ordering::Equal;
+
+        if is_ok_to_concat {
+            self.push_batch(batch);
+            // next time we get input batch might still be ordered, so not build stream yet
+        } else {
+            // no more sorted data, build stream now
+            self.build_sorted_stream()?;
+            self.push_batch(batch);
+        }
+        self.last_value = run_info.last_val;
+        Ok(())
+    }
+
+    /// Get the current working range
+    fn get_working_range(&self) -> Option<TimeRange> {
+        self.all_avail_working_range
+            .get(self.working_idx)
+            .map(|(range, _)| *range)
+    }
+
+    /// Set current working range to the next working range
+    fn set_next_working_range(&mut self) {
+        self.working_idx += 1;
+    }
+
+    /// make `in_progress` as a new `DfSendableRecordBatchStream` and put into `sorted_input_runs`
+    fn build_sorted_stream(&mut self) -> datafusion_common::Result<()> {
+        if self.in_progress.is_empty() {
+            return Ok(());
+        }
+        let data = std::mem::take(&mut self.in_progress);
+
+        let new_stream = MemoryStream::try_new(data, self.schema(), None)?;
+        self.sorted_input_runs.push(Box::pin(new_stream));
+        Ok(())
+    }
+
+    /// Start merging sort the current working set
+    fn start_new_merge_sort(&mut self) -> datafusion_common::Result<()> {
+        if !self.in_progress.is_empty() {
+            return internal_err!("Starting a merge sort when in_progress is not empty")?;
+        }
+
+        self.set_next_working_range();
+
+        let streams = std::mem::take(&mut self.sorted_input_runs);
+        if streams.is_empty() {
+            return Ok(());
+        } else if streams.len() == 1 {
+            self.merge_stream
+                .push_back(streams.into_iter().next().unwrap());
+            return Ok(());
+        }
+
+        let fetch = self.remaining_fetch();
+        let reservation = MemoryConsumer::new(format!("WindowedSortStream[{}]", self.merge_count))
+            .register(&self.memory_pool);
+        self.merge_count += 1;
+
+        let resulting_stream = streaming_merge(
+            streams,
+            self.schema(),
+            &[self.expression.clone()],
+            BaselineMetrics::new(&self.metrics, 0),
+            self.batch_size,
+            fetch,
+            reservation,
+        )?;
+        self.merge_stream.push_back(resulting_stream);
+        // this working range is done, move to next working range
+        Ok(())
+    }
+
+    /// Remaining number of rows to fetch, if no fetch limit, return None
+    /// if fetch limit is reached, return Some(0)
+    fn remaining_fetch(&self) -> Option<usize> {
+        let total_now = self.produced;
+        self.fetch.map(|p| p.saturating_sub(total_now))
+    }
+}
+
+impl Stream for WindowedSortStream {
+    type Item = datafusion_common::Result<DfRecordBatch>;
+
+    fn poll_next(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<datafusion_common::Result<DfRecordBatch>>> {
+        self.poll_next_inner(cx)
+    }
+}
+
+impl RecordBatchStream for WindowedSortStream {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
+/// split batch to sorted runs
+fn split_batch_to_sorted_run(
+    batch: DfRecordBatch,
+    expression: &PhysicalSortExpr,
+) -> datafusion_common::Result<SortedRunSet<Timestamp>> {
+    // split input rb to sorted runs
+    let sort_column = expression.evaluate_to_sort_column(&batch)?;
+    let sorted_runs_offset = get_sorted_runs(sort_column.clone())?;
+    if let Some(run) = sorted_runs_offset.first()
+        && sorted_runs_offset.len() == 1
+    {
+        if !(run.offset == 0 && run.len == batch.num_rows()) {
+            internal_err!(
+                "Invalid run offset and length: offset = {:?}, len = {:?}, num_rows = {:?}",
+                run.offset,
+                run.len,
+                batch.num_rows()
+            )?;
+        }
+        // input rb is already sorted, we can emit it directly
+        Ok(SortedRunSet {
+            runs_with_batch: vec![(batch, run.clone())],
+            sort_column,
+        })
+    } else {
+        // those slice should be zero copy, so supposedly no new reservation needed
+        let mut ret = Vec::with_capacity(sorted_runs_offset.len());
+        for run in sorted_runs_offset {
+            if run.offset + run.len > batch.num_rows() {
+                internal_err!(
+                    "Invalid run offset and length: offset = {:?}, len = {:?}, num_rows = {:?}",
+                    run.offset,
+                    run.len,
+                    batch.num_rows()
+                )?;
+            }
+            let new_rb = batch.slice(run.offset, run.len);
+            ret.push((new_rb, run));
+        }
+        Ok(SortedRunSet {
+            runs_with_batch: ret,
+            sort_column,
+        })
+    }
+}
+
+/// Downcast a temporal array to a specific type
+///
+/// usage similar to `downcast_primitive!` in `arrow-array` crate
+macro_rules! downcast_ts_array {
+    ($data_type:expr => ($m:path $(, $args:tt)*), $($p:pat => $fallback:expr $(,)*)*) =>
+    {
+        match $data_type {
+            arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Second, _) => {
+                $m!(TimestampSecondType $(, $args)*)
+            }
+            arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Millisecond, _) => {
+                $m!(TimestampMillisecondType $(, $args)*)
+            }
+            arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Microsecond, _) => {
+                $m!(TimestampMicrosecondType $(, $args)*)
+            }
+            arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Nanosecond, _) => {
+                $m!(TimestampNanosecondType $(, $args)*)
+            }
+            $($p => $fallback,)*
+        }
+    };
+}
+
+/// Find the slice(where start <= data < end and sort by `sort_column.options`) from the given range
+///
+/// Return the offset and length of the slice
+fn find_slice_from_range(
+    sort_column: &SortColumn,
+    range: &TimeRange,
+) -> datafusion_common::Result<(usize, usize)> {
+    let ty = sort_column.values.data_type();
+    let time_unit = if let DataType::Timestamp(unit, _) = ty {
+        unit
+    } else {
+        return Err(DataFusionError::Internal(format!(
+            "Unsupported sort column type: {}",
+            sort_column.values.data_type()
+        )));
+    };
+    let array = &sort_column.values;
+    let opt = &sort_column.options.unwrap_or_default();
+    let descending = opt.descending;
+
+    let typed_sorted_range = [range.start, range.end]
+        .iter()
+        .map(|t| {
+            t.convert_to(time_unit.into())
+                .ok_or_else(|| {
+                    DataFusionError::Internal(format!(
+                        "Failed to convert timestamp from {:?} to {:?}",
+                        t.unit(),
+                        time_unit
+                    ))
+                })
+                .and_then(|typed_ts| {
+                    let value = Value::Timestamp(typed_ts);
+                    value
+                        .try_to_scalar_value(&value.data_type())
+                        .map_err(|e| DataFusionError::External(Box::new(e) as _))
+                })
+        })
+        .try_collect::<_, Vec<_>, _>()?;
+
+    let (min_val, max_val) = (typed_sorted_range[0].clone(), typed_sorted_range[1].clone());
+
+    // get slice that in which all data that `min_val<=data<max_val`
+    let (start, end) = if descending {
+        // note that `data < max_val`
+        // i,e, for max_val = 4, array = [5,3,2] should be start=1
+        // max_val = 4, array = [5, 4, 3, 2] should be start= 2
+        let start = bisect::<false>(&[array.clone()], &[max_val.clone()], &[*opt])?;
+        // min_val = 1, array = [3, 2, 1, 0], end = 3
+        // min_val = 1, array = [3, 2, 0], end = 2
+        let end = bisect::<false>(&[array.clone()], &[min_val.clone()], &[*opt])?;
+        (start, end)
+    } else {
+        // min_val = 1, array = [1, 2, 3], start = 0
+        // min_val = 1, array = [0, 2, 3], start = 1
+        let start = bisect::<true>(&[array.clone()], &[min_val.clone()], &[*opt])?;
+        // max_val = 3, array = [1, 3, 4], end = 1
+        // max_val = 3, array = [1, 2, 4], end = 2
+        let end = bisect::<true>(&[array.clone()], &[max_val.clone()], &[*opt])?;
+        (start, end)
+    };
+
+    Ok((start, end - start))
+}
+
+macro_rules! array_iter_helper {
+    ($t:ty, $arr:expr) => {{
+        let typed = $arr.as_any().downcast_ref::<PrimitiveArray<$t>>().unwrap();
+        let iter = typed.iter().enumerate();
+        Box::new(iter) as Box<dyn Iterator<Item = (usize, Option<i64>)>>
+    }};
+}
+
+/// Compare with options, note None is considered as NULL here
+///
+/// default to null first
+fn cmp_with_opts<T: Ord>(
+    a: &Option<T>,
+    b: &Option<T>,
+    opt: &Option<SortOptions>,
+) -> std::cmp::Ordering {
+    let opt = opt.unwrap_or_default();
+
+    if let (Some(a), Some(b)) = (a, b) {
+        if opt.descending {
+            b.cmp(a)
+        } else {
+            a.cmp(b)
+        }
+    } else if opt.nulls_first {
+        // now we know at leatst one of them is None
+        // in rust None < Some(_)
+        a.cmp(b)
+    } else {
+        match (a, b) {
+            (Some(a), Some(b)) => a.cmp(b),
+            (Some(_), None) => std::cmp::Ordering::Less,
+            (None, Some(_)) => std::cmp::Ordering::Greater,
+            (None, None) => std::cmp::Ordering::Equal,
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+struct SortedRunSet<N: Ord> {
+    /// sorted runs with batch corresponding to them
+    runs_with_batch: Vec<(DfRecordBatch, SucRun<N>)>,
+    /// sorted column from eval sorting expr
+    sort_column: SortColumn,
+}
+
+/// A struct to represent a successive run in the input iterator
+#[derive(Debug, Clone, PartialEq)]
+struct SucRun<N: Ord> {
+    /// offset of the first element in the run
+    offset: usize,
+    /// length of the run
+    len: usize,
+    /// first non-null value in the run
+    first_val: Option<N>,
+    /// last non-null value in the run
+    last_val: Option<N>,
+}
+
+impl SucRun<Timestamp> {
+    /// Get the time range of the run, which is [min_val, max_val + 1)
+    fn get_time_range(&self) -> Option<TimeRange> {
+        let start = self.first_val.min(self.last_val);
+        let end = self
+            .first_val
+            .max(self.last_val)
+            .map(|i| Timestamp::new(i.value() + 1, i.unit()));
+        start.zip(end).map(|(s, e)| TimeRange::new(s, e))
+    }
+}
+
+/// find all successive runs in the input iterator
+fn find_successive_runs<T: Iterator<Item = (usize, Option<N>)>, N: Ord + Copy>(
+    iter: T,
+    sort_opts: &Option<SortOptions>,
+) -> Vec<SucRun<N>> {
+    let mut runs = Vec::new();
+    let mut last_value = None;
+    let mut iter_len = None;
+
+    let mut last_offset = 0;
+    let mut first_val: Option<N> = None;
+    let mut last_val: Option<N> = None;
+
+    for (idx, t) in iter {
+        if let Some(last_value) = &last_value {
+            if cmp_with_opts(last_value, &t, sort_opts) == std::cmp::Ordering::Greater {
+                // we found a boundary
+                let len = idx - last_offset;
+                let run = SucRun {
+                    offset: last_offset,
+                    len,
+                    first_val,
+                    last_val,
+                };
+                runs.push(run);
+                first_val = None;
+                last_val = None;
+
+                last_offset = idx;
+            }
+        }
+        last_value = Some(t);
+        if let Some(t) = t {
+            first_val = first_val.or(Some(t));
+            last_val = Some(t).or(last_val);
+        }
+        iter_len = Some(idx);
+    }
+    let run = SucRun {
+        offset: last_offset,
+        len: iter_len.map(|l| l - last_offset + 1).unwrap_or(0),
+        first_val,
+        last_val,
+    };
+    runs.push(run);
+
+    runs
+}
+
+/// return a list of non-overlapping (offset, length) which represent sorted runs, and
+/// can be used to call [`DfRecordBatch::slice`] to get sorted runs
+/// Returned runs will be as long as possible, and will not overlap with each other
+fn get_sorted_runs(sort_column: SortColumn) -> datafusion_common::Result<Vec<SucRun<Timestamp>>> {
+    let ty = sort_column.values.data_type();
+    if let DataType::Timestamp(unit, _) = ty {
+        let array = &sort_column.values;
+        let iter = downcast_ts_array!(
+            array.data_type() => (array_iter_helper, array),
+            _ => internal_err!("Unsupported sort column type: {ty}")?
+        );
+
+        let raw = find_successive_runs(iter, &sort_column.options);
+        let ts_runs = raw
+            .into_iter()
+            .map(|run| SucRun {
+                offset: run.offset,
+                len: run.len,
+                first_val: run.first_val.map(|v| Timestamp::new(v, unit.into())),
+                last_val: run.last_val.map(|v| Timestamp::new(v, unit.into())),
+            })
+            .collect_vec();
+        Ok(ts_runs)
+    } else {
+        Err(DataFusionError::Internal(format!(
+            "Unsupported sort column type: {ty}"
+        )))
+    }
+}
+
+/// Left(`start`) inclusive right(`end`) exclusive,
+///
+/// This is just tuple with extra methods
+#[derive(Debug, Clone, Default, Copy, PartialEq, Eq, PartialOrd, Ord)]
+struct TimeRange {
+    start: Timestamp,
+    end: Timestamp,
+}
+
+impl From<&PartitionRange> for TimeRange {
+    fn from(range: &PartitionRange) -> Self {
+        Self::new(range.start, range.end)
+    }
+}
+
+impl From<(Timestamp, Timestamp)> for TimeRange {
+    fn from(range: (Timestamp, Timestamp)) -> Self {
+        Self::new(range.0, range.1)
+    }
+}
+
+impl From<&(Timestamp, Timestamp)> for TimeRange {
+    fn from(range: &(Timestamp, Timestamp)) -> Self {
+        Self::new(range.0, range.1)
+    }
+}
+
+impl TimeRange {
+    /// Create a new TimeRange, if start is greater than end, swap them
+    fn new(start: Timestamp, end: Timestamp) -> Self {
+        if start > end {
+            Self {
+                start: end,
+                end: start,
+            }
+        } else {
+            Self { start, end }
+        }
+    }
+
+    fn is_subset(&self, other: &Self) -> bool {
+        self.start >= other.start && self.end <= other.end
+    }
+
+    /// Check if two ranges are overlapping, exclusive(meaning if only boundary is overlapped then range is not overlapping)
+    fn is_overlapping(&self, other: &Self) -> bool {
+        !(self.start >= other.end || self.end <= other.start)
+    }
+
+    fn intersection(&self, other: &Self) -> Option<Self> {
+        if self.is_overlapping(other) {
+            Some(Self::new(
+                self.start.max(other.start),
+                self.end.min(other.end),
+            ))
+        } else {
+            None
+        }
+    }
+
+    fn difference(&self, other: &Self) -> Vec<Self> {
+        if !self.is_overlapping(other) {
+            vec![*self]
+        } else {
+            let mut ret = Vec::new();
+            if self.start < other.start && self.end > other.end {
+                ret.push(Self::new(self.start, other.start));
+                ret.push(Self::new(other.end, self.end));
+            } else if self.start < other.start {
+                ret.push(Self::new(self.start, other.start));
+            } else if self.end > other.end {
+                ret.push(Self::new(other.end, self.end));
+            }
+            ret
+        }
+    }
+}
+
+/// split input range by `split_by` range to one, two or three parts.
+fn split_range_by(
+    input_range: &TimeRange,
+    input_parts: &[usize],
+    split_by: &TimeRange,
+    split_idx: usize,
+) -> Vec<Action> {
+    let mut ret = Vec::new();
+    if input_range.is_overlapping(split_by) {
+        let input_parts = input_parts.to_vec();
+        let new_parts = {
+            let mut new_parts = input_parts.clone();
+            new_parts.push(split_idx);
+            new_parts
+        };
+
+        ret.push(Action::Pop(*input_range));
+        if let Some(intersection) = input_range.intersection(split_by) {
+            ret.push(Action::Push(intersection, new_parts.clone()));
+        }
+        for diff in input_range.difference(split_by) {
+            ret.push(Action::Push(diff, input_parts.clone()));
+        }
+    }
+    ret
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+enum Action {
+    Pop(TimeRange),
+    Push(TimeRange, Vec<usize>),
+}
+
+/// Compute all working ranges and corresponding working sets from given `overlap_counts` computed from `split_overlapping_ranges`
+///
+/// working ranges promise once input stream get a value out of current range, future values will never be in this range
+///
+/// hence we can merge sort current working range once that happens
+///
+/// if `descending` is true, the working ranges will be in descending order
+fn compute_all_working_ranges(
+    overlap_counts: &BTreeMap<TimeRange, Vec<usize>>,
+    descending: bool,
+) -> Vec<(TimeRange, BTreeSet<usize>)> {
+    let mut ret = Vec::new();
+    let mut cur_range_set: Option<(TimeRange, BTreeSet<usize>)> = None;
+    let overlap_iter: Box<dyn Iterator<Item = (&TimeRange, &Vec<usize>)>> = if descending {
+        Box::new(overlap_counts.iter().rev()) as _
+    } else {
+        Box::new(overlap_counts.iter()) as _
+    };
+    for (range, set) in overlap_iter {
+        match &mut cur_range_set {
+            None => cur_range_set = Some((*range, BTreeSet::from_iter(set.iter().cloned()))),
+            Some((working_range, working_set)) => {
+                // if next overlap range have Partition that's is not last one in `working_set`(hence need to be read before merge sorting), and `working_set` have >1 count
+                // we have to expand current working range to cover it(and add it's `set` to `working_set`)
+                // so that merge sort is possible
+                let need_expand = {
+                    let last_part = working_set.last();
+                    let inter: BTreeSet<usize> = working_set
+                        .intersection(&BTreeSet::from_iter(set.iter().cloned()))
+                        .cloned()
+                        .collect();
+                    if let Some(one) = inter.first()
+                        && inter.len() == 1
+                        && Some(one) == last_part
+                    {
+                        // if only the last PartitionRange in current working set, we can just emit it so no need to expand working range
+                        if set.iter().all(|p| Some(p) >= last_part) {
+                            // if all PartitionRange in next overlap range is after the last one in current working set, we can just emit current working set
+                            false
+                        } else {
+                            // elsewise, we need to expand working range to include next overlap range
+                            true
+                        }
+                    } else if inter.is_empty() {
+                        // if no common PartitionRange in current working set and next overlap range, we can just emit current working set
+                        false
+                    } else {
+                        // have multiple intersection or intersection is not the last part, either way we need to expand working range to include next overlap range
+                        true
+                    }
+                };
+
+                if need_expand {
+                    if descending {
+                        working_range.start = range.start;
+                    } else {
+                        working_range.end = range.end;
+                    }
+                    working_set.extend(set.iter().cloned());
+                } else {
+                    ret.push((*working_range, std::mem::take(working_set)));
+                    cur_range_set = Some((*range, BTreeSet::from_iter(set.iter().cloned())));
+                }
+            }
+        }
+    }
+
+    if let Some(cur_range_set) = cur_range_set {
+        ret.push(cur_range_set)
+    }
+
+    ret
+}
+
+/// return a map of non-overlapping ranges and their corresponding index
+/// (not `PartitionRange.identifier` but position in array) in the input `PartitionRange`s that is in those ranges
+fn split_overlapping_ranges(ranges: &[PartitionRange]) -> BTreeMap<TimeRange, Vec<usize>> {
+    // invariant: the key ranges should not overlapping with each other by definition from `is_overlapping`
+    let mut ret: BTreeMap<TimeRange, Vec<usize>> = BTreeMap::new();
+    for (idx, range) in ranges.iter().enumerate() {
+        let key: TimeRange = (range.start, range.end).into();
+        let mut actions = Vec::new();
+        let mut untouched = vec![key];
+        // create a forward and backward iterator to find all overlapping ranges
+        // given that tuple is sorted in lexicographical order and promise to not overlap,
+        // since range is sorted that way, we can stop when we find a non-overlapping range
+        let forward_iter = ret
+            .range(key..)
+            .take_while(|(range, _)| range.is_overlapping(&key));
+        let backward_iter = ret
+            .range(..key)
+            .rev()
+            .take_while(|(range, _)| range.is_overlapping(&key));
+
+        for (range, parts) in forward_iter.chain(backward_iter) {
+            untouched = untouched.iter().flat_map(|r| r.difference(range)).collect();
+            let act = split_range_by(range, parts, &key, idx);
+            actions.extend(act.into_iter());
+        }
+
+        for action in actions {
+            match action {
+                Action::Pop(range) => {
+                    ret.remove(&range);
+                }
+                Action::Push(range, parts) => {
+                    ret.insert(range, parts);
+                }
+            }
+        }
+
+        // insert untouched ranges
+        for range in untouched {
+            ret.insert(range, vec![idx]);
+        }
+    }
+    ret
+}
+
+/// Get timestamp from array at offset
+fn get_timestamp_from_idx(
+    array: &ArrayRef,
+    offset: usize,
+) -> datafusion_common::Result<Option<Timestamp>> {
+    let time_unit = if let DataType::Timestamp(unit, _) = array.data_type() {
+        unit
+    } else {
+        return Err(DataFusionError::Internal(format!(
+            "Unsupported sort column type: {}",
+            array.data_type()
+        )));
+    };
+    let ty = array.data_type();
+    let array = array.slice(offset, 1);
+    let mut iter = downcast_ts_array!(
+        array.data_type() => (array_iter_helper, array),
+        _ => internal_err!("Unsupported sort column type: {ty}")?
+    );
+    let (_idx, val) = iter.next().ok_or_else(|| {
+        DataFusionError::Internal("Empty array in get_timestamp_from".to_string())
+    })?;
+    let val = if let Some(val) = val {
+        val
+    } else {
+        return Ok(None);
+    };
+    let gt_timestamp = Timestamp::new(val, time_unit.into());
+    Ok(Some(gt_timestamp))
+}
+
+#[cfg(test)]
+mod test {
+    use std::io::Write;
+    use std::sync::Arc;
+
+    use arrow::array::{
+        ArrayRef, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
+        TimestampSecondArray,
+    };
+    use arrow::compute::concat_batches;
+    use arrow::json::ArrayWriter;
+    use arrow_schema::{Field, Schema, TimeUnit};
+    use datafusion::physical_plan::ExecutionMode;
+    use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
+    use futures::StreamExt;
+    use pretty_assertions::assert_eq;
+    use serde_json::json;
+
+    use super::*;
+    #[test]
+    fn test_overlapping() {
+        let testcases = [
+            (
+                (Timestamp::new_second(1), Timestamp::new_millisecond(1001)),
+                (Timestamp::new_second(0), Timestamp::new_millisecond(1)),
+                false,
+            ),
+            (
+                (Timestamp::new_second(1), Timestamp::new_millisecond(1001)),
+                (Timestamp::new_second(0), Timestamp::new_millisecond(1001)),
+                true,
+            ),
+            (
+                (Timestamp::new_second(1), Timestamp::new_millisecond(1001)),
+                (Timestamp::new_second(0), Timestamp::new_millisecond(1002)),
+                true,
+            ),
+            (
+                (Timestamp::new_second(1), Timestamp::new_millisecond(1001)),
+                (
+                    Timestamp::new_millisecond(1000),
+                    Timestamp::new_millisecond(1002),
+                ),
+                true,
+            ),
+            (
+                (Timestamp::new_second(1), Timestamp::new_millisecond(1001)),
+                (
+                    Timestamp::new_millisecond(1001),
+                    Timestamp::new_millisecond(1002),
+                ),
+                false,
+            ),
+            (
+                (Timestamp::new_second(1), Timestamp::new_millisecond(1001)),
+                (
+                    Timestamp::new_millisecond(1002),
+                    Timestamp::new_millisecond(1003),
+                ),
+                false,
+            ),
+        ];
+
+        for (range1, range2, expected) in testcases.iter() {
+            assert_eq!(
+                TimeRange::from(range1).is_overlapping(&range2.into()),
+                *expected,
+                "range1: {:?}, range2: {:?}",
+                range1,
+                range2
+            );
+        }
+    }
+
+    #[test]
+    fn test_split() {
+        let testcases = [
+            // no split
+            (
+                (Timestamp::new_second(1), Timestamp::new_millisecond(1001)),
+                vec![0],
+                (Timestamp::new_second(0), Timestamp::new_millisecond(1)),
+                1,
+                vec![],
+            ),
+            // one part
+            (
+                (Timestamp::new_second(1), Timestamp::new_millisecond(1001)),
+                vec![0],
+                (Timestamp::new_second(0), Timestamp::new_millisecond(1001)),
+                1,
+                vec![
+                    Action::Pop(
+                        (Timestamp::new_second(1), Timestamp::new_millisecond(1001)).into(),
+                    ),
+                    Action::Push(
+                        (Timestamp::new_second(1), Timestamp::new_millisecond(1001)).into(),
+                        vec![0, 1],
+                    ),
+                ],
+            ),
+            (
+                (Timestamp::new_second(1), Timestamp::new_millisecond(1001)),
+                vec![0],
+                (Timestamp::new_second(0), Timestamp::new_millisecond(1002)),
+                1,
+                vec![
+                    Action::Pop(
+                        (Timestamp::new_second(1), Timestamp::new_millisecond(1001)).into(),
+                    ),
+                    Action::Push(
+                        (Timestamp::new_second(1), Timestamp::new_millisecond(1001)).into(),
+                        vec![0, 1],
+                    ),
+                ],
+            ),
+            (
+                (Timestamp::new_second(1), Timestamp::new_millisecond(1001)),
+                vec![0],
+                (
+                    Timestamp::new_millisecond(1000),
+                    Timestamp::new_millisecond(1002),
+                ),
+                1,
+                vec![
+                    Action::Pop(
+                        (Timestamp::new_second(1), Timestamp::new_millisecond(1001)).into(),
+                    ),
+                    Action::Push(
+                        (
+                            Timestamp::new_millisecond(1000),
+                            Timestamp::new_millisecond(1001),
+                        )
+                            .into(),
+                        vec![0, 1],
+                    ),
+                ],
+            ),
+            // two part
+            (
+                (Timestamp::new_second(1), Timestamp::new_millisecond(1002)),
+                vec![0],
+                (
+                    Timestamp::new_millisecond(1001),
+                    Timestamp::new_millisecond(1002),
+                ),
+                1,
+                vec![
+                    Action::Pop(
+                        (Timestamp::new_second(1), Timestamp::new_millisecond(1002)).into(),
+                    ),
+                    Action::Push(
+                        (
+                            Timestamp::new_millisecond(1001),
+                            Timestamp::new_millisecond(1002),
+                        )
+                            .into(),
+                        vec![0, 1],
+                    ),
+                    Action::Push(
+                        (Timestamp::new_second(1), Timestamp::new_millisecond(1001)).into(),
+                        vec![0],
+                    ),
+                ],
+            ),
+            // three part
+            (
+                (Timestamp::new_second(1), Timestamp::new_millisecond(1004)),
+                vec![0],
+                (
+                    Timestamp::new_millisecond(1001),
+                    Timestamp::new_millisecond(1002),
+                ),
+                1,
+                vec![
+                    Action::Pop(
+                        (Timestamp::new_second(1), Timestamp::new_millisecond(1004)).into(),
+                    ),
+                    Action::Push(
+                        (
+                            Timestamp::new_millisecond(1001),
+                            Timestamp::new_millisecond(1002),
+                        )
+                            .into(),
+                        vec![0, 1],
+                    ),
+                    Action::Push(
+                        (Timestamp::new_second(1), Timestamp::new_millisecond(1001)).into(),
+                        vec![0],
+                    ),
+                    Action::Push(
+                        (
+                            Timestamp::new_millisecond(1002),
+                            Timestamp::new_millisecond(1004),
+                        )
+                            .into(),
+                        vec![0],
+                    ),
+                ],
+            ),
+        ];
+
+        for (range, parts, split_by, split_idx, expected) in testcases.iter() {
+            assert_eq!(
+                split_range_by(&(*range).into(), parts, &split_by.into(), *split_idx),
+                *expected,
+                "range: {:?}, parts: {:?}, split_by: {:?}, split_idx: {}",
+                range,
+                parts,
+                split_by,
+                split_idx
+            );
+        }
+    }
+
+    #[test]
+    fn test_compute_working_ranges_rev() {
+        let testcases = vec![
+            (
+                BTreeMap::from([(
+                    (Timestamp::new_second(1), Timestamp::new_second(2)),
+                    vec![0],
+                )]),
+                vec![(
+                    (Timestamp::new_second(1), Timestamp::new_second(2)),
+                    BTreeSet::from([0]),
+                )],
+            ),
+            (
+                BTreeMap::from([(
+                    (Timestamp::new_second(1), Timestamp::new_second(2)),
+                    vec![0, 1],
+                )]),
+                vec![(
+                    (Timestamp::new_second(1), Timestamp::new_second(2)),
+                    BTreeSet::from([0, 1]),
+                )],
+            ),
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![0],
+                    ),
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![0, 1],
+                    ),
+                ]),
+                vec![
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        BTreeSet::from([0]),
+                    ),
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        BTreeSet::from([0, 1]),
+                    ),
+                ],
+            ),
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![0, 1],
+                    ),
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![1],
+                    ),
+                ]),
+                vec![
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        BTreeSet::from([0, 1]),
+                    ),
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        BTreeSet::from([1]),
+                    ),
+                ],
+            ),
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(3), Timestamp::new_second(4)),
+                        vec![0],
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![0, 1],
+                    ),
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![1],
+                    ),
+                ]),
+                vec![
+                    (
+                        (Timestamp::new_second(3), Timestamp::new_second(4)),
+                        BTreeSet::from([0]),
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        BTreeSet::from([0, 1]),
+                    ),
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        BTreeSet::from([1]),
+                    ),
+                ],
+            ),
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(3), Timestamp::new_second(4)),
+                        vec![0, 2],
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![0, 1, 2],
+                    ),
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![1, 2],
+                    ),
+                ]),
+                vec![(
+                    (Timestamp::new_second(1), Timestamp::new_second(4)),
+                    BTreeSet::from([0, 1, 2]),
+                )],
+            ),
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![0, 2],
+                    ),
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![1, 2],
+                    ),
+                ]),
+                vec![(
+                    (Timestamp::new_second(1), Timestamp::new_second(3)),
+                    BTreeSet::from([0, 1, 2]),
+                )],
+            ),
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(3), Timestamp::new_second(4)),
+                        vec![0, 1],
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![0, 1, 2],
+                    ),
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![1, 2],
+                    ),
+                ]),
+                vec![(
+                    (Timestamp::new_second(1), Timestamp::new_second(4)),
+                    BTreeSet::from([0, 1, 2]),
+                )],
+            ),
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![0, 1],
+                    ),
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![1, 2],
+                    ),
+                ]),
+                vec![
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        BTreeSet::from([0, 1]),
+                    ),
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        BTreeSet::from([1, 2]),
+                    ),
+                ],
+            ),
+            // non-overlapping
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![0],
+                    ),
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![1, 2],
+                    ),
+                ]),
+                vec![
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        BTreeSet::from([0]),
+                    ),
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        BTreeSet::from([1, 2]),
+                    ),
+                ],
+            ),
+        ];
+
+        for (input, expected) in testcases {
+            let expected = expected
+                .into_iter()
+                .map(|(r, s)| (r.into(), s))
+                .collect_vec();
+            let input = input.into_iter().map(|(r, s)| (r.into(), s)).collect();
+            assert_eq!(
+                compute_all_working_ranges(&input, true),
+                expected,
+                "input: {:?}",
+                input
+            );
+        }
+    }
+
+    #[test]
+    fn test_compute_working_ranges() {
+        let testcases = vec![
+            (
+                BTreeMap::from([(
+                    (Timestamp::new_second(1), Timestamp::new_second(2)),
+                    vec![0],
+                )]),
+                vec![(
+                    (Timestamp::new_second(1), Timestamp::new_second(2)),
+                    BTreeSet::from([0]),
+                )],
+            ),
+            (
+                BTreeMap::from([(
+                    (Timestamp::new_second(1), Timestamp::new_second(2)),
+                    vec![0, 1],
+                )]),
+                vec![(
+                    (Timestamp::new_second(1), Timestamp::new_second(2)),
+                    BTreeSet::from([0, 1]),
+                )],
+            ),
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![0, 1],
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![1],
+                    ),
+                ]),
+                vec![
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        BTreeSet::from([0, 1]),
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        BTreeSet::from([1]),
+                    ),
+                ],
+            ),
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![0],
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![0, 1],
+                    ),
+                ]),
+                vec![
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        BTreeSet::from([0]),
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        BTreeSet::from([0, 1]),
+                    ),
+                ],
+            ),
+            // test if only one count in working set get it's own working range
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![0],
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![0, 1],
+                    ),
+                    (
+                        (Timestamp::new_second(3), Timestamp::new_second(4)),
+                        vec![1],
+                    ),
+                ]),
+                vec![
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        BTreeSet::from([0]),
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        BTreeSet::from([0, 1]),
+                    ),
+                    (
+                        (Timestamp::new_second(3), Timestamp::new_second(4)),
+                        BTreeSet::from([1]),
+                    ),
+                ],
+            ),
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![0, 2],
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![0, 1, 2],
+                    ),
+                    (
+                        (Timestamp::new_second(3), Timestamp::new_second(4)),
+                        vec![1, 2],
+                    ),
+                ]),
+                vec![(
+                    (Timestamp::new_second(1), Timestamp::new_second(4)),
+                    BTreeSet::from([0, 1, 2]),
+                )],
+            ),
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![0, 2],
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![1, 2],
+                    ),
+                ]),
+                vec![(
+                    (Timestamp::new_second(1), Timestamp::new_second(3)),
+                    BTreeSet::from([0, 1, 2]),
+                )],
+            ),
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![0, 1],
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![0, 1, 2],
+                    ),
+                    (
+                        (Timestamp::new_second(3), Timestamp::new_second(4)),
+                        vec![1, 2],
+                    ),
+                ]),
+                vec![(
+                    (Timestamp::new_second(1), Timestamp::new_second(4)),
+                    BTreeSet::from([0, 1, 2]),
+                )],
+            ),
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![0, 1],
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![1, 2],
+                    ),
+                ]),
+                vec![
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        BTreeSet::from([0, 1]),
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        BTreeSet::from([1, 2]),
+                    ),
+                ],
+            ),
+            // non-overlapping
+            (
+                BTreeMap::from([
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![0, 1],
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        vec![2],
+                    ),
+                ]),
+                vec![
+                    (
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        BTreeSet::from([0, 1]),
+                    ),
+                    (
+                        (Timestamp::new_second(2), Timestamp::new_second(3)),
+                        BTreeSet::from([2]),
+                    ),
+                ],
+            ),
+        ];
+
+        for (input, expected) in testcases {
+            let expected = expected
+                .into_iter()
+                .map(|(r, s)| (r.into(), s))
+                .collect_vec();
+            let input = input.into_iter().map(|(r, s)| (r.into(), s)).collect();
+            assert_eq!(
+                compute_all_working_ranges(&input, false),
+                expected,
+                "input: {:?}",
+                input
+            );
+        }
+    }
+
+    #[test]
+    fn test_split_overlap_range() {
+        let testcases = vec![
+            // simple one range
+            (
+                vec![PartitionRange {
+                    start: Timestamp::new_second(1),
+                    end: Timestamp::new_second(2),
+                    num_rows: 2,
+                    identifier: 0,
+                }],
+                BTreeMap::from_iter(
+                    vec![(
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![0],
+                    )]
+                    .into_iter(),
+                ),
+            ),
+            // two overlapping range
+            (
+                vec![
+                    PartitionRange {
+                        start: Timestamp::new_second(1),
+                        end: Timestamp::new_second(2),
+                        num_rows: 2,
+                        identifier: 0,
+                    },
+                    PartitionRange {
+                        start: Timestamp::new_second(1),
+                        end: Timestamp::new_second(2),
+                        num_rows: 2,
+                        identifier: 1,
+                    },
+                ],
+                BTreeMap::from_iter(
+                    vec![(
+                        (Timestamp::new_second(1), Timestamp::new_second(2)),
+                        vec![0, 1],
+                    )]
+                    .into_iter(),
+                ),
+            ),
+            (
+                vec![
+                    PartitionRange {
+                        start: Timestamp::new_second(1),
+                        end: Timestamp::new_second(3),
+                        num_rows: 2,
+                        identifier: 0,
+                    },
+                    PartitionRange {
+                        start: Timestamp::new_second(2),
+                        end: Timestamp::new_second(4),
+                        num_rows: 2,
+                        identifier: 1,
+                    },
+                ],
+                BTreeMap::from_iter(
+                    vec![
+                        (
+                            (Timestamp::new_second(1), Timestamp::new_second(2)),
+                            vec![0],
+                        ),
+                        (
+                            (Timestamp::new_second(2), Timestamp::new_second(3)),
+                            vec![0, 1],
+                        ),
+                        (
+                            (Timestamp::new_second(3), Timestamp::new_second(4)),
+                            vec![1],
+                        ),
+                    ]
+                    .into_iter(),
+                ),
+            ),
+            // three or more overlapping range
+            (
+                vec![
+                    PartitionRange {
+                        start: Timestamp::new_second(1),
+                        end: Timestamp::new_second(3),
+                        num_rows: 2,
+                        identifier: 0,
+                    },
+                    PartitionRange {
+                        start: Timestamp::new_second(2),
+                        end: Timestamp::new_second(4),
+                        num_rows: 2,
+                        identifier: 1,
+                    },
+                    PartitionRange {
+                        start: Timestamp::new_second(1),
+                        end: Timestamp::new_second(4),
+                        num_rows: 2,
+                        identifier: 2,
+                    },
+                ],
+                BTreeMap::from_iter(
+                    vec![
+                        (
+                            (Timestamp::new_second(1), Timestamp::new_second(2)),
+                            vec![0, 2],
+                        ),
+                        (
+                            (Timestamp::new_second(2), Timestamp::new_second(3)),
+                            vec![0, 1, 2],
+                        ),
+                        (
+                            (Timestamp::new_second(3), Timestamp::new_second(4)),
+                            vec![1, 2],
+                        ),
+                    ]
+                    .into_iter(),
+                ),
+            ),
+            (
+                vec![
+                    PartitionRange {
+                        start: Timestamp::new_second(1),
+                        end: Timestamp::new_second(3),
+                        num_rows: 2,
+                        identifier: 0,
+                    },
+                    PartitionRange {
+                        start: Timestamp::new_second(1),
+                        end: Timestamp::new_second(4),
+                        num_rows: 2,
+                        identifier: 1,
+                    },
+                    PartitionRange {
+                        start: Timestamp::new_second(2),
+                        end: Timestamp::new_second(4),
+                        num_rows: 2,
+                        identifier: 2,
+                    },
+                ],
+                BTreeMap::from_iter(
+                    vec![
+                        (
+                            (Timestamp::new_second(1), Timestamp::new_second(2)),
+                            vec![0, 1],
+                        ),
+                        (
+                            (Timestamp::new_second(2), Timestamp::new_second(3)),
+                            vec![0, 1, 2],
+                        ),
+                        (
+                            (Timestamp::new_second(3), Timestamp::new_second(4)),
+                            vec![1, 2],
+                        ),
+                    ]
+                    .into_iter(),
+                ),
+            ),
+        ];
+
+        for (input, expected) in testcases {
+            let expected = expected.into_iter().map(|(r, s)| (r.into(), s)).collect();
+            assert_eq!(split_overlapping_ranges(&input), expected);
+        }
+    }
+
+    #[test]
+    fn test_find_successive_runs() {
+        impl From<(i32, i32, Option<i32>, Option<i32>)> for SucRun<i32> {
+            fn from((offset, len, min_val, max_val): (i32, i32, Option<i32>, Option<i32>)) -> Self {
+                Self {
+                    offset: offset as usize,
+                    len: len as usize,
+                    first_val: min_val,
+                    last_val: max_val,
+                }
+            }
+        }
+        let testcases = vec![
+            (
+                vec![Some(1), Some(1), Some(2), Some(1), Some(3)],
+                Some(SortOptions {
+                    descending: false,
+                    nulls_first: false,
+                }),
+                vec![(0, 3, Some(1), Some(2)), (3, 2, Some(1), Some(3))],
+            ),
+            (
+                vec![Some(1), Some(2), Some(2), Some(1), Some(3)],
+                Some(SortOptions {
+                    descending: false,
+                    nulls_first: false,
+                }),
+                vec![(0, 3, Some(1), Some(2)), (3, 2, Some(1), Some(3))],
+            ),
+            (
+                vec![Some(1), Some(2), None, None, Some(1), Some(3)],
+                Some(SortOptions {
+                    descending: false,
+                    nulls_first: false,
+                }),
+                vec![(0, 4, Some(1), Some(2)), (4, 2, Some(1), Some(3))],
+            ),
+            (
+                vec![Some(1), Some(2), Some(1), Some(3)],
+                Some(SortOptions {
+                    descending: false,
+                    nulls_first: false,
+                }),
+                vec![(0, 2, Some(1), Some(2)), (2, 2, Some(1), Some(3))],
+            ),
+            (
+                vec![Some(1), Some(2), Some(1), Some(3)],
+                Some(SortOptions {
+                    descending: true,
+                    nulls_first: false,
+                }),
+                vec![
+                    (0, 1, Some(1), Some(1)),
+                    (1, 2, Some(2), Some(1)),
+                    (3, 1, Some(3), Some(3)),
+                ],
+            ),
+            (
+                vec![Some(1), Some(2), None, Some(3)],
+                Some(SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                }),
+                vec![(0, 2, Some(1), Some(2)), (2, 2, Some(3), Some(3))],
+            ),
+            (
+                vec![Some(1), Some(2), None, Some(3)],
+                Some(SortOptions {
+                    descending: false,
+                    nulls_first: false,
+                }),
+                vec![(0, 3, Some(1), Some(2)), (3, 1, Some(3), Some(3))],
+            ),
+            (
+                vec![Some(2), Some(1), None, Some(3)],
+                Some(SortOptions {
+                    descending: true,
+                    nulls_first: true,
+                }),
+                vec![(0, 2, Some(2), Some(1)), (2, 2, Some(3), Some(3))],
+            ),
+            (
+                vec![],
+                Some(SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                }),
+                vec![(0, 0, None, None)],
+            ),
+            (
+                vec![None, None, Some(2), Some(2), Some(1), Some(5), Some(4)],
+                Some(SortOptions {
+                    descending: true,
+                    nulls_first: true,
+                }),
+                vec![(0, 5, Some(2), Some(1)), (5, 2, Some(5), Some(4))],
+            ),
+            (
+                vec![None, None, Some(2), Some(2), Some(1), Some(5), Some(4)],
+                Some(SortOptions {
+                    descending: true,
+                    nulls_first: false,
+                }),
+                vec![
+                    (0, 2, None, None),
+                    (2, 3, Some(2), Some(1)),
+                    (5, 2, Some(5), Some(4)),
+                ],
+            ),
+        ];
+        for (input, sort_opts, expected) in testcases {
+            let ret = find_successive_runs(input.clone().into_iter().enumerate(), &sort_opts);
+            let expected = expected.into_iter().map(SucRun::<i32>::from).collect_vec();
+            assert_eq!(
+                ret, expected,
+                "input: {:?}, opt: {:?},expected: {:?}",
+                input, sort_opts, expected
+            );
+        }
+    }
+
+    #[test]
+    fn test_cmp_with_opts() {
+        let testcases = vec![
+            (
+                Some(1),
+                Some(2),
+                Some(SortOptions {
+                    descending: false,
+                    nulls_first: false,
+                }),
+                std::cmp::Ordering::Less,
+            ),
+            (
+                Some(1),
+                Some(2),
+                Some(SortOptions {
+                    descending: true,
+                    nulls_first: false,
+                }),
+                std::cmp::Ordering::Greater,
+            ),
+            (
+                Some(1),
+                None,
+                Some(SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                }),
+                std::cmp::Ordering::Greater,
+            ),
+            (
+                Some(1),
+                None,
+                Some(SortOptions {
+                    descending: true,
+                    nulls_first: true,
+                }),
+                std::cmp::Ordering::Greater,
+            ),
+            (
+                Some(1),
+                None,
+                Some(SortOptions {
+                    descending: true,
+                    nulls_first: false,
+                }),
+                std::cmp::Ordering::Less,
+            ),
+            (
+                Some(1),
+                None,
+                Some(SortOptions {
+                    descending: false,
+                    nulls_first: false,
+                }),
+                std::cmp::Ordering::Less,
+            ),
+            (
+                None,
+                None,
+                Some(SortOptions {
+                    descending: true,
+                    nulls_first: true,
+                }),
+                std::cmp::Ordering::Equal,
+            ),
+            (
+                None,
+                None,
+                Some(SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                }),
+                std::cmp::Ordering::Equal,
+            ),
+            (
+                None,
+                None,
+                Some(SortOptions {
+                    descending: true,
+                    nulls_first: false,
+                }),
+                std::cmp::Ordering::Equal,
+            ),
+            (
+                None,
+                None,
+                Some(SortOptions {
+                    descending: false,
+                    nulls_first: false,
+                }),
+                std::cmp::Ordering::Equal,
+            ),
+        ];
+        for (a, b, opts, expected) in testcases {
+            assert_eq!(
+                cmp_with_opts(&a, &b, &opts),
+                expected,
+                "a: {:?}, b: {:?}, opts: {:?}",
+                a,
+                b,
+                opts
+            );
+        }
+    }
+
+    #[test]
+    fn test_find_slice_from_range() {
+        let test_cases = vec![
+            // test for off by one case
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([1, 2, 3, 4, 5])) as ArrayRef,
+                false,
+                TimeRange {
+                    start: Timestamp::new_millisecond(2),
+                    end: Timestamp::new_millisecond(4),
+                },
+                Ok((1, 2)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([
+                    -2, -1, 0, 1, 2, 3, 4, 5,
+                ])) as ArrayRef,
+                false,
+                TimeRange {
+                    start: Timestamp::new_millisecond(-1),
+                    end: Timestamp::new_millisecond(4),
+                },
+                Ok((1, 5)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([1, 3, 4, 6])) as ArrayRef,
+                false,
+                TimeRange {
+                    start: Timestamp::new_millisecond(2),
+                    end: Timestamp::new_millisecond(5),
+                },
+                Ok((1, 2)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([1, 2, 3, 4, 6])) as ArrayRef,
+                false,
+                TimeRange {
+                    start: Timestamp::new_millisecond(2),
+                    end: Timestamp::new_millisecond(5),
+                },
+                Ok((1, 3)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([1, 3, 4, 5, 6])) as ArrayRef,
+                false,
+                TimeRange {
+                    start: Timestamp::new_millisecond(2),
+                    end: Timestamp::new_millisecond(5),
+                },
+                Ok((1, 2)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([1, 2, 3, 4, 5])) as ArrayRef,
+                false,
+                TimeRange {
+                    start: Timestamp::new_millisecond(6),
+                    end: Timestamp::new_millisecond(7),
+                },
+                Ok((5, 0)),
+            ),
+            // descending off by one cases
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([5, 3, 2, 1])) as ArrayRef,
+                true,
+                TimeRange {
+                    end: Timestamp::new_millisecond(4),
+                    start: Timestamp::new_millisecond(1),
+                },
+                Ok((1, 3)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([
+                    5, 4, 3, 2, 1, 0,
+                ])) as ArrayRef,
+                true,
+                TimeRange {
+                    end: Timestamp::new_millisecond(4),
+                    start: Timestamp::new_millisecond(1),
+                },
+                Ok((2, 3)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([5, 3, 2, 0])) as ArrayRef,
+                true,
+                TimeRange {
+                    end: Timestamp::new_millisecond(4),
+                    start: Timestamp::new_millisecond(1),
+                },
+                Ok((1, 2)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([5, 4, 3, 2, 0])) as ArrayRef,
+                true,
+                TimeRange {
+                    end: Timestamp::new_millisecond(4),
+                    start: Timestamp::new_millisecond(1),
+                },
+                Ok((2, 2)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([5, 4, 3, 2, 1])) as ArrayRef,
+                true,
+                TimeRange {
+                    end: Timestamp::new_millisecond(5),
+                    start: Timestamp::new_millisecond(2),
+                },
+                Ok((1, 3)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([5, 4, 3, 1])) as ArrayRef,
+                true,
+                TimeRange {
+                    end: Timestamp::new_millisecond(5),
+                    start: Timestamp::new_millisecond(2),
+                },
+                Ok((1, 2)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([6, 4, 3, 2, 1])) as ArrayRef,
+                true,
+                TimeRange {
+                    end: Timestamp::new_millisecond(5),
+                    start: Timestamp::new_millisecond(2),
+                },
+                Ok((1, 3)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([6, 4, 3, 1])) as ArrayRef,
+                true,
+                TimeRange {
+                    end: Timestamp::new_millisecond(5),
+                    start: Timestamp::new_millisecond(2),
+                },
+                Ok((1, 2)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([
+                    10, 9, 8, 7, 6,
+                ])) as ArrayRef,
+                true,
+                TimeRange {
+                    end: Timestamp::new_millisecond(5),
+                    start: Timestamp::new_millisecond(2),
+                },
+                Ok((5, 0)),
+            ),
+            // test off by one case
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([3, 2, 1, 0])) as ArrayRef,
+                true,
+                TimeRange {
+                    end: Timestamp::new_millisecond(4),
+                    start: Timestamp::new_millisecond(3),
+                },
+                Ok((0, 1)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([5, 3, 2])) as ArrayRef,
+                true,
+                TimeRange {
+                    end: Timestamp::new_millisecond(4),
+                    start: Timestamp::new_millisecond(3),
+                },
+                Ok((1, 1)),
+            ),
+            (
+                Arc::new(TimestampMillisecondArray::from_iter_values([5, 4, 3, 2])) as ArrayRef,
+                true,
+                TimeRange {
+                    end: Timestamp::new_millisecond(4),
+                    start: Timestamp::new_millisecond(3),
+                },
+                Ok((2, 1)),
+            ),
+        ];
+
+        for (sort_vals, descending, range, expected) in test_cases {
+            let sort_column = SortColumn {
+                values: sort_vals,
+                options: Some(SortOptions {
+                    descending,
+                    ..Default::default()
+                }),
+            };
+            let ret = find_slice_from_range(&sort_column, &range);
+            match (ret, expected) {
+                (Ok(ret), Ok(expected)) => {
+                    assert_eq!(
+                        ret, expected,
+                        "sort_vals: {:?}, range: {:?}",
+                        sort_column, range
+                    )
+                }
+                (Err(err), Err(expected)) => {
+                    let expected: &str = expected;
+                    assert!(
+                        err.to_string().contains(expected),
+                        "err: {:?}, expected: {:?}",
+                        err,
+                        expected
+                    );
+                }
+                (r, e) => panic!("unexpected result: {:?}, expected: {:?}", r, e),
+            }
+        }
+    }
+
+    #[derive(Debug)]
+    struct TestStream {
+        expression: PhysicalSortExpr,
+        fetch: Option<usize>,
+        input: Vec<(PartitionRange, DfRecordBatch)>,
+        output: Vec<DfRecordBatch>,
+        schema: SchemaRef,
+    }
+    use datafusion::physical_plan::expressions::Column;
+    impl TestStream {
+        fn new(
+            ts_col: Column,
+            opt: SortOptions,
+            fetch: Option<usize>,
+            schema: impl Into<arrow_schema::Fields>,
+            input: Vec<(PartitionRange, Vec<ArrayRef>)>,
+            expected: Vec<Vec<ArrayRef>>,
+        ) -> Self {
+            let expression = PhysicalSortExpr {
+                expr: Arc::new(ts_col),
+                options: opt,
+            };
+            let schema = Schema::new(schema.into());
+            let schema = Arc::new(schema);
+            let input = input
+                .into_iter()
+                .map(|(k, v)| (k, DfRecordBatch::try_new(schema.clone(), v).unwrap()))
+                .collect_vec();
+            let output_batchs = expected
+                .into_iter()
+                .map(|v| DfRecordBatch::try_new(schema.clone(), v).unwrap())
+                .collect_vec();
+            Self {
+                expression,
+                fetch,
+                input,
+                output: output_batchs,
+                schema,
+            }
+        }
+
+        async fn run_test(&self) -> Vec<DfRecordBatch> {
+            let (ranges, batches): (Vec<_>, Vec<_>) = self.input.clone().into_iter().unzip();
+
+            let mock_input = MockInputExec::new(batches, self.schema.clone());
+
+            let exec = WindowedSortExec::try_new(
+                self.expression.clone(),
+                self.fetch,
+                ranges,
+                Arc::new(mock_input),
+            )
+            .unwrap();
+
+            let exec_stream = exec.execute(0, Arc::new(TaskContext::default())).unwrap();
+
+            let real_output = exec_stream.collect::<Vec<_>>().await;
+            let real_output: Vec<_> = real_output.into_iter().try_collect().unwrap();
+            real_output
+        }
+    }
+
+    #[derive(Debug)]
+    struct MockInputExec {
+        input: Vec<DfRecordBatch>,
+        schema: SchemaRef,
+        properties: PlanProperties,
+    }
+
+    impl MockInputExec {
+        fn new(input: Vec<DfRecordBatch>, schema: SchemaRef) -> Self {
+            Self {
+                properties: PlanProperties::new(
+                    EquivalenceProperties::new(schema.clone()),
+                    Partitioning::UnknownPartitioning(1),
+                    ExecutionMode::Bounded,
+                ),
+                input,
+                schema,
+            }
+        }
+    }
+
+    impl DisplayAs for MockInputExec {
+        fn fmt_as(&self, _t: DisplayFormatType, _f: &mut std::fmt::Formatter) -> std::fmt::Result {
+            unimplemented!()
+        }
+    }
+
+    impl ExecutionPlan for MockInputExec {
+        fn as_any(&self) -> &dyn Any {
+            self
+        }
+
+        fn properties(&self) -> &PlanProperties {
+            &self.properties
+        }
+
+        fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+            vec![]
+        }
+
+        fn with_new_children(
+            self: Arc<Self>,
+            _children: Vec<Arc<dyn ExecutionPlan>>,
+        ) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {
+            Ok(self)
+        }
+
+        fn execute(
+            &self,
+            _partition: usize,
+            _context: Arc<TaskContext>,
+        ) -> datafusion_common::Result<DfSendableRecordBatchStream> {
+            let stream = MockStream {
+                stream: self.input.clone(),
+                schema: self.schema.clone(),
+                idx: 0,
+            };
+            Ok(Box::pin(stream))
+        }
+    }
+
+    struct MockStream {
+        stream: Vec<DfRecordBatch>,
+        schema: SchemaRef,
+        idx: usize,
+    }
+
+    impl Stream for MockStream {
+        type Item = datafusion_common::Result<DfRecordBatch>;
+        fn poll_next(
+            mut self: Pin<&mut Self>,
+            _cx: &mut Context<'_>,
+        ) -> Poll<Option<datafusion_common::Result<DfRecordBatch>>> {
+            if self.idx < self.stream.len() {
+                let ret = self.stream[self.idx].clone();
+                self.idx += 1;
+                Poll::Ready(Some(Ok(ret)))
+            } else {
+                Poll::Ready(None)
+            }
+        }
+    }
+
+    impl RecordBatchStream for MockStream {
+        fn schema(&self) -> SchemaRef {
+            self.schema.clone()
+        }
+    }
+
+    #[tokio::test]
+    async fn test_window_sort_stream() {
+        let test_cases = [
+            TestStream::new(
+                Column::new("ts", 0),
+                SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                },
+                None,
+                vec![Field::new(
+                    "ts",
+                    DataType::Timestamp(TimeUnit::Millisecond, None),
+                    false,
+                )],
+                vec![],
+                vec![],
+            ),
+            TestStream::new(
+                Column::new("ts", 0),
+                SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                },
+                None,
+                vec![Field::new(
+                    "ts",
+                    DataType::Timestamp(TimeUnit::Millisecond, None),
+                    false,
+                )],
+                vec![
+                    // test indistinguishable case
+                    // we can't know whether `2` belong to which range
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(2),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([1]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(3),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([2]))],
+                    ),
+                ],
+                vec![
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([1]))],
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([2]))],
+                ],
+            ),
+            TestStream::new(
+                Column::new("ts", 0),
+                SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                },
+                None,
+                vec![Field::new(
+                    "ts",
+                    DataType::Timestamp(TimeUnit::Millisecond, None),
+                    false,
+                )],
+                vec![
+                    // test direct emit
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(3),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            1, 2,
+                        ]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(4),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            2, 3,
+                        ]))],
+                    ),
+                ],
+                vec![
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                        1, 2,
+                    ]))],
+                    // didn't trigger a merge sort/concat here so this is it
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([2]))],
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([3]))],
+                ],
+            ),
+            TestStream::new(
+                Column::new("ts", 0),
+                SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                },
+                None,
+                vec![Field::new(
+                    "ts",
+                    DataType::Timestamp(TimeUnit::Millisecond, None),
+                    false,
+                )],
+                vec![
+                    // test more of cross working range batch intersection
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(3),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            1, 2,
+                        ]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(4),
+                            num_rows: 1,
+                            identifier: 1,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            1, 2, 3,
+                        ]))],
+                    ),
+                ],
+                vec![
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                        1, 1, 2, 2,
+                    ]))],
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([3]))],
+                ],
+            ),
+            TestStream::new(
+                Column::new("ts", 0),
+                SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                },
+                None,
+                vec![Field::new(
+                    "ts",
+                    DataType::Timestamp(TimeUnit::Millisecond, None),
+                    false,
+                )],
+                vec![
+                    // no overlap, empty intersection batch case
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(3),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            1, 2,
+                        ]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(4),
+                            num_rows: 1,
+                            identifier: 1,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            1, 2, 3,
+                        ]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(4),
+                            end: Timestamp::new_millisecond(6),
+                            num_rows: 1,
+                            identifier: 1,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            4, 5,
+                        ]))],
+                    ),
+                ],
+                vec![
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                        1, 1, 2, 2,
+                    ]))],
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([3]))],
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                        4, 5,
+                    ]))],
+                ],
+            ),
+            // test fetch
+            TestStream::new(
+                Column::new("ts", 0),
+                SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                },
+                Some(6),
+                vec![Field::new(
+                    "ts",
+                    DataType::Timestamp(TimeUnit::Millisecond, None),
+                    false,
+                )],
+                vec![
+                    // no overlap, empty intersection batch case
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(3),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            1, 2,
+                        ]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(4),
+                            num_rows: 1,
+                            identifier: 1,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            1, 2, 3,
+                        ]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(3),
+                            end: Timestamp::new_millisecond(6),
+                            num_rows: 1,
+                            identifier: 1,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            4, 5,
+                        ]))],
+                    ),
+                ],
+                vec![
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                        1, 1, 2, 2,
+                    ]))],
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([3]))],
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([4]))],
+                ],
+            ),
+            TestStream::new(
+                Column::new("ts", 0),
+                SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                },
+                Some(3),
+                vec![Field::new(
+                    "ts",
+                    DataType::Timestamp(TimeUnit::Millisecond, None),
+                    false,
+                )],
+                vec![
+                    // no overlap, empty intersection batch case
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(3),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            1, 2,
+                        ]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(4),
+                            num_rows: 1,
+                            identifier: 1,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            1, 2, 3,
+                        ]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(3),
+                            end: Timestamp::new_millisecond(6),
+                            num_rows: 1,
+                            identifier: 1,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            4, 5,
+                        ]))],
+                    ),
+                ],
+                vec![vec![Arc::new(TimestampMillisecondArray::from_iter_values(
+                    [1, 1, 2],
+                ))]],
+            ),
+            // rev case
+            TestStream::new(
+                Column::new("ts", 0),
+                SortOptions {
+                    descending: true,
+                    nulls_first: true,
+                },
+                None,
+                vec![Field::new(
+                    "ts",
+                    DataType::Timestamp(TimeUnit::Millisecond, None),
+                    false,
+                )],
+                vec![
+                    // reverse order
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(3),
+                            end: Timestamp::new_millisecond(6),
+                            num_rows: 1,
+                            identifier: 1,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            5, 4,
+                        ]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(4),
+                            num_rows: 1,
+                            identifier: 1,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            3, 2, 1,
+                        ]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(3),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            2, 1,
+                        ]))],
+                    ),
+                ],
+                vec![
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                        5, 4,
+                    ]))],
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([3]))],
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                        2, 2, 1, 1,
+                    ]))],
+                ],
+            ),
+            TestStream::new(
+                Column::new("ts", 0),
+                SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                },
+                None,
+                vec![Field::new(
+                    "ts",
+                    DataType::Timestamp(TimeUnit::Millisecond, None),
+                    false,
+                )],
+                vec![
+                    // long have subset short run case
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(10),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            1, 5, 9,
+                        ]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(3),
+                            end: Timestamp::new_millisecond(7),
+                            num_rows: 1,
+                            identifier: 1,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            3, 4, 5, 6,
+                        ]))],
+                    ),
+                ],
+                vec![
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([1]))],
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                        3, 4, 5, 5, 6, 9,
+                    ]))],
+                ],
+            ),
+            TestStream::new(
+                Column::new("ts", 0),
+                SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                },
+                None,
+                vec![Field::new(
+                    "ts",
+                    DataType::Timestamp(TimeUnit::Millisecond, None),
+                    false,
+                )],
+                vec![
+                    // complex overlap
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(3),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            1, 2,
+                        ]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(10),
+                            num_rows: 1,
+                            identifier: 1,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            1, 3, 4, 5, 6, 8,
+                        ]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(7),
+                            end: Timestamp::new_millisecond(10),
+                            num_rows: 1,
+                            identifier: 1,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            7, 8, 9,
+                        ]))],
+                    ),
+                ],
+                vec![
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                        1, 1, 2,
+                    ]))],
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                        3, 4, 5, 6,
+                    ]))],
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                        7, 8, 8, 9,
+                    ]))],
+                ],
+            ),
+            TestStream::new(
+                Column::new("ts", 0),
+                SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                },
+                None,
+                vec![Field::new(
+                    "ts",
+                    DataType::Timestamp(TimeUnit::Millisecond, None),
+                    false,
+                )],
+                vec![
+                    // complex subset with having same datapoint
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(11),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+                        ]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(5),
+                            end: Timestamp::new_millisecond(7),
+                            num_rows: 1,
+                            identifier: 1,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                            5, 6,
+                        ]))],
+                    ),
+                ],
+                vec![
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                        1, 2, 3, 4,
+                    ]))],
+                    vec![Arc::new(TimestampMillisecondArray::from_iter_values([
+                        5, 5, 6, 6, 7, 8, 9, 10,
+                    ]))],
+                ],
+            ),
+        ];
+
+        let indexed_test_cases = test_cases.iter().enumerate().collect_vec();
+
+        for (idx, testcase) in &indexed_test_cases {
+            let output = testcase.run_test().await;
+            assert_eq!(output, testcase.output, "case {idx} failed.");
+        }
+    }
+
+    fn new_array(unit: TimeUnit, arr: Vec<i64>) -> ArrayRef {
+        match unit {
+            TimeUnit::Second => Arc::new(TimestampSecondArray::from_iter_values(arr)) as ArrayRef,
+            TimeUnit::Millisecond => {
+                Arc::new(TimestampMillisecondArray::from_iter_values(arr)) as ArrayRef
+            }
+            TimeUnit::Microsecond => {
+                Arc::new(TimestampMicrosecondArray::from_iter_values(arr)) as ArrayRef
+            }
+            TimeUnit::Nanosecond => {
+                Arc::new(TimestampNanosecondArray::from_iter_values(arr)) as ArrayRef
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn fuzzy_ish_test_window_sort_stream() {
+        let test_cnt = 100;
+        let part_cnt_bound = 100;
+        let range_size_bound = 100;
+        let range_offset_bound = 100;
+        let in_range_datapoint_cnt_bound = 100;
+
+        let mut rng = fastrand::Rng::new();
+        rng.seed(1337);
+        let mut bound_val = None;
+        // construct testcases
+        type CmpFn<T> = Box<dyn FnMut(&T, &T) -> std::cmp::Ordering>;
+        let mut full_testcase_list = Vec::new();
+        for _case_id in 0..test_cnt {
+            let descending = rng.bool();
+            fn ret_cmp_fn<T: Ord>(descending: bool) -> CmpFn<T> {
+                if descending {
+                    return Box::new(|a: &T, b: &T| b.cmp(a));
+                }
+                Box::new(|a: &T, b: &T| a.cmp(b))
+            }
+            let unit = match rng.u8(0..3) {
+                0 => TimeUnit::Second,
+                1 => TimeUnit::Millisecond,
+                2 => TimeUnit::Microsecond,
+                _ => TimeUnit::Nanosecond,
+            };
+
+            let mut input_ranged_data = vec![];
+            let mut output_data: Vec<i64> = vec![];
+            // generate input data
+            for part_id in 0..rng.usize(0..part_cnt_bound) {
+                let (start, end) = if descending {
+                    let end = bound_val
+                        .map(|i| i - rng.i64(0..range_offset_bound))
+                        .unwrap_or_else(|| rng.i64(..));
+                    bound_val = Some(end);
+                    let start = end - rng.i64(1..range_size_bound);
+                    let start = Timestamp::new(start, unit.clone().into());
+                    let end = Timestamp::new(end, unit.clone().into());
+                    (start, end)
+                } else {
+                    let start = bound_val
+                        .map(|i| i + rng.i64(0..range_offset_bound))
+                        .unwrap_or_else(|| rng.i64(..));
+                    bound_val = Some(start);
+                    let end = start + rng.i64(1..range_size_bound);
+                    let start = Timestamp::new(start, unit.clone().into());
+                    let end = Timestamp::new(end, unit.clone().into());
+                    (start, end)
+                };
+
+                let iter = 0..rng.usize(0..in_range_datapoint_cnt_bound);
+                let data_gen = iter
+                    .map(|_| rng.i64(start.value()..end.value()))
+                    .sorted_by(ret_cmp_fn(descending))
+                    .collect_vec();
+                output_data.extend(data_gen.clone());
+                let arr = new_array(unit.clone(), data_gen);
+                let range = PartitionRange {
+                    start,
+                    end,
+                    num_rows: arr.len(),
+                    identifier: part_id,
+                };
+                input_ranged_data.push((range, vec![arr]));
+            }
+
+            output_data.sort_by(ret_cmp_fn(descending));
+            let output_arr = new_array(unit.clone(), output_data);
+
+            let test_stream = TestStream::new(
+                Column::new("ts", 0),
+                SortOptions {
+                    descending,
+                    nulls_first: true,
+                },
+                None,
+                vec![Field::new(
+                    "ts",
+                    DataType::Timestamp(unit.clone(), None),
+                    false,
+                )],
+                input_ranged_data.clone(),
+                vec![vec![output_arr]],
+            );
+            full_testcase_list.push(test_stream);
+        }
+
+        for (case_id, test_stream) in full_testcase_list.into_iter().enumerate() {
+            let res = test_stream.run_test().await;
+            let res_concat = concat_batches(&test_stream.schema, &res).unwrap();
+            let expected = test_stream.output;
+            let expected_concat = concat_batches(&test_stream.schema, &expected).unwrap();
+
+            if res_concat != expected_concat {
+                {
+                    let mut f_input =
+                        std::fs::File::create(format!("case_{}_input.json", case_id)).unwrap();
+                    f_input.write_all(b"[").unwrap();
+                    for (input_range, input_arr) in test_stream.input {
+                        let range_json = json!({
+                            "start": input_range.start.to_chrono_datetime().unwrap().to_string(),
+                            "end": input_range.end.to_chrono_datetime().unwrap().to_string(),
+                            "num_rows": input_range.num_rows,
+                            "identifier": input_range.identifier,
+                        });
+                        let buf = Vec::new();
+                        let mut input_writer = ArrayWriter::new(buf);
+                        input_writer.write(&input_arr).unwrap();
+                        input_writer.finish().unwrap();
+                        let res_str =
+                            String::from_utf8_lossy(&input_writer.into_inner()).to_string();
+                        let whole_json =
+                            format!(r#"{{"range": {}, "data": {}}},"#, range_json, res_str);
+                        f_input.write_all(whole_json.as_bytes()).unwrap();
+                    }
+                    f_input.write_all(b"]").unwrap();
+                }
+                {
+                    let mut f_res =
+                        std::fs::File::create(format!("case_{}_result.json", case_id)).unwrap();
+                    f_res.write_all(b"[").unwrap();
+                    for batch in &res {
+                        let mut res_writer = ArrayWriter::new(f_res);
+                        res_writer.write(batch).unwrap();
+                        res_writer.finish().unwrap();
+                        f_res = res_writer.into_inner();
+                        f_res.write_all(b",").unwrap();
+                    }
+                    f_res.write_all(b"]").unwrap();
+
+                    let f_res_concat =
+                        std::fs::File::create(format!("case_{}_result_concat.json", case_id))
+                            .unwrap();
+                    let mut res_writer = ArrayWriter::new(f_res_concat);
+                    res_writer.write(&res_concat).unwrap();
+                    res_writer.finish().unwrap();
+
+                    let f_expected =
+                        std::fs::File::create(format!("case_{}_expected.json", case_id)).unwrap();
+                    let mut expected_writer = ArrayWriter::new(f_expected);
+                    expected_writer.write(&expected_concat).unwrap();
+                    expected_writer.finish().unwrap();
+                }
+                panic!(
+                    "case failed, case id: {0}, output and expected save to case_{0}_*.json file",
+                    case_id
+                );
+            }
+            assert_eq!(
+                res_concat, expected_concat,
+                "case failed, case id: {}",
+                case_id
+            );
+        }
+    }
+}

From e0c4157ad80185dc663c3e9458aec31fca0b15d4 Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Thu, 17 Oct 2024 19:05:12 +0800
Subject: [PATCH 088/128] feat: Seq scanner scans data by time range (#4809)

* feat: seq scan by partition

* feat: part metrics

* chore: remove unused codes

* chore: fmt stream

* feat: build ranges returns smallvec

* feat: move scan mem/file ranges to util and reuse

* feat: log metrics

* chore: correct some metrics

* feat: get explain info from ranges

* test: group test and remove unused codes

* chore: fix clippy

* feat: change PartitionRange end to exclusive

* test: add tests
---
 src/mito2/src/read.rs                    |  37 +-
 src/mito2/src/read/range.rs              | 105 +++-
 src/mito2/src/read/scan_region.rs        | 309 ++---------
 src/mito2/src/read/scan_util.rs          | 182 +++++++
 src/mito2/src/read/seq_scan.rs           | 663 ++++-------------------
 src/mito2/src/read/unordered_scan.rs     | 228 +++-----
 src/mito2/src/sst/file.rs                |   3 +-
 src/mito2/src/sst/parquet/reader.rs      |   6 +-
 src/mito2/src/test_util/memtable_util.rs |  10 +-
 src/store-api/src/region_engine.rs       |   2 +-
 10 files changed, 519 insertions(+), 1026 deletions(-)
 create mode 100644 src/mito2/src/read/scan_util.rs

diff --git a/src/mito2/src/read.rs b/src/mito2/src/read.rs
index d9a6fae746..fbef5bb8ed 100644
--- a/src/mito2/src/read.rs
+++ b/src/mito2/src/read.rs
@@ -22,6 +22,7 @@ pub mod projection;
 pub(crate) mod prune;
 pub(crate) mod range;
 pub(crate) mod scan_region;
+pub(crate) mod scan_util;
 pub(crate) mod seq_scan;
 pub(crate) mod unordered_scan;
 
@@ -57,7 +58,6 @@ use crate::error::{
 use crate::memtable::BoxedBatchIterator;
 use crate::metrics::{READ_BATCHES_RETURN, READ_ROWS_RETURN, READ_STAGE_ELAPSED};
 use crate::read::prune::PruneReader;
-use crate::sst::parquet::reader::{ReaderFilterMetrics, ReaderMetrics};
 
 /// Storage internal representation of a batch of rows for a primary key (time series).
 ///
@@ -738,7 +738,7 @@ impl<T: BatchReader + ?Sized> BatchReader for Box<T> {
 pub(crate) struct ScannerMetrics {
     /// Duration to prepare the scan task.
     prepare_scan_cost: Duration,
-    /// Duration to build parts.
+    /// Duration to build file ranges.
     build_parts_cost: Duration,
     /// Duration to build the (merge) reader.
     build_reader_cost: Duration,
@@ -758,31 +758,17 @@ pub(crate) struct ScannerMetrics {
     num_mem_ranges: usize,
     /// Number of file ranges scanned.
     num_file_ranges: usize,
-    /// Filter related metrics for readers.
-    filter_metrics: ReaderFilterMetrics,
 }
 
 impl ScannerMetrics {
-    /// Sets and observes metrics on initializing parts.
-    fn observe_init_part(&mut self, build_parts_cost: Duration, reader_metrics: &ReaderMetrics) {
-        self.build_parts_cost = build_parts_cost;
-
-        // Observes metrics.
+    /// Observes metrics.
+    fn observe_metrics(&self) {
         READ_STAGE_ELAPSED
             .with_label_values(&["prepare_scan"])
             .observe(self.prepare_scan_cost.as_secs_f64());
         READ_STAGE_ELAPSED
             .with_label_values(&["build_parts"])
             .observe(self.build_parts_cost.as_secs_f64());
-
-        // We only call this once so we overwrite it directly.
-        self.filter_metrics = reader_metrics.filter_metrics;
-        // Observes filter metrics.
-        self.filter_metrics.observe();
-    }
-
-    /// Observes metrics on scanner finish.
-    fn observe_metrics_on_finish(&self) {
         READ_STAGE_ELAPSED
             .with_label_values(&["build_reader"])
             .observe(self.build_reader_cost.as_secs_f64());
@@ -801,6 +787,21 @@ impl ScannerMetrics {
         READ_ROWS_RETURN.observe(self.num_rows as f64);
         READ_BATCHES_RETURN.observe(self.num_batches as f64);
     }
+
+    /// Merges metrics from another [ScannerMetrics].
+    fn merge_from(&mut self, other: &ScannerMetrics) {
+        self.prepare_scan_cost += other.prepare_scan_cost;
+        self.build_parts_cost += other.build_parts_cost;
+        self.build_reader_cost += other.build_reader_cost;
+        self.scan_cost += other.scan_cost;
+        self.convert_cost += other.convert_cost;
+        self.yield_cost += other.yield_cost;
+        self.total_cost += other.total_cost;
+        self.num_batches += other.num_batches;
+        self.num_rows += other.num_rows;
+        self.num_mem_ranges += other.num_mem_ranges;
+        self.num_file_ranges += other.num_file_ranges;
+    }
 }
 
 #[cfg(test)]
diff --git a/src/mito2/src/read/range.rs b/src/mito2/src/read/range.rs
index 8408aad7b2..50cda4aabb 100644
--- a/src/mito2/src/read/range.rs
+++ b/src/mito2/src/read/range.rs
@@ -14,7 +14,9 @@
 
 //! Structs for partition ranges.
 
+use common_time::Timestamp;
 use smallvec::{smallvec, SmallVec};
+use store_api::region_engine::PartitionRange;
 
 use crate::memtable::MemtableRef;
 use crate::read::scan_region::ScanInput;
@@ -48,6 +50,26 @@ pub(crate) struct RangeMeta {
 }
 
 impl RangeMeta {
+    /// Creates a [PartitionRange] with specific identifier.
+    /// It converts the inclusive max timestamp to exclusive end timestamp.
+    pub(crate) fn new_partition_range(&self, identifier: usize) -> PartitionRange {
+        PartitionRange {
+            start: self.time_range.0,
+            end: Timestamp::new(
+                // The i64::MAX timestamp may be invisible but we don't guarantee to support this
+                // value now.
+                self.time_range
+                    .1
+                    .value()
+                    .checked_add(1)
+                    .unwrap_or(self.time_range.1.value()),
+                self.time_range.1.unit(),
+            ),
+            num_rows: self.num_rows,
+            identifier,
+        }
+    }
+
     /// Creates a list of ranges from the `input` for seq scan.
     pub(crate) fn seq_scan_ranges(input: &ScanInput) -> Vec<RangeMeta> {
         let mut ranges = Vec::with_capacity(input.memtables.len() + input.files.len());
@@ -177,7 +199,7 @@ impl RangeMeta {
     }
 
     fn push_seq_mem_ranges(memtables: &[MemtableRef], ranges: &mut Vec<RangeMeta>) {
-        // For non append-only mode, each range only contains one memtable.
+        // For non append-only mode, each range only contains one memtable by default.
         for (i, memtable) in memtables.iter().enumerate() {
             let stats = memtable.stats();
             let Some(time_range) = stats.time_range() else {
@@ -195,6 +217,7 @@ impl RangeMeta {
         }
     }
 
+    // TODO(yingwen): Support multiple row groups in a range so we can split them later.
     fn push_seq_file_ranges(
         num_memtables: usize,
         files: &[FileHandle],
@@ -264,3 +287,83 @@ fn maybe_split_ranges_for_seq_scan(ranges: Vec<RangeMeta>) -> Vec<RangeMeta> {
 
     new_ranges
 }
+
+#[cfg(test)]
+mod tests {
+    use common_time::timestamp::TimeUnit;
+    use common_time::Timestamp;
+
+    use super::*;
+
+    type Output = (Vec<usize>, i64, i64);
+
+    fn run_group_ranges_test(input: &[(usize, i64, i64)], expect: &[Output]) {
+        let ranges = input
+            .iter()
+            .map(|(idx, start, end)| {
+                let time_range = (
+                    Timestamp::new(*start, TimeUnit::Second),
+                    Timestamp::new(*end, TimeUnit::Second),
+                );
+                RangeMeta {
+                    time_range,
+                    indices: smallvec![*idx],
+                    row_group_indices: smallvec![RowGroupIndex {
+                        index: *idx,
+                        row_group_index: 0
+                    }],
+                    num_rows: 1,
+                }
+            })
+            .collect();
+        let output = group_ranges_for_seq_scan(ranges);
+        let actual: Vec<_> = output
+            .iter()
+            .map(|range| {
+                let indices = range.indices.to_vec();
+                let group_indices: Vec<_> = range
+                    .row_group_indices
+                    .iter()
+                    .map(|idx| idx.index)
+                    .collect();
+                assert_eq!(indices, group_indices);
+                let range = range.time_range;
+                (indices, range.0.value(), range.1.value())
+            })
+            .collect();
+        assert_eq!(expect, actual);
+    }
+
+    #[test]
+    fn test_group_ranges() {
+        // Group 1 part.
+        run_group_ranges_test(&[(1, 0, 2000)], &[(vec![1], 0, 2000)]);
+
+        // 1, 2, 3, 4 => [3, 1, 4], [2]
+        run_group_ranges_test(
+            &[
+                (1, 1000, 2000),
+                (2, 6000, 7000),
+                (3, 0, 1500),
+                (4, 1500, 3000),
+            ],
+            &[(vec![3, 1, 4], 0, 3000), (vec![2], 6000, 7000)],
+        );
+
+        // 1, 2, 3 => [3], [1], [2],
+        run_group_ranges_test(
+            &[(1, 3000, 4000), (2, 4001, 6000), (3, 0, 1000)],
+            &[
+                (vec![3], 0, 1000),
+                (vec![1], 3000, 4000),
+                (vec![2], 4001, 6000),
+            ],
+        );
+
+        // 1, 2, 3 => [3], [1, 2]
+        run_group_ranges_test(
+            &[(1, 3000, 4000), (2, 4000, 6000), (3, 0, 1000)],
+            &[(vec![3], 0, 1000), (vec![1, 2], 3000, 6000)],
+        );
+    }
+}
diff --git a/src/mito2/src/read/scan_region.rs b/src/mito2/src/read/scan_region.rs
index 28deb1186e..1c0dd50d0b 100644
--- a/src/mito2/src/read/scan_region.rs
+++ b/src/mito2/src/read/scan_region.rs
@@ -17,14 +17,12 @@
 use std::collections::{BTreeMap, HashSet};
 use std::fmt;
 use std::sync::{Arc, Mutex as StdMutex};
-use std::time::{Duration, Instant};
+use std::time::Instant;
 
 use common_error::ext::BoxedError;
 use common_recordbatch::SendableRecordBatchStream;
 use common_telemetry::{debug, error, tracing, warn};
 use common_time::range::TimestampRange;
-use common_time::Timestamp;
-use datafusion::physical_plan::DisplayFormatType;
 use datafusion_expr::utils::expr_to_columns;
 use parquet::arrow::arrow_reader::RowSelection;
 use smallvec::SmallVec;
@@ -48,7 +46,7 @@ use crate::read::unordered_scan::UnorderedScan;
 use crate::read::{Batch, Source};
 use crate::region::options::MergeMode;
 use crate::region::version::VersionRef;
-use crate::sst::file::{overlaps, FileHandle, FileMeta};
+use crate::sst::file::FileHandle;
 use crate::sst::index::fulltext_index::applier::builder::FulltextIndexApplierBuilder;
 use crate::sst::index::fulltext_index::applier::FulltextIndexApplierRef;
 use crate::sst::index::inverted_index::applier::builder::InvertedIndexApplierBuilder;
@@ -700,73 +698,6 @@ impl ScanInput {
         })
     }
 
-    /// Prunes file ranges to scan and adds them to the `collector`.
-    pub(crate) async fn prune_file_ranges(
-        &self,
-        collector: &mut impl FileRangeCollector,
-    ) -> Result<ReaderMetrics> {
-        let mut file_prune_cost = Duration::ZERO;
-        let mut reader_metrics = ReaderMetrics::default();
-        for file in &self.files {
-            let prune_start = Instant::now();
-            let res = self
-                .access_layer
-                .read_sst(file.clone())
-                .predicate(self.predicate.clone())
-                .time_range(self.time_range)
-                .projection(Some(self.mapper.column_ids().to_vec()))
-                .cache(self.cache_manager.clone())
-                .inverted_index_applier(self.inverted_index_applier.clone())
-                .fulltext_index_applier(self.fulltext_index_applier.clone())
-                .expected_metadata(Some(self.mapper.metadata().clone()))
-                .build_reader_input(&mut reader_metrics)
-                .await;
-            file_prune_cost += prune_start.elapsed();
-            let (mut file_range_ctx, row_groups) = match res {
-                Ok(x) => x,
-                Err(e) => {
-                    if e.is_object_not_found() && self.ignore_file_not_found {
-                        error!(e; "File to scan does not exist, region_id: {}, file: {}", file.region_id(), file.file_id());
-                        continue;
-                    } else {
-                        return Err(e);
-                    }
-                }
-            };
-            if !compat::has_same_columns(
-                self.mapper.metadata(),
-                file_range_ctx.read_format().metadata(),
-            ) {
-                // They have different schema. We need to adapt the batch first so the
-                // mapper can convert it.
-                let compat = CompatBatch::new(
-                    &self.mapper,
-                    file_range_ctx.read_format().metadata().clone(),
-                )?;
-                file_range_ctx.set_compat_batch(Some(compat));
-            }
-            // Build ranges from row groups.
-            let file_range_ctx = Arc::new(file_range_ctx);
-            let file_ranges = row_groups
-                .into_iter()
-                .map(|(row_group_idx, row_selection)| {
-                    FileRange::new(file_range_ctx.clone(), row_group_idx, row_selection)
-                });
-            collector.append_file_ranges(file.meta_ref(), file_ranges);
-        }
-
-        READ_SST_COUNT.observe(self.files.len() as f64);
-
-        common_telemetry::debug!(
-            "Region {} prune {} files, cost is {:?}",
-            self.mapper.metadata().region_id,
-            self.files.len(),
-            file_prune_cost
-        );
-
-        Ok(reader_metrics)
-    }
-
     /// Scans the input source in another task and sends batches to the sender.
     pub(crate) fn spawn_scan_task(
         &self,
@@ -806,10 +737,7 @@ impl ScanInput {
     pub(crate) fn predicate(&self) -> Option<Predicate> {
         self.predicate.clone()
     }
-}
 
-#[cfg(test)]
-impl ScanInput {
     /// Returns number of memtables to scan.
     pub(crate) fn num_memtables(&self) -> usize {
         self.memtables.len()
@@ -819,166 +747,21 @@ impl ScanInput {
     pub(crate) fn num_files(&self) -> usize {
         self.files.len()
     }
+}
 
+#[cfg(test)]
+impl ScanInput {
     /// Returns SST file ids to scan.
     pub(crate) fn file_ids(&self) -> Vec<crate::sst::file::FileId> {
         self.files.iter().map(|file| file.file_id()).collect()
     }
 }
 
-/// Groups of file ranges. Each group in the list contains multiple file
-/// ranges to scan. File ranges in the same group may come from different files.
-pub(crate) type FileRangesGroup = SmallVec<[Vec<FileRange>; 4]>;
-
-/// A partition of a scanner to read.
-/// It contains memtables and file ranges to scan.
-#[derive(Clone, Default)]
-pub(crate) struct ScanPart {
-    /// Memtable ranges to scan.
-    pub(crate) memtable_ranges: Vec<MemtableRange>,
-    /// File ranges to scan.
-    pub(crate) file_ranges: FileRangesGroup,
-    /// Optional time range of the part (inclusive).
-    pub(crate) time_range: Option<(Timestamp, Timestamp)>,
-}
-
-impl fmt::Debug for ScanPart {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(
-            f,
-            "ScanPart({} memtable ranges, {} file ranges",
-            self.memtable_ranges.len(),
-            self.file_ranges
-                .iter()
-                .map(|ranges| ranges.len())
-                .sum::<usize>(),
-        )?;
-        if let Some(time_range) = &self.time_range {
-            write!(f, ", time range: {:?})", time_range)
-        } else {
-            write!(f, ")")
-        }
-    }
-}
-
-impl ScanPart {
-    /// Returns true if the time range given `part` overlaps with this part.
-    pub(crate) fn overlaps(&self, part: &ScanPart) -> bool {
-        let (Some(current_range), Some(part_range)) = (self.time_range, part.time_range) else {
-            return true;
-        };
-
-        overlaps(&current_range, &part_range)
-    }
-
-    /// Merges given `part` to this part.
-    pub(crate) fn merge(&mut self, mut part: ScanPart) {
-        self.memtable_ranges.append(&mut part.memtable_ranges);
-        self.file_ranges.append(&mut part.file_ranges);
-        let Some(part_range) = part.time_range else {
-            return;
-        };
-        let Some(current_range) = self.time_range else {
-            self.time_range = part.time_range;
-            return;
-        };
-        let start = current_range.0.min(part_range.0);
-        let end = current_range.1.max(part_range.1);
-        self.time_range = Some((start, end));
-    }
-
-    /// Returns true if the we can split the part into multiple parts
-    /// and preserving order.
-    pub(crate) fn can_split_preserve_order(&self) -> bool {
-        self.memtable_ranges.is_empty()
-            && self.file_ranges.len() == 1
-            && self.file_ranges[0].len() > 1
-    }
-}
-
-/// A trait to collect file ranges to scan.
-pub(crate) trait FileRangeCollector {
-    /// Appends file ranges from the **same file** to the collector.
-    fn append_file_ranges(
-        &mut self,
-        file_meta: &FileMeta,
-        file_ranges: impl Iterator<Item = FileRange>,
-    );
-}
-
-/// Optional list of [ScanPart]s.
-#[derive(Default)]
-pub(crate) struct ScanPartList(pub(crate) Option<Vec<ScanPart>>);
-
-impl fmt::Debug for ScanPartList {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match &self.0 {
-            Some(parts) => write!(f, "{:?}", parts),
-            None => write!(f, "[]"),
-        }
-    }
-}
-
-impl ScanPartList {
-    /// Returns true if the list is None.
-    pub(crate) fn is_none(&self) -> bool {
-        self.0.is_none()
-    }
-
-    /// Sets parts to the list.
-    pub(crate) fn set_parts(&mut self, parts: Vec<ScanPart>) {
-        self.0 = Some(parts);
-    }
-
-    /// Gets the part by index, returns None if the index is out of bound.
-    /// # Panics
-    /// Panics if parts are not initialized.
-    pub(crate) fn get_part(&mut self, index: usize) -> Option<&ScanPart> {
-        let parts = self.0.as_ref().unwrap();
-        parts.get(index)
-    }
-
-    /// Returns the number of parts.
-    pub(crate) fn len(&self) -> usize {
-        self.0.as_ref().map_or(0, |parts| parts.len())
-    }
-
-    /// Returns the number of memtable ranges.
-    pub(crate) fn num_mem_ranges(&self) -> usize {
-        self.0.as_ref().map_or(0, |parts| {
-            parts.iter().map(|part| part.memtable_ranges.len()).sum()
-        })
-    }
-
-    /// Returns the number of files.
-    pub(crate) fn num_files(&self) -> usize {
-        self.0.as_ref().map_or(0, |parts| {
-            parts.iter().map(|part| part.file_ranges.len()).sum()
-        })
-    }
-
-    /// Returns the number of file ranges.
-    pub(crate) fn num_file_ranges(&self) -> usize {
-        self.0.as_ref().map_or(0, |parts| {
-            parts
-                .iter()
-                .flat_map(|part| part.file_ranges.iter())
-                .map(|ranges| ranges.len())
-                .sum()
-        })
-    }
-}
-
 /// Context shared by different streams from a scanner.
-/// It contains the input and distributes input to multiple parts
-/// to scan.
+/// It contains the input and ranges to scan.
 pub(crate) struct StreamContext {
     /// Input memtables and files.
     pub(crate) input: ScanInput,
-    /// Parts to scan and the cost to build parts.
-    /// The scanner builds parts to scan from the input lazily.
-    /// The mutex is used to ensure the parts are only built once.
-    pub(crate) parts: Mutex<(ScanPartList, Duration)>,
     /// Metadata for partition ranges.
     pub(crate) ranges: Vec<RangeMeta>,
     /// Lists of range builders.
@@ -994,12 +777,11 @@ impl StreamContext {
     pub(crate) fn seq_scan_ctx(input: ScanInput) -> Self {
         let query_start = input.query_start.unwrap_or_else(Instant::now);
         let ranges = RangeMeta::seq_scan_ranges(&input);
-        READ_SST_COUNT.observe(input.files.len() as f64);
-        let range_builders = RangeBuilderList::new(input.memtables.len(), input.files.len());
+        READ_SST_COUNT.observe(input.num_files() as f64);
+        let range_builders = RangeBuilderList::new(input.num_memtables(), input.num_files());
 
         Self {
             input,
-            parts: Mutex::new((ScanPartList::default(), Duration::default())),
             ranges,
             range_builders,
             query_start,
@@ -1010,12 +792,11 @@ impl StreamContext {
     pub(crate) fn unordered_scan_ctx(input: ScanInput) -> Self {
         let query_start = input.query_start.unwrap_or_else(Instant::now);
         let ranges = RangeMeta::unordered_scan_ranges(&input);
-        READ_SST_COUNT.observe(input.files.len() as f64);
-        let range_builders = RangeBuilderList::new(input.memtables.len(), input.files.len());
+        READ_SST_COUNT.observe(input.num_files() as f64);
+        let range_builders = RangeBuilderList::new(input.num_memtables(), input.num_files());
 
         Self {
             input,
-            parts: Mutex::new((ScanPartList::default(), Duration::default())),
             ranges,
             range_builders,
             query_start,
@@ -1024,27 +805,28 @@ impl StreamContext {
 
     /// Returns true if the index refers to a memtable.
     pub(crate) fn is_mem_range_index(&self, index: RowGroupIndex) -> bool {
-        self.input.memtables.len() > index.index
+        self.input.num_memtables() > index.index
     }
 
     /// Creates file ranges to scan.
     pub(crate) async fn build_file_ranges(
         &self,
         index: RowGroupIndex,
-        ranges: &mut Vec<FileRange>,
         reader_metrics: &mut ReaderMetrics,
-    ) -> Result<()> {
-        ranges.clear();
+    ) -> Result<SmallVec<[FileRange; 2]>> {
+        let mut ranges = SmallVec::new();
         self.range_builders
-            .build_file_ranges(&self.input, index, ranges, reader_metrics)
-            .await
+            .build_file_ranges(&self.input, index, &mut ranges, reader_metrics)
+            .await?;
+        Ok(ranges)
     }
 
     /// Creates memtable ranges to scan.
-    pub(crate) fn build_mem_ranges(&self, index: RowGroupIndex, ranges: &mut Vec<MemtableRange>) {
-        ranges.clear();
+    pub(crate) fn build_mem_ranges(&self, index: RowGroupIndex) -> SmallVec<[MemtableRange; 2]> {
+        let mut ranges = SmallVec::new();
         self.range_builders
-            .build_mem_ranges(&self.input, index, ranges)
+            .build_mem_ranges(&self.input, index, &mut ranges);
+        ranges
     }
 
     /// Retrieves the partition ranges.
@@ -1052,35 +834,30 @@ impl StreamContext {
         self.ranges
             .iter()
             .enumerate()
-            .map(|(idx, range_meta)| PartitionRange {
-                start: range_meta.time_range.0,
-                end: range_meta.time_range.1,
-                num_rows: range_meta.num_rows,
-                identifier: idx,
-            })
+            .map(|(idx, range_meta)| range_meta.new_partition_range(idx))
             .collect()
     }
 
     /// Format the context for explain.
-    pub(crate) fn format_for_explain(
-        &self,
-        t: DisplayFormatType,
-        f: &mut fmt::Formatter,
-    ) -> fmt::Result {
-        match self.parts.try_lock() {
-            Ok(inner) => match t {
-                DisplayFormatType::Default => write!(
-                    f,
-                    "partition_count={} ({} memtable ranges, {} file {} ranges)",
-                    inner.0.len(),
-                    inner.0.num_mem_ranges(),
-                    inner.0.num_files(),
-                    inner.0.num_file_ranges()
-                )?,
-                DisplayFormatType::Verbose => write!(f, "{:?}", inner.0)?,
-            },
-            Err(_) => write!(f, "<locked>")?,
+    pub(crate) fn format_for_explain(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let (mut num_mem_ranges, mut num_file_ranges) = (0, 0);
+        for range_meta in &self.ranges {
+            for idx in &range_meta.row_group_indices {
+                if self.is_mem_range_index(*idx) {
+                    num_mem_ranges += 1;
+                } else {
+                    num_file_ranges += 1;
+                }
+            }
         }
+        write!(
+            f,
+            "partition_count={} ({} memtable ranges, {} file {} ranges)",
+            self.ranges.len(),
+            num_mem_ranges,
+            self.input.num_files(),
+            num_file_ranges,
+        )?;
         if let Some(selector) = &self.input.series_row_selector {
             write!(f, ", selector={}", selector)?;
         }
@@ -1110,7 +887,7 @@ impl RangeBuilderList {
         &self,
         input: &ScanInput,
         index: RowGroupIndex,
-        ranges: &mut Vec<FileRange>,
+        ranges: &mut SmallVec<[FileRange; 2]>,
         reader_metrics: &mut ReaderMetrics,
     ) -> Result<()> {
         let file_index = index.index - self.mem_builders.len();
@@ -1131,7 +908,7 @@ impl RangeBuilderList {
         &self,
         input: &ScanInput,
         index: RowGroupIndex,
-        ranges: &mut Vec<MemtableRange>,
+        ranges: &mut SmallVec<[MemtableRange; 2]>,
     ) {
         let mut builder_opt = self.mem_builders[index.index].lock().unwrap();
         match &mut *builder_opt {
@@ -1159,7 +936,7 @@ struct FileRangeBuilder {
 impl FileRangeBuilder {
     /// Builds file ranges to read.
     /// Negative `row_group_index` indicates all row groups.
-    fn build_ranges(&self, row_group_index: i64, ranges: &mut Vec<FileRange>) {
+    fn build_ranges(&self, row_group_index: i64, ranges: &mut SmallVec<[FileRange; 2]>) {
         let Some(context) = self.context.clone() else {
             return;
         };
@@ -1196,7 +973,7 @@ struct MemRangeBuilder {
 impl MemRangeBuilder {
     /// Builds mem ranges to read in the memtable.
     /// Negative `row_group_index` indicates all row groups.
-    fn build_ranges(&self, row_group_index: i64, ranges: &mut Vec<MemtableRange>) {
+    fn build_ranges(&self, row_group_index: i64, ranges: &mut SmallVec<[MemtableRange; 2]>) {
         if row_group_index >= 0 {
             let row_group_index = row_group_index as usize;
             // Scans one row group.
diff --git a/src/mito2/src/read/scan_util.rs b/src/mito2/src/read/scan_util.rs
new file mode 100644
index 0000000000..74e7af42d6
--- /dev/null
+++ b/src/mito2/src/read/scan_util.rs
@@ -0,0 +1,182 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Utilities for scanners.
+
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+use async_stream::try_stream;
+use common_telemetry::debug;
+use futures::Stream;
+use store_api::storage::RegionId;
+
+use crate::error::Result;
+use crate::read::range::RowGroupIndex;
+use crate::read::scan_region::StreamContext;
+use crate::read::{Batch, ScannerMetrics, Source};
+use crate::sst::parquet::reader::ReaderMetrics;
+
+struct PartitionMetricsInner {
+    region_id: RegionId,
+    /// Index of the partition to scan.
+    partition: usize,
+    /// Label to distinguish different scan operation.
+    scanner_type: &'static str,
+    /// Query start time.
+    query_start: Instant,
+    /// Elapsed time before the first poll operation.
+    first_poll: Duration,
+    metrics: ScannerMetrics,
+    reader_metrics: ReaderMetrics,
+}
+
+impl PartitionMetricsInner {
+    fn on_finish(&mut self) {
+        if self.metrics.total_cost.is_zero() {
+            self.metrics.total_cost = self.query_start.elapsed();
+        }
+        self.metrics.build_parts_cost = self.reader_metrics.build_cost;
+    }
+}
+
+impl Drop for PartitionMetricsInner {
+    fn drop(&mut self) {
+        self.on_finish();
+        self.metrics.observe_metrics();
+
+        debug!(
+            "{} finished, region_id: {}, partition: {}, first_poll: {:?}, metrics: {:?}, reader_metrics: {:?}",
+            self.scanner_type, self.region_id, self.partition, self.first_poll, self.metrics, self.reader_metrics
+        );
+    }
+}
+
+/// Metrics while reading a partition.
+#[derive(Clone)]
+pub(crate) struct PartitionMetrics(Arc<Mutex<PartitionMetricsInner>>);
+
+impl PartitionMetrics {
+    pub(crate) fn new(
+        region_id: RegionId,
+        partition: usize,
+        scanner_type: &'static str,
+        query_start: Instant,
+        metrics: ScannerMetrics,
+    ) -> Self {
+        let inner = PartitionMetricsInner {
+            region_id,
+            partition,
+            scanner_type,
+            query_start,
+            first_poll: Duration::default(),
+            metrics,
+            reader_metrics: ReaderMetrics::default(),
+        };
+        Self(Arc::new(Mutex::new(inner)))
+    }
+
+    pub(crate) fn on_first_poll(&self) {
+        let mut inner = self.0.lock().unwrap();
+        inner.first_poll = inner.query_start.elapsed();
+    }
+
+    pub(crate) fn inc_num_mem_ranges(&self, num: usize) {
+        let mut inner = self.0.lock().unwrap();
+        inner.metrics.num_mem_ranges += num;
+    }
+
+    pub(crate) fn inc_num_file_ranges(&self, num: usize) {
+        let mut inner = self.0.lock().unwrap();
+        inner.metrics.num_file_ranges += num;
+    }
+
+    pub(crate) fn inc_build_reader_cost(&self, cost: Duration) {
+        let mut inner = self.0.lock().unwrap();
+        inner.metrics.build_reader_cost += cost;
+    }
+
+    pub(crate) fn merge_metrics(&self, metrics: &ScannerMetrics) {
+        let mut inner = self.0.lock().unwrap();
+        inner.metrics.merge_from(metrics);
+    }
+
+    pub(crate) fn merge_reader_metrics(&self, metrics: &ReaderMetrics) {
+        let mut inner = self.0.lock().unwrap();
+        inner.reader_metrics.merge_from(metrics);
+    }
+
+    pub(crate) fn on_finish(&self) {
+        let mut inner = self.0.lock().unwrap();
+        inner.on_finish();
+    }
+}
+
+/// Scans memtable ranges at `index`.
+pub(crate) fn scan_mem_ranges(
+    stream_ctx: Arc<StreamContext>,
+    part_metrics: PartitionMetrics,
+    index: RowGroupIndex,
+) -> impl Stream<Item = Result<Batch>> {
+    try_stream! {
+        let ranges = stream_ctx.build_mem_ranges(index);
+        part_metrics.inc_num_mem_ranges(ranges.len());
+        for range in ranges {
+            let build_reader_start = Instant::now();
+            let iter = range.build_iter()?;
+            part_metrics.inc_build_reader_cost(build_reader_start.elapsed());
+
+            let mut source = Source::Iter(iter);
+            while let Some(batch) = source.next_batch().await? {
+                yield batch;
+            }
+        }
+    }
+}
+
+/// Scans file ranges at `index`.
+pub(crate) fn scan_file_ranges(
+    stream_ctx: Arc<StreamContext>,
+    part_metrics: PartitionMetrics,
+    index: RowGroupIndex,
+    read_type: &'static str,
+) -> impl Stream<Item = Result<Batch>> {
+    try_stream! {
+        let mut reader_metrics = ReaderMetrics::default();
+        let ranges = stream_ctx
+            .build_file_ranges(index, &mut reader_metrics)
+            .await?;
+        part_metrics.inc_num_file_ranges(ranges.len());
+        for range in ranges {
+            let build_reader_start = Instant::now();
+            let reader = range.reader(None).await?;
+            part_metrics.inc_build_reader_cost(build_reader_start.elapsed());
+            let compat_batch = range.compat_batch();
+            let mut source = Source::PruneReader(reader);
+            while let Some(mut batch) = source.next_batch().await? {
+                if let Some(compact_batch) = compat_batch {
+                    batch = compact_batch.compat_batch(batch)?;
+                }
+                yield batch;
+            }
+            if let Source::PruneReader(mut reader) = source {
+                reader_metrics.merge_from(reader.metrics());
+            }
+        }
+
+        // Reports metrics.
+        reader_metrics.observe_rows(read_type);
+        part_metrics.merge_reader_metrics(&reader_metrics);
+    }
+}
diff --git a/src/mito2/src/read/seq_scan.rs b/src/mito2/src/read/seq_scan.rs
index 6e45c8ce4a..fe4054632e 100644
--- a/src/mito2/src/read/seq_scan.rs
+++ b/src/mito2/src/read/seq_scan.rs
@@ -16,36 +16,29 @@
 
 use std::fmt;
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::Instant;
 
 use async_stream::try_stream;
 use common_error::ext::BoxedError;
 use common_recordbatch::error::ExternalSnafu;
 use common_recordbatch::util::ChainedRecordBatchStream;
 use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream};
-use common_telemetry::{debug, tracing};
+use common_telemetry::tracing;
 use datafusion::physical_plan::{DisplayAs, DisplayFormatType};
 use datatypes::schema::SchemaRef;
-use smallvec::smallvec;
 use snafu::ResultExt;
 use store_api::region_engine::{PartitionRange, RegionScanner, ScannerProperties};
-use store_api::storage::{ColumnId, TimeSeriesRowSelector};
-use table::predicate::Predicate;
+use store_api::storage::TimeSeriesRowSelector;
 use tokio::sync::Semaphore;
 
 use crate::error::{PartitionOutOfRangeSnafu, Result};
-use crate::memtable::MemtableRef;
 use crate::read::dedup::{DedupReader, LastNonNull, LastRow};
 use crate::read::last_row::LastRowReader;
 use crate::read::merge::MergeReaderBuilder;
-use crate::read::scan_region::{
-    FileRangeCollector, ScanInput, ScanPart, ScanPartList, StreamContext,
-};
+use crate::read::scan_region::{ScanInput, StreamContext};
+use crate::read::scan_util::{scan_file_ranges, scan_mem_ranges, PartitionMetrics};
 use crate::read::{BatchReader, BoxedBatchReader, ScannerMetrics, Source};
 use crate::region::options::MergeMode;
-use crate::sst::file::FileMeta;
-use crate::sst::parquet::file_range::FileRange;
-use crate::sst::parquet::reader::ReaderMetrics;
 
 /// Scans a region and returns rows in a sorted sequence.
 ///
@@ -66,6 +59,8 @@ pub struct SeqScan {
 impl SeqScan {
     /// Creates a new [SeqScan].
     pub(crate) fn new(input: ScanInput) -> Self {
+        // TODO(yingwen): Set permits according to partition num. But we need to support file
+        // level parallelism.
         let parallelism = input.parallelism.parallelism.max(1);
         let mut properties = ScannerProperties::default()
             .with_append_mode(input.append_mode)
@@ -102,150 +97,49 @@ impl SeqScan {
 
     /// Builds a [BoxedBatchReader] from sequential scan for compaction.
     pub async fn build_reader(&self) -> Result<BoxedBatchReader> {
-        let mut metrics = ScannerMetrics {
-            prepare_scan_cost: self.stream_ctx.query_start.elapsed(),
-            ..Default::default()
-        };
-        let maybe_reader = Self::build_all_merge_reader(
+        let part_metrics = PartitionMetrics::new(
+            self.stream_ctx.input.mapper.metadata().region_id,
+            0,
+            get_scanner_type(self.compaction),
+            self.stream_ctx.query_start,
+            ScannerMetrics {
+                prepare_scan_cost: self.stream_ctx.query_start.elapsed(),
+                ..Default::default()
+            },
+        );
+        debug_assert_eq!(1, self.properties.partitions.len());
+        let partition_ranges = &self.properties.partitions[0];
+
+        let reader = Self::build_all_merge_reader(
             &self.stream_ctx,
+            partition_ranges,
             self.semaphore.clone(),
-            &mut metrics,
             self.compaction,
-            self.properties.num_partitions(),
+            &part_metrics,
         )
         .await?;
-        // Safety: `build_merge_reader()` always returns a reader if partition is None.
-        let reader = maybe_reader.unwrap();
         Ok(Box::new(reader))
     }
 
-    /// Builds sources from a [ScanPart].
-    fn build_part_sources(
-        part: &ScanPart,
-        sources: &mut Vec<Source>,
-        row_selector: Option<TimeSeriesRowSelector>,
-        compaction: bool,
-    ) -> Result<()> {
-        sources.reserve(part.memtable_ranges.len() + part.file_ranges.len());
-        // Read memtables.
-        for mem in &part.memtable_ranges {
-            let iter = mem.build_iter()?;
-            sources.push(Source::Iter(iter));
-        }
-        let read_type = if compaction {
-            "compaction"
-        } else {
-            "seq_scan_files"
-        };
-        // Read files.
-        for file in &part.file_ranges {
-            if file.is_empty() {
-                continue;
-            }
-
-            // Creates a stream to read the file.
-            let ranges = file.clone();
-            let stream = try_stream! {
-                let mut reader_metrics = ReaderMetrics::default();
-                // Safety: We checked whether it is empty before.
-                let file_id = ranges[0].file_handle().file_id();
-                let region_id = ranges[0].file_handle().region_id();
-                let range_num = ranges.len();
-                for range in ranges {
-                    let mut reader = range.reader(row_selector).await?;
-                    let compat_batch = range.compat_batch();
-                    while let Some(mut batch) = reader.next_batch().await? {
-                        if let Some(compat) = compat_batch {
-                            batch = compat
-                                .compat_batch(batch)?;
-                        }
-
-                        yield batch;
-                    }
-                    reader_metrics.merge_from(reader.metrics());
-                }
-                debug!(
-                    "Seq scan region {}, file {}, {} ranges finished, metrics: {:?}, compaction: {}",
-                    region_id, file_id, range_num, reader_metrics, compaction
-                );
-                // Reports metrics.
-                reader_metrics.observe_rows(read_type);
-            };
-            let stream = Box::pin(stream);
-            sources.push(Source::Stream(stream));
-        }
-
-        Ok(())
-    }
-
     /// Builds a merge reader that reads all data.
     async fn build_all_merge_reader(
-        stream_ctx: &StreamContext,
+        stream_ctx: &Arc<StreamContext>,
+        partition_ranges: &[PartitionRange],
         semaphore: Arc<Semaphore>,
-        metrics: &mut ScannerMetrics,
         compaction: bool,
-        parallelism: usize,
-    ) -> Result<Option<BoxedBatchReader>> {
-        // initialize parts list
-        let mut parts = stream_ctx.parts.lock().await;
-        Self::maybe_init_parts(&stream_ctx.input, &mut parts, metrics, parallelism).await?;
-        let parts_len = parts.0.len();
-
-        let mut sources = Vec::with_capacity(parts_len);
-        for id in 0..parts_len {
-            let Some(part) = parts.0.get_part(id) else {
-                return Ok(None);
-            };
-
-            Self::build_part_sources(part, &mut sources, None, compaction)?;
-        }
-
-        Self::build_reader_from_sources(stream_ctx, sources, semaphore).await
-    }
-
-    /// Builds a merge reader that reads data from one [`PartitionRange`].
-    ///
-    /// If the `range_id` is out of bound, returns None.
-    async fn build_merge_reader(
-        stream_ctx: &StreamContext,
-        range_id: usize,
-        semaphore: Arc<Semaphore>,
-        metrics: &mut ScannerMetrics,
-        compaction: bool,
-        parallelism: usize,
-    ) -> Result<Option<BoxedBatchReader>> {
+        part_metrics: &PartitionMetrics,
+    ) -> Result<BoxedBatchReader> {
         let mut sources = Vec::new();
-        let build_start = {
-            let mut parts = stream_ctx.parts.lock().await;
-            Self::maybe_init_parts(&stream_ctx.input, &mut parts, metrics, parallelism).await?;
-
-            let Some(part) = parts.0.get_part(range_id) else {
-                return Ok(None);
-            };
-
-            let build_start = Instant::now();
-            Self::build_part_sources(
-                part,
-                &mut sources,
-                stream_ctx.input.series_row_selector,
+        for part_range in partition_ranges {
+            build_sources(
+                stream_ctx,
+                part_range,
                 compaction,
-            )?;
-
-            build_start
-        };
-
-        let maybe_reader = Self::build_reader_from_sources(stream_ctx, sources, semaphore).await;
-        let build_reader_cost = build_start.elapsed();
-        metrics.build_reader_cost += build_reader_cost;
-        debug!(
-            "Build reader region: {}, range_id: {}, from sources, build_reader_cost: {:?}, compaction: {}",
-            stream_ctx.input.mapper.metadata().region_id,
-            range_id,
-            build_reader_cost,
-            compaction,
-        );
-
-        maybe_reader
+                part_metrics,
+                &mut sources,
+            );
+        }
+        Self::build_reader_from_sources(stream_ctx, sources, semaphore).await
     }
 
     #[tracing::instrument(level = tracing::Level::DEBUG, skip_all)]
@@ -253,7 +147,7 @@ impl SeqScan {
         stream_ctx: &StreamContext,
         mut sources: Vec<Source>,
         semaphore: Arc<Semaphore>,
-    ) -> Result<Option<BoxedBatchReader>> {
+    ) -> Result<BoxedBatchReader> {
         if stream_ctx.input.parallelism.parallelism > 1 {
             // Read sources in parallel. We always spawn a task so we can control the parallelism
             // by the semaphore.
@@ -286,13 +180,11 @@ impl SeqScan {
             None => reader,
         };
 
-        Ok(Some(reader))
+        Ok(reader)
     }
 
     /// Scans the given partition when the part list is set properly.
     /// Otherwise the returned stream might not contains any data.
-    // TODO: refactor out `uncached_scan_part_impl`.
-    #[allow(dead_code)]
     fn scan_partition_impl(
         &self,
         partition: usize,
@@ -307,28 +199,36 @@ impl SeqScan {
             ));
         }
 
-        let mut metrics = ScannerMetrics {
-            prepare_scan_cost: self.stream_ctx.query_start.elapsed(),
-            ..Default::default()
-        };
         let stream_ctx = self.stream_ctx.clone();
         let semaphore = self.semaphore.clone();
         let partition_ranges = self.properties.partitions[partition].clone();
         let compaction = self.compaction;
-        let parallelism = self.properties.num_partitions();
-        let stream = try_stream! {
-            let first_poll = stream_ctx.query_start.elapsed();
+        let part_metrics = PartitionMetrics::new(
+            self.stream_ctx.input.mapper.metadata().region_id,
+            partition,
+            get_scanner_type(self.compaction),
+            stream_ctx.query_start,
+            ScannerMetrics {
+                prepare_scan_cost: self.stream_ctx.query_start.elapsed(),
+                ..Default::default()
+            },
+        );
 
-            for partition_range in partition_ranges {
-                let maybe_reader =
-                    Self::build_merge_reader(&stream_ctx, partition_range.identifier, semaphore.clone(), &mut metrics, compaction, parallelism)
+        let stream = try_stream! {
+            part_metrics.on_first_poll();
+
+            // Scans each part.
+            for part_range in partition_ranges {
+                let mut sources = Vec::new();
+                build_sources(&stream_ctx, &part_range, compaction, &part_metrics, &mut sources);
+
+                let mut reader =
+                    Self::build_reader_from_sources(&stream_ctx, sources, semaphore.clone())
                         .await
                         .map_err(BoxedError::new)
                         .context(ExternalSnafu)?;
-                let Some(mut reader) = maybe_reader else {
-                    return;
-                };
                 let cache = stream_ctx.input.cache_manager.as_deref();
+                let mut metrics = ScannerMetrics::default();
                 let mut fetch_start = Instant::now();
                 while let Some(batch) = reader
                     .next_batch()
@@ -350,18 +250,10 @@ impl SeqScan {
                     fetch_start = Instant::now();
                 }
                 metrics.scan_cost += fetch_start.elapsed();
-                metrics.total_cost = stream_ctx.query_start.elapsed();
-                metrics.observe_metrics_on_finish();
-
-                debug!(
-                    "Seq scan finished, region_id: {:?}, partition: {}, metrics: {:?}, first_poll: {:?}, compaction: {}",
-                    stream_ctx.input.mapper.metadata().region_id,
-                    partition,
-                    metrics,
-                    first_poll,
-                    compaction,
-                );
+                part_metrics.merge_metrics(&metrics);
             }
+
+            part_metrics.on_finish();
         };
 
         let stream = Box::pin(RecordBatchStreamWrapper::new(
@@ -371,133 +263,6 @@ impl SeqScan {
 
         Ok(stream)
     }
-
-    /// Scans the given partition when the part list is not set.
-    /// This method will do a lazy initialize of part list and
-    /// ignores the partition settings in `properties`.
-    fn uncached_scan_part_impl(
-        &self,
-        partition: usize,
-    ) -> Result<SendableRecordBatchStream, BoxedError> {
-        let num_partitions = self.properties.partitions.len();
-        if partition >= num_partitions {
-            return Err(BoxedError::new(
-                PartitionOutOfRangeSnafu {
-                    given: partition,
-                    all: self.properties.partitions.len(),
-                }
-                .build(),
-            ));
-        }
-        let mut metrics = ScannerMetrics {
-            prepare_scan_cost: self.stream_ctx.query_start.elapsed(),
-            ..Default::default()
-        };
-        let stream_ctx = self.stream_ctx.clone();
-        let semaphore = self.semaphore.clone();
-        let compaction = self.compaction;
-        let parallelism = self.properties.num_partitions();
-
-        // build stream
-        let stream = try_stream! {
-            let first_poll = stream_ctx.query_start.elapsed();
-
-            // init parts
-            let parts_len = {
-                let mut parts = stream_ctx.parts.lock().await;
-                Self::maybe_init_parts(&stream_ctx.input, &mut parts, &mut metrics, parallelism).await
-                    .map_err(BoxedError::new)
-                    .context(ExternalSnafu)?;
-                parts.0.len()
-            };
-
-            for id in (0..parts_len).skip(partition).step_by(num_partitions) {
-                let maybe_reader = Self::build_merge_reader(
-                    &stream_ctx,
-                    id,
-                    semaphore.clone(),
-                    &mut metrics,
-                    compaction,
-                    parallelism
-                )
-                .await
-                .map_err(BoxedError::new)
-                .context(ExternalSnafu)?;
-                let Some(mut reader) = maybe_reader else {
-                    return;
-                };
-                let cache = stream_ctx.input.cache_manager.as_deref();
-                let mut fetch_start = Instant::now();
-                while let Some(batch) = reader
-                    .next_batch()
-                    .await
-                    .map_err(BoxedError::new)
-                    .context(ExternalSnafu)?
-                {
-                    metrics.scan_cost += fetch_start.elapsed();
-                    metrics.num_batches += 1;
-                    metrics.num_rows += batch.num_rows();
-
-                    let convert_start = Instant::now();
-                    let record_batch = stream_ctx.input.mapper.convert(&batch, cache)?;
-                    metrics.convert_cost += convert_start.elapsed();
-                    let yield_start = Instant::now();
-                    yield record_batch;
-                    metrics.yield_cost += yield_start.elapsed();
-
-                    fetch_start = Instant::now();
-                }
-                metrics.scan_cost += fetch_start.elapsed();
-                metrics.total_cost = stream_ctx.query_start.elapsed();
-                metrics.observe_metrics_on_finish();
-
-                debug!(
-                    "Seq scan finished, region_id: {}, partition: {}, id: {}, metrics: {:?}, first_poll: {:?}, compaction: {}",
-                    stream_ctx.input.mapper.metadata().region_id,
-                    partition,
-                    id,
-                    metrics,
-                    first_poll,
-                    compaction,
-                );
-            }
-        };
-
-        let stream = Box::pin(RecordBatchStreamWrapper::new(
-            self.stream_ctx.input.mapper.output_schema(),
-            Box::pin(stream),
-        ));
-
-        Ok(stream)
-    }
-
-    /// Initializes parts if they are not built yet.
-    async fn maybe_init_parts(
-        input: &ScanInput,
-        part_list: &mut (ScanPartList, Duration),
-        metrics: &mut ScannerMetrics,
-        parallelism: usize,
-    ) -> Result<()> {
-        if part_list.0.is_none() {
-            let now = Instant::now();
-            let mut distributor = SeqDistributor::default();
-            let reader_metrics = input.prune_file_ranges(&mut distributor).await?;
-            distributor.append_mem_ranges(
-                &input.memtables,
-                Some(input.mapper.column_ids()),
-                input.predicate.clone(),
-            );
-            part_list.0.set_parts(distributor.build_parts(parallelism));
-            let build_part_cost = now.elapsed();
-            part_list.1 = build_part_cost;
-
-            metrics.observe_init_part(build_part_cost, &reader_metrics);
-        } else {
-            // Updates the cost of building parts.
-            metrics.build_parts_cost = part_list.1;
-        }
-        Ok(())
-    }
 }
 
 impl RegionScanner for SeqScan {
@@ -510,7 +275,7 @@ impl RegionScanner for SeqScan {
     }
 
     fn scan_partition(&self, partition: usize) -> Result<SendableRecordBatchStream, BoxedError> {
-        self.uncached_scan_part_impl(partition)
+        self.scan_partition_impl(partition)
     }
 
     fn prepare(&mut self, ranges: Vec<Vec<PartitionRange>>) -> Result<(), BoxedError> {
@@ -525,24 +290,53 @@ impl RegionScanner for SeqScan {
 }
 
 impl DisplayAs for SeqScan {
-    fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
+    fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
         write!(
             f,
             "SeqScan: region={}, ",
             self.stream_ctx.input.mapper.metadata().region_id
         )?;
-        self.stream_ctx.format_for_explain(t, f)
+        self.stream_ctx.format_for_explain(f)
     }
 }
 
 impl fmt::Debug for SeqScan {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("SeqScan")
-            .field("parts", &self.stream_ctx.parts)
+            .field("num_ranges", &self.stream_ctx.ranges.len())
             .finish()
     }
 }
 
+/// Builds sources for the partition range.
+fn build_sources(
+    stream_ctx: &Arc<StreamContext>,
+    part_range: &PartitionRange,
+    compaction: bool,
+    part_metrics: &PartitionMetrics,
+    sources: &mut Vec<Source>,
+) {
+    // Gets range meta.
+    let range_meta = &stream_ctx.ranges[part_range.identifier];
+    sources.reserve(range_meta.row_group_indices.len());
+    for index in &range_meta.row_group_indices {
+        let stream = if stream_ctx.is_mem_range_index(*index) {
+            let stream = scan_mem_ranges(stream_ctx.clone(), part_metrics.clone(), *index);
+            Box::pin(stream) as _
+        } else {
+            let read_type = if compaction {
+                "compaction"
+            } else {
+                "seq_scan_files"
+            };
+            let stream =
+                scan_file_ranges(stream_ctx.clone(), part_metrics.clone(), *index, read_type);
+            Box::pin(stream) as _
+        };
+        sources.push(Source::Stream(stream));
+    }
+}
+
 #[cfg(test)]
 impl SeqScan {
     /// Returns the input.
@@ -551,266 +345,11 @@ impl SeqScan {
     }
 }
 
-/// Builds [ScanPart]s that preserves order.
-#[derive(Default)]
-pub(crate) struct SeqDistributor {
-    parts: Vec<ScanPart>,
-}
-
-impl FileRangeCollector for SeqDistributor {
-    fn append_file_ranges(
-        &mut self,
-        file_meta: &FileMeta,
-        file_ranges: impl Iterator<Item = FileRange>,
-    ) {
-        // Creates a [ScanPart] for each file.
-        let ranges: Vec<_> = file_ranges.collect();
-        if ranges.is_empty() {
-            // No ranges to read.
-            return;
-        }
-        let part = ScanPart {
-            memtable_ranges: Vec::new(),
-            file_ranges: smallvec![ranges],
-            time_range: Some(file_meta.time_range),
-        };
-        self.parts.push(part);
-    }
-}
-
-impl SeqDistributor {
-    /// Appends memtable ranges to the distributor.
-    fn append_mem_ranges(
-        &mut self,
-        memtables: &[MemtableRef],
-        projection: Option<&[ColumnId]>,
-        predicate: Option<Predicate>,
-    ) {
-        for mem in memtables {
-            let stats = mem.stats();
-            let mem_ranges = mem.ranges(projection, predicate.clone());
-            if mem_ranges.is_empty() {
-                continue;
-            }
-            let part = ScanPart {
-                memtable_ranges: mem_ranges.into_values().collect(),
-                file_ranges: smallvec![],
-                time_range: stats.time_range(),
-            };
-            self.parts.push(part);
-        }
-    }
-
-    /// Groups file ranges and memtable ranges by time ranges.
-    /// The output number of parts may be `<= parallelism`. If `parallelism` is 0, it will be set to 1.
-    ///
-    /// Output parts have non-overlapping time ranges.
-    fn build_parts(self, parallelism: usize) -> Vec<ScanPart> {
-        let parallelism = parallelism.max(1);
-        let parts = group_parts_by_range(self.parts);
-        let parts = maybe_split_parts(parts, parallelism);
-        // Ensures it doesn't returns parts more than `parallelism`.
-        maybe_merge_parts(parts, parallelism)
-    }
-}
-
-/// Groups parts by time range. It may generate parts more than parallelism.
-/// All time ranges are not None.
-fn group_parts_by_range(mut parts: Vec<ScanPart>) -> Vec<ScanPart> {
-    if parts.is_empty() {
-        return Vec::new();
-    }
-
-    // Sorts parts by time range.
-    parts.sort_unstable_by(|a, b| {
-        // Safety: time ranges of parts from [SeqPartBuilder] are not None.
-        let a = a.time_range.unwrap();
-        let b = b.time_range.unwrap();
-        a.0.cmp(&b.0).then_with(|| b.1.cmp(&a.1))
-    });
-    let mut part_in_range = None;
-    // Parts with exclusive time ranges.
-    let mut part_groups = Vec::new();
-    for part in parts {
-        let Some(mut prev_part) = part_in_range.take() else {
-            part_in_range = Some(part);
-            continue;
-        };
-
-        if prev_part.overlaps(&part) {
-            prev_part.merge(part);
-            part_in_range = Some(prev_part);
-        } else {
-            // A new group.
-            part_groups.push(prev_part);
-            part_in_range = Some(part);
-        }
-    }
-    if let Some(part) = part_in_range {
-        part_groups.push(part);
-    }
-
-    part_groups
-}
-
-/// Merges parts by parallelism.
-/// It merges parts if the number of parts is greater than `parallelism`.
-fn maybe_merge_parts(mut parts: Vec<ScanPart>, parallelism: usize) -> Vec<ScanPart> {
-    assert!(parallelism > 0);
-    if parts.len() <= parallelism {
-        // No need to merge parts.
-        return parts;
-    }
-
-    // Sort parts by number of memtables and ranges in reverse order.
-    parts.sort_unstable_by(|a, b| {
-        a.memtable_ranges
-            .len()
-            .cmp(&b.memtable_ranges.len())
-            .then_with(|| {
-                let a_ranges_len = a
-                    .file_ranges
-                    .iter()
-                    .map(|ranges| ranges.len())
-                    .sum::<usize>();
-                let b_ranges_len = b
-                    .file_ranges
-                    .iter()
-                    .map(|ranges| ranges.len())
-                    .sum::<usize>();
-                a_ranges_len.cmp(&b_ranges_len)
-            })
-            .reverse()
-    });
-
-    let parts_to_reduce = parts.len() - parallelism;
-    for _ in 0..parts_to_reduce {
-        // Safety: We ensure `parts.len() > parallelism`.
-        let part = parts.pop().unwrap();
-        parts.last_mut().unwrap().merge(part);
-    }
-
-    parts
-}
-
-/// Splits parts by parallelism.
-/// It splits a part if it only scans one file and doesn't scan any memtable.
-fn maybe_split_parts(mut parts: Vec<ScanPart>, parallelism: usize) -> Vec<ScanPart> {
-    assert!(parallelism > 0);
-    if parts.len() >= parallelism {
-        // No need to split parts.
-        return parts;
-    }
-
-    let has_part_to_split = parts.iter().any(|part| part.can_split_preserve_order());
-    if !has_part_to_split {
-        // No proper parts to scan.
-        return parts;
-    }
-
-    // Sorts parts by the number of ranges in the first file.
-    parts.sort_unstable_by(|a, b| {
-        let a_len = a.file_ranges.first().map(|file| file.len()).unwrap_or(0);
-        let b_len = b.file_ranges.first().map(|file| file.len()).unwrap_or(0);
-        a_len.cmp(&b_len).reverse()
-    });
-    let num_parts_to_split = parallelism - parts.len();
-    let mut output_parts = Vec::with_capacity(parallelism);
-    // Split parts up to num_parts_to_split.
-    for part in parts.iter_mut() {
-        if !part.can_split_preserve_order() {
-            continue;
-        }
-        // Safety: `can_split_preserve_order()` ensures file_ranges.len() == 1.
-        // Splits part into `num_parts_to_split + 1` new parts if possible.
-        let target_part_num = num_parts_to_split + 1;
-        let ranges_per_part = (part.file_ranges[0].len() + target_part_num - 1) / target_part_num;
-        // `can_split_preserve_order()` ensures part.file_ranges[0].len() > 1.
-        assert!(ranges_per_part > 0);
-        for ranges in part.file_ranges[0].chunks(ranges_per_part) {
-            let new_part = ScanPart {
-                memtable_ranges: Vec::new(),
-                file_ranges: smallvec![ranges.to_vec()],
-                time_range: part.time_range,
-            };
-            output_parts.push(new_part);
-        }
-        // Replace the current part with the last output part as we will put the current part
-        // into the output parts later.
-        *part = output_parts.pop().unwrap();
-        if output_parts.len() >= num_parts_to_split {
-            // We already split enough parts.
-            break;
-        }
-    }
-    // Put the remaining parts into the output parts.
-    output_parts.append(&mut parts);
-
-    output_parts
-}
-
-#[cfg(test)]
-mod tests {
-    use common_time::timestamp::TimeUnit;
-    use common_time::Timestamp;
-
-    use super::*;
-    use crate::memtable::MemtableId;
-    use crate::test_util::memtable_util::mem_range_for_test;
-
-    type Output = (Vec<MemtableId>, i64, i64);
-
-    fn run_group_parts_test(input: &[(MemtableId, i64, i64)], expect: &[Output]) {
-        let parts = input
-            .iter()
-            .map(|(id, start, end)| {
-                let range = (
-                    Timestamp::new(*start, TimeUnit::Second),
-                    Timestamp::new(*end, TimeUnit::Second),
-                );
-                ScanPart {
-                    memtable_ranges: vec![mem_range_for_test(*id)],
-                    file_ranges: smallvec![],
-                    time_range: Some(range),
-                }
-            })
-            .collect();
-        let output = group_parts_by_range(parts);
-        let actual: Vec<_> = output
-            .iter()
-            .map(|part| {
-                let ids: Vec<_> = part.memtable_ranges.iter().map(|mem| mem.id()).collect();
-                let range = part.time_range.unwrap();
-                (ids, range.0.value(), range.1.value())
-            })
-            .collect();
-        assert_eq!(expect, actual);
-    }
-
-    #[test]
-    fn test_group_parts() {
-        // Group 1 part.
-        run_group_parts_test(&[(1, 0, 2000)], &[(vec![1], 0, 2000)]);
-
-        // 1, 2, 3, 4 => [3, 1, 4], [2]
-        run_group_parts_test(
-            &[
-                (1, 1000, 2000),
-                (2, 6000, 7000),
-                (3, 0, 1500),
-                (4, 1500, 3000),
-            ],
-            &[(vec![3, 1, 4], 0, 3000), (vec![2], 6000, 7000)],
-        );
-
-        // 1, 2, 3 => [3], [1], [2],
-        run_group_parts_test(
-            &[(1, 3000, 4000), (2, 4001, 6000), (3, 0, 1000)],
-            &[
-                (vec![3], 0, 1000),
-                (vec![1], 3000, 4000),
-                (vec![2], 4001, 6000),
-            ],
-        );
+/// Returns the scanner type.
+fn get_scanner_type(compaction: bool) -> &'static str {
+    if compaction {
+        "SeqScan(compaction)"
+    } else {
+        "SeqScan"
     }
 }
diff --git a/src/mito2/src/read/unordered_scan.rs b/src/mito2/src/read/unordered_scan.rs
index 8a019132d7..992cba9d5c 100644
--- a/src/mito2/src/read/unordered_scan.rs
+++ b/src/mito2/src/read/unordered_scan.rs
@@ -21,24 +21,17 @@ use std::time::Instant;
 use async_stream::{stream, try_stream};
 use common_error::ext::BoxedError;
 use common_recordbatch::error::ExternalSnafu;
-use common_recordbatch::{RecordBatch, RecordBatchStreamWrapper, SendableRecordBatchStream};
-use common_telemetry::debug;
+use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream};
 use datafusion::physical_plan::{DisplayAs, DisplayFormatType};
 use datatypes::schema::SchemaRef;
 use futures::{Stream, StreamExt};
 use snafu::ResultExt;
 use store_api::region_engine::{PartitionRange, RegionScanner, ScannerProperties};
 
-use crate::cache::CacheManager;
-use crate::error::Result;
-use crate::memtable::MemtableRange;
-use crate::read::compat::CompatBatch;
-use crate::read::projection::ProjectionMapper;
-use crate::read::range::RowGroupIndex;
+use crate::error::{PartitionOutOfRangeSnafu, Result};
 use crate::read::scan_region::{ScanInput, StreamContext};
-use crate::read::{ScannerMetrics, Source};
-use crate::sst::parquet::file_range::FileRange;
-use crate::sst::parquet::reader::ReaderMetrics;
+use crate::read::scan_util::{scan_file_ranges, scan_mem_ranges, PartitionMetrics};
+use crate::read::{Batch, ScannerMetrics};
 
 /// Scans a region without providing any output ordering guarantee.
 ///
@@ -85,62 +78,23 @@ impl UnorderedScan {
         Ok(stream)
     }
 
-    /// Fetch a batch from the source and convert it into a record batch.
-    async fn fetch_from_source(
-        source: &mut Source,
-        mapper: &ProjectionMapper,
-        cache: Option<&CacheManager>,
-        compat_batch: Option<&CompatBatch>,
-        metrics: &mut ScannerMetrics,
-    ) -> common_recordbatch::error::Result<Option<RecordBatch>> {
-        let start = Instant::now();
-
-        let Some(mut batch) = source
-            .next_batch()
-            .await
-            .map_err(BoxedError::new)
-            .context(ExternalSnafu)?
-        else {
-            metrics.scan_cost += start.elapsed();
-
-            return Ok(None);
-        };
-
-        if let Some(compat) = compat_batch {
-            batch = compat
-                .compat_batch(batch)
-                .map_err(BoxedError::new)
-                .context(ExternalSnafu)?;
-        }
-        metrics.scan_cost += start.elapsed();
-
-        let convert_start = Instant::now();
-        let record_batch = mapper.convert(&batch, cache)?;
-        metrics.convert_cost += convert_start.elapsed();
-
-        Ok(Some(record_batch))
-    }
-
-    /// Scans a [PartitionRange] and returns a stream.
-    fn scan_partition_range<'a>(
-        stream_ctx: &'a StreamContext,
-        part_range: &'a PartitionRange,
-        mem_ranges: &'a mut Vec<MemtableRange>,
-        file_ranges: &'a mut Vec<FileRange>,
-        reader_metrics: &'a mut ReaderMetrics,
-        metrics: &'a mut ScannerMetrics,
-    ) -> impl Stream<Item = common_recordbatch::error::Result<RecordBatch>> + 'a {
+    /// Scans a [PartitionRange] by its `identifier` and returns a stream.
+    fn scan_partition_range(
+        stream_ctx: Arc<StreamContext>,
+        part_range_id: usize,
+        part_metrics: PartitionMetrics,
+    ) -> impl Stream<Item = Result<Batch>> {
         stream! {
             // Gets range meta.
-            let range_meta = &stream_ctx.ranges[part_range.identifier];
+            let range_meta = &stream_ctx.ranges[part_range_id];
             for index in &range_meta.row_group_indices {
                 if stream_ctx.is_mem_range_index(*index) {
-                    let stream = Self::scan_mem_ranges(stream_ctx, *index, mem_ranges, metrics);
+                    let stream = scan_mem_ranges(stream_ctx.clone(), part_metrics.clone(), *index);
                     for await batch in stream {
                         yield batch;
                     }
                 } else {
-                    let stream = Self::scan_file_ranges(stream_ctx, *index, file_ranges, reader_metrics, metrics);
+                    let stream = scan_file_ranges(stream_ctx.clone(), part_metrics.clone(), *index, "unordered_scan_files");
                     for await batch in stream {
                         yield batch;
                     }
@@ -149,124 +103,68 @@ impl UnorderedScan {
         }
     }
 
-    /// Scans memtable ranges at `index`.
-    fn scan_mem_ranges<'a>(
-        stream_ctx: &'a StreamContext,
-        index: RowGroupIndex,
-        ranges: &'a mut Vec<MemtableRange>,
-        metrics: &'a mut ScannerMetrics,
-    ) -> impl Stream<Item = common_recordbatch::error::Result<RecordBatch>> + 'a {
-        try_stream! {
-            let mapper = &stream_ctx.input.mapper;
-            let cache = stream_ctx.input.cache_manager.as_deref();
-            stream_ctx.build_mem_ranges(index, ranges);
-            metrics.num_mem_ranges += ranges.len();
-            for range in ranges {
-                let build_reader_start = Instant::now();
-                let iter = range.build_iter().map_err(BoxedError::new).context(ExternalSnafu)?;
-                metrics.build_reader_cost = build_reader_start.elapsed();
-
-                let mut source = Source::Iter(iter);
-                while let Some(batch) =
-                    Self::fetch_from_source(&mut source, mapper, cache, None, metrics).await?
-                {
-                    metrics.num_batches += 1;
-                    metrics.num_rows += batch.num_rows();
-                    let yield_start = Instant::now();
-                    yield batch;
-                    metrics.yield_cost += yield_start.elapsed();
-                }
-            }
-        }
-    }
-
-    /// Scans file ranges at `index`.
-    fn scan_file_ranges<'a>(
-        stream_ctx: &'a StreamContext,
-        index: RowGroupIndex,
-        ranges: &'a mut Vec<FileRange>,
-        reader_metrics: &'a mut ReaderMetrics,
-        metrics: &'a mut ScannerMetrics,
-    ) -> impl Stream<Item = common_recordbatch::error::Result<RecordBatch>> + 'a {
-        try_stream! {
-            let mapper = &stream_ctx.input.mapper;
-            let cache = stream_ctx.input.cache_manager.as_deref();
-            stream_ctx
-                .build_file_ranges(index, ranges, reader_metrics)
-                .await
-                .map_err(BoxedError::new)
-                .context(ExternalSnafu)?;
-            metrics.num_file_ranges += ranges.len();
-            for range in ranges {
-                let build_reader_start = Instant::now();
-                let reader = range
-                    .reader(None)
-                    .await
-                    .map_err(BoxedError::new)
-                    .context(ExternalSnafu)?;
-                metrics.build_reader_cost += build_reader_start.elapsed();
-                let compat_batch = range.compat_batch();
-                let mut source = Source::PruneReader(reader);
-                while let Some(batch) =
-                    Self::fetch_from_source(&mut source, mapper, cache, compat_batch, metrics)
-                        .await?
-                {
-                    metrics.num_batches += 1;
-                    metrics.num_rows += batch.num_rows();
-                    let yield_start = Instant::now();
-                    yield batch;
-                    metrics.yield_cost += yield_start.elapsed();
-                }
-                if let Source::PruneReader(mut reader) = source {
-                    reader_metrics.merge_from(reader.metrics());
-                }
-            }
-        }
-    }
-
     fn scan_partition_impl(
         &self,
         partition: usize,
     ) -> Result<SendableRecordBatchStream, BoxedError> {
-        let mut metrics = ScannerMetrics {
-            prepare_scan_cost: self.stream_ctx.query_start.elapsed(),
-            ..Default::default()
-        };
+        if partition >= self.properties.partitions.len() {
+            return Err(BoxedError::new(
+                PartitionOutOfRangeSnafu {
+                    given: partition,
+                    all: self.properties.partitions.len(),
+                }
+                .build(),
+            ));
+        }
+
+        let part_metrics = PartitionMetrics::new(
+            self.stream_ctx.input.mapper.metadata().region_id,
+            partition,
+            "UnorderedScan",
+            self.stream_ctx.query_start,
+            ScannerMetrics {
+                prepare_scan_cost: self.stream_ctx.query_start.elapsed(),
+                ..Default::default()
+            },
+        );
         let stream_ctx = self.stream_ctx.clone();
-        let ranges_opt = self.properties.partitions.get(partition).cloned();
+        let part_ranges = self.properties.partitions[partition].clone();
 
-        let stream = stream! {
-            let first_poll = stream_ctx.query_start.elapsed();
-            let Some(part_ranges) = ranges_opt else {
-                return;
-            };
+        let stream = try_stream! {
+            part_metrics.on_first_poll();
 
-            let mut mem_ranges = Vec::new();
-            let mut file_ranges = Vec::new();
-            let mut reader_metrics = ReaderMetrics::default();
+            let cache = stream_ctx.input.cache_manager.as_deref();
             // Scans each part.
             for part_range in part_ranges {
+                let mut metrics = ScannerMetrics::default();
+                let mut fetch_start = Instant::now();
+
                 let stream = Self::scan_partition_range(
-                    &stream_ctx,
-                    &part_range,
-                    &mut mem_ranges,
-                    &mut file_ranges,
-                    &mut reader_metrics,
-                    &mut metrics,
+                    stream_ctx.clone(),
+                    part_range.identifier,
+                    part_metrics.clone(),
                 );
                 for await batch in stream {
-                    yield batch;
+                    let batch = batch.map_err(BoxedError::new).context(ExternalSnafu)?;
+                    metrics.scan_cost += fetch_start.elapsed();
+                    metrics.num_batches += 1;
+                    metrics.num_rows += batch.num_rows();
+
+                    let convert_start = Instant::now();
+                    let record_batch = stream_ctx.input.mapper.convert(&batch, cache)?;
+                    metrics.convert_cost += convert_start.elapsed();
+                    let yield_start = Instant::now();
+                    yield record_batch;
+                    metrics.yield_cost += yield_start.elapsed();
+
+                    fetch_start = Instant::now();
                 }
+
+                metrics.scan_cost += fetch_start.elapsed();
+                part_metrics.merge_metrics(&metrics);
             }
 
-            reader_metrics.observe_rows("unordered_scan_files");
-            metrics.total_cost = stream_ctx.query_start.elapsed();
-            metrics.observe_metrics_on_finish();
-            let mapper = &stream_ctx.input.mapper;
-            debug!(
-                "Unordered scan partition {} finished, region_id: {}, metrics: {:?}, reader_metrics: {:?}, first_poll: {:?}",
-                partition, mapper.metadata().region_id, metrics, reader_metrics, first_poll,
-            );
+            part_metrics.on_finish();
         };
         let stream = Box::pin(RecordBatchStreamWrapper::new(
             self.stream_ctx.input.mapper.output_schema(),
@@ -302,20 +200,20 @@ impl RegionScanner for UnorderedScan {
 }
 
 impl DisplayAs for UnorderedScan {
-    fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
+    fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
         write!(
             f,
             "UnorderedScan: region={}, ",
             self.stream_ctx.input.mapper.metadata().region_id
         )?;
-        self.stream_ctx.format_for_explain(t, f)
+        self.stream_ctx.format_for_explain(f)
     }
 }
 
 impl fmt::Debug for UnorderedScan {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("UnorderedScan")
-            .field("parts", &self.stream_ctx.parts)
+            .field("num_ranges", &self.stream_ctx.ranges.len())
             .finish()
     }
 }
diff --git a/src/mito2/src/sst/file.rs b/src/mito2/src/sst/file.rs
index 447cd8f576..50d7a57cc1 100644
--- a/src/mito2/src/sst/file.rs
+++ b/src/mito2/src/sst/file.rs
@@ -90,7 +90,8 @@ impl FromStr for FileId {
     }
 }
 
-/// Time range of a SST file.
+/// Time range (min and max timestamps) of a SST file.
+/// Both min and max are inclusive.
 pub type FileTimeRange = (Timestamp, Timestamp);
 
 /// Checks if two inclusive timestamp ranges overlap with each other.
diff --git a/src/mito2/src/sst/parquet/reader.rs b/src/mito2/src/sst/parquet/reader.rs
index ff7fe77e7a..b73026a7a6 100644
--- a/src/mito2/src/sst/parquet/reader.rs
+++ b/src/mito2/src/sst/parquet/reader.rs
@@ -238,9 +238,6 @@ impl ParquetReaderBuilder {
             cache_manager: self.cache_manager.clone(),
         };
 
-        // TODO(yingwen): count the cost of the method.
-        metrics.build_cost = start.elapsed();
-
         let mut filters = if let Some(predicate) = &self.predicate {
             predicate
                 .exprs()
@@ -270,6 +267,9 @@ impl ParquetReaderBuilder {
         );
 
         let context = FileRangeContext::new(reader_builder, filters, read_format, codec);
+
+        metrics.build_cost += start.elapsed();
+
         Ok((context, row_groups))
     }
 
diff --git a/src/mito2/src/test_util/memtable_util.rs b/src/mito2/src/test_util/memtable_util.rs
index 70dec3cad4..ed32a3661e 100644
--- a/src/mito2/src/test_util/memtable_util.rs
+++ b/src/mito2/src/test_util/memtable_util.rs
@@ -35,7 +35,7 @@ use crate::memtable::key_values::KeyValue;
 use crate::memtable::partition_tree::data::{timestamp_array_to_i64_slice, DataBatch, DataBuffer};
 use crate::memtable::{
     BoxedBatchIterator, BulkPart, IterBuilder, KeyValues, Memtable, MemtableBuilder, MemtableId,
-    MemtableRange, MemtableRangeContext, MemtableRef, MemtableStats,
+    MemtableRange, MemtableRef, MemtableStats,
 };
 use crate::row_converter::{McmpRowCodec, RowCodec, SortField};
 
@@ -361,11 +361,3 @@ pub(crate) fn collect_iter_timestamps(iter: BoxedBatchIterator) -> Vec<i64> {
     .map(|v| v.unwrap().0.value())
     .collect()
 }
-
-/// Builds a memtable range for test.
-pub(crate) fn mem_range_for_test(id: MemtableId) -> MemtableRange {
-    let builder = Box::new(EmptyIterBuilder::default());
-
-    let context = Arc::new(MemtableRangeContext::new(id, builder));
-    MemtableRange::new(context)
-}
diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs
index 850b9ad3e2..04bc6df6bc 100644
--- a/src/store-api/src/region_engine.rs
+++ b/src/store-api/src/region_engine.rs
@@ -180,7 +180,7 @@ impl ScannerPartitioning {
 pub struct PartitionRange {
     /// Start time of time index column. Inclusive.
     pub start: Timestamp,
-    /// End time of time index column. Inclusive.
+    /// End time of time index column. Exclusive.
     pub end: Timestamp,
     /// Number of rows in this range. Is used to balance ranges between partitions.
     pub num_rows: usize,

From 2e0e82ddc8df0cf6ea54d0aad4f412e717746d97 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Fri, 18 Oct 2024 12:10:22 +0800
Subject: [PATCH 089/128] chore: update greptime-proto to `b4d3011` (#4850)

---
 Cargo.lock | 2 +-
 Cargo.toml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 46effb6a84..4fc125b3b6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4518,7 +4518,7 @@ dependencies = [
 [[package]]
 name = "greptime-proto"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=0b4f7c8ab06399f6b90e1626e8d5b9697cb33bb9#0b4f7c8ab06399f6b90e1626e8d5b9697cb33bb9"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=b4d301184eb0d01fd4d1042fcc7c5dfb54f3c1e3#b4d301184eb0d01fd4d1042fcc7c5dfb54f3c1e3"
 dependencies = [
  "prost 0.12.6",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index 484cdcf55d..7710c8daae 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -121,7 +121,7 @@ etcd-client = { version = "0.13" }
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "0b4f7c8ab06399f6b90e1626e8d5b9697cb33bb9" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "b4d301184eb0d01fd4d1042fcc7c5dfb54f3c1e3" }
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"

From 74bdba46134d9404447e6b92df6fee75610de148 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Fri, 18 Oct 2024 14:26:41 +0800
Subject: [PATCH 090/128] fix: fix metadata forward compatibility issue (#4846)

---
 src/common/meta/src/rpc/router.rs | 68 +++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)

diff --git a/src/common/meta/src/rpc/router.rs b/src/common/meta/src/rpc/router.rs
index 8dc409c8be..dd7349ae8f 100644
--- a/src/common/meta/src/rpc/router.rs
+++ b/src/common/meta/src/rpc/router.rs
@@ -289,6 +289,7 @@ pub enum LeaderState {
     ///
     /// - The [`Region`] may be unavailable (e.g., Crashed, Network disconnected).
     /// - The [`Region`] was planned to migrate to another [`Peer`].
+    #[serde(alias = "Downgraded")]
     Downgrading,
 }
 
@@ -516,6 +517,73 @@ mod tests {
         assert_eq!(decoded, region_route);
     }
 
+    #[test]
+    fn test_region_route_compatibility() {
+        let region_route = RegionRoute {
+            region: Region {
+                id: 2.into(),
+                name: "r2".to_string(),
+                partition: None,
+                attrs: BTreeMap::new(),
+            },
+            leader_peer: Some(Peer::new(1, "a1")),
+            follower_peers: vec![Peer::new(2, "a2"), Peer::new(3, "a3")],
+            leader_state: Some(LeaderState::Downgrading),
+            leader_down_since: None,
+        };
+        let input = r#"{"region":{"id":2,"name":"r2","partition":null,"attrs":{}},"leader_peer":{"id":1,"addr":"a1"},"follower_peers":[{"id":2,"addr":"a2"},{"id":3,"addr":"a3"}],"leader_state":"Downgraded","leader_down_since":null}"#;
+        let decoded: RegionRoute = serde_json::from_str(input).unwrap();
+        assert_eq!(decoded, region_route);
+
+        let region_route = RegionRoute {
+            region: Region {
+                id: 2.into(),
+                name: "r2".to_string(),
+                partition: None,
+                attrs: BTreeMap::new(),
+            },
+            leader_peer: Some(Peer::new(1, "a1")),
+            follower_peers: vec![Peer::new(2, "a2"), Peer::new(3, "a3")],
+            leader_state: Some(LeaderState::Downgrading),
+            leader_down_since: None,
+        };
+        let input = r#"{"region":{"id":2,"name":"r2","partition":null,"attrs":{}},"leader_peer":{"id":1,"addr":"a1"},"follower_peers":[{"id":2,"addr":"a2"},{"id":3,"addr":"a3"}],"leader_status":"Downgraded","leader_down_since":null}"#;
+        let decoded: RegionRoute = serde_json::from_str(input).unwrap();
+        assert_eq!(decoded, region_route);
+
+        let region_route = RegionRoute {
+            region: Region {
+                id: 2.into(),
+                name: "r2".to_string(),
+                partition: None,
+                attrs: BTreeMap::new(),
+            },
+            leader_peer: Some(Peer::new(1, "a1")),
+            follower_peers: vec![Peer::new(2, "a2"), Peer::new(3, "a3")],
+            leader_state: Some(LeaderState::Downgrading),
+            leader_down_since: None,
+        };
+        let input = r#"{"region":{"id":2,"name":"r2","partition":null,"attrs":{}},"leader_peer":{"id":1,"addr":"a1"},"follower_peers":[{"id":2,"addr":"a2"},{"id":3,"addr":"a3"}],"leader_state":"Downgrading","leader_down_since":null}"#;
+        let decoded: RegionRoute = serde_json::from_str(input).unwrap();
+        assert_eq!(decoded, region_route);
+
+        let region_route = RegionRoute {
+            region: Region {
+                id: 2.into(),
+                name: "r2".to_string(),
+                partition: None,
+                attrs: BTreeMap::new(),
+            },
+            leader_peer: Some(Peer::new(1, "a1")),
+            follower_peers: vec![Peer::new(2, "a2"), Peer::new(3, "a3")],
+            leader_state: Some(LeaderState::Downgrading),
+            leader_down_since: None,
+        };
+        let input = r#"{"region":{"id":2,"name":"r2","partition":null,"attrs":{}},"leader_peer":{"id":1,"addr":"a1"},"follower_peers":[{"id":2,"addr":"a2"},{"id":3,"addr":"a3"}],"leader_status":"Downgrading","leader_down_since":null}"#;
+        let decoded: RegionRoute = serde_json::from_str(input).unwrap();
+        assert_eq!(decoded, region_route);
+    }
+
     #[test]
     fn test_de_serialize_partition() {
         let p = Partition {

From ca6e02980e155b8d5279c677686ae299f6184271 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Fri, 18 Oct 2024 14:31:02 +0800
Subject: [PATCH 091/128] fix: overwrite entry_id if entry id is less than
 start_offset (#4842)

* fix: overwrite entry_id if entry id is less than start_offset

* feat: add `overwrite_entry_start_id` to options

* chore: update config.md
---
 config/config.md                            |  2 ++
 config/datanode.example.toml                | 11 ++++++++++
 config/standalone.example.toml              | 11 ++++++++++
 src/common/wal/src/config/kafka/datanode.rs |  3 +++
 src/log-store/src/kafka/log_store.rs        | 24 ++++++++++++++++++++-
 5 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/config/config.md b/config/config.md
index 6a48ed717e..8e1fb5837a 100644
--- a/config/config.md
+++ b/config/config.md
@@ -83,6 +83,7 @@
 | `wal.backoff_max` | String | `10s` | The maximum backoff delay.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.backoff_base` | Integer | `2` | The exponential backoff rate, i.e. next backoff = base * current backoff.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.backoff_deadline` | String | `5mins` | The deadline of retries.<br/>**It's only used when the provider is `kafka`**. |
+| `wal.overwrite_entry_start_id` | Bool | `false` | Ignore missing entries during read WAL.<br/>**It's only used when the provider is `kafka`**.<br/><br/>This option ensures that when Kafka messages are deleted, the system <br/>can still successfully replay memtable data without throwing an <br/>out-of-range error. <br/>However, enabling this option might lead to unexpected data loss, <br/>as the system will skip over missing entries instead of treating <br/>them as critical errors. |
 | `metadata_store` | -- | -- | Metadata storage options. |
 | `metadata_store.file_size` | String | `256MB` | Kv file size in bytes. |
 | `metadata_store.purge_threshold` | String | `4GB` | Kv purge threshold. |
@@ -409,6 +410,7 @@
 | `wal.backoff_deadline` | String | `5mins` | The deadline of retries.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.create_index` | Bool | `true` | Whether to enable WAL index creation.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.dump_index_interval` | String | `60s` | The interval for dumping WAL indexes.<br/>**It's only used when the provider is `kafka`**. |
+| `wal.overwrite_entry_start_id` | Bool | `false` | Ignore missing entries during read WAL.<br/>**It's only used when the provider is `kafka`**.<br/><br/>This option ensures that when Kafka messages are deleted, the system <br/>can still successfully replay memtable data without throwing an <br/>out-of-range error. <br/>However, enabling this option might lead to unexpected data loss, <br/>as the system will skip over missing entries instead of treating <br/>them as critical errors. |
 | `storage` | -- | -- | The data storage options. |
 | `storage.data_home` | String | `/tmp/greptimedb/` | The working home directory. |
 | `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index 22670f857c..85b7793ef6 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -213,6 +213,17 @@ create_index = true
 ## **It's only used when the provider is `kafka`**.
 dump_index_interval = "60s"
 
+## Ignore missing entries during read WAL.
+## **It's only used when the provider is `kafka`**.
+## 
+## This option ensures that when Kafka messages are deleted, the system 
+## can still successfully replay memtable data without throwing an 
+## out-of-range error. 
+## However, enabling this option might lead to unexpected data loss, 
+## as the system will skip over missing entries instead of treating 
+## them as critical errors.
+overwrite_entry_start_id = false
+
 # The Kafka SASL configuration.
 # **It's only used when the provider is `kafka`**.
 # Available SASL mechanisms:
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index 4c40d2dde6..81f4ee47f0 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -237,6 +237,17 @@ backoff_base = 2
 ## **It's only used when the provider is `kafka`**.
 backoff_deadline = "5mins"
 
+## Ignore missing entries during read WAL.
+## **It's only used when the provider is `kafka`**.
+## 
+## This option ensures that when Kafka messages are deleted, the system 
+## can still successfully replay memtable data without throwing an 
+## out-of-range error. 
+## However, enabling this option might lead to unexpected data loss, 
+## as the system will skip over missing entries instead of treating 
+## them as critical errors.
+overwrite_entry_start_id = false
+
 # The Kafka SASL configuration.
 # **It's only used when the provider is `kafka`**.
 # Available SASL mechanisms:
diff --git a/src/common/wal/src/config/kafka/datanode.rs b/src/common/wal/src/config/kafka/datanode.rs
index 27f6932040..405d13d7d4 100644
--- a/src/common/wal/src/config/kafka/datanode.rs
+++ b/src/common/wal/src/config/kafka/datanode.rs
@@ -46,6 +46,8 @@ pub struct DatanodeKafkaConfig {
     pub create_index: bool,
     #[serde(with = "humantime_serde")]
     pub dump_index_interval: Duration,
+    /// Ignore missing entries during read WAL.
+    pub overwrite_entry_start_id: bool,
 }
 
 impl Default for DatanodeKafkaConfig {
@@ -60,6 +62,7 @@ impl Default for DatanodeKafkaConfig {
             auto_create_topics: true,
             create_index: true,
             dump_index_interval: Duration::from_secs(60),
+            overwrite_entry_start_id: false,
         }
     }
 }
diff --git a/src/log-store/src/kafka/log_store.rs b/src/log-store/src/kafka/log_store.rs
index f084f20f02..23493812a4 100644
--- a/src/log-store/src/kafka/log_store.rs
+++ b/src/log-store/src/kafka/log_store.rs
@@ -49,6 +49,8 @@ pub struct KafkaLogStore {
     max_batch_bytes: usize,
     /// The consumer wait timeout.
     consumer_wait_timeout: Duration,
+    /// Ignore missing entries during read WAL.
+    overwrite_entry_start_id: bool,
 }
 
 impl KafkaLogStore {
@@ -64,6 +66,7 @@ impl KafkaLogStore {
             client_manager,
             max_batch_bytes: config.max_batch_bytes.as_bytes() as usize,
             consumer_wait_timeout: config.consumer_wait_timeout,
+            overwrite_entry_start_id: config.overwrite_entry_start_id,
         })
     }
 }
@@ -205,7 +208,7 @@ impl LogStore for KafkaLogStore {
     async fn read(
         &self,
         provider: &Provider,
-        entry_id: EntryId,
+        mut entry_id: EntryId,
         index: Option<WalIndex>,
     ) -> Result<SendableEntryStream<'static, Entry, Self::Error>> {
         let provider = provider
@@ -225,6 +228,25 @@ impl LogStore for KafkaLogStore {
             .client()
             .clone();
 
+        if self.overwrite_entry_start_id {
+            let start_offset =
+                client
+                    .get_offset(OffsetAt::Earliest)
+                    .await
+                    .context(GetOffsetSnafu {
+                        topic: &provider.topic,
+                    })?;
+
+            if entry_id as i64 <= start_offset {
+                warn!(
+                "The entry_id: {} is less than start_offset: {}, topic: {}. Overwriting entry_id with start_offset",
+                entry_id, start_offset, &provider.topic
+            );
+
+                entry_id = start_offset as u64;
+            }
+        }
+
         // Gets the offset of the latest record in the topic. Actually, it's the latest record of the single partition in the topic.
         // The read operation terminates when this record is consumed.
         // Warning: the `get_offset` returns the end offset of the latest record. For our usage, it should be decremented.

From 081c6d9e74a540541e32b323c72eee0213efdac9 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Fri, 18 Oct 2024 15:21:35 +0800
Subject: [PATCH 092/128] fix: flush metric metadata region (#4852)

* fix: flush metric metadata region

* chore: apply suggestions from CR
---
 src/metric-engine/src/engine.rs         |  6 ++-
 src/metric-engine/src/engine/catchup.rs |  3 +-
 src/metric-engine/src/engine/flush.rs   | 52 +++++++++++++++++++++++++
 src/store-api/src/region_request.rs     |  2 +-
 4 files changed, 59 insertions(+), 4 deletions(-)
 create mode 100644 src/metric-engine/src/engine/flush.rs

diff --git a/src/metric-engine/src/engine.rs b/src/metric-engine/src/engine.rs
index ab789db661..a136ed3c76 100644
--- a/src/metric-engine/src/engine.rs
+++ b/src/metric-engine/src/engine.rs
@@ -17,6 +17,7 @@ mod catchup;
 mod close;
 mod create;
 mod drop;
+mod flush;
 mod open;
 mod options;
 mod put;
@@ -145,7 +146,7 @@ impl RegionEngine for MetricEngine {
                     .alter_region(region_id, alter, &mut extension_return_value)
                     .await
             }
-            RegionRequest::Flush(_) | RegionRequest::Compact(_) => {
+            RegionRequest::Compact(_) => {
                 if self.inner.is_physical_region(region_id) {
                     self.inner
                         .mito
@@ -157,10 +158,11 @@ impl RegionEngine for MetricEngine {
                     UnsupportedRegionRequestSnafu { request }.fail()
                 }
             }
+            RegionRequest::Flush(req) => self.inner.flush_region(region_id, req).await,
             RegionRequest::Delete(_) | RegionRequest::Truncate(_) => {
                 UnsupportedRegionRequestSnafu { request }.fail()
             }
-            RegionRequest::Catchup(ref req) => self.inner.catchup_region(region_id, *req).await,
+            RegionRequest::Catchup(req) => self.inner.catchup_region(region_id, req).await,
         };
 
         result.map_err(BoxedError::new).map(|rows| RegionResponse {
diff --git a/src/metric-engine/src/engine/catchup.rs b/src/metric-engine/src/engine/catchup.rs
index 576ca54cfa..4b1268c049 100644
--- a/src/metric-engine/src/engine/catchup.rs
+++ b/src/metric-engine/src/engine/catchup.rs
@@ -47,9 +47,10 @@ impl MetricEngineInner {
             .await
             .context(MitoCatchupOperationSnafu)?;
 
+        let data_region_id = utils::to_data_region_id(region_id);
         self.mito
             .handle_request(
-                region_id,
+                data_region_id,
                 RegionRequest::Catchup(RegionCatchupRequest {
                     set_writable: req.set_writable,
                     entry_id: req.entry_id,
diff --git a/src/metric-engine/src/engine/flush.rs b/src/metric-engine/src/engine/flush.rs
new file mode 100644
index 0000000000..5afadd54b6
--- /dev/null
+++ b/src/metric-engine/src/engine/flush.rs
@@ -0,0 +1,52 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use snafu::ResultExt;
+use store_api::region_engine::RegionEngine;
+use store_api::region_request::{AffectedRows, RegionFlushRequest, RegionRequest};
+use store_api::storage::RegionId;
+
+use crate::engine::MetricEngineInner;
+use crate::error::{MitoFlushOperationSnafu, Result, UnsupportedRegionRequestSnafu};
+use crate::utils;
+
+impl MetricEngineInner {
+    pub async fn flush_region(
+        &self,
+        region_id: RegionId,
+        req: RegionFlushRequest,
+    ) -> Result<AffectedRows> {
+        if !self.is_physical_region(region_id) {
+            return UnsupportedRegionRequestSnafu {
+                request: RegionRequest::Flush(req),
+            }
+            .fail();
+        }
+
+        let metadata_region_id = utils::to_metadata_region_id(region_id);
+        // Flushes the metadata region as well
+        self.mito
+            .handle_request(metadata_region_id, RegionRequest::Flush(req.clone()))
+            .await
+            .context(MitoFlushOperationSnafu)
+            .map(|response| response.affected_rows)?;
+
+        let data_region_id = utils::to_data_region_id(region_id);
+        self.mito
+            .handle_request(data_region_id, RegionRequest::Flush(req.clone()))
+            .await
+            .context(MitoFlushOperationSnafu)
+            .map(|response| response.affected_rows)
+    }
+}
diff --git a/src/store-api/src/region_request.rs b/src/store-api/src/region_request.rs
index cad251988f..2453b83340 100644
--- a/src/store-api/src/region_request.rs
+++ b/src/store-api/src/region_request.rs
@@ -639,7 +639,7 @@ impl From<v1::ChangeColumnType> for ChangeColumnType {
     }
 }
 
-#[derive(Debug, Default)]
+#[derive(Debug, Clone, Default)]
 pub struct RegionFlushRequest {
     pub row_group_size: Option<usize>,
 }

From c02b5dae9338efff55f5ec6c0282f1b5e1e3b02b Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Fri, 18 Oct 2024 16:07:13 +0800
Subject: [PATCH 093/128] chore: bump version to 0.9.5 (#4853)

---
 Cargo.lock | 138 ++++++++++++++++++++++++++---------------------------
 Cargo.toml |   2 +-
 2 files changed, 70 insertions(+), 70 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 4fc125b3b6..ea4bb40007 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -214,7 +214,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c"
 
 [[package]]
 name = "api"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "common-base",
  "common-decimal",
@@ -775,7 +775,7 @@ dependencies = [
 
 [[package]]
 name = "auth"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -1385,7 +1385,7 @@ dependencies = [
 
 [[package]]
 name = "cache"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "catalog",
  "common-error",
@@ -1393,7 +1393,7 @@ dependencies = [
  "common-meta",
  "moka",
  "snafu 0.8.5",
- "substrait 0.9.4",
+ "substrait 0.9.5",
 ]
 
 [[package]]
@@ -1420,7 +1420,7 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
 [[package]]
 name = "catalog"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arrow",
@@ -1759,7 +1759,7 @@ checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
 
 [[package]]
 name = "client"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arc-swap",
@@ -1789,7 +1789,7 @@ dependencies = [
  "serde_json",
  "snafu 0.8.5",
  "substrait 0.37.3",
- "substrait 0.9.4",
+ "substrait 0.9.5",
  "tokio",
  "tokio-stream",
  "tonic 0.11.0",
@@ -1819,7 +1819,7 @@ dependencies = [
 
 [[package]]
 name = "cmd"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-trait",
  "auth",
@@ -1876,7 +1876,7 @@ dependencies = [
  "similar-asserts",
  "snafu 0.8.5",
  "store-api",
- "substrait 0.9.4",
+ "substrait 0.9.5",
  "table",
  "temp-env",
  "tempfile",
@@ -1922,7 +1922,7 @@ checksum = "55b672471b4e9f9e95499ea597ff64941a309b2cdbffcc46f2cc5e2d971fd335"
 
 [[package]]
 name = "common-base"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "anymap2",
  "async-trait",
@@ -1940,7 +1940,7 @@ dependencies = [
 
 [[package]]
 name = "common-catalog"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "chrono",
  "common-error",
@@ -1951,7 +1951,7 @@ dependencies = [
 
 [[package]]
 name = "common-config"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "common-base",
  "common-error",
@@ -1974,7 +1974,7 @@ dependencies = [
 
 [[package]]
 name = "common-datasource"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "arrow",
  "arrow-schema",
@@ -2011,7 +2011,7 @@ dependencies = [
 
 [[package]]
 name = "common-decimal"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "bigdecimal 0.4.5",
  "common-error",
@@ -2024,7 +2024,7 @@ dependencies = [
 
 [[package]]
 name = "common-error"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "snafu 0.8.5",
  "strum 0.25.0",
@@ -2033,7 +2033,7 @@ dependencies = [
 
 [[package]]
 name = "common-frontend"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -2048,7 +2048,7 @@ dependencies = [
 
 [[package]]
 name = "common-function"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arc-swap",
@@ -2088,7 +2088,7 @@ dependencies = [
 
 [[package]]
 name = "common-greptimedb-telemetry"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-trait",
  "common-runtime",
@@ -2105,7 +2105,7 @@ dependencies = [
 
 [[package]]
 name = "common-grpc"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arrow-flight",
@@ -2131,7 +2131,7 @@ dependencies = [
 
 [[package]]
 name = "common-grpc-expr"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "common-base",
@@ -2149,7 +2149,7 @@ dependencies = [
 
 [[package]]
 name = "common-macro"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "arc-swap",
  "common-query",
@@ -2163,7 +2163,7 @@ dependencies = [
 
 [[package]]
 name = "common-mem-prof"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "common-error",
  "common-macro",
@@ -2176,7 +2176,7 @@ dependencies = [
 
 [[package]]
 name = "common-meta"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "anymap2",
  "api",
@@ -2233,11 +2233,11 @@ dependencies = [
 
 [[package]]
 name = "common-plugins"
-version = "0.9.4"
+version = "0.9.5"
 
 [[package]]
 name = "common-pprof"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "common-error",
  "common-macro",
@@ -2249,7 +2249,7 @@ dependencies = [
 
 [[package]]
 name = "common-procedure"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -2276,7 +2276,7 @@ dependencies = [
 
 [[package]]
 name = "common-procedure-test"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-trait",
  "common-procedure",
@@ -2284,7 +2284,7 @@ dependencies = [
 
 [[package]]
 name = "common-query"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -2310,7 +2310,7 @@ dependencies = [
 
 [[package]]
 name = "common-recordbatch"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "arc-swap",
  "common-error",
@@ -2329,7 +2329,7 @@ dependencies = [
 
 [[package]]
 name = "common-runtime"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-trait",
  "common-error",
@@ -2351,7 +2351,7 @@ dependencies = [
 
 [[package]]
 name = "common-telemetry"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "atty",
  "backtrace",
@@ -2379,7 +2379,7 @@ dependencies = [
 
 [[package]]
 name = "common-test-util"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "client",
  "common-query",
@@ -2391,7 +2391,7 @@ dependencies = [
 
 [[package]]
 name = "common-time"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "arrow",
  "chrono",
@@ -2407,7 +2407,7 @@ dependencies = [
 
 [[package]]
 name = "common-version"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "build-data",
  "const_format",
@@ -2418,7 +2418,7 @@ dependencies = [
 
 [[package]]
 name = "common-wal"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "common-base",
  "common-error",
@@ -3227,7 +3227,7 @@ dependencies = [
 
 [[package]]
 name = "datanode"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arrow-flight",
@@ -3277,7 +3277,7 @@ dependencies = [
  "session",
  "snafu 0.8.5",
  "store-api",
- "substrait 0.9.4",
+ "substrait 0.9.5",
  "table",
  "tokio",
  "toml 0.8.19",
@@ -3286,7 +3286,7 @@ dependencies = [
 
 [[package]]
 name = "datatypes"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -3892,7 +3892,7 @@ dependencies = [
 
 [[package]]
 name = "file-engine"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -4003,7 +4003,7 @@ dependencies = [
 
 [[package]]
 name = "flow"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arrow",
@@ -4060,7 +4060,7 @@ dependencies = [
  "snafu 0.8.5",
  "store-api",
  "strum 0.25.0",
- "substrait 0.9.4",
+ "substrait 0.9.5",
  "table",
  "tokio",
  "tonic 0.11.0",
@@ -4122,7 +4122,7 @@ checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa"
 
 [[package]]
 name = "frontend"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arc-swap",
@@ -5170,7 +5170,7 @@ dependencies = [
 
 [[package]]
 name = "index"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-trait",
  "asynchronous-codec",
@@ -6001,7 +6001,7 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
 
 [[package]]
 name = "log-store"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -6321,7 +6321,7 @@ dependencies = [
 
 [[package]]
 name = "meta-client"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -6347,7 +6347,7 @@ dependencies = [
 
 [[package]]
 name = "meta-srv"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -6425,7 +6425,7 @@ dependencies = [
 
 [[package]]
 name = "metric-engine"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "aquamarine",
@@ -6528,7 +6528,7 @@ dependencies = [
 
 [[package]]
 name = "mito2"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "aquamarine",
@@ -7264,7 +7264,7 @@ dependencies = [
 
 [[package]]
 name = "object-store"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "anyhow",
  "bytes",
@@ -7555,7 +7555,7 @@ dependencies = [
 
 [[package]]
 name = "operator"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -7600,7 +7600,7 @@ dependencies = [
  "sql",
  "sqlparser 0.45.0 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=54a267ac89c09b11c0c88934690530807185d3e7)",
  "store-api",
- "substrait 0.9.4",
+ "substrait 0.9.5",
  "table",
  "tokio",
  "tokio-util",
@@ -7850,7 +7850,7 @@ dependencies = [
 
 [[package]]
 name = "partition"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -8151,7 +8151,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "pipeline"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "ahash 0.8.11",
  "api",
@@ -8313,7 +8313,7 @@ dependencies = [
 
 [[package]]
 name = "plugins"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "auth",
  "common-base",
@@ -8587,7 +8587,7 @@ dependencies = [
 
 [[package]]
 name = "promql"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "ahash 0.8.11",
  "async-trait",
@@ -8823,7 +8823,7 @@ dependencies = [
 
 [[package]]
 name = "puffin"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-compression 0.4.13",
  "async-trait",
@@ -8945,7 +8945,7 @@ dependencies = [
 
 [[package]]
 name = "query"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "ahash 0.8.11",
  "api",
@@ -9012,7 +9012,7 @@ dependencies = [
  "stats-cli",
  "store-api",
  "streaming-stats",
- "substrait 0.9.4",
+ "substrait 0.9.5",
  "table",
  "tokio",
  "tokio-stream",
@@ -10446,7 +10446,7 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
 [[package]]
 name = "script"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arc-swap",
@@ -10740,7 +10740,7 @@ dependencies = [
 
 [[package]]
 name = "servers"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "ahash 0.8.11",
  "aide",
@@ -10851,7 +10851,7 @@ dependencies = [
 
 [[package]]
 name = "session"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arc-swap",
@@ -11172,7 +11172,7 @@ dependencies = [
 
 [[package]]
 name = "sql"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "chrono",
@@ -11233,7 +11233,7 @@ dependencies = [
 
 [[package]]
 name = "sqlness-runner"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-trait",
  "clap 4.5.19",
@@ -11453,7 +11453,7 @@ dependencies = [
 
 [[package]]
 name = "store-api"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "aquamarine",
@@ -11622,7 +11622,7 @@ dependencies = [
 
 [[package]]
 name = "substrait"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "async-trait",
  "bytes",
@@ -11821,7 +11821,7 @@ dependencies = [
 
 [[package]]
 name = "table"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "async-trait",
@@ -12087,7 +12087,7 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
 
 [[package]]
 name = "tests-fuzz"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "arbitrary",
  "async-trait",
@@ -12129,7 +12129,7 @@ dependencies = [
 
 [[package]]
 name = "tests-integration"
-version = "0.9.4"
+version = "0.9.5"
 dependencies = [
  "api",
  "arrow-flight",
@@ -12191,7 +12191,7 @@ dependencies = [
  "sql",
  "sqlx",
  "store-api",
- "substrait 0.9.4",
+ "substrait 0.9.5",
  "table",
  "tempfile",
  "time",
diff --git a/Cargo.toml b/Cargo.toml
index 7710c8daae..846ab2d938 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -65,7 +65,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.9.4"
+version = "0.9.5"
 edition = "2021"
 license = "Apache-2.0"
 

From 2ee1ce2ba1ecba515761a87f1bad17c8fd626176 Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Fri, 18 Oct 2024 16:42:01 +0800
Subject: [PATCH 094/128] docs: change cpu/mem panel to time-series (#4844)

* docs: change cpu/mem panel to time-series

* docs: update version
---
 grafana/greptimedb.json | 114 ++++++++++++++++++++++++++++++----------
 1 file changed, 87 insertions(+), 27 deletions(-)

diff --git a/grafana/greptimedb.json b/grafana/greptimedb.json
index 1096ca92ed..7c6dfb0751 100644
--- a/grafana/greptimedb.json
+++ b/grafana/greptimedb.json
@@ -409,7 +409,39 @@
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "thresholds"
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
           },
           "fieldMinMax": false,
           "mappings": [],
@@ -438,18 +470,16 @@
       },
       "id": 27,
       "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
         },
-        "text": {},
-        "textMode": "auto",
-        "wideLayout": true
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
       },
       "pluginVersion": "10.2.3",
       "targets": [
@@ -467,7 +497,7 @@
         }
       ],
       "title": "CPU",
-      "type": "stat"
+      "type": "timeseries"
     },
     {
       "datasource": {
@@ -477,7 +507,39 @@
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "thresholds"
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
           },
           "decimals": 0,
           "fieldMinMax": false,
@@ -503,18 +565,16 @@
       },
       "id": 28,
       "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "auto",
-        "orientation": "auto",
-        "reduceOptions": {
-          "calcs": ["lastNotNull"],
-          "fields": "",
-          "values": false
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
         },
-        "text": {},
-        "textMode": "auto",
-        "wideLayout": true
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
       },
       "pluginVersion": "10.2.3",
       "targets": [
@@ -532,7 +592,7 @@
         }
       ],
       "title": "Memory",
-      "type": "stat"
+      "type": "timeseries"
     },
     {
       "collapsed": false,
@@ -3335,6 +3395,6 @@
   "timezone": "",
   "title": "GreptimeDB",
   "uid": "e7097237-669b-4f8d-b751-13067afbfb68",
-  "version": 15,
+  "version": 16,
   "weekStart": ""
 }

From a50eea76a60e1cb215920e5d847d5685824a7af9 Mon Sep 17 00:00:00 2001
From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com>
Date: Mon, 21 Oct 2024 16:18:30 +0800
Subject: [PATCH 095/128] chore: bump greptime-meter (#4858)

chore/bump-greptime-meter: Add meter-core package and update meter-core dependency across various packages to
new git revision.
---
 Cargo.lock | 18 ++++++++++++++----
 Cargo.toml |  2 +-
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index ea4bb40007..fc57d67046 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6415,12 +6415,22 @@ dependencies = [
  "parking_lot 0.12.3",
 ]
 
+[[package]]
+name = "meter-core"
+version = "0.1.0"
+source = "git+https://github.com/GreptimeTeam/greptime-meter.git?rev=a10facb353b41460eeb98578868ebf19c2084fac#a10facb353b41460eeb98578868ebf19c2084fac"
+dependencies = [
+ "anymap2",
+ "once_cell",
+ "parking_lot 0.12.3",
+]
+
 [[package]]
 name = "meter-macros"
 version = "0.1.0"
 source = "git+https://github.com/GreptimeTeam/greptime-meter.git?rev=80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd#80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd"
 dependencies = [
- "meter-core",
+ "meter-core 0.1.0 (git+https://github.com/GreptimeTeam/greptime-meter.git?rev=80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd)",
 ]
 
 [[package]]
@@ -7585,7 +7595,7 @@ dependencies = [
  "futures-util",
  "lazy_static",
  "meta-client",
- "meter-core",
+ "meter-core 0.1.0 (git+https://github.com/GreptimeTeam/greptime-meter.git?rev=a10facb353b41460eeb98578868ebf19c2084fac)",
  "meter-macros",
  "moka",
  "object-store",
@@ -8988,7 +8998,7 @@ dependencies = [
  "humantime",
  "itertools 0.10.5",
  "lazy_static",
- "meter-core",
+ "meter-core 0.1.0 (git+https://github.com/GreptimeTeam/greptime-meter.git?rev=a10facb353b41460eeb98578868ebf19c2084fac)",
  "meter-macros",
  "num",
  "num-traits",
@@ -10862,7 +10872,7 @@ dependencies = [
  "common-telemetry",
  "common-time",
  "derive_builder 0.12.0",
- "meter-core",
+ "meter-core 0.1.0 (git+https://github.com/GreptimeTeam/greptime-meter.git?rev=a10facb353b41460eeb98578868ebf19c2084fac)",
  "snafu 0.8.5",
  "sql",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index 846ab2d938..a2998ce7db 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -127,7 +127,7 @@ humantime-serde = "1.1"
 itertools = "0.10"
 jsonb = { git = "https://github.com/datafuselabs/jsonb.git", rev = "46ad50fc71cf75afbf98eec455f7892a6387c1fc", default-features = false }
 lazy_static = "1.4"
-meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd" }
+meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "a10facb353b41460eeb98578868ebf19c2084fac" }
 mockall = "0.11.4"
 moka = "0.12"
 notify = "6.1"

From 5d28f7a91230cfc44f14d3bac6c291187e76b85e Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Mon, 21 Oct 2024 21:52:47 +0800
Subject: [PATCH 096/128] feat: yields empty batch after reading a range
 (#4845)

* feat: add empty batch to end of range stream

* feat: add batch validation

* fix: validate batch order

* fix: not yield empty batch in compaction

* fix: empty record batch

* feat: add a flag to enable empty batch
---
 src/catalog/src/system_schema/memory_table.rs |   2 +-
 src/common/recordbatch/src/recordbatch.rs     |  14 ++-
 src/mito2/src/read.rs                         | 110 ++++++++++++++++++
 src/mito2/src/read/projection.rs              |   5 +
 src/mito2/src/read/seq_scan.rs                |  26 +++++
 src/mito2/src/read/unordered_scan.rs          |  25 ++++
 src/store-api/src/region_engine.rs            |  14 +++
 7 files changed, 191 insertions(+), 5 deletions(-)

diff --git a/src/catalog/src/system_schema/memory_table.rs b/src/catalog/src/system_schema/memory_table.rs
index dd026010bb..f5c675832e 100644
--- a/src/catalog/src/system_schema/memory_table.rs
+++ b/src/catalog/src/system_schema/memory_table.rs
@@ -74,7 +74,7 @@ impl MemoryTableBuilder {
     /// Construct the `information_schema.{table_name}` virtual table
     pub async fn memory_records(&mut self) -> Result<RecordBatch> {
         if self.columns.is_empty() {
-            RecordBatch::new_empty(self.schema.clone()).context(CreateRecordBatchSnafu)
+            Ok(RecordBatch::new_empty(self.schema.clone()))
         } else {
             RecordBatch::new(self.schema.clone(), std::mem::take(&mut self.columns))
                 .context(CreateRecordBatchSnafu)
diff --git a/src/common/recordbatch/src/recordbatch.rs b/src/common/recordbatch/src/recordbatch.rs
index 420901902b..1459b87fac 100644
--- a/src/common/recordbatch/src/recordbatch.rs
+++ b/src/common/recordbatch/src/recordbatch.rs
@@ -17,6 +17,7 @@ use std::slice;
 use std::sync::Arc;
 
 use datafusion::arrow::util::pretty::pretty_format_batches;
+use datatypes::prelude::DataType;
 use datatypes::schema::SchemaRef;
 use datatypes::value::Value;
 use datatypes::vectors::{Helper, VectorRef};
@@ -58,13 +59,18 @@ impl RecordBatch {
     }
 
     /// Create an empty [`RecordBatch`] from `schema`.
-    pub fn new_empty(schema: SchemaRef) -> Result<RecordBatch> {
+    pub fn new_empty(schema: SchemaRef) -> RecordBatch {
         let df_record_batch = DfRecordBatch::new_empty(schema.arrow_schema().clone());
-        Ok(RecordBatch {
+        let columns = schema
+            .column_schemas()
+            .iter()
+            .map(|col| col.data_type.create_mutable_vector(0).to_vector())
+            .collect();
+        RecordBatch {
             schema,
-            columns: vec![],
+            columns,
             df_record_batch,
-        })
+        }
     }
 
     pub fn try_project(&self, indices: &[usize]) -> Result<Self> {
diff --git a/src/mito2/src/read.rs b/src/mito2/src/read.rs
index fbef5bb8ed..f7ed4ca958 100644
--- a/src/mito2/src/read.rs
+++ b/src/mito2/src/read.rs
@@ -491,6 +491,116 @@ impl Batch {
         // Safety: sequences is not null so it actually returns Some.
         self.sequences.get_data(index).unwrap()
     }
+
+    /// Checks the batch is monotonic by timestamps.
+    #[cfg(debug_assertions)]
+    pub(crate) fn check_monotonic(&self) -> bool {
+        if self.timestamps_native().is_none() {
+            return true;
+        }
+
+        let timestamps = self.timestamps_native().unwrap();
+        let sequences = self.sequences.as_arrow().values();
+        timestamps.windows(2).enumerate().all(|(i, window)| {
+            let current = window[0];
+            let next = window[1];
+            let current_sequence = sequences[i];
+            let next_sequence = sequences[i + 1];
+            if current == next {
+                current_sequence >= next_sequence
+            } else {
+                current < next
+            }
+        })
+    }
+
+    /// Returns true if the given batch is behind the current batch.
+    #[cfg(debug_assertions)]
+    pub(crate) fn check_next_batch(&self, other: &Batch) -> bool {
+        // Checks the primary key and then the timestamp.
+        use std::cmp::Ordering;
+        self.primary_key()
+            .cmp(other.primary_key())
+            .then_with(|| self.last_timestamp().cmp(&other.first_timestamp()))
+            .then_with(|| other.first_sequence().cmp(&self.last_sequence()))
+            <= Ordering::Equal
+    }
+}
+
+/// A struct to check the batch is monotonic.
+#[cfg(debug_assertions)]
+#[derive(Default)]
+pub(crate) struct BatchChecker {
+    last_batch: Option<Batch>,
+}
+
+#[cfg(debug_assertions)]
+impl BatchChecker {
+    /// Returns true if the given batch is monotonic and behind
+    /// the last batch.
+    pub(crate) fn check_monotonic(&mut self, batch: &Batch) -> bool {
+        if !batch.check_monotonic() {
+            return false;
+        }
+
+        // Checks the batch is behind the last batch.
+        // Then Updates the last batch.
+        let is_behind = self
+            .last_batch
+            .as_ref()
+            .map(|last| last.check_next_batch(batch))
+            .unwrap_or(true);
+        self.last_batch = Some(batch.clone());
+        is_behind
+    }
+
+    /// Formats current batch and last batch for debug.
+    pub(crate) fn format_batch(&self, batch: &Batch) -> String {
+        use std::fmt::Write;
+
+        let mut message = String::new();
+        if let Some(last) = &self.last_batch {
+            write!(
+                message,
+                "last_pk: {:?}, last_ts: {:?}, last_seq: {:?}, ",
+                last.primary_key(),
+                last.last_timestamp(),
+                last.last_sequence()
+            )
+            .unwrap();
+        }
+        write!(
+            message,
+            "batch_pk: {:?}, batch_ts: {:?}, batch_seq: {:?}",
+            batch.primary_key(),
+            batch.timestamps(),
+            batch.sequences()
+        )
+        .unwrap();
+
+        message
+    }
+
+    /// Checks batches from the part range are monotonic. Otherwise, panics.
+    pub(crate) fn ensure_part_range_batch(
+        &mut self,
+        scanner: &str,
+        region_id: store_api::storage::RegionId,
+        partition: usize,
+        part_range: store_api::region_engine::PartitionRange,
+        batch: &Batch,
+    ) {
+        if !self.check_monotonic(batch) {
+            panic!(
+                "{}: batch is not sorted, region_id: {}, partition: {}, part_range: {:?}, {}",
+                scanner,
+                region_id,
+                partition,
+                part_range,
+                self.format_batch(batch),
+            );
+        }
+    }
 }
 
 /// Len of timestamp in arrow row format.
diff --git a/src/mito2/src/read/projection.rs b/src/mito2/src/read/projection.rs
index 375248d418..9ba5f6eccf 100644
--- a/src/mito2/src/read/projection.rs
+++ b/src/mito2/src/read/projection.rs
@@ -160,6 +160,11 @@ impl ProjectionMapper {
         self.output_schema.clone()
     }
 
+    /// Returns an empty [RecordBatch].
+    pub(crate) fn empty_record_batch(&self) -> RecordBatch {
+        RecordBatch::new_empty(self.output_schema.clone())
+    }
+
     /// Converts a [Batch] to a [RecordBatch].
     ///
     /// The batch must match the `projection` using to build the mapper.
diff --git a/src/mito2/src/read/seq_scan.rs b/src/mito2/src/read/seq_scan.rs
index fe4054632e..8d867d7d92 100644
--- a/src/mito2/src/read/seq_scan.rs
+++ b/src/mito2/src/read/seq_scan.rs
@@ -203,6 +203,7 @@ impl SeqScan {
         let semaphore = self.semaphore.clone();
         let partition_ranges = self.properties.partitions[partition].clone();
         let compaction = self.compaction;
+        let distinguish_range = self.properties.distinguish_partition_range();
         let part_metrics = PartitionMetrics::new(
             self.stream_ctx.input.mapper.metadata().region_id,
             partition,
@@ -230,6 +231,8 @@ impl SeqScan {
                 let cache = stream_ctx.input.cache_manager.as_deref();
                 let mut metrics = ScannerMetrics::default();
                 let mut fetch_start = Instant::now();
+                #[cfg(debug_assertions)]
+                let mut checker = crate::read::BatchChecker::default();
                 while let Some(batch) = reader
                     .next_batch()
                     .await
@@ -240,6 +243,20 @@ impl SeqScan {
                     metrics.num_batches += 1;
                     metrics.num_rows += batch.num_rows();
 
+                    debug_assert!(!batch.is_empty());
+                    if batch.is_empty() {
+                        continue;
+                    }
+
+                    #[cfg(debug_assertions)]
+                    checker.ensure_part_range_batch(
+                        "SeqScan",
+                        stream_ctx.input.mapper.metadata().region_id,
+                        partition,
+                        part_range,
+                        &batch,
+                    );
+
                     let convert_start = Instant::now();
                     let record_batch = stream_ctx.input.mapper.convert(&batch, cache)?;
                     metrics.convert_cost += convert_start.elapsed();
@@ -249,6 +266,15 @@ impl SeqScan {
 
                     fetch_start = Instant::now();
                 }
+
+                // Yields an empty part to indicate this range is terminated.
+                // The query engine can use this to optimize some queries.
+                if distinguish_range {
+                    let yield_start = Instant::now();
+                    yield stream_ctx.input.mapper.empty_record_batch();
+                    metrics.yield_cost += yield_start.elapsed();
+                }
+
                 metrics.scan_cost += fetch_start.elapsed();
                 part_metrics.merge_metrics(&metrics);
             }
diff --git a/src/mito2/src/read/unordered_scan.rs b/src/mito2/src/read/unordered_scan.rs
index 992cba9d5c..c51f349cbd 100644
--- a/src/mito2/src/read/unordered_scan.rs
+++ b/src/mito2/src/read/unordered_scan.rs
@@ -129,6 +129,7 @@ impl UnorderedScan {
         );
         let stream_ctx = self.stream_ctx.clone();
         let part_ranges = self.properties.partitions[partition].clone();
+        let distinguish_range = self.properties.distinguish_partition_range();
 
         let stream = try_stream! {
             part_metrics.on_first_poll();
@@ -138,6 +139,8 @@ impl UnorderedScan {
             for part_range in part_ranges {
                 let mut metrics = ScannerMetrics::default();
                 let mut fetch_start = Instant::now();
+                #[cfg(debug_assertions)]
+                let mut checker = crate::read::BatchChecker::default();
 
                 let stream = Self::scan_partition_range(
                     stream_ctx.clone(),
@@ -150,6 +153,20 @@ impl UnorderedScan {
                     metrics.num_batches += 1;
                     metrics.num_rows += batch.num_rows();
 
+                    debug_assert!(!batch.is_empty());
+                    if batch.is_empty() {
+                        continue;
+                    }
+
+                    #[cfg(debug_assertions)]
+                    checker.ensure_part_range_batch(
+                        "UnorderedScan",
+                        stream_ctx.input.mapper.metadata().region_id,
+                        partition,
+                        part_range,
+                        &batch,
+                    );
+
                     let convert_start = Instant::now();
                     let record_batch = stream_ctx.input.mapper.convert(&batch, cache)?;
                     metrics.convert_cost += convert_start.elapsed();
@@ -160,6 +177,14 @@ impl UnorderedScan {
                     fetch_start = Instant::now();
                 }
 
+                // Yields an empty part to indicate this range is terminated.
+                // The query engine can use this to optimize some queries.
+                if distinguish_range {
+                    let yield_start = Instant::now();
+                    yield stream_ctx.input.mapper.empty_record_batch();
+                    metrics.yield_cost += yield_start.elapsed();
+                }
+
                 metrics.scan_cost += fetch_start.elapsed();
                 part_metrics.merge_metrics(&metrics);
             }
diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs
index 04bc6df6bc..86e96a161b 100644
--- a/src/store-api/src/region_engine.rs
+++ b/src/store-api/src/region_engine.rs
@@ -203,6 +203,9 @@ pub struct ScannerProperties {
     /// Total rows that **may** return by scanner. This field is only read iff
     /// [ScannerProperties::append_mode] is true.
     total_rows: usize,
+
+    /// Whether to yield an empty batch to distinguish partition ranges.
+    distinguish_partition_range: bool,
 }
 
 impl ScannerProperties {
@@ -224,12 +227,19 @@ impl ScannerProperties {
         self
     }
 
+    /// Sets distinguish partition range for scanner.
+    pub fn with_distinguish_partition_range(mut self, distinguish_partition_range: bool) -> Self {
+        self.distinguish_partition_range = distinguish_partition_range;
+        self
+    }
+
     /// Creates a new [`ScannerProperties`] with the given partitioning.
     pub fn new(partitions: Vec<Vec<PartitionRange>>, append_mode: bool, total_rows: usize) -> Self {
         Self {
             partitions,
             append_mode,
             total_rows,
+            distinguish_partition_range: false,
         }
     }
 
@@ -244,6 +254,10 @@ impl ScannerProperties {
     pub fn total_rows(&self) -> usize {
         self.total_rows
     }
+
+    pub fn distinguish_partition_range(&self) -> bool {
+        self.distinguish_partition_range
+    }
 }
 
 /// A scanner that provides a way to scan the region concurrently.

From 6d503b047a131e8edf50e124fc1872c1eec8fe79 Mon Sep 17 00:00:00 2001
From: ZonaHe <zonahe@qq.com>
Date: Tue, 22 Oct 2024 10:34:09 +0800
Subject: [PATCH 097/128] feat: update dashboard to v0.6.0 (#4861)

Co-authored-by: ZonaHex <ZonaHex@users.noreply.github.com>
---
 src/servers/dashboard/VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/servers/dashboard/VERSION b/src/servers/dashboard/VERSION
index 8ea9cc1eb1..60f6343282 100644
--- a/src/servers/dashboard/VERSION
+++ b/src/servers/dashboard/VERSION
@@ -1 +1 @@
-v0.5.4
+v0.6.0

From 0c01532a37d5deb4820d88d0e5edee48749e2a06 Mon Sep 17 00:00:00 2001
From: discord9 <55937128+discord9@users.noreply.github.com>
Date: Wed, 23 Oct 2024 15:01:55 +0800
Subject: [PATCH 098/128] feat: Sort within each PartitionRange (#4847)

* feat: PartSort

* chore: rm unused

* chore: typo

* chore: mem pool df

* chore: add location to arrow error

* refactor: test_util

* refactor: per review

* chore: rm unused

* chore: more cases

* chore: test&buffer clear

* fix: remove fetch

* chore: fmt

* chore: per review

* chore: rm unused
---
 src/query/src/lib.rs         |   5 +-
 src/query/src/part_sort.rs   | 545 +++++++++++++++++++++++++++++++++++
 src/query/src/test_util.rs   | 135 +++++++++
 src/query/src/window_sort.rs | 146 ++--------
 4 files changed, 707 insertions(+), 124 deletions(-)
 create mode 100644 src/query/src/part_sort.rs
 create mode 100644 src/query/src/test_util.rs

diff --git a/src/query/src/lib.rs b/src/query/src/lib.rs
index e4e527a477..7eb82a1e60 100644
--- a/src/query/src/lib.rs
+++ b/src/query/src/lib.rs
@@ -27,6 +27,7 @@ pub mod executor;
 pub mod metrics;
 mod optimizer;
 pub mod parser;
+mod part_sort;
 pub mod physical_wrapper;
 pub mod plan;
 pub mod planner;
@@ -36,8 +37,10 @@ mod range_select;
 pub mod region_query;
 pub mod sql;
 pub mod stats;
-mod window_sort;
+pub(crate) mod window_sort;
 
+#[cfg(test)]
+pub(crate) mod test_util;
 #[cfg(test)]
 mod tests;
 
diff --git a/src/query/src/part_sort.rs b/src/query/src/part_sort.rs
new file mode 100644
index 0000000000..965dfa9c5a
--- /dev/null
+++ b/src/query/src/part_sort.rs
@@ -0,0 +1,545 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+use arrow::compute::{concat, take_record_batch};
+use arrow_schema::SchemaRef;
+use common_recordbatch::{DfRecordBatch, DfSendableRecordBatchStream};
+use datafusion::common::arrow::compute::sort_to_indices;
+use datafusion::execution::memory_pool::{MemoryConsumer, MemoryReservation};
+use datafusion::execution::{RecordBatchStream, TaskContext};
+use datafusion::physical_plan::coalesce_batches::concat_batches;
+use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet};
+use datafusion::physical_plan::{
+    DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
+};
+use datafusion_common::{internal_err, DataFusionError};
+use datafusion_physical_expr::PhysicalSortExpr;
+use futures::Stream;
+use itertools::Itertools;
+use snafu::location;
+
+use crate::error::Result;
+
+/// Sort input within given PartitionRange
+///
+/// Input is assumed to be segmented by empty RecordBatch, which indicates a new `PartitionRange` is starting
+///
+/// and this operator will sort each partition independently within the partition.
+#[derive(Debug, Clone)]
+pub struct PartSortExec {
+    /// Physical sort expressions(that is, sort by timestamp)
+    expression: PhysicalSortExpr,
+    input: Arc<dyn ExecutionPlan>,
+    /// Execution metrics
+    metrics: ExecutionPlanMetricsSet,
+    properties: PlanProperties,
+}
+
+impl PartSortExec {
+    pub fn try_new(expression: PhysicalSortExpr, input: Arc<dyn ExecutionPlan>) -> Result<Self> {
+        let metrics = ExecutionPlanMetricsSet::new();
+        let properties = PlanProperties::new(
+            input.equivalence_properties().clone(),
+            input.output_partitioning().clone(),
+            input.execution_mode(),
+        );
+
+        Ok(Self {
+            expression,
+            input,
+            metrics,
+            properties,
+        })
+    }
+
+    pub fn to_stream(
+        &self,
+        context: Arc<TaskContext>,
+        partition: usize,
+    ) -> datafusion_common::Result<DfSendableRecordBatchStream> {
+        let input_stream: DfSendableRecordBatchStream =
+            self.input.execute(partition, context.clone())?;
+
+        let df_stream = Box::pin(PartSortStream::new(context, self, input_stream)) as _;
+
+        Ok(df_stream)
+    }
+}
+
+impl DisplayAs for PartSortExec {
+    fn fmt_as(&self, _t: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "PartSortExec {}", self.expression)
+    }
+}
+
+impl ExecutionPlan for PartSortExec {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.input.schema()
+    }
+
+    fn properties(&self) -> &PlanProperties {
+        &self.properties
+    }
+
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+        vec![&self.input]
+    }
+
+    fn with_new_children(
+        self: Arc<Self>,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {
+        let new_input = if let Some(first) = children.first() {
+            first
+        } else {
+            internal_err!("No children found")?
+        };
+        Ok(Arc::new(Self::try_new(
+            self.expression.clone(),
+            new_input.clone(),
+        )?))
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        context: Arc<TaskContext>,
+    ) -> datafusion_common::Result<DfSendableRecordBatchStream> {
+        self.to_stream(context, partition)
+    }
+
+    fn metrics(&self) -> Option<MetricsSet> {
+        Some(self.metrics.clone_inner())
+    }
+}
+
+struct PartSortStream {
+    /// Memory pool for this stream
+    reservation: MemoryReservation,
+    buffer: Vec<DfRecordBatch>,
+    expression: PhysicalSortExpr,
+    produced: usize,
+    input: DfSendableRecordBatchStream,
+    input_complete: bool,
+    schema: SchemaRef,
+}
+
+impl PartSortStream {
+    fn new(
+        context: Arc<TaskContext>,
+        sort: &PartSortExec,
+        input: DfSendableRecordBatchStream,
+    ) -> Self {
+        Self {
+            reservation: MemoryConsumer::new("PartSortStream".to_string())
+                .register(&context.runtime_env().memory_pool),
+            buffer: Vec::new(),
+            expression: sort.expression.clone(),
+            produced: 0,
+            input,
+            input_complete: false,
+            schema: sort.input.schema(),
+        }
+    }
+}
+
+impl PartSortStream {
+    /// Sort and clear the buffer and return the sorted record batch
+    ///
+    /// this function should return None if RecordBatch is empty
+    fn sort_buffer(&mut self) -> datafusion_common::Result<Option<DfRecordBatch>> {
+        if self.buffer.iter().map(|r| r.num_rows()).sum::<usize>() == 0 {
+            return Ok(None);
+        }
+        let mut sort_columns = Vec::with_capacity(self.buffer.len());
+        let mut opt = None;
+        for batch in self.buffer.iter() {
+            let sort_column = self.expression.evaluate_to_sort_column(batch)?;
+            opt = opt.or(sort_column.options);
+            sort_columns.push(sort_column.values);
+        }
+
+        let sort_column =
+            concat(&sort_columns.iter().map(|a| a.as_ref()).collect_vec()).map_err(|e| {
+                DataFusionError::ArrowError(
+                    e,
+                    Some(format!("Fail to concat sort columns at {}", location!())),
+                )
+            })?;
+
+        let indices = sort_to_indices(&sort_column, opt, None).map_err(|e| {
+            DataFusionError::ArrowError(
+                e,
+                Some(format!("Fail to sort to indices at {}", location!())),
+            )
+        })?;
+
+        // reserve memory for the concat input and sorted output
+        let total_mem: usize = self.buffer.iter().map(|r| r.get_array_memory_size()).sum();
+        self.reservation.try_grow(total_mem * 2)?;
+
+        let full_input = concat_batches(
+            &self.schema,
+            &self.buffer,
+            self.buffer.iter().map(|r| r.num_rows()).sum(),
+        )
+        .map_err(|e| {
+            DataFusionError::ArrowError(
+                e,
+                Some(format!(
+                    "Fail to concat input batches when sorting at {}",
+                    location!()
+                )),
+            )
+        })?;
+
+        let sorted = take_record_batch(&full_input, &indices).map_err(|e| {
+            DataFusionError::ArrowError(
+                e,
+                Some(format!(
+                    "Fail to take result record batch when sorting at {}",
+                    location!()
+                )),
+            )
+        })?;
+
+        // only clear after sorted for better debugging
+        self.buffer.clear();
+        self.produced += sorted.num_rows();
+        drop(full_input);
+        // here remove both buffer and full_input memory
+        self.reservation.shrink(2 * total_mem);
+        Ok(Some(sorted))
+    }
+
+    pub fn poll_next_inner(
+        mut self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<datafusion_common::Result<DfRecordBatch>>> {
+        loop {
+            if self.input_complete {
+                if self.buffer.is_empty() {
+                    return Poll::Ready(None);
+                } else {
+                    return Poll::Ready(self.sort_buffer().transpose());
+                }
+            }
+            let res = self.input.as_mut().poll_next(cx);
+            match res {
+                Poll::Ready(Some(Ok(batch))) => {
+                    if batch.num_rows() == 0 {
+                        // mark end of current PartitionRange
+                        return Poll::Ready(self.sort_buffer().transpose());
+                    }
+                    self.buffer.push(batch);
+                    // keep polling until boundary(a empty RecordBatch) is reached
+                    continue;
+                }
+                // input stream end, sort the buffer and return
+                Poll::Ready(None) => {
+                    self.input_complete = true;
+                    continue;
+                }
+                Poll::Ready(Some(Err(e))) => return Poll::Ready(Some(Err(e))),
+                Poll::Pending => return Poll::Pending,
+            }
+        }
+    }
+}
+
+impl Stream for PartSortStream {
+    type Item = datafusion_common::Result<DfRecordBatch>;
+
+    fn poll_next(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Option<datafusion_common::Result<DfRecordBatch>>> {
+        self.poll_next_inner(cx)
+    }
+}
+
+impl RecordBatchStream for PartSortStream {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::sync::Arc;
+
+    use arrow::json::ArrayWriter;
+    use arrow_schema::{DataType, Field, Schema, SortOptions, TimeUnit};
+    use common_telemetry::error;
+    use common_time::Timestamp;
+    use datafusion_physical_expr::expressions::Column;
+    use futures::StreamExt;
+    use store_api::region_engine::PartitionRange;
+
+    use super::*;
+    use crate::test_util::{new_ts_array, MockInputExec};
+
+    #[tokio::test]
+    async fn fuzzy_test() {
+        let test_cnt = 100;
+        let part_cnt_bound = 100;
+        let range_size_bound = 100;
+        let range_offset_bound = 100;
+        let batch_cnt_bound = 20;
+        let batch_size_bound = 100;
+
+        let mut rng = fastrand::Rng::new();
+        rng.seed(1337);
+
+        for case_id in 0..test_cnt {
+            let mut bound_val: Option<i64> = None;
+            let descending = rng.bool();
+            let nulls_first = rng.bool();
+            let opt = SortOptions {
+                descending,
+                nulls_first,
+            };
+            let unit = match rng.u8(0..3) {
+                0 => TimeUnit::Second,
+                1 => TimeUnit::Millisecond,
+                2 => TimeUnit::Microsecond,
+                _ => TimeUnit::Nanosecond,
+            };
+
+            let schema = Schema::new(vec![Field::new(
+                "ts",
+                DataType::Timestamp(unit.clone(), None),
+                false,
+            )]);
+            let schema = Arc::new(schema);
+
+            let mut input_ranged_data = vec![];
+            let mut output_data = vec![];
+            // generate each input `PartitionRange`
+            for part_id in 0..rng.usize(0..part_cnt_bound) {
+                // generate each `PartitionRange`'s timestamp range
+                let (start, end) = if descending {
+                    let end = bound_val
+                        .map(|i| i.checked_sub(rng.i64(0..range_offset_bound)).expect("Bad luck, fuzzy test generate data that will overflow, change seed and try again"))
+                        .unwrap_or_else(|| rng.i64(..));
+                    bound_val = Some(end);
+                    let start = end - rng.i64(1..range_size_bound);
+                    let start = Timestamp::new(start, unit.clone().into());
+                    let end = Timestamp::new(end, unit.clone().into());
+                    (start, end)
+                } else {
+                    let start = bound_val
+                        .map(|i| i + rng.i64(0..range_offset_bound))
+                        .unwrap_or_else(|| rng.i64(..));
+                    bound_val = Some(start);
+                    let end = start + rng.i64(1..range_size_bound);
+                    let start = Timestamp::new(start, unit.clone().into());
+                    let end = Timestamp::new(end, unit.clone().into());
+                    (start, end)
+                };
+                assert!(start < end);
+
+                let mut sort_data = vec![];
+                let mut batches = vec![];
+                for _batch_idx in 0..rng.usize(1..batch_cnt_bound) {
+                    let cnt = rng.usize(0..batch_size_bound) + 2;
+                    let iter = 0..rng.usize(1..cnt);
+                    let data_gen = iter
+                        .map(|_| rng.i64(start.value()..end.value()))
+                        .collect_vec();
+                    sort_data.extend(data_gen.clone());
+                    let arr = new_ts_array(unit.clone(), data_gen.clone());
+
+                    let batch = DfRecordBatch::try_new(schema.clone(), vec![arr]).unwrap();
+                    batches.push(batch);
+                }
+                assert!(batches.iter().all(|i| i.num_rows() >= 1));
+
+                let range = PartitionRange {
+                    start,
+                    end,
+                    num_rows: batches.iter().map(|b| b.num_rows()).sum(),
+                    identifier: part_id,
+                };
+                input_ranged_data.push((range, batches));
+
+                if descending {
+                    sort_data.sort_by(|a, b| b.cmp(a));
+                } else {
+                    sort_data.sort();
+                }
+
+                output_data.push(sort_data);
+            }
+
+            let expected_output = output_data
+                .into_iter()
+                .map(|a| {
+                    DfRecordBatch::try_new(schema.clone(), vec![new_ts_array(unit.clone(), a)])
+                        .unwrap()
+                })
+                .collect_vec();
+
+            assert!(!expected_output.is_empty());
+            run_test(case_id, input_ranged_data, schema, opt, expected_output).await;
+        }
+    }
+
+    #[tokio::test]
+    async fn simple_case() {
+        let testcases = vec![
+            (
+                TimeUnit::Millisecond,
+                vec![
+                    ((0, 10), vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8, 9]]),
+                    ((5, 10), vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8]]),
+                ],
+                false,
+                vec![
+                    vec![1, 2, 3, 4, 5, 6, 7, 8, 9],
+                    vec![1, 2, 3, 4, 5, 6, 7, 8],
+                ],
+            ),
+            (
+                TimeUnit::Millisecond,
+                vec![
+                    ((5, 10), vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8, 9]]),
+                    ((0, 10), vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8]]),
+                ],
+                true,
+                vec![
+                    vec![9, 8, 7, 6, 5, 4, 3, 2, 1],
+                    vec![8, 7, 6, 5, 4, 3, 2, 1],
+                ],
+            ),
+        ];
+
+        for (identifier, (unit, input_ranged_data, descending, expected_output)) in
+            testcases.into_iter().enumerate()
+        {
+            let schema = Schema::new(vec![Field::new(
+                "ts",
+                DataType::Timestamp(unit.clone(), None),
+                false,
+            )]);
+            let schema = Arc::new(schema);
+            let opt = SortOptions {
+                descending,
+                ..Default::default()
+            };
+            let input_ranged_data = input_ranged_data
+                .into_iter()
+                .map(|(range, data)| {
+                    let part = PartitionRange {
+                        start: Timestamp::new(range.0, unit.clone().into()),
+                        end: Timestamp::new(range.1, unit.clone().into()),
+                        num_rows: data.iter().map(|b| b.len()).sum(),
+                        identifier,
+                    };
+
+                    let batches = data
+                        .into_iter()
+                        .map(|b| {
+                            let arr = new_ts_array(unit.clone(), b);
+                            DfRecordBatch::try_new(schema.clone(), vec![arr]).unwrap()
+                        })
+                        .collect_vec();
+                    (part, batches)
+                })
+                .collect_vec();
+
+            let expected_output = expected_output
+                .into_iter()
+                .map(|a| {
+                    DfRecordBatch::try_new(schema.clone(), vec![new_ts_array(unit.clone(), a)])
+                        .unwrap()
+                })
+                .collect_vec();
+
+            run_test(0, input_ranged_data, schema.clone(), opt, expected_output).await;
+        }
+    }
+
+    async fn run_test(
+        case_id: usize,
+        input_ranged_data: Vec<(PartitionRange, Vec<DfRecordBatch>)>,
+        schema: SchemaRef,
+        opt: SortOptions,
+        expected_output: Vec<DfRecordBatch>,
+    ) {
+        let (_ranges, batches): (Vec<_>, Vec<_>) = input_ranged_data.clone().into_iter().unzip();
+
+        let batches = batches
+            .into_iter()
+            .flat_map(|mut cols| {
+                cols.push(DfRecordBatch::new_empty(schema.clone()));
+                cols
+            })
+            .collect_vec();
+        let mock_input = MockInputExec::new(batches, schema.clone());
+
+        let exec = PartSortExec::try_new(
+            PhysicalSortExpr {
+                expr: Arc::new(Column::new("ts", 0)),
+                options: opt,
+            },
+            Arc::new(mock_input),
+        )
+        .unwrap();
+
+        let exec_stream = exec.execute(0, Arc::new(TaskContext::default())).unwrap();
+
+        let real_output = exec_stream.map(|r| r.unwrap()).collect::<Vec<_>>().await;
+
+        // a makeshift solution for compare large data
+        if real_output != expected_output {
+            {
+                let mut buf = Vec::with_capacity(10 * real_output.len());
+                for batch in &real_output {
+                    let mut rb_json: Vec<u8> = Vec::new();
+                    let mut writer = ArrayWriter::new(&mut rb_json);
+                    writer.write(batch).unwrap();
+                    writer.finish().unwrap();
+                    buf.append(&mut rb_json);
+                    buf.push(b',');
+                }
+                let buf = String::from_utf8_lossy(&buf);
+                error!("case_id:{case_id}, real_output: [{buf}]");
+            }
+            {
+                let mut buf = Vec::with_capacity(10 * real_output.len());
+                for batch in &expected_output {
+                    let mut rb_json: Vec<u8> = Vec::new();
+                    let mut writer = ArrayWriter::new(&mut rb_json);
+                    writer.write(batch).unwrap();
+                    writer.finish().unwrap();
+                    buf.append(&mut rb_json);
+                    buf.push(b',');
+                }
+                let buf = String::from_utf8_lossy(&buf);
+                error!("case_id:{case_id}, expected_output: [{buf}]");
+            }
+            panic!("case_{} failed, opt: {:?}", case_id, opt);
+        }
+    }
+}
diff --git a/src/query/src/test_util.rs b/src/query/src/test_util.rs
new file mode 100644
index 0000000000..77a2dee1de
--- /dev/null
+++ b/src/query/src/test_util.rs
@@ -0,0 +1,135 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::any::Any;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+use arrow::array::{
+    ArrayRef, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
+    TimestampSecondArray,
+};
+use arrow_schema::{SchemaRef, TimeUnit};
+use common_recordbatch::{DfRecordBatch, DfSendableRecordBatchStream};
+use datafusion::execution::{RecordBatchStream, TaskContext};
+use datafusion::physical_plan::{
+    DisplayAs, DisplayFormatType, ExecutionMode, ExecutionPlan, PlanProperties,
+};
+use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
+use futures::Stream;
+
+pub fn new_ts_array(unit: TimeUnit, arr: Vec<i64>) -> ArrayRef {
+    match unit {
+        TimeUnit::Second => Arc::new(TimestampSecondArray::from_iter_values(arr)) as ArrayRef,
+        TimeUnit::Millisecond => {
+            Arc::new(TimestampMillisecondArray::from_iter_values(arr)) as ArrayRef
+        }
+        TimeUnit::Microsecond => {
+            Arc::new(TimestampMicrosecondArray::from_iter_values(arr)) as ArrayRef
+        }
+        TimeUnit::Nanosecond => {
+            Arc::new(TimestampNanosecondArray::from_iter_values(arr)) as ArrayRef
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct MockInputExec {
+    input: Vec<DfRecordBatch>,
+    schema: SchemaRef,
+    properties: PlanProperties,
+}
+
+impl MockInputExec {
+    pub fn new(input: Vec<DfRecordBatch>, schema: SchemaRef) -> Self {
+        Self {
+            properties: PlanProperties::new(
+                EquivalenceProperties::new(schema.clone()),
+                Partitioning::UnknownPartitioning(1),
+                ExecutionMode::Bounded,
+            ),
+            input,
+            schema,
+        }
+    }
+}
+
+impl DisplayAs for MockInputExec {
+    fn fmt_as(&self, _t: DisplayFormatType, _f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        unimplemented!()
+    }
+}
+
+impl ExecutionPlan for MockInputExec {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn properties(&self) -> &PlanProperties {
+        &self.properties
+    }
+
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+        vec![]
+    }
+
+    fn with_new_children(
+        self: Arc<Self>,
+        _children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {
+        Ok(self)
+    }
+
+    fn execute(
+        &self,
+        _partition: usize,
+        _context: Arc<TaskContext>,
+    ) -> datafusion_common::Result<DfSendableRecordBatchStream> {
+        let stream = MockStream {
+            stream: self.input.clone(),
+            schema: self.schema.clone(),
+            idx: 0,
+        };
+        Ok(Box::pin(stream))
+    }
+}
+
+struct MockStream {
+    stream: Vec<DfRecordBatch>,
+    schema: SchemaRef,
+    idx: usize,
+}
+
+impl Stream for MockStream {
+    type Item = datafusion_common::Result<DfRecordBatch>;
+    fn poll_next(
+        mut self: Pin<&mut Self>,
+        _cx: &mut Context<'_>,
+    ) -> Poll<Option<datafusion_common::Result<DfRecordBatch>>> {
+        if self.idx < self.stream.len() {
+            let ret = self.stream[self.idx].clone();
+            self.idx += 1;
+            Poll::Ready(Some(Ok(ret)))
+        } else {
+            Poll::Ready(None)
+        }
+    }
+}
+
+impl RecordBatchStream for MockStream {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
diff --git a/src/query/src/window_sort.rs b/src/query/src/window_sort.rs
index c2cc9b2389..25dd4b5fb8 100644
--- a/src/query/src/window_sort.rs
+++ b/src/query/src/window_sort.rs
@@ -31,6 +31,7 @@ use arrow_schema::{DataType, SchemaRef, SortOptions};
 use common_error::ext::{BoxedError, PlainError};
 use common_error::status_code::StatusCode;
 use common_recordbatch::{DfRecordBatch, DfSendableRecordBatchStream};
+use common_telemetry::error;
 use common_time::Timestamp;
 use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool};
 use datafusion::execution::{RecordBatchStream, TaskContext};
@@ -376,9 +377,11 @@ impl WindowedSortStream {
 
                         if sort_column.options.unwrap_or_default().descending {
                             if cur_range.end > working_range.end {
+                                error!("Invalid range: {:?} > {:?}", cur_range, working_range);
                                 internal_err!("Current batch have data on the right side of working range, something is very wrong")?;
                             }
                         } else if cur_range.start < working_range.start {
+                            error!("Invalid range: {:?} < {:?}", cur_range, working_range);
                             internal_err!("Current batch have data on the left side of working range, something is very wrong")?;
                         }
 
@@ -1123,20 +1126,17 @@ mod test {
     use std::io::Write;
     use std::sync::Arc;
 
-    use arrow::array::{
-        ArrayRef, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray,
-        TimestampSecondArray,
-    };
+    use arrow::array::{ArrayRef, TimestampMillisecondArray};
     use arrow::compute::concat_batches;
     use arrow::json::ArrayWriter;
     use arrow_schema::{Field, Schema, TimeUnit};
-    use datafusion::physical_plan::ExecutionMode;
-    use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
     use futures::StreamExt;
     use pretty_assertions::assert_eq;
     use serde_json::json;
 
     use super::*;
+    use crate::test_util::{new_ts_array, MockInputExec};
+
     #[test]
     fn test_overlapping() {
         let testcases = [
@@ -2423,95 +2423,6 @@ mod test {
         }
     }
 
-    #[derive(Debug)]
-    struct MockInputExec {
-        input: Vec<DfRecordBatch>,
-        schema: SchemaRef,
-        properties: PlanProperties,
-    }
-
-    impl MockInputExec {
-        fn new(input: Vec<DfRecordBatch>, schema: SchemaRef) -> Self {
-            Self {
-                properties: PlanProperties::new(
-                    EquivalenceProperties::new(schema.clone()),
-                    Partitioning::UnknownPartitioning(1),
-                    ExecutionMode::Bounded,
-                ),
-                input,
-                schema,
-            }
-        }
-    }
-
-    impl DisplayAs for MockInputExec {
-        fn fmt_as(&self, _t: DisplayFormatType, _f: &mut std::fmt::Formatter) -> std::fmt::Result {
-            unimplemented!()
-        }
-    }
-
-    impl ExecutionPlan for MockInputExec {
-        fn as_any(&self) -> &dyn Any {
-            self
-        }
-
-        fn properties(&self) -> &PlanProperties {
-            &self.properties
-        }
-
-        fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
-            vec![]
-        }
-
-        fn with_new_children(
-            self: Arc<Self>,
-            _children: Vec<Arc<dyn ExecutionPlan>>,
-        ) -> datafusion_common::Result<Arc<dyn ExecutionPlan>> {
-            Ok(self)
-        }
-
-        fn execute(
-            &self,
-            _partition: usize,
-            _context: Arc<TaskContext>,
-        ) -> datafusion_common::Result<DfSendableRecordBatchStream> {
-            let stream = MockStream {
-                stream: self.input.clone(),
-                schema: self.schema.clone(),
-                idx: 0,
-            };
-            Ok(Box::pin(stream))
-        }
-    }
-
-    struct MockStream {
-        stream: Vec<DfRecordBatch>,
-        schema: SchemaRef,
-        idx: usize,
-    }
-
-    impl Stream for MockStream {
-        type Item = datafusion_common::Result<DfRecordBatch>;
-        fn poll_next(
-            mut self: Pin<&mut Self>,
-            _cx: &mut Context<'_>,
-        ) -> Poll<Option<datafusion_common::Result<DfRecordBatch>>> {
-            if self.idx < self.stream.len() {
-                let ret = self.stream[self.idx].clone();
-                self.idx += 1;
-                Poll::Ready(Some(Ok(ret)))
-            } else {
-                Poll::Ready(None)
-            }
-        }
-    }
-
-    impl RecordBatchStream for MockStream {
-        fn schema(&self) -> SchemaRef {
-            self.schema.clone()
-        }
-    }
-
     #[tokio::test]
     async fn test_window_sort_stream() {
         let test_cases = [
@@ -3045,21 +2956,6 @@ mod test {
         }
     }
 
-    fn new_array(unit: TimeUnit, arr: Vec<i64>) -> ArrayRef {
-        match unit {
-            TimeUnit::Second => Arc::new(TimestampSecondArray::from_iter_values(arr)) as ArrayRef,
-            TimeUnit::Millisecond => {
-                Arc::new(TimestampMillisecondArray::from_iter_values(arr)) as ArrayRef
-            }
-            TimeUnit::Microsecond => {
-                Arc::new(TimestampMicrosecondArray::from_iter_values(arr)) as ArrayRef
-            }
-            TimeUnit::Nanosecond => {
-                Arc::new(TimestampNanosecondArray::from_iter_values(arr)) as ArrayRef
-            }
-        }
-    }
-
     #[tokio::test]
     async fn fuzzy_ish_test_window_sort_stream() {
         let test_cnt = 100;
@@ -3067,6 +2963,7 @@ mod test {
         let range_size_bound = 100;
         let range_offset_bound = 100;
         let in_range_datapoint_cnt_bound = 100;
+        let fetch_bound = 100;
 
         let mut rng = fastrand::Rng::new();
         rng.seed(1337);
@@ -3088,6 +2985,11 @@ mod test {
                 2 => TimeUnit::Microsecond,
                 _ => TimeUnit::Nanosecond,
             };
+            let fetch = if rng.bool() {
+                Some(rng.usize(0..fetch_bound))
+            } else {
+                None
+            };
 
             let mut input_ranged_data = vec![];
             let mut output_data: Vec<i64> = vec![];
@@ -3119,7 +3021,7 @@ mod test {
                     .sorted_by(ret_cmp_fn(descending))
                     .collect_vec();
                 output_data.extend(data_gen.clone());
-                let arr = new_array(unit.clone(), data_gen);
+                let arr = new_ts_array(unit.clone(), data_gen);
                 let range = PartitionRange {
                     start,
                     end,
@@ -3130,7 +3032,10 @@ mod test {
             }
 
             output_data.sort_by(ret_cmp_fn(descending));
-            let output_arr = new_array(unit.clone(), output_data);
+            if let Some(fetch) = fetch {
+                output_data.truncate(fetch);
+            }
+            let output_arr = new_ts_array(unit.clone(), output_data);
 
             let test_stream = TestStream::new(
                 Column::new("ts", 0),
@@ -3138,7 +3043,7 @@ mod test {
                     descending,
                     nulls_first: true,
                 },
-                None,
+                fetch,
                 vec![Field::new(
                     "ts",
                     DataType::Timestamp(unit.clone(), None),
@@ -3158,8 +3063,7 @@ mod test {
 
             if res_concat != expected_concat {
                 {
-                    let mut f_input =
-                        std::fs::File::create(format!("case_{}_input.json", case_id)).unwrap();
+                    let mut f_input = std::io::stderr();
                     f_input.write_all(b"[").unwrap();
                     for (input_range, input_arr) in test_stream.input {
                         let range_json = json!({
@@ -3181,8 +3085,7 @@ mod test {
                     f_input.write_all(b"]").unwrap();
                 }
                 {
-                    let mut f_res =
-                        std::fs::File::create(format!("case_{}_result.json", case_id)).unwrap();
+                    let mut f_res = std::io::stderr();
                     f_res.write_all(b"[").unwrap();
                     for batch in &res {
                         let mut res_writer = ArrayWriter::new(f_res);
@@ -3193,21 +3096,18 @@ mod test {
                     }
                     f_res.write_all(b"]").unwrap();
 
-                    let f_res_concat =
-                        std::fs::File::create(format!("case_{}_result_concat.json", case_id))
-                            .unwrap();
+                    let f_res_concat = std::io::stderr();
                     let mut res_writer = ArrayWriter::new(f_res_concat);
                     res_writer.write(&res_concat).unwrap();
                     res_writer.finish().unwrap();
 
-                    let f_expected =
-                        std::fs::File::create(format!("case_{}_expected.json", case_id)).unwrap();
+                    let f_expected = std::io::stderr();
                     let mut expected_writer = ArrayWriter::new(f_expected);
                     expected_writer.write(&expected_concat).unwrap();
                     expected_writer.finish().unwrap();
                 }
                 panic!(
-                    "case failed, case id: {0}, output and expected save to case_{0}_*.json file",
+                    "case failed, case id: {0}, output and expected output to stderr",
                     case_id
                 );
             }

From 5d42e63ab054eb94dfe129ac1171e382e2acf800 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Wed, 23 Oct 2024 16:20:34 +0800
Subject: [PATCH 099/128] fix!: replace timeout_millis and
 connect_timeout_millis with Duration in DatanodeClientOptions (#4867)

* fix: correct options struct

* fix: fix unit test
---
 src/cmd/src/frontend.rs             |  2 ++
 src/cmd/tests/load_config_test.rs   | 14 +++++++++++++-
 src/meta-srv/src/metasrv.rs         | 14 +++++++++-----
 src/meta-srv/src/metasrv/builder.rs | 11 +++--------
 4 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs
index 2f10b722c7..38fa75a3a0 100644
--- a/src/cmd/src/frontend.rs
+++ b/src/cmd/src/frontend.rs
@@ -342,6 +342,8 @@ impl StartCommand {
         // Some queries are expected to take long time.
         let channel_config = ChannelConfig {
             timeout: None,
+            tcp_nodelay: opts.datanode.client.tcp_nodelay,
+            connect_timeout: Some(opts.datanode.client.connect_timeout),
             ..Default::default()
         };
         let client = NodeClients::new(channel_config);
diff --git a/src/cmd/tests/load_config_test.rs b/src/cmd/tests/load_config_test.rs
index 7a50234431..bf4871e0bd 100644
--- a/src/cmd/tests/load_config_test.rs
+++ b/src/cmd/tests/load_config_test.rs
@@ -20,7 +20,7 @@ use common_config::Configurable;
 use common_grpc::channel_manager::{
     DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE, DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
 };
-use common_telemetry::logging::{LoggingOptions, DEFAULT_OTLP_ENDPOINT};
+use common_telemetry::logging::{LoggingOptions, SlowQueryOptions, DEFAULT_OTLP_ENDPOINT};
 use common_wal::config::raft_engine::RaftEngineConfig;
 use common_wal::config::DatanodeWalConfig;
 use datanode::config::{DatanodeOptions, RegionEngineConfig, StorageConfig};
@@ -159,8 +159,20 @@ fn test_load_metasrv_example_config() {
                 level: Some("info".to_string()),
                 otlp_endpoint: Some(DEFAULT_OTLP_ENDPOINT.to_string()),
                 tracing_sample_ratio: Some(Default::default()),
+                slow_query: SlowQueryOptions {
+                    enable: false,
+                    threshold: Some(Duration::from_secs(10)),
+                    sample_ratio: Some(1.0),
+                },
                 ..Default::default()
             },
+            datanode: meta_srv::metasrv::DatanodeOptions {
+                client: meta_srv::metasrv::DatanodeClientOptions {
+                    timeout: Duration::from_secs(10),
+                    connect_timeout: Duration::from_secs(10),
+                    tcp_nodelay: true,
+                },
+            },
             export_metrics: ExportMetricsOption {
                 self_import: Some(Default::default()),
                 remote_write: Some(Default::default()),
diff --git a/src/meta-srv/src/metasrv.rs b/src/meta-srv/src/metasrv.rs
index 5c53a1abbb..ee02518fa3 100644
--- a/src/meta-srv/src/metasrv.rs
+++ b/src/meta-srv/src/metasrv.rs
@@ -184,22 +184,26 @@ pub struct MetasrvInfo {
 // Options for datanode.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
 pub struct DatanodeOptions {
-    pub client_options: DatanodeClientOptions,
+    pub client: DatanodeClientOptions,
 }
 
 // Options for datanode client.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct DatanodeClientOptions {
-    pub timeout_millis: u64,
-    pub connect_timeout_millis: u64,
+    #[serde(with = "humantime_serde")]
+    pub timeout: Duration,
+    #[serde(with = "humantime_serde")]
+    pub connect_timeout: Duration,
     pub tcp_nodelay: bool,
 }
 
 impl Default for DatanodeClientOptions {
     fn default() -> Self {
         Self {
-            timeout_millis: channel_manager::DEFAULT_GRPC_REQUEST_TIMEOUT_SECS * 1000,
-            connect_timeout_millis: channel_manager::DEFAULT_GRPC_CONNECT_TIMEOUT_SECS * 1000,
+            timeout: Duration::from_secs(channel_manager::DEFAULT_GRPC_REQUEST_TIMEOUT_SECS),
+            connect_timeout: Duration::from_secs(
+                channel_manager::DEFAULT_GRPC_CONNECT_TIMEOUT_SECS,
+            ),
             tcp_nodelay: true,
         }
     }
diff --git a/src/meta-srv/src/metasrv/builder.rs b/src/meta-srv/src/metasrv/builder.rs
index 31c17bb8f2..781839a193 100644
--- a/src/meta-srv/src/metasrv/builder.rs
+++ b/src/meta-srv/src/metasrv/builder.rs
@@ -14,7 +14,6 @@
 
 use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, Mutex, RwLock};
-use std::time::Duration;
 
 use client::client_manager::NodeClients;
 use common_base::Plugins;
@@ -250,13 +249,9 @@ impl MetasrvBuilder {
         let memory_region_keeper = Arc::new(MemoryRegionKeeper::default());
         let node_manager = node_manager.unwrap_or_else(|| {
             let datanode_client_channel_config = ChannelConfig::new()
-                .timeout(Duration::from_millis(
-                    options.datanode.client_options.timeout_millis,
-                ))
-                .connect_timeout(Duration::from_millis(
-                    options.datanode.client_options.connect_timeout_millis,
-                ))
-                .tcp_nodelay(options.datanode.client_options.tcp_nodelay);
+                .timeout(options.datanode.client.timeout)
+                .connect_timeout(options.datanode.client.connect_timeout)
+                .tcp_nodelay(options.datanode.client.tcp_nodelay);
             Arc::new(NodeClients::new(datanode_client_channel_config))
         });
         let cache_invalidator = Arc::new(MetasrvCacheInvalidator::new(

From fcde0a48744f8ec44a1f839d679e1e2db2969a1e Mon Sep 17 00:00:00 2001
From: localhost <xpaomian@gmail.com>
Date: Thu, 24 Oct 2024 13:55:57 +0800
Subject: [PATCH 100/128] feat: Add functionality to the Opentelemetry write
 interface to extract fields from attr to top-level data.  (#4859)

* chore: add otlp select

* chore: change otlp select

* chore: remove json path

* chore: format toml

* chore: change opentelemetry extract keys header name

* chore: add some doc and remove useless code and lib

* chore: make clippy happy

* chore: fix by pr comment

* chore: fix by pr comment

* chore: opentelemetry logs select key change some type default semantic type
---
 src/frontend/src/instance/otlp.rs             |   1 -
 src/pipeline/src/etl.rs                       |  27 +-
 .../src/etl/transform/transformer/greptime.rs |   6 +-
 src/pipeline/src/lib.rs                       |   3 +-
 src/servers/src/error.rs                      |  10 +-
 src/servers/src/http/header.rs                |   1 +
 src/servers/src/http/otlp.rs                  |  35 +-
 src/servers/src/otlp/logs.rs                  | 359 ++++++++++++++++--
 tests-integration/tests/http.rs               |  61 ++-
 9 files changed, 438 insertions(+), 65 deletions(-)

diff --git a/src/frontend/src/instance/otlp.rs b/src/frontend/src/instance/otlp.rs
index 7ed1609715..a2245d8fb8 100644
--- a/src/frontend/src/instance/otlp.rs
+++ b/src/frontend/src/instance/otlp.rs
@@ -114,7 +114,6 @@ impl OpenTelemetryProtocolHandler for Instance {
             .get::<OpenTelemetryProtocolInterceptorRef<servers::error::Error>>();
         interceptor_ref.pre_execute(ctx.clone())?;
         let (requests, rows) = otlp::logs::to_grpc_insert_requests(request, pipeline, table_name)?;
-
         self.handle_log_inserts(requests, ctx)
             .await
             .inspect(|_| OTLP_LOGS_ROWS.inc_by(rows as u64))
diff --git a/src/pipeline/src/etl.rs b/src/pipeline/src/etl.rs
index bec06cab8b..9bd47a899e 100644
--- a/src/pipeline/src/etl.rs
+++ b/src/pipeline/src/etl.rs
@@ -317,8 +317,33 @@ pub(crate) fn find_key_index(intermediate_keys: &[String], key: &str, kind: &str
         .context(IntermediateKeyIndexSnafu { kind, key })
 }
 
+/// SelectInfo is used to store the selected keys from OpenTelemetry record attrs
+#[derive(Default)]
+pub struct SelectInfo {
+    pub keys: Vec<String>,
+}
+
+/// Try to convert a string to SelectInfo
+/// The string should be a comma-separated list of keys
+/// example: "key1,key2,key3"
+/// The keys will be sorted and deduplicated
+impl From<String> for SelectInfo {
+    fn from(value: String) -> Self {
+        let mut keys: Vec<String> = value.split(',').map(|s| s.to_string()).sorted().collect();
+        keys.dedup();
+
+        SelectInfo { keys }
+    }
+}
+
+impl SelectInfo {
+    pub fn is_empty(&self) -> bool {
+        self.keys.is_empty()
+    }
+}
+
 pub enum PipelineWay {
-    Identity,
+    OtlpLog(Box<SelectInfo>),
     Custom(std::sync::Arc<Pipeline<crate::GreptimeTransformer>>),
 }
 
diff --git a/src/pipeline/src/etl/transform/transformer/greptime.rs b/src/pipeline/src/etl/transform/transformer/greptime.rs
index 11d63c4503..3b43696b5a 100644
--- a/src/pipeline/src/etl/transform/transformer/greptime.rs
+++ b/src/pipeline/src/etl/transform/transformer/greptime.rs
@@ -207,11 +207,11 @@ impl Transformer for GreptimeTransformer {
 /// As you traverse the user input JSON, this will change.
 /// It will record a superset of all user input schemas.
 #[derive(Debug, Default)]
-struct SchemaInfo {
+pub struct SchemaInfo {
     /// schema info
-    schema: Vec<ColumnSchema>,
+    pub schema: Vec<ColumnSchema>,
     /// index of the column name
-    index: HashMap<String, usize>,
+    pub index: HashMap<String, usize>,
 }
 
 fn resolve_schema(
diff --git a/src/pipeline/src/lib.rs b/src/pipeline/src/lib.rs
index 1043006b73..e9082de83c 100644
--- a/src/pipeline/src/lib.rs
+++ b/src/pipeline/src/lib.rs
@@ -18,10 +18,11 @@ mod metrics;
 
 pub use etl::error::Result;
 pub use etl::processor::Processor;
+pub use etl::transform::transformer::greptime::SchemaInfo;
 pub use etl::transform::transformer::identity_pipeline;
 pub use etl::transform::{GreptimeTransformer, Transformer};
 pub use etl::value::{Array, Map, Value};
-pub use etl::{error as etl_error, parse, Content, Pipeline, PipelineWay};
+pub use etl::{error as etl_error, parse, Content, Pipeline, PipelineWay, SelectInfo};
 pub use manager::{
     error, pipeline_operator, table, util, PipelineInfo, PipelineRef, PipelineTableRef,
     PipelineVersion,
diff --git a/src/servers/src/error.rs b/src/servers/src/error.rs
index 9d7b2f9f8a..9e4ef0976b 100644
--- a/src/servers/src/error.rs
+++ b/src/servers/src/error.rs
@@ -538,6 +538,13 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+    #[snafu(display("Unsupported json data type for tag: {} {}", key, ty))]
+    UnsupportedJsonDataTypeForTag {
+        key: String,
+        ty: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -603,7 +610,8 @@ impl ErrorExt for Error {
             | ParseJson { .. }
             | UnsupportedContentType { .. }
             | TimestampOverflow { .. }
-            | OpenTelemetryLog { .. } => StatusCode::InvalidArguments,
+            | OpenTelemetryLog { .. }
+            | UnsupportedJsonDataTypeForTag { .. } => StatusCode::InvalidArguments,
 
             Catalog { source, .. } => source.status_code(),
             RowWriter { source, .. } => source.status_code(),
diff --git a/src/servers/src/http/header.rs b/src/servers/src/http/header.rs
index 2d621db9e7..f5b833f673 100644
--- a/src/servers/src/http/header.rs
+++ b/src/servers/src/http/header.rs
@@ -47,6 +47,7 @@ pub mod constants {
     pub const GREPTIME_LOG_PIPELINE_NAME_HEADER_NAME: &str = "x-greptime-log-pipeline-name";
     pub const GREPTIME_LOG_PIPELINE_VERSION_HEADER_NAME: &str = "x-greptime-log-pipeline-version";
     pub const GREPTIME_LOG_TABLE_NAME_HEADER_NAME: &str = "x-greptime-log-table-name";
+    pub const GREPTIME_LOG_EXTRACT_KEYS_HEADER_NAME: &str = "x-greptime-log-extract-keys";
 }
 
 pub static GREPTIME_DB_HEADER_FORMAT: HeaderName =
diff --git a/src/servers/src/http/otlp.rs b/src/servers/src/http/otlp.rs
index ddd318acc7..51f2683e1b 100644
--- a/src/servers/src/http/otlp.rs
+++ b/src/servers/src/http/otlp.rs
@@ -34,11 +34,12 @@ use opentelemetry_proto::tonic::collector::trace::v1::{
     ExportTraceServiceRequest, ExportTraceServiceResponse,
 };
 use pipeline::util::to_pipeline_version;
-use pipeline::PipelineWay;
+use pipeline::{PipelineWay, SelectInfo};
 use prost::Message;
 use session::context::{Channel, QueryContext};
 use snafu::prelude::*;
 
+use super::header::constants::GREPTIME_LOG_EXTRACT_KEYS_HEADER_NAME;
 use super::header::{write_cost_header_map, CONTENT_TYPE_PROTOBUF};
 use crate::error::{self, Result};
 use crate::http::header::constants::{
@@ -181,13 +182,41 @@ where
     }
 }
 
+pub struct SelectInfoWrapper(SelectInfo);
+
+#[async_trait]
+impl<S> FromRequestParts<S> for SelectInfoWrapper
+where
+    S: Send + Sync,
+{
+    type Rejection = (StatusCode, String);
+
+    async fn from_request_parts(parts: &mut Parts, _state: &S) -> StdResult<Self, Self::Rejection> {
+        let select = parts.headers.get(GREPTIME_LOG_EXTRACT_KEYS_HEADER_NAME);
+
+        match select {
+            Some(name) => {
+                let select_header =
+                    pipeline_header_error(name, GREPTIME_LOG_EXTRACT_KEYS_HEADER_NAME)?;
+                if select_header.is_empty() {
+                    Ok(SelectInfoWrapper(Default::default()))
+                } else {
+                    Ok(SelectInfoWrapper(SelectInfo::from(select_header)))
+                }
+            }
+            None => Ok(SelectInfoWrapper(Default::default())),
+        }
+    }
+}
+
 #[axum_macros::debug_handler]
-#[tracing::instrument(skip_all, fields(protocol = "otlp", request_type = "traces"))]
+#[tracing::instrument(skip_all, fields(protocol = "otlp", request_type = "logs"))]
 pub async fn logs(
     State(handler): State<OpenTelemetryProtocolHandlerRef>,
     Extension(mut query_ctx): Extension<QueryContext>,
     pipeline_info: PipelineInfo,
     table_info: TableInfo,
+    SelectInfoWrapper(select_info): SelectInfoWrapper,
     bytes: Bytes,
 ) -> Result<OtlpResponse<ExportLogsServiceResponse>> {
     let db = query_ctx.get_db_string();
@@ -218,7 +247,7 @@ pub async fn logs(
         };
         pipeline_way = PipelineWay::Custom(pipeline);
     } else {
-        pipeline_way = PipelineWay::Identity;
+        pipeline_way = PipelineWay::OtlpLog(Box::new(select_info));
     }
 
     handler
diff --git a/src/servers/src/otlp/logs.rs b/src/servers/src/otlp/logs.rs
index 4c47da29c8..5faaced461 100644
--- a/src/servers/src/otlp/logs.rs
+++ b/src/servers/src/otlp/logs.rs
@@ -12,7 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::{BTreeMap, HashMap as StdHashMap};
+use std::mem;
 
 use api::v1::column_data_type_extension::TypeExt;
 use api::v1::value::ValueData;
@@ -23,12 +24,14 @@ use api::v1::{
 use jsonb::{Number as JsonbNumber, Value as JsonbValue};
 use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
 use opentelemetry_proto::tonic::common::v1::{any_value, AnyValue, InstrumentationScope, KeyValue};
-use opentelemetry_proto::tonic::logs::v1::LogRecord;
-use pipeline::{Array, Map, PipelineWay, Value as PipelineValue};
-use snafu::ResultExt;
+use opentelemetry_proto::tonic::logs::v1::{LogRecord, ResourceLogs, ScopeLogs};
+use pipeline::{Array, Map, PipelineWay, SchemaInfo, SelectInfo, Value as PipelineValue};
+use snafu::{ensure, ResultExt};
 
 use super::trace::attributes::OtlpAnyValue;
-use crate::error::{OpenTelemetryLogSnafu, Result};
+use crate::error::{
+    IncompatibleSchemaSnafu, OpenTelemetryLogSnafu, Result, UnsupportedJsonDataTypeForTagSnafu,
+};
 use crate::otlp::trace::span::bytes_to_hex_string;
 
 /// Convert OpenTelemetry metrics to GreptimeDB insert requests
@@ -44,8 +47,8 @@ pub fn to_grpc_insert_requests(
     table_name: String,
 ) -> Result<(RowInsertRequests, usize)> {
     match pipeline {
-        PipelineWay::Identity => {
-            let rows = parse_export_logs_service_request_to_rows(request);
+        PipelineWay::OtlpLog(select_info) => {
+            let rows = parse_export_logs_service_request_to_rows(request, select_info)?;
             let len = rows.rows.len();
             let insert_request = RowInsertRequest {
                 rows: Some(rows),
@@ -278,7 +281,7 @@ fn build_otlp_logs_identity_schema() -> Vec<ColumnSchema> {
             SemanticType::Field,
             None,
             Some(ColumnOptions {
-                options: HashMap::from([(
+                options: StdHashMap::from([(
                     "fulltext".to_string(),
                     r#"{"enable":true}"#.to_string(),
                 )]),
@@ -298,13 +301,14 @@ fn build_otlp_logs_identity_schema() -> Vec<ColumnSchema> {
     .collect::<Vec<ColumnSchema>>()
 }
 
-fn build_identity_row(
+fn build_otlp_build_in_row(
     log: LogRecord,
-    resource_attr: JsonbValue<'_>,
+    resource_attr: JsonbValue<'static>,
     scope_name: Option<String>,
     scope_version: Option<String>,
-    scope_attrs: JsonbValue<'_>,
+    scope_attrs: JsonbValue<'static>,
 ) -> Row {
+    let log_attr = key_value_to_jsonb(log.attributes);
     let row = vec![
         GreptimeValue {
             value_data: scope_name.map(ValueData::StringValue),
@@ -319,9 +323,7 @@ fn build_identity_row(
             value_data: Some(ValueData::BinaryValue(resource_attr.to_vec())),
         },
         GreptimeValue {
-            value_data: Some(ValueData::BinaryValue(
-                key_value_to_jsonb(log.attributes).to_vec(),
-            )),
+            value_data: Some(ValueData::BinaryValue(log_attr.to_vec())),
         },
         GreptimeValue {
             value_data: Some(ValueData::TimestampNanosecondValue(
@@ -355,35 +357,320 @@ fn build_identity_row(
                 .map(|x| ValueData::StringValue(log_body_to_string(x))),
         },
     ];
-
     Row { values: row }
 }
 
-fn parse_export_logs_service_request_to_rows(request: ExportLogsServiceRequest) -> Rows {
-    let mut result = Vec::new();
-    for r in request.resource_logs {
-        let resource_attr = r
-            .resource
-            .map(|x| key_value_to_jsonb(x.attributes))
-            .unwrap_or(JsonbValue::Null);
-        for scope_logs in r.scope_logs {
-            let (scope_attrs, scope_version, scope_name) = scope_to_jsonb(scope_logs.scope);
-            for log in scope_logs.log_records {
-                let value = build_identity_row(
-                    log,
-                    resource_attr.clone(),
-                    scope_name.clone(),
-                    scope_version.clone(),
-                    scope_attrs.clone(),
-                );
-                result.push(value);
+fn extract_field_from_attr_and_combine_schema(
+    schema_info: &mut SchemaInfo,
+    log_select: &SelectInfo,
+    jsonb: &jsonb::Value<'static>,
+) -> Result<Vec<GreptimeValue>> {
+    if log_select.keys.is_empty() {
+        return Ok(Vec::new());
+    }
+    let mut append_value = Vec::with_capacity(schema_info.schema.len());
+    for _ in schema_info.schema.iter() {
+        append_value.push(GreptimeValue { value_data: None });
+    }
+    for k in &log_select.keys {
+        let index = schema_info.index.get(k).copied();
+        if let Some(value) = jsonb.get_by_name_ignore_case(k).cloned() {
+            if let Some((schema, value)) = decide_column_schema(k, value)? {
+                if let Some(index) = index {
+                    let column_schema = &schema_info.schema[index];
+                    ensure!(
+                        column_schema.datatype == schema.datatype,
+                        IncompatibleSchemaSnafu {
+                            column_name: k.clone(),
+                            datatype: column_schema.datatype().as_str_name(),
+                            expected: column_schema.datatype,
+                            actual: schema.datatype,
+                        }
+                    );
+                    append_value[index] = value;
+                } else {
+                    let key = k.clone();
+                    schema_info.schema.push(schema);
+                    schema_info.index.insert(key, schema_info.schema.len() - 1);
+                    append_value.push(value);
+                }
             }
         }
     }
-    Rows {
-        schema: build_otlp_logs_identity_schema(),
-        rows: result,
+    Ok(append_value)
+}
+
+fn decide_column_schema(
+    column_name: &str,
+    value: JsonbValue,
+) -> Result<Option<(ColumnSchema, GreptimeValue)>> {
+    let column_info = match value {
+        JsonbValue::String(s) => Ok(Some((
+            GreptimeValue {
+                value_data: Some(ValueData::StringValue(s.into())),
+            },
+            ColumnDataType::String,
+            SemanticType::Tag,
+            None,
+        ))),
+        JsonbValue::Number(n) => match n {
+            JsonbNumber::Int64(i) => Ok(Some((
+                GreptimeValue {
+                    value_data: Some(ValueData::I64Value(i)),
+                },
+                ColumnDataType::Int64,
+                SemanticType::Tag,
+                None,
+            ))),
+            JsonbNumber::Float64(_) => UnsupportedJsonDataTypeForTagSnafu {
+                ty: "FLOAT".to_string(),
+                key: column_name,
+            }
+            .fail(),
+            JsonbNumber::UInt64(u) => Ok(Some((
+                GreptimeValue {
+                    value_data: Some(ValueData::U64Value(u)),
+                },
+                ColumnDataType::Uint64,
+                SemanticType::Tag,
+                None,
+            ))),
+        },
+        JsonbValue::Bool(b) => Ok(Some((
+            GreptimeValue {
+                value_data: Some(ValueData::BoolValue(b)),
+            },
+            ColumnDataType::Boolean,
+            SemanticType::Tag,
+            None,
+        ))),
+        JsonbValue::Array(_) | JsonbValue::Object(_) => UnsupportedJsonDataTypeForTagSnafu {
+            ty: "Json".to_string(),
+            key: column_name,
+        }
+        .fail(),
+        JsonbValue::Null => Ok(None),
+    };
+    column_info.map(|c| {
+        c.map(|(value, column_type, semantic_type, datatype_extension)| {
+            (
+                ColumnSchema {
+                    column_name: column_name.to_string(),
+                    datatype: column_type as i32,
+                    semantic_type: semantic_type as i32,
+                    datatype_extension,
+                    options: None,
+                },
+                value,
+            )
+        })
+    })
+}
+
+#[derive(Debug, Clone, Copy)]
+enum OpenTelemetryLogRecordAttrType {
+    Resource,
+    Scope,
+    Log,
+}
+
+fn merge_schema(
+    input_schemas: Vec<(&SchemaInfo, OpenTelemetryLogRecordAttrType)>,
+) -> BTreeMap<&String, (OpenTelemetryLogRecordAttrType, usize, &ColumnSchema)> {
+    let mut schemas = BTreeMap::new();
+    input_schemas
+        .into_iter()
+        .for_each(|(schema_info, attr_type)| {
+            for (key, index) in schema_info.index.iter() {
+                if let Some(col_schema) = schema_info.schema.get(*index) {
+                    schemas.insert(key, (attr_type, *index, col_schema));
+                }
+            }
+        });
+    schemas
+}
+
+fn parse_export_logs_service_request_to_rows(
+    request: ExportLogsServiceRequest,
+    select_info: Box<SelectInfo>,
+) -> Result<Rows> {
+    let mut schemas = build_otlp_logs_identity_schema();
+    let mut extra_resource_schema = SchemaInfo::default();
+    let mut extra_scope_schema = SchemaInfo::default();
+    let mut extra_log_schema = SchemaInfo::default();
+    let parse_infos = parse_resource(
+        &select_info,
+        &mut extra_resource_schema,
+        &mut extra_scope_schema,
+        &mut extra_log_schema,
+        request.resource_logs,
+    )?;
+
+    // order of schema is important
+    // resource < scope < log
+    // do not change the order
+    let final_extra_schema_info = merge_schema(vec![
+        (
+            &extra_resource_schema,
+            OpenTelemetryLogRecordAttrType::Resource,
+        ),
+        (&extra_scope_schema, OpenTelemetryLogRecordAttrType::Scope),
+        (&extra_log_schema, OpenTelemetryLogRecordAttrType::Log),
+    ]);
+
+    let final_extra_schema = final_extra_schema_info
+        .iter()
+        .map(|(_, (_, _, v))| (*v).clone())
+        .collect::<Vec<_>>();
+
+    let extra_schema_len = final_extra_schema.len();
+    schemas.extend(final_extra_schema);
+
+    let mut results = Vec::with_capacity(parse_infos.len());
+    for parse_info in parse_infos.into_iter() {
+        let mut row = parse_info.values;
+        let mut resource_values = parse_info.resource_extracted_values;
+        let mut scope_values = parse_info.scope_extracted_values;
+        let mut log_values = parse_info.log_extracted_values;
+
+        let mut final_extra_values = vec![GreptimeValue { value_data: None }; extra_schema_len];
+        for (idx, (_, (attr_type, index, _))) in final_extra_schema_info.iter().enumerate() {
+            let value = match attr_type {
+                OpenTelemetryLogRecordAttrType::Resource => resource_values.get_mut(*index),
+                OpenTelemetryLogRecordAttrType::Scope => scope_values.get_mut(*index),
+                OpenTelemetryLogRecordAttrType::Log => log_values.get_mut(*index),
+            };
+            if let Some(value) = value {
+                // swap value to final_extra_values
+                mem::swap(&mut final_extra_values[idx], value);
+            }
+        }
+
+        row.values.extend(final_extra_values);
+        results.push(row);
     }
+    Ok(Rows {
+        schema: schemas,
+        rows: results,
+    })
+}
+
+fn parse_resource(
+    select_info: &SelectInfo,
+    extra_resource_schema: &mut SchemaInfo,
+    extra_scope_schema: &mut SchemaInfo,
+    extra_log_schema: &mut SchemaInfo,
+    resource_logs_vec: Vec<ResourceLogs>,
+) -> Result<Vec<ParseInfo>> {
+    let mut results = Vec::new();
+    for r in resource_logs_vec {
+        let resource_attr = r
+            .resource
+            .map(|resource| key_value_to_jsonb(resource.attributes))
+            .unwrap_or(JsonbValue::Null);
+        let resource_extracted_values = extract_field_from_attr_and_combine_schema(
+            extra_resource_schema,
+            select_info,
+            &resource_attr,
+        )?;
+        let rows = parse_scope(
+            extra_scope_schema,
+            extra_log_schema,
+            select_info,
+            r.scope_logs,
+            resource_attr,
+            resource_extracted_values,
+        )?;
+        results.extend(rows);
+    }
+    Ok(results)
+}
+
+struct ScopeInfo {
+    scope_name: Option<String>,
+    scope_version: Option<String>,
+    scope_attrs: JsonbValue<'static>,
+}
+
+fn parse_scope(
+    extra_scope_schema: &mut SchemaInfo,
+    extra_log_schema: &mut SchemaInfo,
+    select_info: &SelectInfo,
+    scopes_log_vec: Vec<ScopeLogs>,
+    resource_attr: JsonbValue<'static>,
+    resource_extracted_values: Vec<GreptimeValue>,
+) -> Result<Vec<ParseInfo>> {
+    let mut results = Vec::new();
+    for scope_logs in scopes_log_vec {
+        let (scope_attrs, scope_version, scope_name) = scope_to_jsonb(scope_logs.scope);
+        let scope_extracted_values = extract_field_from_attr_and_combine_schema(
+            extra_scope_schema,
+            select_info,
+            &scope_attrs,
+        )?;
+        let rows = parse_log(
+            extra_log_schema,
+            select_info,
+            scope_logs.log_records,
+            &resource_attr,
+            ScopeInfo {
+                scope_name,
+                scope_version,
+                scope_attrs,
+            },
+            &resource_extracted_values,
+            &scope_extracted_values,
+        )?;
+        results.extend(rows);
+    }
+    Ok(results)
+}
+
+fn parse_log(
+    extra_log_schema: &mut SchemaInfo,
+    select_info: &SelectInfo,
+    log_records: Vec<LogRecord>,
+    resource_attr: &JsonbValue<'static>,
+    ScopeInfo {
+        scope_name,
+        scope_version,
+        scope_attrs,
+    }: ScopeInfo,
+    resource_extracted_values: &[GreptimeValue],
+    scope_extracted_values: &[GreptimeValue],
+) -> Result<Vec<ParseInfo>> {
+    let mut result = Vec::with_capacity(log_records.len());
+
+    for log in log_records {
+        let log_attr = key_value_to_jsonb(log.attributes.clone());
+
+        let row = build_otlp_build_in_row(
+            log,
+            resource_attr.clone(),
+            scope_name.clone(),
+            scope_version.clone(),
+            scope_attrs.clone(),
+        );
+
+        let log_extracted_values =
+            extract_field_from_attr_and_combine_schema(extra_log_schema, select_info, &log_attr)?;
+
+        let parse_info = ParseInfo {
+            values: row,
+            resource_extracted_values: resource_extracted_values.to_vec(),
+            scope_extracted_values: scope_extracted_values.to_vec(),
+            log_extracted_values,
+        };
+        result.push(parse_info);
+    }
+    Ok(result)
+}
+
+struct ParseInfo {
+    values: Row,
+    resource_extracted_values: Vec<GreptimeValue>,
+    scope_extracted_values: Vec<GreptimeValue>,
+    log_extracted_values: Vec<GreptimeValue>,
 }
 
 /// transform otlp logs request to pipeline value
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index 8541adc37a..131ba8363f 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -1585,34 +1585,57 @@ pub async fn test_otlp_traces(store_type: StorageType) {
 
 pub async fn test_otlp_logs(store_type: StorageType) {
     common_telemetry::init_default_ut_logging();
-    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_otlp_traces").await;
+    let (app, mut guard) = setup_test_http_app_with_frontend(store_type, "test_otlp_logs").await;
 
+    let client = TestClient::new(app);
     let content = r#"
 {"resourceLogs":[{"resource":{"attributes":[{"key":"resource-attr","value":{"stringValue":"resource-attr-val-1"}}]},"schemaUrl":"https://opentelemetry.io/schemas/1.0.0/resourceLogs","scopeLogs":[{"scope":{},"schemaUrl":"https://opentelemetry.io/schemas/1.0.0/scopeLogs","logRecords":[{"flags":1,"timeUnixNano":1581452773000009875,"observedTimeUnixNano":1581452773000009875,"severityNumber":9,"severityText":"Info","body":{"value":{"stringValue":"This is a log message"}},"attributes":[{"key":"app","value":{"stringValue":"server"}},{"key":"instance_num","value":{"intValue":1}}],"droppedAttributesCount":1,"traceId":[48,56,48,52,48,50,48,49,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48],"spanId":[48,49,48,50,48,52,48,56,48,48,48,48,48,48,48,48]},{"flags":1,"timeUnixNano":1581452773000000789,"observedTimeUnixNano":1581452773000000789,"severityNumber":9,"severityText":"Info","body":{"value":{"stringValue":"something happened"}},"attributes":[{"key":"customer","value":{"stringValue":"acme"}},{"key":"env","value":{"stringValue":"dev"}}],"droppedAttributesCount":1,"traceId":[48],"spanId":[48]}]}]}]}
 "#;
 
     let req: ExportLogsServiceRequest = serde_json::from_str(content).unwrap();
     let body = req.encode_to_vec();
+    {
+        // write log data
+        let res = send_req(
+            &client,
+            vec![
+                (
+                    HeaderName::from_static("x-greptime-log-table-name"),
+                    HeaderValue::from_static("logs"),
+                ),
+                (
+                    HeaderName::from_static("x-greptime-log-extract-keys"),
+                    HeaderValue::from_static("resource-attr,instance_num,app,not-exist"),
+                ),
+            ],
+            "/v1/otlp/v1/logs?db=public",
+            body.clone(),
+            false,
+        )
+        .await;
+        assert_eq!(StatusCode::OK, res.status());
 
-    // handshake
-    let client = TestClient::new(app);
+        let expected = r#"[["","",{},{"resource-attr":"resource-attr-val-1"},{"customer":"acme","env":"dev"},1581452773000000789,1581452773000000789,"30","30",1,"Info",9,"something happened",null,null,"resource-attr-val-1"],["","",{},{"resource-attr":"resource-attr-val-1"},{"app":"server","instance_num":1},1581452773000009875,1581452773000009875,"3038303430323031303030303030303030303030303030303030303030303030","30313032303430383030303030303030",1,"Info",9,"This is a log message","server",1,"resource-attr-val-1"]]"#;
+        validate_data(&client, "select * from logs;", expected).await;
+    }
 
-    // write traces data
-    let res = send_req(
-        &client,
-        vec![(
-            HeaderName::from_static("x-greptime-log-table-name"),
-            HeaderValue::from_static("logs"),
-        )],
-        "/v1/otlp/v1/logs?db=public",
-        body.clone(),
-        false,
-    )
-    .await;
-    assert_eq!(StatusCode::OK, res.status());
-
-    let expected = r#"[["","",{},{"resource-attr":"resource-attr-val-1"},{"customer":"acme","env":"dev"},1581452773000000789,1581452773000000789,"30","30",1,"Info",9,"something happened"],["","",{},{"resource-attr":"resource-attr-val-1"},{"app":"server","instance_num":1},1581452773000009875,1581452773000009875,"3038303430323031303030303030303030303030303030303030303030303030","30313032303430383030303030303030",1,"Info",9,"This is a log message"]]"#;
-    validate_data(&client, "select * from logs;", expected).await;
+    {
+        // write log data
+        let res = send_req(
+            &client,
+            vec![(
+                HeaderName::from_static("x-greptime-log-table-name"),
+                HeaderValue::from_static("logs1"),
+            )],
+            "/v1/otlp/v1/logs?db=public",
+            body.clone(),
+            false,
+        )
+        .await;
+        assert_eq!(StatusCode::OK, res.status());
+        let expected = r#"[["","",{},{"resource-attr":"resource-attr-val-1"},{"customer":"acme","env":"dev"},1581452773000000789,1581452773000000789,"30","30",1,"Info",9,"something happened"],["","",{},{"resource-attr":"resource-attr-val-1"},{"app":"server","instance_num":1},1581452773000009875,1581452773000009875,"3038303430323031303030303030303030303030303030303030303030303030","30313032303430383030303030303030",1,"Info",9,"This is a log message"]]"#;
+        validate_data(&client, "select * from logs1;", expected).await;
+    }
 
     guard.remove_all().await;
 }

From 9d3ee6384a6eb1d8a4c67696afa6c3dc1b37cdc3 Mon Sep 17 00:00:00 2001
From: pa <1020401660@qq.com>
Date: Thu, 24 Oct 2024 15:30:24 +0800
Subject: [PATCH 101/128] feat: Limit CPU in runtime (#3685) (#4782)

feat: add throttle runtime (#3685)
---
 Cargo.lock                                    |  30 ++
 Cargo.toml                                    |   2 +
 src/client/Cargo.toml                         |   2 +-
 src/common/runtime/Cargo.toml                 |  15 +
 src/common/runtime/README.md                  |  60 ++++
 src/common/runtime/resources/rdme-exp.png     | Bin 0 -> 231854 bytes
 src/common/runtime/src/bin.rs                 | 205 +++++++++++++
 src/common/runtime/src/error.rs               |   8 +
 src/common/runtime/src/global.rs              |   1 +
 src/common/runtime/src/lib.rs                 |   2 +
 src/common/runtime/src/repeated_task.rs       |   1 +
 src/common/runtime/src/runtime.rs             | 116 ++++---
 src/common/runtime/src/runtime_default.rs     |  77 +++++
 .../runtime/src/runtime_throttleable.rs       | 285 ++++++++++++++++++
 src/common/telemetry/Cargo.toml               |   2 +-
 src/datanode/src/tests.rs                     |   1 +
 src/meta-srv/Cargo.toml                       |   2 +-
 src/meta-srv/src/procedure/utils.rs           |   1 +
 .../puffin_manager/stager/bounded_stager.rs   |   1 +
 src/script/src/python/utils.rs                |   1 +
 src/servers/Cargo.toml                        |   2 +-
 src/servers/src/grpc/greptime_handler.rs      |   1 +
 src/servers/src/grpc/region_server.rs         |   1 +
 src/servers/src/mysql/server.rs               |   1 +
 src/servers/src/postgres/server.rs            |   1 +
 src/servers/tests/grpc/mod.rs                 |   1 +
 src/servers/tests/mysql/mysql_server_test.rs  |   1 +
 src/servers/tests/postgres/mod.rs             |   1 +
 tests-integration/src/cluster.rs              |   1 +
 tests-integration/src/test_util.rs            |   5 +-
 tests-integration/tests/grpc.rs               |   1 +
 31 files changed, 787 insertions(+), 41 deletions(-)
 create mode 100644 src/common/runtime/README.md
 create mode 100644 src/common/runtime/resources/rdme-exp.png
 create mode 100644 src/common/runtime/src/bin.rs
 create mode 100644 src/common/runtime/src/runtime_default.rs
 create mode 100644 src/common/runtime/src/runtime_throttleable.rs

diff --git a/Cargo.lock b/Cargo.lock
index fc57d67046..f9e0c1849c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1808,6 +1808,17 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "clocksource"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "129026dd5a8a9592d96916258f3a5379589e513ea5e86aeb0bd2530286e44e9e"
+dependencies = [
+ "libc",
+ "time",
+ "winapi",
+]
+
 [[package]]
 name = "cmake"
 version = "0.1.51"
@@ -2332,16 +2343,24 @@ name = "common-runtime"
 version = "0.9.5"
 dependencies = [
  "async-trait",
+ "clap 4.5.19",
  "common-error",
  "common-macro",
  "common-telemetry",
+ "futures",
  "lazy_static",
  "num_cpus",
  "once_cell",
+ "parking_lot 0.12.3",
  "paste",
+ "pin-project",
  "prometheus",
+ "rand",
+ "ratelimit",
  "serde",
+ "serde_json",
  "snafu 0.8.5",
+ "tempfile",
  "tokio",
  "tokio-metrics",
  "tokio-metrics-collector",
@@ -9205,6 +9224,17 @@ dependencies = [
  "rand",
 ]
 
+[[package]]
+name = "ratelimit"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c1bb13e2dcfa2232ac6887157aad8d9b3fe4ca57f7c8d4938ff5ea9be742300"
+dependencies = [
+ "clocksource",
+ "parking_lot 0.12.3",
+ "thiserror",
+]
+
 [[package]]
 name = "raw-cpuid"
 version = "11.2.0"
diff --git a/Cargo.toml b/Cargo.toml
index a2998ce7db..ab78bbd90f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -140,6 +140,7 @@ opentelemetry-proto = { version = "0.5", features = [
     "with-serde",
     "logs",
 ] }
+parking_lot = "0.12"
 parquet = { version = "51.0.0", default-features = false, features = ["arrow", "async", "object_store"] }
 paste = "1.0"
 pin-project = "1.0"
@@ -148,6 +149,7 @@ promql-parser = { version = "0.4.1" }
 prost = "0.12"
 raft-engine = { version = "0.4.1", default-features = false }
 rand = "0.8"
+ratelimit = "0.9"
 regex = "1.8"
 regex-automata = { version = "0.4" }
 reqwest = { version = "0.12", default-features = false, features = [
diff --git a/src/client/Cargo.toml b/src/client/Cargo.toml
index 469d7d1a7e..501c2f82d8 100644
--- a/src/client/Cargo.toml
+++ b/src/client/Cargo.toml
@@ -28,7 +28,7 @@ enum_dispatch = "0.3"
 futures-util.workspace = true
 lazy_static.workspace = true
 moka = { workspace = true, features = ["future"] }
-parking_lot = "0.12"
+parking_lot.workspace = true
 prometheus.workspace = true
 prost.workspace = true
 query.workspace = true
diff --git a/src/common/runtime/Cargo.toml b/src/common/runtime/Cargo.toml
index e5fa276c4b..c249ba221e 100644
--- a/src/common/runtime/Cargo.toml
+++ b/src/common/runtime/Cargo.toml
@@ -4,21 +4,36 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 
+[lib]
+path = "src/lib.rs"
+
+[[bin]]
+name = "common-runtime-bin"
+path = "src/bin.rs"
+
 [lints]
 workspace = true
 
 [dependencies]
 async-trait.workspace = true
+clap.workspace = true
 common-error.workspace = true
 common-macro.workspace = true
 common-telemetry.workspace = true
+futures.workspace = true
 lazy_static.workspace = true
 num_cpus.workspace = true
 once_cell.workspace = true
+parking_lot.workspace = true
 paste.workspace = true
+pin-project.workspace = true
 prometheus.workspace = true
+rand.workspace = true
+ratelimit.workspace = true
 serde.workspace = true
+serde_json.workspace = true
 snafu.workspace = true
+tempfile.workspace = true
 tokio.workspace = true
 tokio-metrics = "0.3"
 tokio-metrics-collector = { git = "https://github.com/MichaelScofield/tokio-metrics-collector.git", rev = "89d692d5753d28564a7aac73c6ac5aba22243ba0" }
diff --git a/src/common/runtime/README.md b/src/common/runtime/README.md
new file mode 100644
index 0000000000..a4214350e4
--- /dev/null
+++ b/src/common/runtime/README.md
@@ -0,0 +1,60 @@
+# Greptime Runtime
+
+## Run performance test for different priority & workload type
+
+```
+# workspace is at this subcrate
+cargo run --release -- --loop-cnt 500
+```
+
+## Related PRs & issues
+
+- Preliminary support cpu limitation
+  
+  ISSUE: https://github.com/GreptimeTeam/greptimedb/issues/3685
+  
+  PR: https://github.com/GreptimeTeam/greptimedb/pull/4782
+
+## CPU resource constraints (ThrottleableRuntime)
+
+
+To achieve CPU resource constraints, we adopt the concept of rate limiting. When creating a future, we first wrap it with another layer of future to intercept the poll operation during runtime. By using the ratelimit library, we can simply implement a mechanism that allows only a limited number of polls for a batch of tasks under a certain priority within a specific time frame (the current token generation interval is set to 10ms).
+
+The default used runtime can be switched by 
+``` rust
+pub type Runtime = DefaultRuntime;
+```
+in `runtime.rs`.
+
+We tested four type of workload with 5 priorities, whose setup are as follows:
+
+``` rust
+impl Priority {
+    fn ratelimiter_count(&self) -> Result<Option<Ratelimiter>> {
+        let max = 8000;
+        let gen_per_10ms = match self {
+            Priority::VeryLow => Some(2000),
+            Priority::Low => Some(4000),
+            Priority::Middle => Some(6000),
+            Priority::High => Some(8000),
+            Priority::VeryHigh => None,
+        };
+        if let Some(gen_per_10ms) = gen_per_10ms {
+            Ratelimiter::builder(gen_per_10ms, Duration::from_millis(10)) // generate poll count per 10ms
+                .max_tokens(max) // reserved token for batch request
+                .build()
+                .context(BuildRuntimeRateLimiterSnafu)
+                .map(Some)
+        } else {
+            Ok(None)
+        }
+    }
+}
+```
+
+This is the preliminary experimental effect so far:
+
+![](resources/rdme-exp.png)
+
+## TODO
+- Introduce PID to achieve more accurate limitation.
diff --git a/src/common/runtime/resources/rdme-exp.png b/src/common/runtime/resources/rdme-exp.png
new file mode 100644
index 0000000000000000000000000000000000000000..3bf0aa2dc0ea5a562264f16b67cd421341cc7fec
GIT binary patch
literal 231854
zcmZ5|1y~eq+cqG|0!m9rgOn)Uu{21Rbc+axbjK0`BB7{&G)p6mG}0|0OC#M<0t+m=
z^!~Fx&-;Gg_rt-lv$L~v-!u0W=XIXf5Ur!7a*yBv0Tve4J=Ld*dRSPuE`UE0&>dh;
zvN(eh@Ph5Bry`G4F+#TkywR~UR(+|diS-EB24Ue~lVRP$90L4eMLA;O{@upHVh3J<
z(_~}+&!^cq|M~RRMfR=#Zj)dR{I*;J1x{k(_{`YLSW`p7*43HE=7sBXJ03r0H_QQ8
zQhpM^rn8-w4U?bqYZp%mKWUb~M@RtMnBBZAOn(pYa*}2-*3@BAaP_cb65-+F;bQ?4
zFflPndAxWjp{J<)@7IBM(ku>MUTzY+yuQA^JidZFt{(Qh{Nm!`ynF(@0s>EfBc6Er
zyLj37J#q15{pTeAIgg^9r>%#ho0p@j3lrwNHqTwXy`))KFa!Pf=bw3cIlla#kz73g
zy)57Yc`={x^7HWV{`cI#SEVp}CA2*p?SPRn=Lhpk{XO#k?E7~dDPGLr|7$Y;T<PDv
zz@>r-q<H^(+rR`gmaBMJSh85EigM5Vu>UOLHM%Hyi^82_$ZVg*Nh=@re*MOh`)l)g
z8PjuoV;wo&v23z$u>_Pj@02N?b#NMgL&dN>%zb`%N9i^#l1zy|E$C3R{r*k=>S;!I
z6ICN>Fw;NldL+20%;aQQux2LBMDUWO=6B)h9h_TXVX_JxZYvid=tHz-$VE|zX^Qy<
z3l*1D<0blf8J&5vp?&iLo7ZOCGj`MLshEq1Bkp*8=x76!-shLi#ejBj^(eBZK=UX;
zJ5XvpPO#85<Z^zZ#Gtfr$m-@;NhN*WF4>|Gf}AZH&*fcq1BUC6wPxZYp$6T>6|uv;
zd97b@_4N|i1rNmKd`71<<UPn$l&BYST51KEdtLKWE&2kAK8F$n!UC5BTTjbfGe@LO
zhE=7Moc78aDh9B<0+bF=Y;nz+G}wc_6ux5lt&jsxXnNb_;-R7Wc#1|$?Al=ndtw{#
zP0b=`aKIQz`%jVi$!i^W_#7M$bNT<w=hq}Db<9oS5{~jU9*#Y$tg6IbSxsR>>%Q~E
z1oQ^6Q)+d+i`WXe*_yE94V>LMiVwM}op)(Y!fBte4Aksr2z<?DK~WiNiO8M!q{1LB
zy{j9a#&Er@qMh#AcCPbXxzv;bd$84qvXo?hYdpCPfdJ0xW!f5W>a}OK{CzH6$Piwj
z&XAx1Js-9&Bo2m$PwWGCa;HzU?dZ)~$VB>*m(6{~pZsIBoL8)F|K2~$>S=<)a<*ON
zi36W`GgZ;yP;jYDCP1$?24Z$fUNS(AVy%KM#^Qs8r1wl<k8b~|ozP0RyqufSnrS`o
z-V-3T7Dlc!qR+f~NL4h<UVrQLYQJ1`y{M6)RT%Oxc8zhT@Nw3d&+gIx!G&P|WQnX!
zp6VH|3_dO0$wIYk)$Z5NPd`jJE4$JM*63<WyPw{7!E2^@$va?Mf$=+5o{%m)>T^5R
zP$Lbn*ndq8xYy*$)=$^d;oXb|j35Na_-DvE?XnnTOL93CShq_JR2!<*sV*rRp{qhK
zOdou<YR3Z^GF$SHSP8vVxMzjFup8GNllh%uTet;AHBHPf-ynB*e4ErY2@tjY*RohR
z>~A?YdDNs(p?Ff()hy`mGp#NGz1)`kkc$Z@k5Vv98kJ#KH&B@WlOW{Sj*JOC_W8?R
z_ap18|Cu3hcYpPe0M@{c>`?K>!%=630|N(K{7?Z4EN&}{rZJqP^lJduc{+%jM)vYP
z(mg(K0nyEP;})=^ZY8n!=cT9_UwmPlNiTX^MaEQZjyL!ycE=;Rc|or?4z5WN#<_eH
zNL}Gw?qrY-n9zP<IFW>d#Gk6}!u5EWw)*ftEhi&&(ux88g1R5^LnBa#zU%+G%-<!a
z4_ju;Y%ONx-<1JoN^1S!`loBK=l2os3PFp_M@C$mSv0jtleA0lw{_pOR~zCb1|N7j
z9uOEc%9q2@D|j-TCx0~Q26eB!WjjPZnJw-bPDm~tec0)^MVz4}eLCqFFO5uLxLU!}
zUcOvHQO;!`zgiUDt5DEo9r{1>Ae6>Y7;7JuS<~$d>lh@>mJ&Cwv=UE}9Gt?$0KMw!
z61COpL(>a$aarfhf)9ze!q5t`wBv;gNAFZL7;oTFMS#7y2LOW@1tOh^-7O=KN%)0f
z^Z#d%u#Pp*U8N9rbLDMW>jZ^R0av@5SgWfw1Yoy~yHIIZS4Nx`{$&pb*q!Kg*)d>-
zGSIf;q)LsD>zNQ~i$Bk)xb6PXMfX9&I+~b1)>9^2Ia!{hoeD?UgHKYYco>mYQByMv
zBay@J82*=$0w&-9y4tz<dGFz0yA4g!!^_F^8dsf2wa*f_*ikb|3r00BYjQw?JoZz4
zKF;xmoDWHq9kPoysdAkfS{L}A4_Sd*cMU4j>gQgS?pUI}&v-?vn?Fx6jAyTVj_dIp
zY?Yq-C*2Pd;$F?aL;4J;TUKkF@p`5XY50Dh$lNU@=H{_}NcX?zR29wa_K98RdYQjB
z-MANHnYY3`S!4%1UbR8Sh<-P(-LEuOOP)BRui6T&)eb&R@TOi~24eV(>Q2ZJRR}-i
z<^&3DgGcf{%Tgc`e-ZFQfuS1iSnK(P@8%vZrAmhrry5uZ8rub<D$<m=`}y`j0WV}n
z?~1BOME-vTN=)E>n=3Pi37a|~ox(qzsPbb4V(D6q)AAD2fsmD3jAGLJj@sOtuYRTp
zPxQ;&90|aRvZL+C+lP3XC!=>BV@Igi(87su4P#_~XP(bJJFL~V3336qX-gjbpK)IA
zVpd+?6+7?dR#**bwyfRhWL6M#Xx+svoK^z}jPI%1!dwidc5nK@$ZtFC=*#w^pQ6h_
zb2d@*>`*yRMz#>n3;SD=E%HoiK`T$qE=PRZ^w}?xTmRT`t2J`(Keawk2xnAkcWuC@
zb>`(H4q<Xzm1VM(l^NRtLvFfshje5ACI`107CwsW6DpXTGZS3py==r+PBqdBy<$%5
zF0S)&^rqF#=9^h>C|KyT@7XPEkb|J3$`zi}3bt^x*Gc+vP&E~LZl=u#axQ1K20Cs+
z$)Kd~t!D7qYgQ@JxB!OM2xl5?PrpFLJ=jHc-}^UCGGd05>3I<WWJwcTPSw`uQ_JMF
zxw1;LX?SXW<M&pb7&4ERi>N2wzUecl44B%vTt)}eulT)pGS4HLZC>1W5p{0AMotv)
zco*Mh2JfrV?%lp#h@C<95m<eszmZ+(3J(n)>77}&*g;cerF}^IC;Xid1K9)`VoaoV
zD@-N;ms1UwV^FWs_!K7zaz)j;twe?U*q_Dg6cvK7jw8r81R><V2kzU&KB@mYfzZTq
zFdGSS`;n(DytMD)o9;93+^{1t)1*~C>Et_9$1gMy4D<bs2gC|l>u~05t1W3EnBtPb
z$?7`#ugCxY890nZL$^K{xPux+Splyyy$&KEMH%`%pl8T!#=t+{CNS%rJYKi?#cZ16
z6cbG~shWTF#+XJ_V;{&efe_Oy*Cm$*6&^_gYUt+W>w_I2Lc<Z&$*r$1rW@LXuXl~w
zOZJ)PCPn673+}g~mL{BCJkw3jMQO672LT6&mO(^~GWJBtkTkk6uh94ItnX~CTX?Pp
z^yP_QP{Vm_hYkW}C52ZP*1;%@lURx|$cauoM)MeO=kB0U+i@)nd^tIADRaG}?WA27
zegQIe*P2`QU50RvWL<-7q8V?1bTy|@Vf<bP5vhBOM!KTw+2g^89AeGGpe@Ow1Z}Xt
z#D48WA@ur>eE|afbNNcM{1`}h4N#58<(91htEI$qkJ66t)-dylBn(4eRHD;H#Ma|>
zfLPzv)meF+zZzLu|Id77S6WyxX8d6_8?ZSYCRsU&gU0nj74Y$=y4-;4J*##@P4tNj
zx<;dzSmp}8Q&$Pu2Lc|4CPleHso~Tvx(Iw)yu-?P{!|mx0tAt=#&gIoDuPgDwh=)b
z!<F(j6`C!p{Z~e%N&e8+`P=p^vNo8h?(Gq;)k->ls*4`T$7H05pDsQ%1I)pOSCxLf
z+>{}<pLe8W3gxVss&rZ0)}lGIv?06tMSB*90aIJ9{zGOw8pzYTv<hni3=fAx)`?`?
zRp*O#x+&F@`TQ>e&o9tT#I38uf1Tzu8d#7Sj=8+Duny(8j=xFwTRh|s0$7V`L-?V0
zWnhvl0st=d5*r^CUhxVm?QQ49XHA!x2wMCWNapf%d^yeQ*GX8&`a$gZoF`1W2$Le}
z=Bi1JysPT5N#2qC^tEH-8El0Kn|?c_XHL}_8Q(}>zWR*-;$$&<-Vy*^rrLs!9^)YA
zoGM>64@gGrG4WpkD}xu-0N~FF=>zxp2a>NjnlGj*!OnG4h7%to55%0C_v`D*In;tE
z+|Zg*Cy`CsbTTt~2@`8gXMIml+je*N`qLxgD?`lv<`%yW$gafGgvngmz6^$g0SuiH
z0y;BiIC11INDRWuHi5MEu<`JlsEV&fkPikyv+s8*w<L^XQmv6?;k130|8Fl$p200S
ziEozM=?aydyN8`B`=@<KtsQMX{sP@v8**wCVo>Tgw?QQNhwmDbOqWh8d^J_*y(bZ=
zu%N=nI-uq8zE>+<p)+5*0k;KEd)&PxaiZPY_kYNtXFZ4}7^Gq(?&$Ob0KkL2XVv<2
z>kGDW6IHN9Enk!k_&v1f`xK`>0IeN&J06Z=ld?5qqm%gugx_L;bI?}((M15Kts-TX
z9cy^?>QGQFI#glX-IcfY62My(1s!JBKa||QUv2VQk>f6(^@$GF1{6<ydjIu%c+zQ{
zY{c3bX-nOG=O?QXFPt?~8o>*#(Hz`pAYAk(I<uw*skNP&*Xb?*Be-F+u1M#FoIsD_
z>@|Q&CAsJtIbX@1Mn7^6SiW<>8aY%<TxwcTxD?6=!1xIh`%Hs#feLE<(bdryS1Awt
z1ei^2QHF<FCjMV^$PB||TnU*T$_6|zJghFM0~dt>1#00w?AQYSCXsOf0>NAvHVHNN
z1uYgrcSF|NRxy<96o^d~NEz}B&|@B`MDtlMuO$j8r}8w07b*lm#?4p4&}_wyXqE>4
zX>WL~4aum;I8QBYa{ZbItYDEXvnpRxN(QI-`W#~B#b`QyPZY3<t0UIzcF&83Z=D2f
z*B2BnR-rk+LV)DYd;9tc;8Ppl54<ZS7tcq0v+4o-nMUGP_2*@}v*EhJHZ}LC;2qs<
zjN{H23MIVD*vBS@Y%3Hk`YyK)26vHX>CDeRcaIps)H&6!0w?nr3=)=TU_F0_1ya>R
z01+;K+>Wot!=WX`iJ-qcI580{!;h6!^c%pHh1L7Hv*UAK9p6<dRdHBQVA^+Rb6x<V
zaRri`@N+Gr^P~Mrs+0sP;{$~dCTZY8%o|Dh*k8QX(LXH1MDV(TD5caIa0=qmBkKM+
z#=$4rujkvRRv~7tt*2h4W{a=BcheuZ5q&%+@?6={&pJ+=Bj{24Nf~w+g1(%<#-r0g
z0X5(SB5kInu)a<lcQPj%DC>-z%;qtWGA)RYIaFnxCZzG7m14$Jgl`o?5~WFgy(3{t
zA#n-5T4zi?Z=Eu-S*U0}kXYPc7fm&+OF#y04z&)cffAJElLtaAvFG2Fh#AjzfE=h=
z$3JA=B9^L}Bg~OFx~k1NJ2}6oh17j+Bc?@Rte#O6X9wN-`EW4-y(Bq|be5iXEXpjz
zME|-FGyv*P5I3mycEhVo;S(TT1qd*QTJ{m3>MQ60i#{zN4!ka3lcbIC2-(hw(rV*e
zzfl(bKDI$9)JB=Bu@%{Jb6N?_Le>@~35V`GFXjCUF5Y8fD#p9oHgKTM0g*KR>g5r!
z0BXXD*7!Cvt&h>gr(eumOakGobfgt54{a7v@7sZJKe^=SwGmsqsjO4i*`*9b4s~45
z$Bx<>ZkImy4GRwEQ-yFZmm*(Yfn63b;h5?%&c%z1&7gKekBcgy*;ff|b47>CR87eo
zjtbm1T?ssn0n5=>JW;+k>oPa%h;YWM?BRU~e;<eBW%Zayg$@G3K3#&&16*%FQd<Ig
zY+Hzip|4;GsNDN#+J%)n)dt}CIiFd(<Q?Fv*3R^B9`o0+=?K4PQ42FaS!7NeJvtnA
z+n;_R75d9GP%5ot=uPCRDz9K%kPAqD_8+Koim2Y^S}W-=*D6(g9<fISy@dI`r~e|7
zc%_-sTV_T-@r9+9?0FNYVXcnKXpUOue7bz{lM{g0#M((y4wEn3F1#-)1Wp$*-a;5t
zv2UZX)(+hA8D>mLgJOsJzCT_n5+=%?jmL8c=zjriLC1=3ntG$~wee2**HO`6BF)Xn
zigJ6vvVDghb=W|2o-A30*pL43j5S-xJ<i^j2|4*GS}8<y)Gb+bv*rmiQohM+P82kO
ze@<?BQN6|3YCL_K!jt41@lOI|2>X`1oW~D<=yX0D3|dpKa%&_N<;@4NIw;Djwz$2V
z!K3%m*$=OE)m<P_V-hqc8D$58NlqD@ztp?_mec1Upu|sNm#=McRB`l;R`=TmFfm{x
ze4y?03%2DFQBR-2>hQxg&V@AyQ9|`Tmh|t_t_rtE?#ZQd4CFi|I^r3;Wb-;V?;|}Q
z9Zvb|@%uWsg)bG-FS+Dyqk*qk>jAw=+|!Vvb`s!XvgqSE?};qh&}>8x?12d`n~v_b
zWB8W3&6SmKVldvdKX{BD1CXgfk0XK9o-F?oxKI&(T$ZY@J|WB9va-w$iaMV@)C}4P
zL)Fc%X@ERJ6jqygHa7^q{G(rLZ{}QQV3H`70c)*K^7*wKkDS7mZ~+1n2{weE2Ziiu
zKYH(42Kh77*eCo*0)oUC8{$_#xE>tK6({fdxa#xGj&?}EMF$>O2XA@5s+(vE#Wg&x
z+Ks8PQ~yBG&Fz=X7l#~&*^!}5v;+aidHv_h-0KHihZ+a;6~28m_LKRx<L8M`L5IZ6
zs`mLMWW5r{`se#+l|h@_A#Tc=gGs+$-J~xe3L{ard|dLLGK0@IJ4&#=<gmh!^R8B#
z!*(<w{}e>IAoKitbnT4ARGI=fm|dFKVIVnD%gI^z%5L`?C0<#!axeF5NY~r;9hQ&C
z&pw~(>v}28CI_?7Wpbbs-1EzYEpXjM-{UU{+G&lW1L(hyiS`aQBt3Un`sOGenst(~
zr2+6p98OS4HiPzpyXieOKZgJ=202%{wwY(n=-xw`5FI*IYBW=Pp4_s*Ms*QLVR5+)
zsa!(xm7Z|Z+!A=%lic4-^mAdIc2OF&P-8g0-xM;QkoC-V3{wN{0O;W*y;$IGanZDi
zF!Ev^2-5KqyI%+FQFO+E!~FdkM&HMi9%%&)8iYjM?t8$ro>RS*DP~5KvR~u#LoveZ
z&>t2^&~PsGZ?#~27qIqr>PM`vnX2$vNNm)m${+Fx#xZR?4CcOTpXqIgZ-PrhV2n%O
z6*q*AaDGGb;Sp~*7It#Sg92H}v6ol&-(in7HWIXXlvXgL7zX7`5zMz9LJsu^G(b<e
ztrBDGVrX?SLHc%_Y@%^*BWmd#j`M9vwel*_meqUSqot}X0LV`&__LMS{l-4sxjq&7
z%QpURm7(~lUdBS|JC7|MVd%{sUDubut#`!tq`p&8wvoX^{i^~mg_hj=>5ZVHfm5ZH
zb$lb7-2<1%v-FQ=Q6Xnhrc)>)AyR8voTUSRG@=OM>i63OOa1DuEQ9Nbs$cn`2e%f}
zPjKL2zyGCE3ovzZ!PEC%x55a;aY$v$SafzVL1kv^&7a~cz=%z|DCHC+O1ZQ)+b1#)
zo^e;swEPxUKZ@7s<H)}&N|K@yibdNgjZ>8rwETU#E#>CIJV|7`qhd=2eSYf5ji*G`
z!-WgBQu@K*{F-6$D1@o%hGf)AaQfs@`ebKo+)b;t$XaGwjySC%<a!TkD{m|#f?Wk5
zi>ZOrMCYP~JerHtg%hZP=%$e@iuJwfjv!2`X30pa4^`YUs-}0|3KRISuxr>*qJK&s
zfdZ<us;>8%0KX``hX`Je*FH&&^P3&AtnYx<C<Ph9@X;e|pr4}#S%K38!^fFQp<f1u
zCb^}Koaz0J6QEM`Ay-G<*39Mw3PvXXCcGp}EuF+7Jbn*X1Pcc@0mK^P%h*ok_o(Wc
z1ea|2^1dIBjDzI8S+!k&fLxeUNv%E~WNqO$5u$MmxGaPdTEgFu`7r<>S{uX5as&z-
zk$_<ZKS*1nnfZ9nO4_O-1m}IJ^IeSv*QLKfP#1r3@GLCY=OP|`7Qd69pp|A=Zfa2K
zI1*7s_Vs6~-`eln=LG-2-rH|v(ZS<UCt7P9JT~HfhawG4KgE}bqqi=Vr6U)DebJsK
zy@lJq+q-w;2q~T)J|w7Z>BDxEK{dgq0{<lHYZXtqtq%ey&ddzY7&kk?*EE#$rF_R(
zxQdW5+nFNNy)Q<_cjJ?a7KXHS@<xlcpKC(;4huzFcL1xGoWgyPcy%tY5wf^vDGDSE
z-bW{aX4DMFAEPE;G=&phq6$3(3&r;tt;`*!p+|y$Q>kGn5Q~kl#-9q1{Q3y;YibUV
zi~2ZD_-L&OWOm|CN)o}l33^KOU@ioGtsYvfase>d?&Uhh7TyGWs}Zv-IejrvkQ6PQ
z;8j8^WA?X~`wdq6PUa<Xq+k-`yV(1J3C|Ajn*ip(0{=JB?sI|C@>%FbByUiTA(~!q
zGe7teQfF`(B4|@(Oapj6kc!cyqgJQQoFzA&`0CK%<}HEbrr|eTJUoYw&i@Rdpt(a$
zzILxyXo)d|wdSE;zvQm;W@}@K=FKAd22~j@{r-O7VXLp^v-FQ&ti;PAG?Zn2^T!tt
zM8LT%XM;w1I;YKCEe_ifxuqN{0;Wo*#c|ntGigG%g4kp)CPn?vR<RcWeCPK4df!zB
zBZ&0I-U1!jt&wGJsQddL!X_#r*6shMuna-~8p}|HDg)(>0t*Rq1bjv5@WpciBDpXY
zie#^GMQc@R85AkQ=Sozngv8T#OsBl)W8R4o$wTwsW4Mg&%?IC|@bxwL7|%NJrgz#k
z9+bD4rdHuiln~^wm6A$dyUUOm6JYO4;*<+Hc6T|Gx;q{=i+UQBdKva<@HxgJf0*8`
zB@o3_K7jK+l`+V@YzaZkGXgBHxBph7p|D+sEA+d<PCg94oYw%&K2wL|D!>#|F0>>^
z*SmNFN$%cF`KDj{7vMNvo4GXEY2ui^7iHd;F4p)WcNn;Qsn+x|i&yJh8a;kG5^|d2
z+qG#s?pi?a>bIFsH8bJCXc{E7yv~67DyTZyJ)Ms!D^7*OAgINL$9SWl^0I?V-VoS{
zl<U9oJw;`OSmoO}#oxD8a-{Fz5%^Jw{H5&xz~`H#3OQSingPFzI&~uzVDSU7aw=t;
zYCUCoA^z+gN$vUv$!W)SL+TB8<FsQ9tf0&J<%%haE*hIC6IM^f!f52FU@4soQ<VtF
zp_24@UBgwpTP$VHTs`S>XT1y(QSZHhVLrV8r|^~Ql+AO!bn5PYDhXJBnKTXrI$<Ra
zfgkv`2b%cq)EIVVcFJ+p-&0{2k?iRmP`4@imG<jo*_cFWPICAXM$p)b!eA~<HOX&K
zvcpj0m}AN1^O3Zm`Zun2%a&+N9Ihkag<hkiq5h(4;_F1gU947VXMP^5lH??-R(>IF
zmR<S`55|9a5+@f)ZE{TjJm1ZC`+^RMfwrrx<cz?NEe=GOn(r<^r*;^RS#jyHF3MWp
zVkTi`#<|1XLGT-J`JZIZvu+SYGhi7!>{;z?PpRbZlLeF%MIb%g$W1z%zE^}{T<1w_
z?bVz1mQRgy{|H*-ZxeE)l$+EPvJ(U;|7h!ix0<K2x-gO8vb=C|GM~%31U0R(q?io^
zcHCm%TLg%YauF_zi+7iQ1;x|w&~KKE-X`CCT;2vA62-VKF<(@ai>l6Vk^6Sxb3C&`
z-X<EM?kKRD-*QER)vHyc)<_AX(6jaHiNP_gzNE2&hwSG`on6B={v%NL#<Jt4mb@#`
zG3VS+f9WBim_}4~^%3*}9%<J?)_=|`H5dZb3fleXW^)YCOQL1Y?jy}M#T(h+FQEFN
zf}7$fsejWzot!o2g;V%6u#(0wkRs=g*rkvUJB-W`T<y})0NLd<<hA73r;(AVp=HgC
zAd&jai=ZsMlhZ#aeLE`}AYvU8pm1jT95$_tscCMj^T7#NFtTQl3j2}cTfp~-u=lwJ
zL%nWL&R<<!Nz3XaQR8;Yu7leX9IwwXAmThew5^c$H@$mh;G&c11Up0=l7IuFg4wgN
zKfV^*X_3?*t76aF_Hc*-F9${swAI)iTdFsSHqZWiG%Pucf77+Onz-4~%x!rT<2|C`
zr)ufQc`=gq?ahJE6$V)n9*$uYN@0_ah#dNhVk<wT_;l^$tMSe2-6bQ%EDXuf06_G?
zwT?&au-!=T&E~yUbRIbPAAH$_VFo?>9rCF`ulaC~bD{Pf!7uPW0{6s<yR$cXgXmF>
z=$MlA_vyL{gTXe}OT8=4SA3?V)B&C0p|)&&ZCil5s86J^m)Qxw)nVCZj-u0hLsE*3
zW8gr#0J4c_wA?Pn1>&Ha2#%vGwqkR@bWK)W6>ozP!A1RY!%RWfm|~?7litW*J(iFN
z`&w|ljr^eeEy{+g=@7c+xie8V8PwV<+23Pv(IjNPBzKUCiuGk%`t?cZYKcvAP`hpE
zaAID5<uM8Q_ENOr8P{t4@F~X%2eCvep@q$X;I~z<V*-1l>R#qvsIJ*IT~O?@F(`ua
zKXtwypl7t}<tu*=>$oEy`jKoEi;&b5goWgg{Phm&_^e;14J?PmkcdIc7t-e&2~Htv
zfWAGrFt!nzbnk=g6d|bw{eEfCu0B*^zi!5|F2oJUYL!WM-@q`e!cK~hF%(9=8R6NY
zQ<ida*;aL*{N-oqc7tDMG$0{d44Fg4JoAM-L1XR{+I^hNYdsd$DwLCi$Vo<bGXj3i
zV~opLWq#pIv4c<w&bX0~!RHue%T!$py3k^%jfho&x;B4MeN8@(T(Tejh-_T{z#Ta^
z2rzf(^Bq8r8juJ3w?b2HYNXUt2i+O1{OfNo0L)KCH+cUVMbnO9TLEzyQ8@RxFKxt!
znH=kwUFN~bsV4P}rWWE<JM%M#mqsqPLVy;7qxBO_?L4yo@K|?o+LKGZs0!s`1hV(U
z*u2*uLqmnKd=l9|3J<q+uFA{!ylZ*KZWIcyDM~?ik3EA3zF&tCt=n>~cq-xYl4EHJ
z`mAJmC59N{9CY7G2oB_hXjAh7zWST=ooqV%szIJIOxkAdKGp3vuFqr0w?2R2&4AV>
zIs6rBsl;$D%DE3e-GFT%GWUN!zgA*Kxp*`c{fbEd`y(8a-C}sy7B@sDko^GZdg?FD
zegG)+e=sulWb+Uu7W<jjwb965&92Ju!R2M@MqWdrt>%jKHn#vEHtcTPrJHuIuQjpr
z1!dn4HoF^iehUq*p|XG6<unLiZXFn&<ghvZ%FXeZx?}7zWy+i)DB;*Pj{9pL43l3e
zG1ZztJJiD>HjZ@%vnQD3DHR)t1pFt=Yqv4{N6Q&GqYLOhDG0=D$d>U|`&_0q^Vva_
z)ZnKehwp0<s=ff7^F=t691%$>L08q#O(dBZs&u~9!v(YoTudVXJ-@b$kPh^eb7$x6
z4d66$cZ(p6+LFwpWNze)`Ja^~<fU@burkUyb$R4DY7;{K2B6#FvKOx|zUpz#*k`ya
zpCjw{pcyx+pH9U!=7&>?%xtI*o&^&$qK57P>e`^-GBha+iIL!?Pf}qS0J8~wCUMBF
z))?&q&9WQ&9T76v1i5^C8_?b2rv_W1T?ziORkyQt3$L%>Q8JHXg{EJ(@z@3pG*dvJ
z9f3dww)BEKzv*mtHK_R_$lp<-1vz4tS|_{MB`foY5AkZ{286ql$S2t2>aog$TinF)
zgFouU9iwOg2e^YX#LTCGN>;G%0TBFiToy6wF=^~BCE2HR`u&-{OHH_>r8J4|^28iZ
zRNieL;+ul!lcTX?rZ7odVhs8*44f>mNS+N6+t&H@mKOB*t?Zl~&~VXJD>sNOI14m)
z6iqD!9nKk&96eSp%MuUi5+N;hDbs%~Y%@s9;XG&G1;lrNiv(DZ{vA*Mct04O-%6N4
zHf#<*e*7ZFs3mtzLu^Q0bt_1&&#>li$H>j)p%tZVmx2{d1_IO2puSo(Ii!+P*eL0^
z%nT_Tjbqa|H%ZOO2?|`!96n6E9DR}tRR3y0L2B)=0KMPNww@o}qrMIW9*e{LwEz!2
zXJ^qckJ$Q1d<V$>D5vA=9&+un%7w29s`&O;Vu2h98=VHh<PWRl<`v)W2mGh(VZxL>
z*+Sz0FYAE|BE<~{nc*gaxQrW0<oJ&>mr@S&e{3f3bQv33rRba8vx(u&QvYs=rApI9
z7LTbc3rL;@s->@z|8N#QKTd8pk7>bwBwusFqQN3w75~|7+Qi;KQbpmx^rI+xA7f3>
zV}m6SCR|N3;;^p=vBPj{f_KOJ%iqbY;%k2ezdl_G{IS7jADOYn^AkhQ3^s}#vhmoy
z4I~dVT;y6b>}_=gc-^kJO9SV~#hh>K*zY-(JhjG_UzL^*{t2o1DWdRFHEJeKldNe=
zk2vrobGIF+Uo?v{eN0nNYEmR`&en@C)tQ~Q;@IJGuTu8)3d#z7YE_inezob%n6}gm
zkg4gh_GW*cms5ZzI}LHo;IBQY|Dl)4C;&#ZlA`z%fC)b_%RYU_7gh&SAY}+(W+rI$
zI;Wsnm49HgmIpt{Lqz#(N4CN9wW%dkIf5I*WaB%YJBRnabLMPASbN7Z-2kFTgW)Eq
z#c(ZU-t-g@@(-S|R}ih|QW$R@J3b2ZsFO3Adtr6Z_&2bN;Ubz4(vX=pKj;s2tI0!W
zQSy=D%wB3w??b}m@)skJZ-+}czC^*&K<`V8jeo(aKrl7OpDAs>wX-F&Qlm<q-eonh
zxjqW+%~ipKg>jhV`6Vj#Y?Ad*F{sL;#%LM$0+7*WswF(YAD+0ZieDX%@zJ<pszCq}
zx)x#tnz|U6=?+E{KwuiVMmpVzu*!_tBVd!{ng8^Lb<8pGeOa2mQ~t>ZBVD{3U1e)u
z^qHSp@|P!7!Fm^ZpP8vE&g}zKXpM}$mD?Tv@LDYhslENM4Xw@e4Z2(sE}*9cZEq*8
z#3$t4Zq&Ys;E;x{^szawZMZ{Q#y2eecvc6(m2WsgZa+REf6&^lIPQr{F1D$w?Dv_;
zYk{L>i}i}_vEyf6q4SWtT7jAtwTwY##YG#EY?nY;UDEosK83_9by>h@5hx^_Tq~0x
z+?IyNP@vXJ($V$<JX@wUl<m8^xZe%XiIdg_RB6fW-&QGu|0ww|mQD7fdy0au(fRxH
zPsgA9j<R&9`chS`cJwqbiFb!G&t2~AZ0;v~D4oyr$>&QrG{EYklm)dzH=!XiDsV|-
zfGcZA-2ZofZa!5qxF{shB`X<!6X9#8Yh71b1T;WQHQ45UpxUYMD&W?{Sf&$BXjfcC
zY%G*NoLAfLGrRkN_z^p92FE(s#(N^T%hTZ`vVeXdA+U7&A`<!4RW13LuDa-M=~Wx9
zb+fLO<#yMvtqb^TgJp4?L<r^GK5;Y3QTO0Z1;&ff9YjroXg=Mc^ncn10Hj;CU3t%x
zbswk%m`H5GJ7muX)ZQh7bHa2;ZDi-lBqF9j^y7*;@8Hb$9vuIml_2%VPcNeS9<;NW
zw{h6899;Ln_Hb3W(*GxTmT_T-ylp{}=_5&R?I)e20Mek8R|3l~OMcqf-LV=Myzjkj
z?HA9RmF-)sZc&2NFg@3l9{WT@j5H%#0~I1&yL`oq$}mW|;9~#-Sh$f+dH0RIvBzNT
zgZI9r@`D@eMZRwM?QmUfv2|CV>@Uyv_c^+}qFsl;IHb%Y0?d>!Z3L#w-J&&+OGmET
z>{vHD(3|3tTda+6Mu4Z#z-T2-&@z~nbL;7JNa(869Yc!!BRkXAPBD;9I`Ho!D6F~d
zru*HoEI(hE&9R2$(@T8Daff+u@_^SYdeRo`d^~)qTB!l+l!MMCPboooAVNF8Ppov%
zN%os3T-yF>r-798PSa(@Yf?q<?yYh5UD%np5w7(a&{<YNp|(|FEN8FPo-slBMzA13
z0G7Cr`k?P+!sy(mw$}^7VIBU;iMIZAx|^FcG1|1Au#H7y$CiY?`%zkh+9@qv7CGT~
zS2Wy<W<O^N3sy&m8y{S(CGNG_b=fTaZ63-37+EUKLqfATd0wFe4#<7*U%g7Z-9sZ^
zJp9E}i9wQ4$hv$~<ePoaUy8>(p&xl?i}-9KYu|Slir{knoBL(0#W5I{P6DXmsz{ku
zA`ds0E)T@pxN&q6S9+qZdsLpnW!{i>Dh9Z7seq42AOGHoblE2T%GsfH$EQ6bfuj~*
zW_Ed7hc$|u^tkMb<Tc2Ff@lz)#=d=tfL6h^J_aE27%c)@x&!@^`V_G{-}|$1zq9a}
zEr`&$iu5I?z>pSd1Bht$m`x9&ctDf4RmaaW&k8yo7-HkiA!A=Pc`nb&rXNgjk6edG
zstG3Oepq^E@S(B(8~vBK)1c(yW+~b{!jRq6?-z-A`uVF_e8#$T`KUjdNQB?lbToz=
zk(P{dEzm5AE7)C5X4bsu)7zP`7h5{f;L_2enh~jkP7<Oo#O($*lERzlHta6Lql?6L
zK6BWd1g~Az7hm~vEUIn)+>+!)ALX0F@U(BXW_5GRg3?yL*_$rTh=yzxxX=o?^5uIu
z8J)M2WgXu=yT}j}(lB)a5);4#Z`FNI{h1yr@M$F-(G}!jz*rbMKa<tHaXdWk``+{5
z(&urzmufottQRlY@9-YSy5vLR5O(U8TN#?R+}L&jH}uh20-A=@Xw8frGl>6qk6qb)
zMk%s>VqW&$xP1rYDti)YDey3|o__9Vvp-(?l86b?Czz#sC}uPM(1Miy9|3*E9MiMd
zge8;<XdGnNybvCo_sK!H_^aoKSxh8hztAad%0G6Eu0B^%Jy_r{zwzIE_N!&@B%(6w
zcTfPDT>Fzv{SU;PeK=g3I{(*dYL_;AXs0iNfrER#-|<i+?1?;S8dBNxx<iysgN5Xg
z=AA}R{*4*i0sZUEWP3KP3y_~~2mDjO&k3E%oAdK{-yP4NMRM`vhhUth%TYD)al|=!
z+qd&o$hYsESsoqHMvPr;fvdQ#)cN;{T#);OFaqnY<l}H(J_Q*Y^-YuI)!kdfEw>31
zEJrw2tXVT|$2}n~5+%1XqYtVd7Qur=rMq(67ej@A@S@1iHv;1&i|t&X@!PY5?6_*S
zv`ub26FIqaRTv=%FQBuuiQ+njT4soa<Z30I|A(j5PL1Zg2D78!&6deGH`6J@a*SR#
zU7vyBR;KC?04_@lBN*>xw_tiq3yht%9CFv`S3XJ{-e%v2(1Jo9n`dGAT`_G1UYO?8
zoS!ibZ_xQ1(#3PPO}E!nGmg6b8Q9r1m`Vc8`?SF7Ty0VnD^r&_T<t>ayrT)Q`{tfl
z2N;dVW=Vx;Pf*-`ZFUA9VL?g4&h-2^zAgGOWPX}}{l@SjGAbSp23kmfBEroDbQN4H
z*^tN@3=oROhakmYOtp>iI?AJVBd_XUYCV|t5;8rImc}w<`UkipMccH%twda3wzSA0
zvM>-Q29ii5tWE>D^0?0tWGv2{@`F~+u}SL8<B@>A@(yG?K9z?#PKQ#$FS~=}?n%=_
zZ~NV>*3aYL%RXEpK3<AB+Ss@@2tXKTIpTu8Kfy6M(}X1WazXVT0=@E`Zd_4}j^UnF
zRBt~VuX{ix-k~(|&Q66kl9$-Bpg}ne{O6sVS8JNog!jHM4ewqbPWUUE_&+2WqAj|?
zeLhl>8Pti(5G#HZYvHEe1az5O+4wI(F7>E@-ibJ0EvaBg?1JiVOs|D=dXB7)&)Sz#
zeYm_n&0|Z>rw5w(1y4Gq9Fy~s1z^OC+VxISfDrM>Ba4?R3!dZr6ff2o%OLlt<{^%o
zr>@rf_nq+}E?Q}ARs~NG>mTgpK0$g|8FXPIS>EJzhE>jl{7d@gCQAPG6z0zJ)UJPC
zQo82u^2hrRzrk5NQ9k-9LS)+*kgr)4%zqSKxo5s5A&mXvwfO0zJ~uHq>#T=ky3<!{
zduzOY(Mg*wfQ!6*Ayyw9Cs*7awAu}U>+pKNaPCTtYjK`u=)Aie_#-TEslo4=6}f*-
zWWB13J!QuopE5GN85XhNe|n_}F#b#+`#|*$;Ll+oR?Fi9d@L}p;?ZUVX%^>kq8*g0
zT8}5AO3u%Xglsjo`VweRuExFN?v6|N29$6NB9BPpoKO`#R5{cCg1~LvfNH0s*(u<T
zj8#%vHc0yCmxB5#SB;}TQIY!=2pSRp#Plf(6z-E%lb_GNxDmW*U7%1+xNT@~<}@b~
zI!bI6v^l`x!c}B$ciO^}WGEKk_9;sLEhMi?0q+Sq+`^CK5bWNW%Ih4jziq@!yQ94u
z);c+4U&~oCR~XPq(1-gRI#6BSx$$f_qh+_!BCwBQA`9s;m8=a7UF>^na<*D@@74T)
zO(=>1uXJ`!<aQr(I7McZ%+;#*QjZ5UZg=8p`w^2NMb*YQ)F{T;PJ%HQUJ$=B(8|5f
z7StJ;pcS~|Ui=QT&U5Xhj0#^gf_x(Ev25Frsau~!QbkxscBNwlUT*RF52z20NAxCv
z%Qa1AeYBJZlV=3Irh70NPugRWzEfNKWul(aTgzYA&BWpMxkDR9>`|k9j`>!8TfZLt
zu4-<J&l#KOOGf1kkeu>$e#Ez&FNBv=`Yr2Bj;)Qlud=qN($3Xeh(DcdqAhN2VdvNU
z<uq|1Zbkd;8v9Ekq!hgk$ZZ(1cKQyc<oGW{V}1j-u3Ku8k^ih8yzl)ml+RHLF{&_>
zG4={O>`l{h$xHbG3cUdK;|R{u-+<5}c7V?iLDp3_bqgn|YBS^NjFp5xzUZ}E4y}w$
z5Dra^7EXo-JQmj2rh$rJEWILfVr3#-VG$dcNG6qg7Bv22=;1`n_6)$~3#1O`_-w0l
zFeal<?LR?B6PNDWXr1Y4nL~NJKZCs%P^Tbi8}pmyC|atl-W*5o`G+6(>}4)Z3do6g
z@$Ql3a!rk+5Tj$tpc=R^=`GLw4#OCO{n;we&I!dWPZ_@gFTJ_*Z9Oe|H#@TaheOf(
z*~)EvW<u~SYi{Q*n{Wyv=h59ZXUe%oe5{uZiRV-HAtLGzKAekykEm}FV-NJSTEKaW
zYpG1GQ?M+{7F{|UrO^+kgr@#r@IJ?r*9g{WHg)kWL*w2_UTcT@0zo`U9Geg6tx_aT
z+X5VnkPPYDH4lhIuSp6sRAu%7u0l7ker=yz<ckguJipROv@IaJeM9hVCw~hkqtCc$
z($6satoJp1f0AMO>c{z6gW-|Y1-4=2!BV!cPC<T`oX%R4bdt9!+`VTxh#=s}iZWzQ
zoNLfJt#_J#m2MW2h>Bd!!SBOOsXZ)N>fBXt0?7c2!HJ5mt(nY0O^l4LjONgF_0_n=
zvrE;HIvb%+h8x>nqu@3<%ut`(tT4d@>po-WiJD2?Nk@JO2t<n2mO)}yUVIA0xi}Z;
zVe?2ErcHVfOa?`c7Da~pxjV4G$2a8On=9;AKc#^^u?wge?=q!c&aV_%h1}+au={Z-
zm~asX_HmgR#}Utc+;WkOcUPug%o`f--4G=V$7t9S-3-X8N96bHnym|5Ijy8?X<_TY
zQ#&5g2~w_J*2S$($rmw60fi{P;rriL=W7zOj$K+#q{cRmA5Tuz2$1GVuh6!$BRBc`
z^5!LBRBcs!TqTQcM1ghG+1E5qtN5(fj}GzXbLl>C5<%Sux6y?CPJM$d98xm2KPb&!
z>)P*7$2MiDcv0pV;rC;RGq*4d4NXY5rVHe|MbaOH6GP*h1zMo$7};+eVkFd@a31zh
zL(LRiG7pv@af*T6nENuzT<kBll(GasmL~4ss<p*Ps5+gCO|uomUNX2cO6MpUZy%pP
zNt~Lo<x)JWB|0B=nfdJw#bigx5Evg`AaXvPT_r1naqv})V{#c8671Q)0bHVXkCS;%
zwX@D^@vZJ|^&E9OZTG6g=N<@pYvmNxl5N8&<>GYIQarPvBHaUr$HTSw=RrOnJk(NI
zQwIB6rbP&U=i!>SL_pyxKWeoX9#Sa9!GG#1*VbW}s_mYV9#0IXQ4&P1O=OUFcQv(I
ziTdR$bw?II?*5Z*LUK=zEhkT0lR@$VDITd!bPF-LV`4!YvM${(myhfPS9pX~3`2q~
zPEeIRVF>a<ubz{?{*BwXn?v(fjcm>{F*i7-XGODzO_R-XgX1d768f+<@BzhxU++#9
z#v88eC0-wQ<wTCR06nYaN8x%3IiW?ae(CRWBIdQ7ac&{MRpB!R`LEbj3g7+_@xfl(
z#(0Wi)NEmmvd&|Uv7Yj#ZVc$mX{4ZxiT&1>qQoUwoTt4uR#G~^#a-FQjADPBzcq<{
z;J8e5RJ%KHTatiSoyi^P8TdQv8)UIqd)%-OsgNTQJec=>pgmDvL?VZc-j4iBqEf=k
zISwT&#veJ+lKZv#Hjm3--0g@94sejYrIqd2^J68M?6UzCfVm#(DMDct@#I8_ZayGh
zDFjnUBI3f^L~OjxzHcP>GQ_hiTqBOSxW}T0sK5C6xi)QnG2l4lS$}h6l+Is~o?S2h
zfhwjrfL5W_QLW-|BGA`P;7QOO^fhX^ox&rg!kv(@`LHdpjy0-LVnyGqMtJyA11fw@
z9>ox>fusM`rXVvQO6go{aYMHPl@zQzd)g#rTXgg@<z-pA?`o>6hWlCR)=QQ!@)557
z)~%plHc?5{@fA<*f!Si}-7U}?`Mr5hu<Cc*FXALT`aG}YFEEXPeX5S9p0*J!YjN0s
zrVn|gA^h8O?Re>42vPj$DfW8c5_y)dgIH{bpY7)l>TL||0FK3|NdZcLmbG{PK=+rd
z^*$*)_aU{eSNYq_C5<ysoKxC)6W<%Y-l*)gO_!-RpBXzlhf=att*iB9#gEsvTH-9m
z)82C=M#t+tIaITpjCfPFF+iRMaUT0tqR_$2N9qF-*qT~3@L#Q~Ob|#);&x=|$(X4Y
zRlK?J-^dF6^YIambWTc_k|Rr%BX1%x&H+83s$bACwQWvSWaw#HC&j<{-P94q$HB;!
zw5Ec^K3u3FWc^cjn)7H5dS;-xllM5!6{kb6NRjNLI1pD?$-6X7xhC?vZ#4ai{<zKY
zN9vm<V6!)!QB0M2kZR=_57rpZlaE9z#A+AM5WmI`OEPd%`&ONbV|JeVP6wwfqL1{Y
zvvB9FoKyihZ2iMgBi`DcZpK37F7j98k{YD8DKYQvh@e)c^;ek}EpRJ3r8$O|XKRt;
zOz)@K{Im3k=72~m$qqB5<hq(89G;|qS(VHAjn3n1S~Oj)XHHUJf~cF2BaWG)IDzxH
zK?jSBA8eQJHSnmzA^hWv-IS?GqNJfhR``~~XF`Tfo{dP8R;74Bajf(Ly1C;Xu7oZX
zQUP5>n*6U_P@KxcREKw;>kiv4DMN~#9D0jJX(m=x#eIKp4kOqgy7Li|caKtoij$8z
z@(DIg0{M<s>9gq-bCjS7qUZY1x})%BWX@ysYIiXdMNSxL{3$i!V_*Kewk?wj*X5QQ
z7Xf4Gy4!8OS@<N^Qu`|DGr^|{b`f|9Z`p<7O!!2{l;wnXK55rJVC`mnkoYP_XgD&j
zfPtMhu^f^HSJa12XL0VG3E`jQQnh|%XV(;>^BR>i!1GoibZt_hBHryf>W&f{@w85|
zeiOb)A{Zh?eG6jImmh&D^E!5@oC|lm?s`ZR#S^y{mk#G63hYmR{HF3~@iMC*66~x;
z7Q71;7=;J(H5E%8_4(NOf0_FlEko`#$0n5o{odJ;gkE+0OH%~MXM1=2<sIKsgdwXl
zo95J;#a?p@)v!yA%-y{y!{@p`gIi`|b~{B_?$xwqVGVMv-0fV<-$Ii=wGKA%GmS}i
zHCTQbLjKTxU-8l;@DnVpL%@mm$fSmnq%L2b#yQe@nxg4ZdpY6>eA>RpHQnLs@jP6_
z2@hl_v=jiokQ!PXnjmsb6Yg}Hj>wt?THvcbN(qsV8h*FeG=B7FE=FaQjp+T$arFo3
z!Zv=>xJNHc#$CJC%eZ_A{l94V`6%-3P6&t99*eHNo~D=qzZ$z_&%PgW_8Mne9R8-&
z{`aitDrA}hVM6|N@4VNQJ8tSF@YtR2GqVozIH~n|c}XXBcnv$&%n4=T6QNc2X^KV@
z{SO2Ug^d(T#qnDM!g5~I!(DqcLV~j<W=&ajQ?sc)(UeQ+j(cTX4|{FNf7xp~SbPol
zqN7XeGV)5x6>>_z>m>%)Dg>LYKCw(=l8^+Rnk>m2h~+SHz&9JN8;FVAT4b+(={E~J
z>rhH`{3s6}QB(iibko7;=QH0h@NmA9RR|}z0ib(L^hpSY+N5^U-iVSPrq(6Zz4GjA
z)oz|_-TGySMOeeWKt;@Dm9=t6vvcX^Jh)KCwLtiAb3&v?IMQjlet_Z&9PzrA__WiA
z{C8<t;jdvWAk61hwph=POc{45S~;o7kICXyWWC?2Y@3Vop<&>kq=+>U^sMl6?i8CI
zp-d6JUF;UsN+>--y>-5_6+y1`l~2a+)fe2Fmk%s!)AXMCk1J2&-Ls>mO4qYEbxjbT
z>Kl8Sx!~#SU7z&{{30VXR!F@G%!4|E08hDC83T_E>@Fo9@7|iLa95UoQ|V#7#9p>b
zQ#<gcWPVJAu{AVWLfFo^{?$~8TAs`+R_VTNp-sQSyRaP1-1|PAbzGjnO#(Zqy|&@j
zOs7Z{<}S6{i*-4K@1Iu!{St$($Vu-%k-9>7t6Rd}PRwkEH6Ujnk5cwA{1T1))<<zV
zs{MZ1Je}_AaoU=t?X&bh+>h&=oMB?@?ORLie0wG(dBM2L^}-I*hS{-Sw%8$plDuUb
zGx3Pj`WQ>Ev?S$%8N)Gm3*a%Ebm7}AS@h<0R_jdxD+;sjj<i)PS1`M~ccoj9j&mD5
z@@5WJ(fCcbALJCkc2C@1CpZ!AB<9eAh~JJ27qq!`=OOchcb{0)#FU^svvmDj$%$%!
z=1!_g=e5^1=5K3|&Q~ZpgaNDsqU9DAUmH2;&-{D=BgO6A3o8&6s3&#!+a%x3I%?TP
zShKocU?I3`_<nAvhR0@~&5Zl_pjSA@#zvS~xaxQ!;!|X6m~&z=Cqi6kcs#vav^oB*
z)c6zc0glWTaA|FzV0!I%RQ%|D7=BI6pkf0T10nm{gM@mrithL;&97fG{7#oQmtsC8
zX>hZS3$4}~c2DP|yk}A0jiNyr>RB(~j+zSBS5BQ&rTtL5bzpeA93*_(8QbaXoo97T
z(oImGR?3vTbMN3+2;49+Wy#cLZ$9(5eAr~4tj$c2quz8jqW=-zJ#OTP%{zZ-%cH`e
z5eWH%qQZw$minauhDNE^uR@CPmctSr<JHno-lbz$gMWxG$Ph?kghtmtd14r;|KYk2
z!*4s?CUbV~Cy*&JcS*MJ;fhP4H|r(iDMuCrA#9%a5`TH@X3ebWK#}8p)>$MlI;IVA
zogd5)aJH>7E&9+|fPXMW^yi5W35P}<q4~lT?SK9kfDi)zC49Fgq$YnQ?O2aHNRx*1
z!FB+W$RMGAS3D>Mw0`?{87^JbfV_%y`LQ^?bR#Y~+Cp>StAYTr*h;ABQBpq}(N9m$
zoS_*X_{Vy(2lzD)h^UPD!{4e`>3#^DCn>w~m`lFNTT*Z8IdzU^yPptn4QksB<G9*R
zs28DOO+u|vwKqPE`XF;H{rxd6qR4u9Mzn0Qd+^a?!n9g{f9r2nX3lRLj%4TS>l7`M
zr>v(5>hm{JIceXw1=(<W58fl*otB}s6SAwyY7pCzx+I??sBw=CZvcd%<E|{Fh$NdY
z5_6SPFTBLOtUKi2PLY<<L1Y_)AGND5*XzX>f*-w~3M%|PZW?B9=N@2IFs<V3ds5+$
zu<Bh%*aF{#8ye!n<<uXT<%_{g=wct*<~uvq*SHP7W7Hsf`{mL)%WWsJ+_bLb;v2et
zu!<qHMqSGM$W<8H`)(hA0TN9Y0!JWid;G7Z>eoIC?)}wmM^FQ;5o{fzBJ<xjhG-Mk
zUj*aqyrtKj#@8k>Rg@lWZ-U^Ti5as*8DYgvqzwNcF@=?d<*$41lXOS_A?ZlyxQ>x{
zBkf|X@9=3z1zN4w7syvy-*MTH;`GFN;<M8Ee<H_8WNU?=SIdibv0#7hdipL=#I89R
znukmJn+&G-iST{F;8(?=s~=I?2_P^mB1_;rf+^+^+c>&jK=%#X&2(Lhov{h8uD>P0
z-X7MiGvqJS-P_3<$)e}1q1{7D)GECGpoL|Jz+YU6xEe8;VuR#%GaK)HRA&cH41N4;
z(lmcO)F4nuCxpDo!M_!xqntYQor;lzkvuT4Xn)oIZ45)NNJXW7*YE3E`|Hp*LyF0J
z?l}+7zcgHby-0k*I}AMv=Ot&-Kx#g%HJH&W?AZ}@PZYOdmq>93U^y--Us{yWa(3wP
zlTDRC(ND7ecGKm=BC#1{UHV^Y503Um`EaDS&P&LZMh!uT$yf!W#nhv@8ZG`;UziNu
z@^|Jd)vM*xd9$3QYdn9zAm}+r-J6^s;@tfr`x-UCyL5&nlY(zG7^ytIqROyZA=STO
zKR#@p`o;%nAWuC0GwV5CA{jZ*7T`U_F3)NgIo-kwwj1j9H`Mrj%6iRz4Qij`Wn1(V
zWZ3eLaM^vKj@ygBFO|IVT)S?-*ctk~xlq%5a7&=+MY}|s#+-nicE(kPSkb6H(-h5e
zG$EtY-a2XK*}?y#=`7ru@cy^22*N}_0Y%!PBn2eL5R@)~4<#`WVT5!J#t1>ARYt=A
zDV3Cxj!`mVbjJpaj?tr^`TbqbU$AS}&dzzqeZOukMY{@wo6{?Kg7a^Av+0lZD}9@+
zm&m*EUWZ?i5e*dz(bDT{ieOolVR+!gFV4$^pBXF=sjK~TKH^bn52QS7m}9R=WN@wg
zMMo}Y#{(BTRhORdW2%A-zpGR#vE|m`*Rh`HfulvjC*_jRGfrffTDW$)=z*@btmtoS
zIt|opXpz%<yCY99LWO^g=*W`Ky^3Y(UGiw1@UTL;04u_*y!=UB8F})q0tZ74w?bG>
zrcCX4W|h&Qq($l-Zw>`BU-bVO05<4?=QX%CN_%yO{;5<H$s0aPH!<-md=;7My|aPW
z_=|5)E%w%HX!qz&O9BPAY9!OY`ttb=H|Ri%z9@!_W6HqN$aWQ%vT;Gh(w$a``y*P^
z7I%?M{=z1^n2qd@j0d?|VJvz7{@XE`GM^?Jz|Jn;;1mUJc+N~+GCRI(nOTD}K6i)t
zt~QqY%XFa&zR>8v>hsNPJ-8dfeJj>djrS?gF_6F&sxr~RFjqu;<6KlURWBGgWpO?{
z{VO$sd~KleG{1rBGp;QV9qo-aX?)LzeGQHOPi=nO?b0G6FNgf~Djld59T@rhirmkJ
zzG1h}X|h5iJB#-YsyKRO0EE|7#PVIFa=+j3Sn1HcHC#Q|nz?eD|Jbmoc;e5NNQYQc
zEzyv*xMeY;La|ghSzwTJB!SHKuwN{zs4O{)K=jNUwdVhY@aAvNmW@kOW4iU05wi7(
z{$U1W)!k(Wi2p0smB~r>?A_N)P~6!PKD_iG@QTlwF@<4TnjWEpiKhQn*{MmnhU>dk
zpqAe*sCTaTMFQc83l??w-Uhb|=*W%mRNH{Axe#7RK%H^2_KaEPw{)&lv(dvl2Ydme
zVhr5HJ<<SV{H1-!tH9eA$WM@W__u-4B;_9=fq1HXb<P{k_{w-ge&OJHyl)v~H`pav
zpF5$rPinq(DgL88>DS0VPPxRN=-S1&wNk_iIHw0-Im|DfCS%PfeF>XY;6p=>^Ds5X
z5Cxen^JY<n_?7Gohw-DHMI*lw;kNeBOBec)`WH}s+@yV2TbO>+QShgxEt6k79PRcz
z+&tgLa&Mn&f|A8UW7JgR<+#l=ni;}xGRZ?LQ$qQ3d^)uYPq+;btYOG>oB%xxHBRox
z#!`Z{(%h9leC~$g0-gL%cT)uZx4MMya^%sFkKGEsR9X*Pv(b`z`i7qS)K`6!v`nR*
zSK=iMaWmdQNQnc47xoon=fnGk`vUPAOrZPw49eJts$6oxD7k`O0?Q*lSZ)zies>pO
z5~{47f>0n6d^8`?0-EhSFfbdcp`!zQR3YykD7n4G17&FXuP%4z0^|}X-1p8#J%<fG
zgxP;4tn}&4mU|F`)%dd)9swv&YvRJo2Qk!Ft1@;Gu1;QYEkfFjRQc&=R5j#2{4a@?
z3aEy;M*(|xzFTRYd`VJNoGb`>`zR5_ppScU_+7PUQJz5`)8EMdfOj}Bk{;hHN72`=
zI1K?u!q@Me?!)bNck|`58O(-{mYx_dCM~=s76|-pYaME*pkUb%sZ14=U%^U3+f8W@
z7AdYHbcPG3by*sd%Y^lm<R4{wqzmoon>UAn>VE6I#dWoWV&Qh&I9ho4eLR=pYO%52
z!JyZzKd(A@b;-TsZG4Z-4F3S03LUn_otYV?Yq71xaeLL|y2Www53T2Nq&M(;4Si3d
zmb2P@RVd5=no5YEL`UjusR`4t@&4`PUaOq$I9=2`&->tc*qSaN4BUDYKt*HoosJXO
z4mh4yve%SrKR&jD(%R|&BJR0QB6DM)+TBUw*_Z;6J!$xmVatpT=MD(Zm9j@)QY?k&
z|JC?Ag5K#dMm8$qkHosPYm<))*F=4UO=KH$=0Pq`!e}f2!r;JYa;W$!(EUe2`N-_}
z4c?PEF4V}r2KZEO;+G6y#aJ>ch8uFbTk4(rt$gnL1k2lfVt2OZj9*T6H2*fpq^0c?
zioe0=ilIy_K@<ZQ!{G@)S9grh56fzgzl)-GzxZFMH4aKFr}(-Oa=hQevXcuMfGTcm
zNJ5;;f!~}WryTB-H2lJqg%-+qRI6eo@v3bA4;Ql|%}NSu&ZNro|24BTA6w8*V`<u^
zc}a+#?k8WEa+!WINpfYXIT^k_W%?-&UGr0m>Ip=(M>r|9YseOwtVZ|<F6$k=a|6V4
zE51kf2A+S`VF7W4TvPeW^+hcnwy^s;SxssVRp5&w`3qjbW62!2eCO0K-8v8BjgqfQ
z2!(7oG*9K7=mk{Hm7Qu@!#UoJGZPr90z0T2pej8dxht>6WbsPuunaT$R9Uv>rz=+r
zJrXcYQg0-bEB0)%%u1mf8>T4o#vLBmsK{2J$Zy!lK1%)mJ}&})t$^ljaL@ibWi(DJ
zS%Q)W1(XT3$F<5g6vprA&s$tqf7|5Qnul?8e|&Y)`Xf2z3_t^Ph=LjIEpcRS^97Q7
zk_jk)3WxueKep_6d3`rUN}Uo1mN}cl$3-;zGd+^B>HA&13Ffar@)g9g$(Y&IGu@xN
zSV5#{yzO72%Ii*|{-750Xl^iZeqX}={;>I>jx@KYqRP|(g=ZLq|226aZeM6C5Cr48
z+C}3GCZ}#PHD6X{5>}h#=KpCp?tL)4Z8%e<A`UCjxd7^nJ^HEdO4~>T=u9)lBE6j-
z8?ZkAjUTo}i_AfVPJ^~?omx*R*mdsMhhKHTtcW9$yLZ_t4UP$;wx<pDKIku99!oPh
zRVCWY=q19d-Ri5krEdnOykkvwUP{lB{4Y_$uw(%-lLP?2X>&9msnxiZuZ6l5cu|w3
zDaA2d{SFIeo%^R-(u*JB*967J$)B&r?LA{{1X3FF^*pWFQb9Omws|gxMsT8qc}gQF
z<H6|93@$Is%;g|>(hCd4k_B$P0v$tXVevWkK6`4ynNh<@<$Wk)`@Gpu^0(YZCfU99
zh+ximMjWZm*+E=SBmAPt<~Q$T1Evh%QTph3VcfPp-P+LdH}C%qU`|!A$4^F2h5Zqv
zaXVEJ1(`*)#RTJ9MOAW-r+eSF!F3ju0G<zTZLBOH6kNhfRlM-Q+-w(v=xcpE;^A?`
zyR-zW7jf8k2_br~e|+i3gM`i**xZG+Q!Rp2fp1X&al73MQk81;fJd`olzDA8<3=Wx
z%EIyqeLkE0uEI~X)PaoSj&M4+J>BnNtqLi=I26Q}nq%lSvFBOksoSa}Q?GD}2Yop^
ztx_5HO9rxHQTn$5E+#2^e|4ZdQYFrXfgmrN??Ly1?maYC>w>(E1d~1ApyIdgZ3le5
zo4osu9{sC%6mn)${QddYs*qkyL-qlj9t1;oh1M(iSV!O%%!{RSfz{J5Tck$~aMraJ
z-7;BFnGBt%ybBEhz}<CL6B5gI=(pwPp9Y-*B!60ce^t|1Vz?MOU!8KSvfBaP6K)+V
zZ{2gx4%UG!QaUA-<*NA+U4K5A`4BJi{Pp$aN*^bIgV0H^i%l%OqT6jbvOsjaHqy*D
zm||Rh{Fc<2N@`O&$eHWdmqysmjcC%ZRv`aCOvi;m?C7!o5rQXt?k8stXFR7|Y&M^O
z&5_l1m~5`5?0Br!59yjaau;m0P@&(hgxets>v!t-4|h4kK{BV-tLsUv6mh6CKi}%T
z1FP&m5^&<mV%yS}1k06c$(Fa}xA*}{hG%y>?fF~p9dWB@B355@vR~)~8X4jEsACOV
z2CXuzV7eSVj8;=X4xl6S%nb|`VqhPNDEOgaeY<MCM0mp15-18U_eH<Y6)(!g-iOKg
z8{R@ng9i1KiHDqEZn8=RYd5<GoNN+yg>gNRjUMuQ(u&h&1Lf;;+LfhnN4D^&2<1ee
zs}A&hh94|**m?tgk}m%9<Eh5R&?o7&XtyD|Qf^-yevi`}bWYzH&cOeORt~Q!_CG6=
z)g5BvNWB%BGmn-uo8W)h(j$RNXX0UM#&BdSpMsEI6fEy9<I~PBu8{reJA-=MB)QG$
zP8J`DYg6Q77zq$U=-wh9ww(9q1Y?Z$LYtd}?CO^8ZX-Eup5h3pK632GBr1luRScGS
zYIpF91d!gBDs@30LjZx1R|{r*X*#X+`9Lyzwp2Gh{RhJaYlJ)SCo^0bFvLUdTFp0M
z`clPn^#1`G5zt#Q9Tq2y9BWhwhMWMO%VY6b)4X^NC)<3(PAWgcS6heq;&;*kS-#j!
z08eQ79m~eAH)`6mJ1DH}M=9A4Z_`2(aXcqW_X@UlKPu@AUo^zapHTgodWd5U5W?5~
zIknZJbVB(v2H3v}A0Rcj3z}AyN+DU=3fW7~O=&UvO`>C+-I?WWE;<z}?B;GLq8d-;
zsjs&H3-2ekhOpzc4^^LooHtcLJg)jdREK<Gux^)f;pA4QIW^YHl|{onc_6owZ!hT%
z#828PJ{M)S{r0uoCa&)M6ftUJ{aMCsZ^<G>$Z8+7shQ=s%#CwDt!-HNW@PG)q`>ov
zhkU%+SO^Iu+C0ltW8L!k=}y3F#*aYE(b`BXxeCVoD$f`tYef|ICRlwyjX#a*Zk~fm
z+9ef*Nm&;Is1J>+91OzjeYhE-TM@S5y(S$i(LSBxi!sl|;+ctGbQ+}(Dg$g%2-aV%
ziiT)4y}v^`%uJ6P5Mgv)PMKrC@bYL5o7ss1%z+bVf;S1!sP=H$TKmJ8*QE!e<D$z^
zW!v8hF{JC+PIF+)B)4ib-&Bh5+XsavqP|JF8GM*LxKL%K>|**U1eRpM#={Pjw)Ulc
zd#_Z8__}SWI3oH)bopu}g@?_2>)rp&!yXa><pSCea_a1)CBI?NON*I{sNo)IoGNh#
zzooRT#F~^;dv*8bsqcwY<=AMph8p+h8Y)KIi-W<$*oRhJAHQSqJNqh#+!~~$70FN@
z@r0<Ww{$PbePlRbJ<VOeln4+Gfm~r#KxEIZfBo`Hz*s#KXm{r{*^Zu9Iz(py!~N$+
zIPzT0@m=Ug)<+sC5lJ`Y@5Z*9=!O@IqnAQ7)-8e|SE^liU#8ss-?WmONdH(Gf5-jA
zaJ}LeQc4^smdH|uqv{&!e)ZQ|#PrC=PspxkMGq25XOom=-;w6t7oj6}Lhi`#N8ln?
zVg4qu_D$IM(<g$Iy?KxHiKT`sBw#=m+ELG@jhCmGfBhQK(sY|N`Q~es`^#N|rJ(kQ
zXpx86?y1!a3BeAbVNhkW1^qdg^bnV`SK4au@9K`cdDVE&B3|rkw_I}Q9620A>Ew34
zq-1K^pU8$0vz<0jJ!Y_|&#J03FhnN}USZ1~Ji@Cj@{^_OtmzW@G#7w8`DfWF2RQvX
zDLmE6$ZSPySR84w!^Cn}aPh`MY%IdGXr6mZ{d^>a(~r^SVz|PKkTz-{biTLi!bV_|
zh8=kQYN;YBh~axaaVA~Tcjy80QY+MZe?<*<5m;E}==L0vgp!$`NL$DFGdO~Js63p+
zZiy+495M6nf^2+A+G^w?K?*bT{z<`Y^&)^qf1)s^KlxXwY|s$%y=v^{JfqVE7x3Az
z9Mfh4#`yR>iXPe8Do05@=2`Ru#@;qjs2T8V(UI$e{A&2kI&L*hvmb5$9AAELZVk(K
znudlu=NK^q9_w#TMmUOjv-S7xdK?#7ndkaDBt(;eOnjXiUuoAPH~aC0Tx#Wie5rrl
zlYbPG(++=x#2sUP@?~(YMe1GUVo?!~YC6$Ejv<X#yjibJs~($8u&&$asI7w=hIlY<
zpO0~Ix7;bGBEnFyObz@bH<7}}cT0M7z~`pXdl7#?#qV;Tj6BCwrWVa*3(AE%GlH-5
zOK0}+5e%7EpjS#?LTDIfhNsbzb=i;1(T{#i%d1n_Bs$zvZc3;@LinW&XYmH5Y}yH@
z{H7UZhSY%Xe8YgXg){Gm>^$lEhC*vI8jB`(TlO<Wx#L8sLI5isj2>VQY;Mj7fx%_^
z20zV}+!LpgxFi<?21Y)%viN*b?W5D;GbF(gXMg5w{dTrMi~ZwW+=Se1h80l5A=D#t
zg}1RzS{(C~oI|Qf+>n)>mYe+%yl;|$YxUS3ZzbA0Dd}h+T!=u>Rt~7jd4QS6aFmTB
zaXae9+q+C0_i)IErH#_OI@4{AMQTfFn2$P#S>k>Y)sD%x%Tw(%lo3!UpM!5!nep!t
zZ9b1v14O;okM(aRgnTW1Rm8L@P9Q-|razZs-<f{{p|lt)Q*QjGG`(mpjw>7R<Zv`G
zz+?7^Px~@iN`Fq1d|8^(bkU=_H22D0BVBx@gD+}Pf23${Vf3kCb^QCHH;Fxb+ajt>
zmD#t9!#QojKS7++2TT2U=9Cl;?bYpza`k`g@@tze@Z#pUxcfux>({;fSktunvx>E+
za@s^<GUwbtgKHO-q6j>_#Q<JYF=bWP1v6gL5&5d2n5+c*UXMDlh`yb*Ua_FmQGRa8
z<QOC7+q1|KW8ehzl=uYVnjzn=Ao0jpqRzY0JaHY371WkXEt;nHXZ){x<NA}n?w#<B
z`=W=mL9v-IJDGNa<cJ&b9%#?sLAymW{`XI*cwFoV!-*ZO;AG(WOcd)s1IENPs!DCC
z|GnpTN&AHF9qi4GH4v>VWYxt{MfJDmj>0o049T{AFqg5=lSmi!=?^5ZU<EgC?^I7U
z%zkQpyJ(9&<AqJ}P<Dc&d115hWd|y?%sck+)a&*lz}=2a8SzKXW1*EM42TT}G^z;T
zR>A!rW(ucSkw0T-p2kcp;>+!j4AJsUd4Uf*5Ot*N>$4mo&wEDAX+|iN_qd9$=G&>L
zrZ=*|76wq(pg(yL?f9!Js#h9)ol&PnDHlT6q;fswox-zsyM58{F+}LI3P5>9HJOo@
zkuBPZd%q$-*?`5Sx~(YEaEk1Tgf>CGh~G7lJ@g+V$1bqK%RVR?;qAE_!*m(UAX?9#
z9W6O2T92xn*`XkKBQV}kWh-r=g8M+IGDqU~bJF5$_?ETTw|qbv=}vSEIV|FPX>gOp
zmoP8($;FhW@MwzGpPyr(CA=^L*7U~6rdsWziD_%0OiOmggq$O#XP@_cz_H16lB^>H
z@L$saRi~XGKJ|?`boXOYepcU9Em;Nkxlu;ip1k82Lq-16{~A0=?)YQWKJ^yjC!5o1
z^`AzkL$Nh;B*cuk7udx4wC^1}?E~+JK}Yo?X{3K9$Awvdlm$M1en0G|1fo#OvNx4r
zVX`z%32d_s)5RF1IozjyQKDLTb$MpQe)!^|jIWY9;C1GQ6^Z|#c6Jb_Q6d}ch1jZz
ztW*(kaIC+tN-ch&mJe6Kpe==C>&8tODl<(9SyY;C&Dlh6unUanvVp`Tens&=z%}xH
zuaKDh&)(l`(Dg-Lc_&-4t-y;CiTIW8)<r{~M4j#0wz8TW9H-LC54t2whAN~S)i;Z@
z6dC269%rLvR6;7QD!U`XK*v8+<bEmccJHR;p%kZIBvu)0nvk}{M^3<jl{5fRaoVnM
zDj}bW<W)K4`$*@p!%SpmI%Hq;GF1~k?IUkGZFXI})~v;G58gRsyQ^os8t#<P<ohA=
zJ>Kd5)6wR^F^g$(O}C?9I#|}6wUfK=7~`&A?9}heac)Y7^TGVIce0LSH1|GzT3Ju2
z2U2Ti%#n1-{i$9GqpfGUC(Ktwrv;x+O+LjdG1}8dy3j-atzZxNxUX0CVnYW3V6!uI
zmC<?Gm-0&B<{_Qv!mX$W21to%?$?oYPCQ0wLFT;jFy8R}X2EO!HIA6fyWBc2%D7(k
zMjq29JDwbqU+sgMybw4bOcqh8HwEQ*Tbpgqi*-UVjZR%@_Hl&Q^*#Z!X9N;vfMkUu
zP$fJ4-(UCh-BnUoc)FEUMgI)&(#b2!q9~M&OwIIBW|dA%!kbCMpOC%}b&QtBq4$LS
z)fvK_LmA+K@|;Wx+-nQrk_V^fwM<RP4D-G7kiLa1BKOO?Pe+2UO&H3mo7nMpQllcz
zZ1toa`D>NvX(Hj%rWRyut)Cf?g=C3DH5RXj*Zx^ms2lFYP61}P-Z0FhLH!bfbt3Or
zeI>B5XfXc%AfohHL~}Kckk1xbA*yA0=aTR^MilUdFFx+r`;NI)sIqB5#tViey&uj#
z_0xT+UP=J;LMrsUsCe!_cyxE^T%Wk7!gISqv%}`%&waec<Zm#a3HjSj6h%|XM?rz=
zH|kPuzoCVPK{=3l-jTF;Z8@qVWX_&+NUhp4FUR>@;XDPFN?voMZ0hC8kRjFIzRGIk
ztI+KM0aVnvoh*XPvyv5v(`puhzI1y;h*AN|ia!t8)=Pq9(c+l38LPa!__~%pdS{h{
z?s~bc-;~7!@A(}Js*ER`RXe=B^;wkfbiMepuc&^iVK=8Z)R1&`9#nY01joqv4vk^+
z8cQr1>!v7gYqq?RkoJ@gX*zv9=mPfpLBPX#Pr1W+nTWw2WjM$;x;a8?c=h`CUXK9e
z3`qgXy!D2Cgb8yR(apRy;~{CdO`?nBl@&OJS;$VfXS;mwX6~`1a6HE{OH3ik58}MP
z`|o~ga-%(J%vojiAmm=W?@dq+&`<PSe`YTGsA;IL;5j$15et6p5BFL$pGY40A55M>
zgW9|J>piu=E&{f>`}QI^TUA9NX{jBSb9qbOis0EgPHIi#$48H{j5x(RYjazp8rF8!
zEx&&m>eFqvB0)F8F{zVRMa2$AC(uN0K~0vzC(%_tZbDl_{&#M=Ax3Pus)O!Ll?`g=
zg=KmV_J94UcH=57;kDM&)mJ1aYLN*U;@vjXZ9TC%(VZSy{gfupxG~LOv?wE6)G_?p
zea_Cu4B1<o!KmwT+)-0jpE~F`MoKfnbnyq6T;Wm^&;J@@7EcAuu)i!i?;VOt<WQ=_
z3#Qbdz`outiY@hOrs{r}xh1~}vLBG0+KBzzga$@hxwyrDqWv1|-3Q^jRgx_vcV*1I
zU<q<YN3EB<`du#K42rfjnpqj|Hbu&g_KJ62iUoUq`SNiXrX0hUBlmd=Z^8Xddlr)F
z9LKpod!%>y(cw>$M;Mn7{%nUn)89>vrvXr7lYdP4I*G~>wPyL4heqtV&D%A?6Mv1g
z7F~bgv}<7h40gtG#>y$mZbS`0(x8-3{`zju%>oe7Fc$Od`FJR4D9kh3W@@s)?l%&e
zivs5N{yZ+H5WM@71AP!}nRTi8CguA=jC7fl%`;54x7Kw&hlP*LjUC#INpU96JdBq+
zv_1<orCFNtto<21IR$wk`Q}D@02f-WB~B05dqXq%#01jq`ow1`s40E%BA_+41IU$x
z$7=PIVS{}f(e75aV&9gSQ;1c{SiW;aWx@T<zQ}hDWOf1mrm{B94xBG|bQqrY<c>E5
zZr!-x7D)QZKWvE6t*eX~$UM$-Yn`_+@iL&Jd|jQ;g~~`NtMdmf)K&%@i)i|<l-lQC
zXzBjcVVU|<3NDH25z}H`4;cqmROK-PQ_BqsR>OtBW>YTb?UG!@w7-ULriF0VRKq5~
zMZ$h#p!wC5ZfcF>=U&bXF^4I21OW@v-az<v6ydRa01J%eikL2yEXIGt{4(>yFu00o
zD@3emvOrJ=?M{c0;M1MHbs*heFk<Eul5?6__hKV?>U-N#lh(DwwHm+B@BuAW*Ps;m
z2x3O`qeSea4vjosTFZjfiwL&ktzV%DxMJX(QhNfjf(hB@xp7$pj{ad^&Bsj5;hUSR
z+3Xn1Ud$bcM`A_gaWtRMy52;QIdPkFwt#jqF<aFnnF+nl6Bi9;h5*V?KS-<Q)qc~v
z#O;$)RJ4zYehbq&kA+FK)x@)5#L2Hr56b9FWY}(};lt&#PGx5Uv6QX?L7JaYztQ(u
zw=yQ1i{DStgJ;dq&I|?ylAh_X(&Fc}Mv_T;(f2xUpHn0u;!ibLS2&c@>sTmvv*i4f
zqT(?g<IQXb_S4A{_5yvAymEa5$YcEI&5!pk2b~!7Myq{hy~a`K2Zg-^RvDy0+F!o8
zN-pbC$k=_3^^8z?u;VTiE%DRXT4{>^(w)=qe-H$ctC-22{q#ya;?QWB*Z!Xa<`G$4
z0TUN4(y>ri-_-sYJ`$yElK5#sXVreWuc{!K@czomqWrx2nc=*qO4lQmyUOQG^K=w^
z!%^v25)its&R*VPZpdw$9r2oIGD`ldTF;gCG~vgYZX&<<QqFssrC5>tCc8U`X|yH)
zezbH1e9V&>aUG>&3gA<E<x6yLMo<?wZKK$TkT?ko?6accEW|an2z~9cjj*{qw8=9z
z^Q1k#K63L1&*Jq1Ly`jy5O*+-4%M5e-mKHfpCwE`3jm-H{4j=XzU;EsPfCQrO>D#h
zZEi-YX~x$fLeX0q`PX#w3E3Y)GIXYK7|9>IK`!=+nR|=Z1h4plo$w@Q!zqkahFq1z
zu+O`fKU*aTIIOzHzz&+G^f}CCyAuLv2U#ntR$nofx<NrFoS7}!*Y&|DJ{tvpF|`He
zHxChmhV&X=41LaXg;tkSa`^fKZ{6Y=e@bmZC%yCqC5tZ^G15kVyly3P_LWc0af6q(
zY{*%QD$IK)XU*i7d`CtZ9n%)q7C#=nGU-q#guaIy)~OP6qSOvB9~dmDdV*J7sTDj-
zU#c3<7z*QTaI{990e!PQu}bxO0=e0nKW`+hg!#biauC04<@BthUNH{d$7EY$$|K0!
z`dM7dSdn?znlfo^uT@S%M9rt-BrI?1f@@aWxHqH0M;RNC&SAFP#0IB(l}Eac8r)i>
z#TA)$iqKj}jvAj$a_Cn>Gh(UcmWpXOwGFG5RBELj-Qf%7f&g9jK@n{M{J;t#nDT0O
z1G<p2G8iC2*GPbXfxyMw(N93f0D3K2+fItLv*$ga{eRfgIj7U6YAiGrD{=EdR5Vjr
z{P+XAp)Vdf!AF`gTj<WD(;AoMJ$*5J=8;SI4)d!Hyr|kc_jb6qb=c+m$NzD@V79;Q
zcISJ9h|>!v@smNLSOQ{YgAqLfKA>0>EHJlEE5k>7q*nkmu`Pew-&ah$Z83+f)~_7c
zfS(e2V^mzl1{XGI4&yD-%tq(N2`K(cplmX$q~+MKfzE{%=U9I3U}BZW7Ugm$q;erd
z?EjD^Hy#eiEd7rSryTpzKRc0+jiK)7UB+|Adt(-8YHs(Vr79-Rl+_njStxx$`?Qz;
zac64Xl~qX>U(LO}JfrpS*Ts-l7M};Y`k81&X?f!T#_Ac%DcZyNnunUdWRiznjZ{<l
z;__7iTrJHLw%lU2_f)XCz$9WBzSPa7?srJ?Dc`b_4=_Wk)e+U|B|vZ2gOz81jptpj
z*6+zfiE=TaJT42wo`lJ9F=%LQa^s)24TpKqFpd-w0vY3uMr$%?O)GEqIisvnTqUx9
zPYvvDMTc?UNt$%lPw&Y!GEL~pi?NqH%01&8XRMmywz~2_dWn1~(Q@wH#M{hmG>~h<
zh_o4VOi9H2+|hu&19)aZGaV;tpp~*2FlXR$R_P>fr*4tyRJ^@=m&sy3(CGr%HKH@|
z&NECcUWTuyQRW!h_DWyXfwYK*o^(Hf{f428)(199m-IhHMa?HAh&0sGfrq9&*ebla
zhXT@lukd1X_8+HyU-UTHG%S>>%uBJ&Eor0#Bzy13Gu1T;>tvSF3+)XLQwhQ|CGA6N
z<K#27iSM?I>+;N2xj<L?wy6Evdx3MAnDGL-W~gm|#hTQ#{8XrsQ?Xs-rXu}CGhb?j
zL~z9$$pf==cPFji#VunQp&gR<eGRwtv#Y#<?ETLgEadi&q#JKJ7EV=}ccidSUGwlW
zl}KFxFK^5h_C;>9oed{!sLRb_>XACfLkk_gj{b<>q4S7yhbSk#tc6$p&r{KMiPV)_
zQz2fY5|e3`Bd%jFEFbd4A3C$byc9^X@EaY$QbEnc|7|o6!-G=({GU+wL2$$C6I#m0
zZ{8f7CH+n+lk;~P%|~#Ckd0G&hBA=dzMf#ZiJvxv*!CfQ4`q$ot``DP&ZeV3+_&;*
z&Be`}@|#217-(o|>Ajs}Meg7pd45w#FZ5|73>4_dRJsd;j^$!5Enr|cXVUq?Y!_Dp
z9d%5u$xjmLINZo8J40j_pz-#q&2)iStAjjO=^6c#0@BJH4_HTto>t;C%(ekJj4n-W
zs7fU#H5Nrrv3tdBOw+%#t1ts9N#9-Ke!$#(?FwV7z?mT(Tsppo%>jEqr=DGB7hW)2
zAhfH=XJP`4rfS?nOzn8$&QDYK?~L@3Q?*_-)F7tEJ<Qj_KAomU??+Fmo6Yj`B>&Rc
zj_<1Ya9@o2ET`=wY^ac52{q>#4#NkPEa{w{uk?&tCy7U$;CS4+ldgzVhP3t`zF8Mz
ze5IJoa;ulMlE~W4ecD;V{nI@uU_bNyA(`_&g%60nY6r9UUqk9lN&awjoUp)dH<Q4b
z_yPeHp<{N*(|<J(CC|wXTsu+g+S$RI_Tv=b@IAHp;k7FsAV#H$cNiIb90rd+E%<Qa
z2lo)>pjnG^Rxb?LB>7Gsjw-WnJCMEPJb+FA+7J0&WbWtU2X{Jb=s#TTp)O@~q|B6v
z3QxWLP5FddEW^!`Z1{D)2sDIZKKILWO_{~yGNeyss9U#b;8dJz(&fz@;675>bXNFu
zD!Yw<5|cj<wK%sJl+->te;Rl02I#qJ=K^BN<Cs<=d#*@#KrZwDpHBK8NCw^O;B|jk
zrbj;ITK1)~5!loFt^0X=4>u!X(#-!PshizHf7z{fBl)VT%^OWl7juLX)Yr=Q@1?y+
zii@y7zsyv7$e>_KwpnF8)Sdf{`H_BGb);$Kw3U&~9+OP+<{H*o`j2rF^5)zg=j@2C
zF+<S(Ug<$yL-hSpj-g{K)XFR{PA!u_cnDsY!GyQDnwy~&WWBH?m+9K#48JLh*L%Zm
zz_C1LNRFvV@l`VLgmY8Rc2zw|X0I)ZNO&l4Fq=70jS$ImQU|Bp6jK=;YZ~nR<izm1
zGyxMm0_h%5>gfnunTdkjEM^y7Ob|M3qg-h&zJYd8Z<Ds;*5XP(`wEi!tUg6p2o&0F
zMKv~DPK%;oRe+T$@C9)(uC(}?)=6>keY>y{7{AwEnB*~9U&jfA@z1OLMjLlp>LIc@
zt{l%NJq$i`NU1H53v;=Od301FOC37t?h6Q9U5rWj?RG=!?B{f(eF{QMsXFEcZFapx
z@>8e1d=ShE9cvxgS2PV}G(*OFY6bJ}yD#RQat?#$KUYVsc_LlVxUT0h#}}>;1>yn6
zu>J8AtG=()AuJSfGDuTQMCr%+Loev)BH}V3LNWXk^>G^;&jf%^wJ!Nzz121&rs)^~
zmf@!|5@9aEbd~0I2gl`SrCatYr;&nmy_+d9IS|v^VlJkDgI$BZ&~#zW@bQuHKq!e5
z`CcRBdB>c4{we4>WI7JAz!2B@*fZvpW?w>IgPEVH<$ZtWDas1i*``y{0%2jUma+PN
zRViG6kOb*6sxE%esjzsT3AUkgEN?uyca3hxXxbwg1=;vgOm@n3Irtm@l0Dj|I>wbB
z>YPw^k(Dj94?Wj0CiVrc;WA+B0Le8eT#3{PJrm2qDNs%Je+KM}M<jFFVfZ1b_$(Wv
zTH@O1P+NTjWU!o4{|_L}NKF>snAsHb0Ss!5be+<cC~fSt7$67C-{zT+iRSZc`*Oui
z1uM-cTbf>@<txAH@0o*ZQ(nLOee;UvOQpexOcpK8$t!*6%JSnoPcs^#YCBLQ@JWd<
zllH|`huy8HheP{!{UUa*j*t0r3L1@#Ih#iUe7Dw)f!Q1DZMk6zJ0Z0-Gd!TmLEIi^
zUnm{~)puUCzd>>x6V)DAX-~U-$riiEdoXqP(HN9+0nm%c^=1j&edOYcI|hD!0_5+x
z#{<+xo?{xXaDr>g7l`=sj)G{A%fc&DnFDi1|A5v>s-p%pBo4$y_Nkj6<6g1?u4@uF
z#zC6;dj5P99qAF9EM@0aiP)7tIk2`4hHqzew`Uid1*E^ze1r>)&}a~;WCPkMc7Yrz
z;Vj|27M+jJeoq&SuRoNs$H!1}UQ%bq9rtZl%e?LOpZlhhrtbd&(9KMlLD`zUW=6S~
z^KoyJLQ+C+HFP>^M>@5Ab%U`EQ2%zCU;F0_DNUo`e^$nSaQ8_5FeYxCRtiq=*bb|k
zisdQs3Gfw3F5qu0t2uF}pF0~0wLtl^05s~(!Du5U`>KwCt$>qU00;NcW}5u(44TCm
zvVe|c%OQ0cAsg9Vec8c4-l;iR*ZBTX>m|+W;UcM_3e6@aBRHs1*@f!P<W$to-23=8
z=E3~cG~oHw<}7t(o|*v<QCMsPAn2Ks^%>nr#?|=Cys0iqH__dQEGFj06AZIcHa)e1
z88G!7ukO@iK}NwJ6ptT^zBG7;Bm3`73Ns{C@yClJhu=5XEQTrv9PcGe#VHsqel*OA
zL<1`V-L|jZFa7jzp6f+-)k2&90wtS--*v@)W|!sg?}{ZNzG|GvQRlQ|!@j8O9c>E8
z7NlcQK59A<K{~w8QA~mg7uVauge-YydH=4*f7iO3T}olJUf9292w&u-XpW*79~)z7
zYT5s28hT(hKZYU`7Q)CyyK0-6E!@cT;{n0#5-Nk4GQDqauEsGw;B)onb3J$1-CUMi
zme8}zZIR%$q{oChy6I<L+DmfR3q%r}cHcvSvb6VihQt*cO=q@VyWTf<rmt4*;j&9f
z??_7)bQb`YW6N7q*&g*#7~Ut`?>v<Lod{03TTP*TWApOBW;=ulWb0NYQclLy!L#}L
z(%bE(4))bf!!evLZG|W|<DL0Oz23l#;6izG^KlL6!=*H}vi;zzG}~5Q0jCc*=kPSk
zSUSKFj3^MarC6Jg8nkn?f>npR(v$HfZCbWEx~YonSN!!F)F;0U^%&AM17H+^HGdrB
z?c@JIoa+gpFYUJ;GKc{A)6A<+g^tQBlHLL~*2Md86JR&E89KWk`Q&{hH7zc?ihxic
zMtGkB)(+Jw@>hp%HaWcf?EWY_s)GtI)UZEFajSiakF%se1N{u>bw~Nym__FUl=P#{
z?c-uD>~D^cB(E>50|52zS8`a4|N4U&A=WUSL!BnRaA2OA->$cLj8(etqCr4)k%{=k
z^Sq$F!&|6a>ZA1vcP+K-S(43r?qga=0@p2g(@$rpq~Uk#g|l*gF#3VKXRquGkDqpG
zv1LccAu$v$2ghnNj<+3RS{dZ=J^q$q<2);$PR9Z&$Qa__jPfwgVaVV>jly*$6J_1@
zXMJ0Tq}08A7ESQj01Ifbo$g|QuBvA_w~M3?5)#Y%`<%yNql!M#B=0V&>h}>f2dVVX
z<z{j#SA1C1^^?xxTdDM*f7}z-g$u<b4xfc+8^%!I&wK1ewWAnX&^fx4W4)av>_*qS
zPKtQ+14`lX6=sVieo3YpVu10DiSyyEV_XmR#*<mt)MaC!cIqG}pNJk-KsW1V&R`7b
z^p$;Q{1FC`jVti2`r(r233xNRDR(yKs<pfY^k{0_)U0q>8e5W=b)@T5vLk_v9H;&a
z;|@Po+$X0g?HOpPIZ)v++oOzMd8?}9aD8^qhx_>o+_7~YwcJcMdnYE)TY$o0a9SwY
zgL~R!oN#--cC4Juex9aumS@OOmWQ?ZK87L5D0l<f?_oAOGSe@+s7ruz-L4c#I*<;J
zx<VBU;cLCurYKCST$?C037NEDIB9)fUw~pKxA_MA+2LOw#BZZHsf`e>a}U<JQukd=
z0N!Adw?8}nRCczwO?FU}apvt^6I$qfa{_NsH9RkrBBqTHnQPjZTYQ#XSXBTo)?Cn2
zZaM=wT$krlQi$Teo|kduQ!*>1AEL(ku|-N}tDj<i)*Cr5y;XxfuuwFlPGmB+NvbB#
z_D=eE#zxRMud1p#vQRW%WCBv%*W^QT>ja(-PqW|Xw|5jyD)?D{9>B_uNs#uA96BG6
zg$ccZo?}Ih7IbVj7QdBHY@_7uRUjkg?|#ghCz_x$y#qgLSRAHJH9E_$98`t<zD{Qj
z#V2fOC(|$R<mDm<f#PEvZrNVG^x!rpp%7tc7>1-A2q`$O5=3^0ZChc<z+=Mkr)Kj@
zq(B@yyn6b%^r47Lo$O2V8iJ-J0=Jyzf<$z@OoN!)u7g+`em0;>XtxJkcY=O*n7zG~
zubf0r;Q0q|joXkF%%KQ8Fbe2Sk6n2cPLX9jpc?#@Y(<)aBx<V2eZySA%;)-y@pFFD
zeQGU7@S^VU`E>1{itQrM@PC<uzx{8{m84|Axr_RX)i?BCdaE6cpG>~Z>W_RJTd1U}
zVM;w;J^YOT3VeI=mRPBC)}6^*UC~UMn|o`q_$vFr-@G(+^oULQ!~SQ}E{q9w3dqE<
zW^R@{x`a>n-<zwf07p&RvTP6g5di|AeoU>A?CHX}!&e!x-2@#?r@Tc}ns@u=uQNuD
z%bQVGewWSw?KaMu3q(pqPIATzT)GsVPLtY<z_W{v=rnUB;8fWgbHdG=Lf3YNA~lfI
z<@O5R6YP1G9oFM_8a#W0wmzJ(CAz(qhm$}-gZ&b+e_X~5K=y@1bZ2WeaIO`494+Kc
z_?LS_mxr~VXZ4Au5}wHGg~G=;xh811K4830a2;R*YQ-Z$;Kfp(L(AlcdL-Ws;z+n!
zqcznwqh-d!#!O!#!PfEw?tNhLw6&ByBq=UVZ)J%A2*NGb+@OBuD+P7SR`Z{<2~-5n
z+xaYgd=Z8jwXdk;lYuTPx>O!(`HpdQ#{Ox*3vPM1p1?Sa*IG1KQ>j-%7mb?R^R|YC
zrpkTx$O}31J`g9QP>Ml$!A)O*CSgR{05lSI;+o)|Z8Cs!G`vwncDkq91)P_9hB;M#
zS>dW%f~XRfZ9v61ZD;uQT62gYzFQG$H7`7!oM`^av9ZAk&t$11|L+c|=X3qg(OkbK
zG6v6WQP+DI-;A;u{1z6`ad-G!bK6A3Sk?H$S$=6ZGV7s$_9?bcQMsYxy3O_Bjj3!q
zvP9pqZr1cL8V}W|_bUe}rlL8Oe{mu4@^u*+*Uz(xtE3hBYHgQu*pRN1bJ=wh(XK1s
zL|1Gx_-I_DqarDHyGhHvru_GjYu-OXRA4t`rP^G4J&}P3r3V%VPtM22c~T$Z6ou>&
zwJYOG9xfMHP5B^U%vv1Z&iNT^ZYWH7AsNBo%11k*%Ptj?E4E^{6D=ks68d$D&ne_T
zm&h6VzKcQSsn@9p>d%n=<9a?vDtzyRdG*P^9GQuU2|JtX<HKgDSY1jCOy2b;6LNR^
zM^KKkX2mT3yCT=zAKuE-KI<E{1lwgAmsR?CXrTIQbtPMDEV-H2sDak1LO3EO&RMMs
z<AJ6K^p%F{KCU?sfN`-kRND7o?-pn;;A7^=dX6=qE!*5lb>S{_<9Jd!t0rMk%2QE8
ze8D5z)UJ01c_)gKT~N)}4q##GHub&0Cr+b*<kD>FmffQ67i%E*P>EvG@F;po0(#(8
z{oIfjEpKS0SXP>DUC@-w1;;Ccz3nrtZ@03i9@wn+2xUZhnR}{b+ptM})tFSI+{s}j
zJs6fgdf18eK%cR|qC?f-go2>%sn;25vJ^*1@PfZQH6=A~6hOOSML;x{vg=RIZt?tS
z@z%0s(oQYiL!mVyJiX$aT(I8B+gSj+h-IDaFL>#j)B85OYD%e`p~z`B$Xqz&M@Bvq
zDO6DlYqUt9GzLQ+F4Z3NYE4c;DJ3yahQf4q3y4b;AfdI*n(q4WUA88=^R<Jsh7hA#
zeTY@I$pX^};Rk<IsEqeOfAq>XO`HvC+WTTtCI{@o_eA)XG*eWqWSh{cC=(8$mn#fG
zCRjK23r#Bp7?@9ViFT!2Zj{tJd*7(r%bg329igM$8tI~@E`n$SJDyNKOr#L%Cxwem
zPeD(qUI%N*4SljvmNRqfxYzeA^coeVRpdP?i*HO5CkLb^4w$2~6Y8<vLZJ;^*?3e!
zcZ?LhiGeY*44D%s&4<8)LGPF|Yeq^|?pXzIbkjfZ<)vv2QW}S*rdkt%&7G6LM@g#v
zZAVaF+NMgQg+~4xmt`J0R1mFi<=pJVkt$&^shuL=%tr-i`{BuwiF_{A=X;ua%i+x<
zPYAFb#^BX%y=ya~Oe{~$;~Y&=pTlxnRD<`^$ht7F6(94isTjryH^#s1#n_a}6S&Fe
zCAoy2dUHVj@co;i-{DLGWhQrg|NM0)Zz%~jr@`ERz$O*NObgde&Je7x_MS=R>g1bV
z^#)N4jggB+xsV$6VCuDoX{UD<nXdNbcK2)*ws|8meh_SlLd3?dL#rVkAxs2iCgddI
zm}92ZN9KGfzPx441lhQ;ct$u`8Bxl#igjE`)c`R<yvNvsEy)^nWj8@`wgm|HmRKGQ
z0D=4f`bgsBcZ(&BYmK*wdE^fmJX?P+SITuV{dvejE3Lf#NjoLzFlK&Ou1rw(Ld|?n
z2#n^m{VTMR1AJH265IjK)<EUDEww0xhw?I|=)$GGz4yudPN}^X4_;3?#@>SCEYH7z
zQF_9*V<6_g=vGJ{mEA_(ok7|r&u_qwuQ<BL_nqDC5-($K#zx+;HK*$N{$y)lxUL{h
zJl&VH-2@~$sFA*-fC8a$x0Ax3aX2<`Reo966^-JO$Eb0FjrZ^$?{ztJn|0%fpH@cb
z%rGjg;hB6?y-AjSl=4zI&0Y3jQQ>-lH^%7eox%jyrIfIY#TT(zucgOUo?oTA=gsWz
zE}s2+ck9E+a<T%Ef4sQwj*1i0avz5tVp7<|CIy~o#Fd73w-1Hmp(ZT5JwqAlEX2e-
z(?rag_Ko@ShQtJ?Lh%5zd5imhy(QpR$AQg)LNJ-XF%9=+D=*0@P=CU{c7%o{h5crK
z9+fL)qXh*}_kK9jx%jM6djiG0Q%(*e(n!t>+=qDk`g`;(iS1CXz<kl7oZA<k`N1wG
z7Cn2D&tht9))Mg^b;BPFnrv*QpJ}1PcQ(`SZU$`kDjH{OK5dHvx41H!5>)P|gz6h|
zMcm+Nth8+L3(pCxju(`t4uowLl`cLbM#Y1CQE_weE;rs1Ak2cf=F8u>PlBmo=O<;k
zCZS@Bg2P_2L9wWV8i3epJ{?0{YiK@#A915UXCI)CaF1fc5$P_mbgA!N&L^tafxsYC
zlQ6Vpr_*2Y#e<VQeVsLZxYIKs*VbYIWV~(Dhau_p;jVCb`mvRt7XZC&x~x#o|3dAh
ztQv%P(-NMa-%vr*!(j5h?DnotTb}3rX};JJYtq;Dk*{fP8K;WQW1DMl>o;^klnDze
zh`AB^hkq*dUwl1&oFpm~DVt%B)v>T|L-^~?0W?w@j+^#c@2Wgkvdo{!xw9*+cuq|z
z^9j;I9glphR9|;>ZC-vw8k2(CJmJNQ+^^jat_6mL>_&uXm?I{Z>qcmt?Z0Uwqlk*W
zV9GT=xTYUmM|g*TquDR0K}gzTKg9(($2RSDvZi(1=4Y|C!?w6Wj|DUXd+$o4CHH+S
zTMx9SKo_tKzX{+m0T9(}018eyPRhIfN1#++jdVghd-kMf?Xjkb<K^W}3Co^IE+&)J
zXh`8+v3kZ(cv6Cj)Eve)N4tJ9Cgf7c{=)Niugmp5*f+uoj}Uv?{;FX(*;EH6$O&hK
zE$@LS9FS5Xi#h;22h+2A8RoJ#d)HD7Avgu&a_qZPsX)HesiQ5-)65Ew<Y2u9*rq$L
z1~{J(?Q+lbbI*K=l1R?<B;#MjLFY1!!}Il<E7fIO0>$G(E8(OZPuo;2+!NZ&+^t>A
zKeQnCk7<MISGNj$o4~%ez`Bel+x6^@_xW6LrGy9ZMutx{>i}It;ljF8A*M)EL07|U
zB|Vpr{O1$NAI7z72L}qOHXTGCare5e`3%*)j{=+(9|B}iAS=tSPZ+bpF&WZvE-H7o
zE(REd<S|$({(iEaVdy)k-u)?id`noh!0lwrnMEx-8pHu7bp+|+G^-Yux;%!jJ6~o}
zweusc?YS+fnj{SS6M0M)Uf@NqbZte#^95*}1_#|fV_?f-F~U_b><+N-K=z-6Zh4hE
zv>d<*J04DX&!y6G4Y^lo2MLuMKPwqn7)Ai)t5LMReQYluEx>7I=X^WdL=KMQKN(P7
zekX`Z@`YFW_84cUTZZ*?5wR)nrYU@vc_Wq2pT&AncZ~nbE|-u{oByM>UYt(ID53_*
z9?NYZ2J1N+vTyhvV1X4RR4#&r^W?9}6ht}(!g!A{*-o`~3C<9pKJWe~tkz5>vj6h)
zOm|{nC?{XpFkjZyMqnr1MeOkW0ds&bs|8jRx#ka@TRW6b*xTVFD>~}+d{)IW%%&17
z{=D<pgMas52HMSdH9qPfV%MWpi$t%)rNQ8S4jrzT_JVKD3B&2?8Re>9B*YE7fKlW5
za@*}yEfj(DI=twR_wYA}+iaIu9y%QA^tcI_-7Ma%k_u(2IUyLR=4ck0HR<zBHx}y?
zHP1qM5}s^rN&}zv%1Tj#`#tEU$&$_i7K1~sD0&WX4H3Isa9b`gWFupR%fR9Pvj7gF
z%=X?g`p68E2d*;Az?!M<pyO+DBZSRh=!V<0=c38EfIh$U2o3l?+;Szr)t>Ss-)VVx
zMb4(je<$<DZMmWq1cf$6ZAZ=q5G_-X4l5DXSZE1P3Mmc0cR!t`n++M-vE%kMdVXa#
z+#fV+ueUs+!BJ3{p%=#i*2%1Vg<Z;68Hq3bu$&Uj3H>CwtHyf1dsh9?e-f4e!=?DR
z3c3&FIzw+V!Ef~=l+2JPSR#OdbG$$chWf5bwJ9o3h+4pzEg3$FaW8g1(a!r|oe}Fc
z{d2&@B-OIxkL5Y=Oc#B+PDuehwLfRTQP1rUa{3%#x(v705wt<wV%LXQs3A9%{gM6O
zA_66SKF50U?<XG(fV|~5zlhk7P)w*g(m+)Itv7D-HjMOS1Wwy%rTfYXfL!owLJTU}
zzLqG?pEVs}>?NBG6VkdjahjrAb+2=t&4k<n`93ZpA79-uyJe>@zvTj%aIWx)<-Am*
zscIGhwn}yqH<4}-WLLXbTh)Gel1aSCE6Se9pB4LI7=l2iy|BkYT;Tbl)3IY<fT+8*
zbx!M9JPdN3|1`*36SE~Gd)&8V(yXhJgqv*iCxwbmc*nnz1gR)=eOK`Z{!3oV*aiP*
zaG+Y(-ZQP^iyRJY&DH9_H_CnJ4C~yl$<vp(gdx=eT>@-EGBr+!jW5GLc`H}!->NR<
z=f9T7OG(RS7p1rs<F&2nYS!N2aKRNG051|T{uy+f2XM<)0ZTwjereQ)!GF$oCrTbF
znKn$H?O|KW6kkco4htAc1*CU`K4FWEv1`s9hYwMb=te~ybEM=j$wCG&QZ-GTHSgNS
zgZpGcS0l=6`rhp(*H5I?zwEU(n)Aq?_mR>bKibS@!_qMZN8Bf<rF)m;nW6zl6`4*R
zGMN+%NQwg3vQgJ7<DHg_I&QG9ilm<9AKix+nvI7PJ7e5qci1-g1GwA-U6ZbuDK5-g
zu05E;ETJT8_cA6uOSB3^?gADUgdc0ocb^i6Gj48IB#cj@SN_|`b*lV&WG&(FwcGO~
z)v+W6Ftw@^wKh`{cAY^PKD75QZTU}Tm_QUQ^)1>tEYr;-if;|c4@B?vK7I;&Ct`E6
zSMwGfRNKN8^(R5rR9rp*hB*6gsMtN#-;~FzjbZB42#z;O1;~pnV@KyKPcy@+FhcYL
zg2rv8q+0EI)cD-_vZUmEtsT=q?Tv>GY5?rn6r5o4nxACOgZ$0#sFFI0aU{FJp*ck=
zJ9>P+%FbF(l*5)Xq?UWpCGu<a-cJ!9Pik}S>iD7<Qum&OD3CsZW;g&9l!l6-j54!J
zKgZI>&?oKXUaoa5Snoa&a@sU88=`~^VHQpQD+xWcr@zm6sU&pLm&8*>$_Ux_IxNu+
z)s&TId_#=&@I>iSEj|I|-pLT=OlQo;QbGPjT(CyFHXCprBfjdF0t<SfF4sH)m)As?
z<l62v;2yaW7L3T6nx?7IjL#8cAO_|ML_y$qzN`L!`kilrm`#Vq@k}epsk^Osuh=S)
zfTX!biOkcHg16E$R<BaZdP(CJV7HikYpTl$dfS*1zYL5WP_BX4NO&`=4Bj-yhJPGl
zh@&U&1NxT!B+%`MN`jFs#fmI8CiuM!^~p||ur_U%X%q`mE0ClB|1a}&RPr#X9U{<u
z_M3jIyx<*naEt8L$kQQ*?&h`pH15$y++NCVdjF56vv6qo4ZFUyv<ME6`Y9#SAq)nL
z5Rn#8=@Jo;+UOWvN~)u4G$KehND9b6Mt6;FHgfbkpZ9tGhI{wD?{%(o&WFwBaSXU=
zJWoN8lqaCJWP#!=z3n_5ZcMDXy}qR;m}Z>f?POT*pz{$bQk>aAx>$5%8jVmRCxaWC
z{(bs%uH36OTBcZ|<zal#PsNe#it84?GJ=}$BZr&`Z*}Gbdv=R>KiGm1!{%%i2mi-+
zPO^rbTlvl{Sqbej#YaW(CS8BMkn4P*KWSq&MzM?-70mHUj0n`D)mM*kPwOLgejz-R
z(vjR8ZVc4<I9ZNX16p|Vk@Eb*7kV;SDbt3R&0kFvX)Gbe8t2sb#(6wSuJWCyoKN@_
z3&sVnum8d0-s4}9lM|Us*<urPeW~9j$&+(ZgXNk~Y))7Xb6Jcu`AZO~su>fo*kxWv
zeJQin0K=nYc@!XXwe;E(btqtEuLo*?%hb!QeEofNS>@MOyxu<Jk(rP!&#y;r2Ct&A
zUonqYWuFOy*M)Vf%^QjGseJ6G++SjB+ZM9Xk{%b9(3<MIn7KnEOK*CJJ{&}B#1BPt
zG+@k@Ls-i6Wb0e?qS`*6s;9{#WO>`Fcv4!HAC#w*{`~oOT~wrKduh*p^BjZjg6rLR
z25$N0`-k^rf3#rt$cj83;e9|38n24qdO@-4PO(XJL2Mg?sb&{tPtI+TJA+c=Ujn1D
zkvSWR_@mmU?!})UCCq#Z`^pA~C6JxDCx>%wh;oT(wUa2GZxI%=b|6&al{yoX)28^V
z+HAXKVo%|e$<VcS^6FWuBBUR6N^h+kuy8~uuZm@PnTzL>M7}f7b2yRmOG@pGnaztW
zFI6^P2$7_|8lt0$QncwPlo0<}Vs+JSjf*(4f1;{-^*g#t=NicAr1dA(7OtIQ+sfhH
zVq=R6f4v|LWi_05>`md)L^ls2>oMTEGAzkw9eJ<m5X66_8hFx8;Xk;~Ntt$Vo^`-H
zvGYxt@jo$9uga9xQsE}ZUFxAU5|qx(s*WUokj<O1#n=aNM-_l2CBN%)B79Lg<h!R5
zYwt^wzFFS>vX(=nm6OxY0<~FRMT?p`ecyh;`kM3$BX(IlmW4G%|B^W6@N{oU+G*zA
zLNpk2Y@IAhH*fkPiX#<L7<ahVX}||}trCMiS_~Fjv^BCSeSQhP>H<4T<uy+u3c@8k
z4bHh+p1}i~&l~UDD=*OOJ1~EJ5X0-Ss(j8y4Xw;6H~Q^VA3G*qku`cy8;+5u+OIKK
zq*=lQUi=gd3gs-@0HH9;`u6B$%8g5C#|Cpu<@P^lz^l$1h=b8Pk?Vtg+Wq%t+g}eG
zKS|gGYk03a9t3an&k74c7tr@2QHIBxZOIk^=P?0l;uoAGp8O=oUyt)zQCnxG%T=Te
z_5Rne?4wG$6WI5^hLL`+6jVR;SF3&ZB4?ZcpNXs-HRc2<s{7dtRf$wv+P+1(W6`vM
z-qE(6#?9AScWirBdF($FLoxRfK5kffuX1uskL3RpzbA$Y#KY=loiA$bY!_Ta?UuNf
zKsvkP;MV?Ye7fN$AK}tv>>@7q^z&1zF#0p1=N(`0Cu*yA$7M-Qe)4Dl#sqAc={mND
zn>Xb@Sbsuig}^BOVVEr9oLknCCGi^Z?QsEg1AdQixp*5)7c2Kz`Nu$h+ybR;{Myd^
z#I0WV3s`1A<CMaJ)c+ilSH(7^TbAH^*;e2_tv!lEYOP@~bwWb1j?a+6urxlX<|O%H
z8-}uNXz;dQd1f_yf^{+eTZmEZDMk%^CeUDIYD5n4Pd}0UCP}f-j*XR`k+ibBbmcN-
zy70Drtc@<#Tv*%lCW?$RB7RzwK1&{CFm9_F>(Tmf*^Xijxnq9Tlyrk<3AyVFbgaP|
zn;)?<KFZenIG7A_w`VR};yA%fgL=(}vIQWd9>;RNQS(nE7=9Jidf8uI_!=zMY`|E3
zo5>N(a%*BX6gG+O7uC&tb7}GAe6NL?cdL4&;xb62z$grFJZ|{{vNqOpL_@U(7gop3
zvn02mU;0t%CxsRJEp&&!Ajew)^Qkj{5|^J5xX+U84YU7wd?TIP#6J?A6!{#REIu#j
z{mTOn5MKO$UnmCF*IgMNuqhIRz77CT5<jA}LnKzMtj4r-8_8>|X9x$+6%`2-B1@YJ
z>a$$|GeU~;<&(k^_d+Y)N1~jQ!-c~Y0AijP!4M;ohM=fBXxNYLM~2O%1{VGvCcFG_
zvs`G<Je@$@Po-tKmU~%WKLy#<Ppx?mrbM9zD^+T7DwMBb1Ui79-U)|)xZ(28#u1Z=
z!oj2OCYfPTWx%aWmvEP1pY8UaX$W8b<arGbuHD;Xo!etfjIC_maHllH|4pE>^2uoL
zaUfC-Q8~too<VM*W>hWKhmcBtv?bAV+;;d$tO)X`{h}rV<?&D)DbYq=*BU+eS2#CJ
z8a4Q%cfa*gHbzChYx5`wsfIxT6Ps)uUwNVA0>e=-wx0cJuX7$*%ppydA+T|_J@ein
zflkV=n~R-bcD&F}M}Tnc%cTETf^F<GPXCLKxrVHSJc<zaM1MHqkN}o?jyep!eeOi?
z;SH<rM}BY!_y(euDdT_U6?o>D%;B$JUh}oaE&(I!r^Aw}VUA~pODZOkP~6y5DsOQK
zw=8$6wY<w%8k)An1?+DGlwpA|dkY`83rb)V!PXO3!qGc#S9OpHGGTAA-Iv9T7tqhj
z%4q!X7Pg?9L~;=I+D)M6pK1gO!mCDYL=Wu+miG%4@USi)&|{%Xtp9xcnA0j?1pxM}
zdUD779BDu^aQ8fnB-$%Z;eAIp)lj#Yk;wbu4_4@DZA3Ub!JP$x7ga%=pBnDPT7Ec-
z|3)x}hfg*n4JysjIJ(d@HJiBDPRhe_ud$Kc&sweJI+cFEdz}Il0sI7<IM^Ki%?(@b
z%a!$hMO2Qq$8?gX{X>UFq&qhiWBKSE2?199$y_vv_Kjw34&}e<&<9Yy=&LII@2ogz
zqM5~w!0~VwmKbyrKn(LwBZ_~!xw#>2P1x2eWpN;bTJ+rJ+zeIasNor#8TKqj1Ppvu
zRkjjxII$sWp8CgQvt7H#fqs7OU!RePAMJZM0rCQI&Vu6h_NnU8k(^HanXYFxI%c@-
zP0GhYT1Wen&H3TH+NrD@@4ntMO<=_bN{e^<;17tb7LwWV*|zpvs3wxlezQMHm`^4)
z#~-vwbV$ZBlW$tGTC~r=Mn=o-s4Dp+Vn_QE8lUuF*)EHgJl$!IbGbJ-E`Izr`G#4p
zJF*qaXJAO3!a3r(&Z+A7Bsk#IHn5IQYk);)F(1<Iz%VsZA+d$!u~6kV&HdzM{VYiq
z#hp1EEcVTYydYInt#Ns*+Z(X<%3GzOCWp@A_Bm0Md`(=Hyu|lQoisxJ2GziSg6~&C
z6FfV|;JHa6EQW=zLgR_0XnX3hAYsOIt($sE{g?V|OCcAi4ez*iqDo}ukBp7liV73t
z^QpM(aK0A0sNa<RA)Mzv$>p-R;5?wE=oghr*xr61u8YGtdNtpHgyVMCkm7jHZgL`-
zJ;hG7qH=2R9f!cpe_Z}}xS1m)%&l+`2k%nG33;D;`jZ<I$N!Vhv^FE*7;?sputknr
zEQbrQGaAqp_7XJDU<g5PEIGIH<5cl7UA^qlFlnseGm(O`-DDZR^&yfyaY=y^9`s`=
zuTTP1O{R$Em+H|hvtjwbD0!#9A2;#n2bXhx{>f3G{kL9YKJU97#91sZEl_VftYW(7
zu@a@eQ*-S=H+rHF@)VAcJS!{1AXOWn-MWw07)S?&L)Z<YbdbD=Clc~e%KRJE(%vEP
z1Kt7Pb_ofwi8W*QH}MP2qP@8iMq?&WDvZCmVu6p+6v_7R1Y2@ky1;sqGhoH;T=t*r
zMOFAYq2T=RJ9(?vV6wZbcI5$RMenh6jb@)h5e3z2aaGQ{=P3Vuw^^E-j_r+gnw3wS
z>{}}f^CwDEyaV)6a7GUQ;jF|SS=v8mrZU&NMFHDIX}Mt!C>)Y7rgno;7Lf+`<2<v*
zhs9fo^5R&zUoMB|&Ewzon1AbgGm4my8&AGxov{o+GKN3z6ehB?jx$J_!oQe-(ZocQ
zkFbD7PtB~;nxa~gbdu1cuY>Pq&}R$6Q-|7i7A4`ngNG(_Au9q(?M4ms3If^HONoeX
zRb(NnQFiU`6$G0a0j%I}Ew@#WTFp_v=hE)6^m3(n;i_l>i9?6Rmm2zy!Q1H*p1ylm
z^xq!`J$mV#j$%j~S{pSQG1+w$P&Q1tcUZ928J$JWo1p1A>=~thbAB@%sq#4&yrr?Z
z%}AgC<~|PLggLEcczvK&3>tODKn}$Ln_Y)=W#vT&AL;Slk!RMLU>YttaEKHxe^T4i
zbe=qWk9;WZVybey?jLt|7D`3Y`)_r$_gHBwG4G5`)OGQ${A1XxTHA6I|3kzRu%UJ;
zn=&R(!u2djWNm9JV<Ug>s8cyXSNCNHWm%DGXa0_T`CRaWWLSH7n!e=PiM|w##j6%O
zenLsSY+Qh<9>9#E_)G>sGt@4>DY#nTzWNM6bwwXS;j~xnz-1dMY^)#86@%O|hLS^;
zJD8It5*ZcwO|bz{M-}*_+SY3MTRFyUEkRXor0t^WsXme^O;@vtQPE=GN6o*6GMfEt
zm$eK8F%>B;%fLM;H*Uw-{{L&to83N95=PI(o~MdGYG5@(g@Q<9ug@v{e(!UUXo>(p
zLnryEr{?mvIF5#};2eR9FV^I*5Q86+2#95(={pIJw3}rCyaVKLjzyZb@SOQKN`L<C
zZ!iXc(LqO(?3MToor!(4^3^IE7`i}y?zwq=7lZq{b@Vv0SHO<kxRl@tRTvtqM4Gko
z`1K!+s}+u0+{#d2+s4+)V}o6iSf#esIl8QIEJW~_yfF^BX}zi7%{$a2yZ_u7KX;{W
ziRbYew*P&f`QJXQDboSB+Fc-<wYROeO&6j(x}cK^mxPY|V6oM#?AqU;WQ#i$*{I*y
zN?&g~0=13C5uO0^^=VYe5~f9olEdy_rQTdh7jng8Y?0q;ep!wsQ!d=dq3kKQuuMRk
zWw)%3ptG2`;=Hq^&x`keZgQOH4>pYTW#oX*E0_PX+djf6%ral{O<>IY?^n(DjZ8BB
zT<S4MP!@PGbk`r~y|>VuEqbJf?3nyuX{xT)*j2zL8>$u{w-6RAZJ^EEDaB`N0kFCt
zj3!Yv8Hy~3S`p|0VmFD&7U>vNR}FO=T~yo1=mU14@hDD`e|K^!3je<gR?Xa_Dh&R-
zn0K$|FYTY;ScF6{%5Q|bB@pFisT%0@>Oa|0VP`!z*+=qFqt~BxWLgnG`DBu+jr_`#
zicdU4;%hbSZUkwNjeWsZQ<~MfZC-io=h)sg7wEf<zs@n4Zz$7-Gr<h)9E1C1$?bcc
z-Ld!CyTHuw<&s)k@J1&sUQ`}Tn;G}`nmXSu@WGEF++dI40!qGe8Qy!@K-xf_zLB|9
z<gA%)B!~+kjV9HW8;<{C^Hpal*N!rl4`{eq@g>_G+d43<=z(Qdp{8VetAJ@>3?3>p
z<1^gd=e4k@j@Cf;8iz6;J|@%YVL%0pxE?_}6Q=)eYhGrK7q(;%Z9(&63(jH~F{~vt
zsswj~_ibKzNw`T{hxuYMYFOnpGR_i&v7HU_W+Pm;>~(=jgMP@Yv$4g?ry!d%eRGdb
zM*^mxf}c-zQ!<(rP>hdo3$+@E1##bMSw&SOSHno*{%{Z&S$HlM<km(N@FBRfASQUt
z?Pw)l<Q&gC%n%{07JkWf#TbZXq@RbOJ%`yMSMqhUmK=gjuB?V%JStikX`dW@)8H%p
zv{HA4cde*3KIxwZNHI89#L2P+_p(Sz09y$u+IWPn`=<#;$-IbVFl44UzAAmS5KB9e
z!4SW-v8BNok1!sM4GB=K*TkbF5o%8@NvF;UnCRcNxAVx`*X{6{VcMB{wC6mbP9$U-
zo1QfIyXUgNXg(AtT2lMzUR-3ytn{M_6Bv^Kxgkq!`yVIV$SDkv;cQ;#wj%C3?Lo9-
zd&l#r)veBxgkA_7%}|e-wXep}v}P)=Rpv)OxhG%Q^`0@cA_9Sl1MZbxpC4UuEyPt&
z)?^?&9ei;5=F0_p{%D9G1$@ECrNx=?p)-x+p^lI7ZyH>eYYz?Y8ngIC!eZ(N|1T9N
zr`L-%)WM{ktg%Bp&I`9+InKW(IqQ0mJCx&T?K8uXCENQn0D4d3J+7I8@#NHzE>zeo
z9pwt?8lhFR$lv>+qn@TSRP(is&_~+mjXoTX8DiCUtHd*odY1IPaOji4$%Q4pr&*XV
zUJ3dD3a+)cQr<D~Xl0bQ1nl8lAJ~R&U;2dHg=eliU+72P64y&Qr8SQJPh()*rKpV0
zPkhI9hHugSf<7$@S}Hv>Daz)>P;mzu!EQG4B8SM=Z$emZWqg%8!`=^PBX4m|K1w>`
zsy1s5xJr$;I2F=q(S&l-9uQwAn8={lN9tQoehQj;qB$Uz`6lc4X=x{uRW8Q<%BoF&
z5U}~X{PLSwfCT`m^V!E(encZ#Y*h}2O_Na>V$5<C;$LBs*RvddB7&;k;>RsC9HdYL
zBQ)9!Uz2UUsLvqm{jj+v(#%8Z7H%b$O|c^O$F`k1G=!3fR1$N&1Oj6=<O2HhZ{Fup
z2B7&yWgSOiZqBs`t`fT?%quqCm22k7Hn!}+sjn|H=)@Hmt)z=S8f4X3JlryS_uil@
zT+e&Fe8b_2Vo>Qfjewhll!eJv^A~k?&s{+=%sU)AN=zBnL1{kJ*&V-sm95azkwIR+
z6OOnaNA@HU6I*eP?FXUT^Y@ZP^?<y;x_@{P63Wu0tSaBDiQW`<adxLg=}OQkWfIjh
zy=;(}`w^eW$Lj`@cyj}U0&c>bC<6<32F2Z$k6l0~Lb7hQr1W0bZeNwjK&N3TEY1x%
zd&|wNbXmt%gw?HOY~ePJLyJZXc8q&)Nit5ktb2^6(T6&Qmfzks)Wx0XaKr)n@E=jn
zU%rF>PW!|iTD{7U$`Hc<9y!NDy|$GbR+TK!tSS^Lk*G$x4{&YP0NK~Jzs<^emK97f
zuR`SU%Tjy7hYq~j^H*bJnVl<wt}3_Lp;b?%7@Gw6%e*N}!AGAi@B2;`*`|gF&-Dg*
z+H3eUPNA?uB5FiE)g1i~{Im4<SE68bxjQhn!>mJ}NDycpNxv}kcKLc09a(<D>%?xZ
zF1HTlrQI4!UNc$m{oM!JqJJ^%v6g61C;avw@WTmF>j4gd4$jM9_-qeDLn+zMak<1r
z8R%a_O)4>g_<37_Vl%zx4)_Wo?6^s^MFy%c-l+Sd@@E*I5h9)ZY+*`0wO2_xNzArE
zNrs};yk%YbxlA~ExfE*PJ#r>ebWy5Z`$@D}3hW{K24>wPt`l4X&idZrq9+xr^U(@a
ztmdyrS*}Jd2fu&HYx53NJX^zC+dsabfy+t=5-DbDR1?hLsSQdUE1Syq{2sUB_Uwx)
zPN*gM?6=^NpSMUx#qgdZMgFhxT%qN(glB*92ef!=nGjfwndV56gI@`n@EQNC1lR9a
zGjbdH6)npyDGwp9WrOCEYHuiahW0kS8C+~JYfQ;g@55?ryu*f%UdZ#6TbJ`Bm(;QP
z2HUM~Qh|wP+vd#=;P*3(=jriPCqdfCF&{nKQQlgN{Tn)VCZ}xSWCbxkhtGl~Ph+8Z
z@^7Sy=dZ}qn87h@HY!5_GjCHw^NF|AQvaY1)zi5TYAlSdG}X~lGxq(2uiuXl3wdlo
zqv^9M+J7@5#J<zcI669NsvC(EcQ56KRjFk<kWOT{`_Hs=StKV)-tcjclsYbJe3s$k
zk5(hsf#0X%waJm-3s+r1&4fEih{q*CCM;{UFg-dqCELj%-g?MNHKTA=Rm4wZiSuT}
zjXe>j*p1^Y#rM8{gitmAD36Fhc4y|S84pClJ=tQG3ln3XtRHY&WOro0unpP`ds2LI
zaxzhb{83QwfP?kO-cQ!U-AOORZ>9L6DeQgkF#mynnryO|y<XuJys4t%@34;V2%M*&
zJ$=k341V9ycu|Y_yr)~{bYvadbPUmpDo!%!cfS(Xvn12~4>r#@onjnSs>a6r5oX@2
zV(TyJUl@5-dTYds^I@OIGB9}^gXA<MzN_S>Mob^Q6*A_5993NVWv#drq!Pk$H$Ek<
zG0+Vul8-?Svglk+#&m4z;)GKD-s`F9`p^SO>*ni$f<0eR@$Sh1jK6U?piWS(0RsTz
zy1a5eT!eXvcmh9mjC#@=Qcg8ig6K^x^HY22VR>zCPqL}I80BNvdi)oA<V_iWS5g$K
zK%2XQ?K91|Wi2<R<Y)%6=3DouaBLp(Nws7S@ZIc_D|D%3R&F0-76Mf{9bVV<k#~}}
zHDg-nQMuVv<YH<?+r@af^M^)hJi`7-M!*A^utB@cjohaDe@t#w#Dyxd_U_IYeONfS
zKk(C({Has&h>GpZU+p@#NdclEfH`O<?nFe#wwB&a&1P$s%JBktw9-1~e|~Y;{z%A!
zenYOIC6A2EVw=SqlEcVBwR6w*diX8}PJ)G>_NqgTr42RfhVjda<SWnx4c-A?$X`yQ
zoO|C45iNp3I$LISZW^V{127Tn)YzyStdX3*l<j7sS>q^ae^jP7CWmk8T+&S{67=AB
zqdnx{=nu$ZQ5CcjVdnH!s1(^y(`;MOp8;E5M}IRw(w0S{n$7UdT!w`FT-PtsmM9mw
z5DR3Zc$O|2Zh_+HT(Z?&9;E8!gb5`SL{aj0ccs=~V=GzviTnZOanAYe+W89JEyB$K
z)-5O2wf9S2i)=;9UEyHw{tbly3FLqMs5_Y=ed`D!&a%M4JDHzfG^q$d{9%Vd<p?04
zfTL>=Off(khA5`|YZ|sX@lNSWxHOY33>($|LxA<}`QqiT#o0~_&5=0-a6S-1e$k1|
zhqSEuXQ6&WA(uw4KqfC6MDCcpFP+T&`d<%~q-?-V?eF~(-5K-0nWLVsf2TS#s-X!&
z-T>0j9I_n@^SNyCOT0X%@_7-$=z$bK)H1V`>2ka7ePE<-wopma>5N0V<zrUaBPS(*
z4I?~2RWl`x*B??|g{X=cT)4jba&z;s?D5*rSIQVK+X=Q6#RD4}R)Prl+rY#@VeOj$
zdu5a*zJI@B*_R0#Ogq=YA-i;yz>l)zWU{%sL(9{GmeXeP9%LgP_%knehg63&iB|I9
zP(vPaSzHYffhn<9+aWxyo64y}da+zmIngNICD;7NHp}Hreq(v;&v4zvXP&GwVISnQ
zl)*sRN#`2~K7apgfy#O7P{7G9O=3o+#V72G%l<b~Ir#o_8{%X!9<z~;8wt`rq!yM>
z_~%7eUdH$_KY8Ae@N7-3>F3pwX4X|JRR{AGOx?#T`<0y@7u{)I3>`;Cf*S-%ci0_X
zZ}R)!EJ%~B;N2wrL&OcBxLB!+r*F$jUw_a|lfTBnbpD+)t5)Z8QV&fV0-vM5QuQp|
znUikpWAB&VJ&Yy*Jz4_VU&ECCbPIbL{c2v?RS<mJ;TE0p40!h=UcSGaiAzzO%RZ1@
zuR`fP3tK@PB@v=~F)=EI^Q4Rr>11Ru^ebbQJL%#eUyg})06-paA7JHq)+hjIZ$CKX
z0+ok7S=5vqqDY<)`jM$G<*_mMaRjZ8B(k;HyPC1zWb_oQ#-if#6lM%MjGU*?HPIS>
zu%Dk;7{3HXg5jj7dpzczC$}=Yfx9;|!L5NM;0B{Kiemd_!1qzCIPgxVe&M>)oSRPu
ziHrBw_+Ck5+L}@AitqQqirHzYx1>_6z0bQkc`g4F@SDl9<m^s$E#_a0*GLJl@0F(!
zRGMthE6;otKz!bQ7BIp?<)6QW{iMN!pk$+`q2)Crj=}{y-J!gBsR6a35-qfAK6kYd
z_$F(U;DvBO=HlTWl_;-1^n+fKKnqDLI})Cg*JKC5^?5qO?^rokc>{q3Uz4%JgZ%x)
z`5LKy-u7?*o*Z|8H=$xSFV5P#9?W2s^>1coj6><!<-ZCct&Y`a+sWhfb}m~FE{$b?
z?UZLL-wqz9MU6UpHlO1Lgn>B!iy8w=wA&=LMwnQRiw!9praz(LLn$u*u&9(;{4KZB
z)mT^a#!$Z-(q2vtpH5RF{3WR#_%#k>yi$;B=8bOc)V;3yL)2TN^D1boq@*08mu#)g
zU1&W(L51p#dM2N#OR;o${PQ^YnCu{+C;m85+&JZk>HNv2S8R006_B2^qbC<oor>oi
zggu%uPd=a0`tLO~WqhAPtB%s|E|}Db>Z9&J%MgjZd^sUwa?6redJW6p)UrkiQmPDM
zslu6oGC>i$%#{n8;Yjjimu321oh<(;T+0Q+9|hYX@pFA3nnh@DFE?M{LQwYN^h{td
zmjJ<I2Ez#oV19&0Q3l_Sr1hUca+?o4HS76J%*0CXgjjZ^qoX8Bj-+dNQGdjDua$y>
zX;dJ@?2hVJHS&)FT$s_Om0mKr&EPJnvU1u*?B1fN?CMTy>qNGngMgU)LwJ<Mnc)}!
z1<=ttogm~ml&k$-FCzCecj0;)TYY<i(lCN_V=ykT=iKSq7pezBiXIjnfPpNftArx(
zPvl{Lm&Mh&t;tmWM5CYZ_wFv9v_r|sSn?d>l>La$AcL}lW}*MwR_dS@4FOT7nma}>
z-Wy2UFIVpm^EAG9ENN;l;-C+S;rMlr%PdS@Bm=9EBHEFXqe)lBnG!6{UODG|d%3t6
z$0Cwx+fi9pHv#$NH@{;p#rmUv@9sqp_o8LF-}4#ZH>`SVYPHxz2a3<lE_Xkmn3ls_
z1c{$fi(}|S^oxf6HoG6vCBed(7lBb8P)j(C+>(t-_<CGEq(j)>(&RVAjjsHf>G~Ln
z+|b~|PpI&+tM+4$JwzAp2zfFOO`EIbQfd+NR9JW!1m~k8k5obLcB->*CQL@|*o8L=
zUWHEXe$fXSo1ANMwI-|1m>MlnX$_bOgf@)LyB(gi{OQ<<rV8jcjn6C6MW%Z@$o775
zWhfT5U^)AECF7fqJ0<LdolHss9%@8qb-6n);vsjh{f4Sa-%6^s%V63{?KXgODW)fO
zrIMNSr@P&G<3bNwHVCc1RZR+)`i1-mi(HPh9UaW6jXJXmju)1<mvdReM0+Mq2=6t$
zNrT5~80odjqcG%P2ygVQdpp0$fu4|+|Fm9YmiF>B;VgR-`14LkPp{LfrK!6u)JyYs
zLcV5DLQjYH3&c%4iU%qxD~AE5JdB_dK)sQrTAuavx$Ok<`MB1Pn$b=$CYV+toItV9
zsm!xSq#g*TSql!2NAaY_<kx1`-AU+q{6mQLBdEc1%>7G1E`}yq#752brRuU<=i01S
z>HJV3=vSOSNRQp<4zA~yn?IUp??AIAc(JoyXpLoJa?%zaSH}xnasu%#p}Q&3jP`yD
zf4JT$Mn{sn36SIcfWX<kkS>3MIEt%Nms`)v<NFJHUR+)>*Is_-msrjgJFEVTMH(hX
z_Nx1{)kWSV#FtGJKXJt@@$KVnoT}=UPaoa>`5Gm~^MH1K-1`7GKHGYKlbn>%O!kd)
zr`jKvHNIE<ZI^Q!%g9Pk>iC&d!J7xPWrt2I&z~JT39(Sj^kaQqFG;&yWZ(3tZ{VSH
zs(Cpg;3LAcZ9fL&65iW>pxRN0&}|TOTROJ3J#@i4<AYs;+SMOzXo{#j3w}cQ_M*+*
zlJ7r53h8*Em=%5&LyHaXn<E5xF5q&{N#Gmf8g9qp_QEP}T{Ft5S9Tz=ht1<;P~hC`
zz}Fx&Sp2**9kzSp8wW{g3C|(n=Im}rKlA_iifzJSfYL%wCJ66{mEDkoA_XkKNWBn%
zA(oGpBLKa0JtPP`&%ng=30jieKA%>8lp6=lAuA9hFlMngKu-vk{vJxo<vgJl47F|!
zu%l!bI*vVxiGyrNJFrx&KbnyD)|_=<^ysjdz9x+fG_(V5zmWbM?~kNe?84*H5luw`
zS2bQ1bqQF_%Rccn{j3-9l4TFd_Zt14{xMLDDUPTs?yN~Vrnk0?Wqi<5zCA^Xe7j`r
z<X_ky=!z>b_4}bS-uBB+sJKp-(b`nb+qvuw<iZe|{y->@R3vui=Cno`HFWRga@bkT
zDeD3eD)UosD5dELsjV_3?Ka=ke|k09_3jx>Tp&;Xh=JBkKp(B(aZXqPY|D%M02+g`
zjrh0d3o4WN-F8vZHZzdn?~?cesM*-sy1*0Vz8N;k)BHUWqHX7uRZBfEKAf^|9ndM0
z;~$v%!KPQi7rrf6KQ{tklDAzQ$p=P}7S8aPbtXlByC8XSsI#Hg<X#s5p(k5#aR=eu
zzG9(u)#?ro4qDkf?<)^0{y?G3-3&MXc~8cUbw!&W=l1Y>ckRa<6$#1r)7|$DWqBr0
z_zjOR$25|(xwjVW#z-EGXT=MA$2QN;n-)Hwe!w2);0}TlGF#x7HN*nK0uL9K9jY^e
zERt2Ef=zzn2S!JYLdo6N-X<B-2IWH({8)Qj=e!rq?Zx8NjwgUtzO5X#WI%(sokh_G
z9-X%_vmcMs{xB=m`>>z7-xDo->a-%*u`7S}(`B#DESsXd8v@^|X3zM#kXsSZ@&K|u
zgs{>6H9TH@^W7z<#MbJ9%eJ*L+MfMUjj-ED)AO(G`$9yZtcx1aE<-F47(>(*OhlHp
zOcR%&C|^pR^1b}$&u2DXhcDZB?AmbM!p?;Cq9pQbVuKYQFOQ4BaxlDN?t)ZAf_)|g
z(OQoT>(lD1Pg?uR;`%^77}~@KLC{_!AAsgBx7W&<ra~Wi5tQ@!dGF5}hi~Y;xAOyn
z*Yqn={JL`<bW>b3Pz|@Rr5%MLzHzY4lYKe8J2=#H`9ntH7Eh`kNf62TRrM4^Qt8r6
zpKl~pBmb+<jX``u=wvN+VqzprC`+fOfGuCG`Q9J$eecYmrN_UsZY*o5Rp_$XHH0#|
zO%iBlLJ7a+4dmX-o+~V^=&6#owpzYwrX<pjsBzFR47`xZ>_62!uL(YR(w1h2n{c^J
zLN9CVq#$2W3`zEA^%8fbRg6@BgU_Z!&x_AJ3s-pBs2=d3;zP34$T|(o@TBzK5vi15
zJ#&8Dm5Gfc!eL#+&v8vjS!8R)Km8P&bzMa>Lvc{mfZxd&lKNGlk$@KO4nOeCF*d8Q
z9pPwRM7S9L;E-0(*ObqILsA~$L*Y2DF$lZiPYl&oe0lLNY2-;x+>rt4vRW1q0-{W1
zzNy5ry;=O^fql(WPWn3VuGxTD$CMegGQI}Dr)g<FoEoEYSEJ}}_ea2s!5fo=zX&`J
z^Kh9mU07Kr6_e1cBQ9SqL{gKCP?4+lNn-cxH~eDAfaZv1I@tH|K1yGl1C$J2e6|%_
z_L~NPQN9z^56dvhkTarMTeNq#PzDLyt;BQPH-M;XoEH_rU)GSzmuJZEQmQFai@=G6
zT`e8eR&-5HGAMA{R|I}`Wfp56ST3N``YR!Ug(aR@sCDyQ#Ca{zn`qkGz4b{V0N;ZN
zh}ZQ^9qur({3<!t1rry3VR5r&F@D*4pX=5Q>z{g1f5;T=K=%lWzSuknbZ=N))`%p7
zjT1SLIUu7U`%b+CPxTd_0qR<$EQ*3pT;no*TX$`Q(@(mGKmm5t&RI9>Buc=Yte?dG
z0NF4oD1B#Vr>m#uMSAny*8-!vVe}QO)X`tU1P(_@duW^kZs=cNH7j`U(Puq*WK`5x
z5r^3o>n<s4`Z0fLRm4gY@=)dUUlK#|QI<*&<<yw&FLBa7gu?bd?UB9kW^}c<!$Lw)
zrm<}v`C=&`KW8(6kssvdp`*XG!WykHA9fjRBbXEOOPRJ|NKej;saH+-uQrqcyiT^#
zctSDbw!IMtK+-2YE(Ol*bVgCNy;^FBNmRLG1!{r1Y{xp!d@eVYR2!nVNH6N7%@Gb5
zuZxq>1KRny>3B;)_~54!f&a6kwxEIzXIUpCJeR{4x2sNgWUc5$U|w{hYRX}Kqt{Xk
ze7Y{a8=wdm*zb$6clJ-uwWIX}fPDRhV<m>QFO*=?Yk!yb#m!dryDYk8nu_*Mir-aX
zmOmcbTi@yiSG(%0Q2gk~H%}Kn!}wklIMI^=SDv;O3DEppDY7WKKKQCg^6GAtK!<hd
zizniEK=y3arhDmR&*Op&NL#`)h;5a?QG<BgU0SDMJ;ZXa4^1q+<b`b#{()Z{HsGxk
z|8ey0xh=KUhcM1&79RLb3l|g7r{K<1{}0zX0f+EeuGF~XZUTbO)9If{ORFs*8rGAY
ztM*=RnOaO`ewz8ym$T_(Sg)dGq9s1ZbHMoEV!WjU@qusC2vhSJ?sFL7^q8jQic*%T
zN(VOP83qPM9>@M?DLNQN$KeWC@KkI$Yl+}kx*N}w(G_SZ*TtW;bFO-xGC89wiuO6^
zpSvFu=*dbl(lfI#`B31u;1#)N>R4@ccUzt|0BzBPm;Z!VPsh)P{cF2gPlw7`7A}&c
zXctd551z7jTI<8z@&4(W_n;;h9ddisk5VV7sJ5r=o$+rJt}i<oTP3cV1221HZBs9!
zoqb%&SQd^P4%qh3&i28whv`D)DgIrVs;#o7E#*B!f4i-e4FWAo=1c;Nc;1G%Ee}2O
z=f74{yuZ$j$D9OkWCf@XO~&i2-mM}Yjg5`v+5G)2hH$B{>G#mNkBFzMQMzd0dV<Ey
zMe9ZkSyp0iv*;&1luaf6R#3;)PXB8W=gDWG9k8c6>=GZ;s^)^&#%8`n<%C{@%Ola!
z5`RL5L()75Nywr>Nm|Gsrg7cVjRgv#r_+s|yg@*pd@9f2^2YUtMl}%ikd)n{;p9W4
zcY39su~owU1o37$jL+{X!d>MytTigKEMS{^ZYHcsyZINI+QrU|T;Cfy^Cy}5iD7%T
z(>_!>@khkls~Y|iaS9u&8HMQ@eny$<hU_IMmaTv|faqXyQY$UY1az(*3yZPd&#_X2
z2XrS-%-LM8XIux*yvk7s;ylK}>dX%F4bw&}XPo_Joa;rk=HE;Ned=|1ZFBN^c10C=
z{A=~*N0DfC$e`kRy|OU3@X7PUW~A}juO+4M%n5soztp{^C7Z|Nr-2+ERNv)t(gRBV
zI+fb%>|6kywtI8CjWbT}+T^%z&K;SL2ps<GjV0^R-vR<VP3+rL>~$omgw~d;pMDcA
z`LRlgxCl{0q#X_b!epkE90|$?bELY<J%VF3uRo0(Bl*i6SNItGQ|lF^Uu5#&Wsd;W
zmuwcjk7TEzQpM@_b}2T8Qbw75>0{h30U{&!@>8UCf&hA*4I-_!^gKsmnL|RKNbYwB
z6LdP4dN(vu(WD01kqds*gz$8R^%oudxpuRB*ts9yQ69OI+td3XUt#erb368{8NaHg
zt54oMcNAB%a9o-p)hxC>8K40uB{i|##X7NGv2p~!;3SOn*T2=u!9kRPkvi3X3QGxX
zE6(sK*TSIFZd%yHOV3VxsRnOYmoSeTsAs@#k0xM^=*P8igN|U;{tvT@wud)eMK|xg
zCxUJ3kK_GzS0XvROj;)6esvce9fxmV5H}lEc&L2s$9f*<L@PE$Bxp^15!G;py*-PM
zhzz)%q^3QODCZDzNY0(uSUEL&y>a5up=)j3#cNbhYaaHlq&sJ`8a-A#ULvNvc+Bgw
zVTMq**tz`fzCbUm1@X!9@&uDRd%C1f^hdvP;GGl75x)=!HN3)&kZL?~<4A^~bsCSM
zN=zFsnKxVqTia~et3SnU$+9xH<v-%;vM<v*|K)cYs&f-?NErYn_K8walH+g4SEh$}
zhKxsg&?_QHf_-h#wB{*@jUfwjOwj@|Re}$+7?P;;EV&N9=Qpw`3dj&MamyDBpJir<
zqka54kEAJz_s;B<Ncq(889oOM8kae9_n0IK4-xUX@JWh9RH4C$xm-w%(1<58r7m4q
zmBV@AxFF5M<yS45u*83@%Lp$0cx&TRn$E}%;Sy~mq1(={pDz7&>4AE}9{Ee{vv!yQ
zP#bXWjr){a&5Vt+LV5zXzPL|$CZ@~l>IAa}auD>*FMaS7r3RX*`8$AmY{)VbEttBy
zcqZS4sikcU{QKy{ChjdMi@{jk#Mqlz$xrp!0Fc48{nYv*CgQYqu<%dLpPCoLahncQ
zexvCNWBZiS-Wbc-yP|br6X@m`aIeiPVFoSyo8n|9#IeU2+kzr4Z|^S-6@J230=qhX
zfGrI%a@Z;=Eg4Kw)_H6AKekG*s!p+<3eP%nn_s@)<K9e*7{5SOznPd%Lg7^$!LmG&
z@JE{|cLcaHNNOr<)L%?_tq(CvZnW;2T7H1*c!~*Ti|H2VBb8JcdLw(^_5ZeV6}{W>
zRU^@zNhmWIk(qS&g7g6d16705l<PD9)+0reJ_3V8gIzxNg84NF_STDj)fAkLo_dEY
z?MtG4cn2RReFn3)n*io<)a<|p%nOLd#`N;kS^pZR3o5~AfI~d>C9S0W9)#``Q|z8M
zd(JRaH-o4K)>Zr`P&gl}62{j|>3Lc9)ZyzbS~s4(b-DG7<>tcW#<yWFUS8(P#U-#G
z(|kL5K;z<fIU5n^elA68arx{-?a#Y588sQ!YuDK5T4(#V^9QEX!JWM6pvyO3PUoD{
zoY&H*7NOl<R2&z74jynCm_DdJCQSTJ+~r#jbc}A_di6ASL?_hPJ3K=pxs~zasYd6q
zsDI_bncX1C0w_4LAa#CYfR_6ErLw}%ftyaulW1Y<psTFIgd&M<gv;>kqpFujwW%&Q
z>n_gNLxS|GdsIhRViY_5wq?XFbZNkYCFzP_(c1!n<ped@cAqM(kWa;{?9w@WO#m8G
zsA(x%lFX?INQK-b>(yb;g2&t93468z8V?CA4yjujRv!rf<;s63Q#yPBdej~D)tCI#
z&S(nb=+<?v$c4<UEXXu;(Qn4+TsPqDAz$ikF*syP;Z<th?@Mkczs;L=+sw5hYBa@Q
zerSKS_aL1%1$C5w61e1!dMUo6BuUuN+pqpfX*_s(BLN=rxCW;ac#4Q<?kD{5yN^$1
zyv+4pL{a>+x`{6Ro1!xsuQ_v7@rWjgs_b}ROIt{px=ECw&817%gZL&t+<c(Nn?z3;
zJg+t|uIDF(Q!6m*)PKB2+0It*U_QPee%?>kY1aykz9E_r=#f*;WStBbmQC|q<IO<L
zBvpZ|jv$>IUA|drrpgsdItUBPbB2PrLtn{-EygVkObMt>r|r)5NVa@=Yl{Z`YvO2f
zB#G)vqq$dex?*stC;Emi(+vOZrdK!ogt$DIydNJ}rxgrUS@?<!`V#GbR}i2lbPTPe
zh5J()6V|xBPZ<}i;Zb1|+|pEH{xWLxO=a*Ph?<@Sa#t5$#)P7!@m^CVA8!7A1Q$kd
z6L!SF4Gvdq(PJyj2!nJ&|4Tmm1;#`fDBxHWOdP4YOt>=d#-aLotv9?4!w%QFnxq|3
z(@^}NU{Rv-YEVOx81I+UA&_ksKy;4&W>Y12Uunf!Lo`vE`f{V2-{Rzrh3HlF!8{l?
z3qDuv62INxV3I#o<q*!<0n^Y13@H~ji`BlEr4Mi{8NCpvKjn}7`^>cRgY+H1c$R}=
z1G5x0qScT|@|^uHQ~e#ss>`gW`k)=7!rPM7kqsPkbD}V%Bx<W@%l0D27K~&}0wnlH
zGX#uu32I?@f1Rzi=@N?TYxD|d{FLIjzCztz378@8$om{&31tOF1ia~mE#49^%=(-C
zCW~k%Q6&zrUv7si%h9(F81>D{I`RIz9sK3`-u%dMPTzn6x7tU`y67S=eBMbP9^s|E
zfaxx9^w~Ib-|sTO8qFt83cJ5?sreB8$Z>((x;9i&l7BzeB!XkGM(vi>TecRjOO(po
z1;s@iWBgMuATZ2M+s)1%Zsq3s($;cQ<Y4zx)t9`1?Bl{aP{)xpS_D1u+F#eW5G{GR
zEQ}%|5D7+1eqOjd4MqVs^z$UruOZUioJkcHA9lvW@vd&VuH=J@$T#wc8F6Fn2OoHm
zN4ver*~VmzLQ|KwZ#FJnLUr=@8zD1ivGXSr0Zy;K_eK?a`Rn53X3Gg#ZkIP|C@=C5
zqMUDEEDab1Ok4^`8&XP7bOpOeEbdauwT2x@?<G_U?Qwubmc9w?r2*G843TNaYLk~f
zN#?C`>}#5^cqtJ<*A9*f)9utlj5IbFOJjivwW3>%V?ZkG5Fq)`r9DIQP(8r+{it`M
z33b5^?*IFJ&p3KI&?^zkY#HWAMC_Dp7EpiEl+hFRUo7HShJ5iI7n=xMHy1K^55(pj
zC){$N(0KuYFu#)RJprzr^+>f=+LwnHtrThkHZ&osUN`#xT~&WCL_d@5G-^=Q$Z)sB
z&@)*9rqD(CjQgYz!jPk?0pzz2x~<kzgZT<Ql`%S=a6V-zNy(W?E?L(a`ONxy`NT2L
zDTha=?PRbM%<3)nGSc9>-yyQ?YR;n8%$HyKL%u4~d{G4kaI?uU)2Qo?u2UI9=9b^r
zVQhIY^g@>zlsA(>qOM~2<&!YS)eqoxgt88eKm7#Z`VCiHIW2n`r=?IVCq%%F*nAz!
z@GLG+jUO@aXsgKcim#UZGLA$JAN6toiSd~nCh3fH{cUvP!LBnMv4%I~fj#Zus+LPN
z@E$r=X)ESP--aIj^c7klPH<rSb0;g?7`IP;OV4BP7z60SB}|N?aA0XkKLE0%55OI<
z!Ym1S1#GCDi0Y&8Q8VHq?d{(Di2}b1ce|0%ez(#)gZW|xMnm|d^<W=jXz6b7N$k8L
z!kt?JXu;O4HxtFp5)JUE!h2>$Uw}`o!4o)c4}=THgp1Ipu*otmal7VAA-;T5&Qf4c
z4gPmN=ml~P;!TXFG;ghFL2bC@dTo94j8J8V@Ki=lYp76rsH(>n3)O?f8V@jQeMD)7
zruD}UrfOhnO;1~w4L<i%prcRDj?%mJRQJr?LPRsu$Hh-Rfn&37OMWUVqgY!30AC8;
zFM10NXQ8gDixoNO5Iu*he@=R~Xm_fp#|iJ?wfVx}sqR7so8iqvUH_*-PPR^~=68{g
zuOnGIlzt2Ma;_B%U~nN{`%fJw%mU8WV$U$FHpGu+nD&^;v1xBIfj*+?UVVk2Hr8r)
zRD@V>V@X&6+|S5yb<TSwwmRChtJyM<Na*A&=8UtM*kI|h^}_(!!KO|r{1-n<_mo$_
zhL_|;s)crMB_hG*SVuEhcj&<Gz3Zbf>vvQGSNT$MW6XlAb})M{Gi?$Y@!8n3(mYoq
z0dKs^iI_r)b?XjBgWq&!2_Dy<%@9rS!NyXIPCKWNlEI2K@b>2Va~kzWLMnGp{Z(+&
z5u6achza$IE1zdP_ofh**$`e3-cshax(I@x&|0n4F>&|Xh0UinJtav-B4*|u&y+Fs
zQ^OlhrQU{AHi#~0g%hn%+*H04lz%eOeCvO!=zm^T_|Wl*<-u3upo@=H-1Z&g;CDBD
z5USVH2-ll{qki8}*{35$<9YN_Qv9mdGZ0=SW^JCL5HOK;VETde&kWut>F)j<Vb&#;
z8I6@I4*(JC#6J1AJb|r{Xmtupo}Yl4K`+vJS#eJ;xuXDUQg|R^*i#nypy;IsEsqdh
zU~d6-Y`m;rvIOkBeTzxYTZ`!t;^2l(RjnaGe}NV5FXH*FVd;TO<@}L=EVAoe{9)%c
zC`@ik{~#j&#(R=>&Lx|bI2dS?_+gyQ*j_3rYJxo{lX7t=0;7actRCwV2Zo4qM;-0w
zoNSer*OCX{b+&+hJ}RS1_#^DUrIDVU``7fzR^-80H%)UL_jh~VJ(VQ`+9W&hkvLc!
zcR#kNz5b8AG8i8}Ht~)biU+6oT*e4?GY6xDMBHpBoIjwiFi)@gRu+aRXgnZR>j*}a
z9EO0_(R#gR_f2Hap?+(*V4>s>8b4Ug*nR*t%x@<|-owmvUWE%-Vije>4(Vtu&yz57
zqSzTxyfeDZ#&1A>!w->%(LcUULamw2X7dKL6Y1KiJhG$CYK$R-H#S;74s$&Vc**+<
zs^TlVPUu4Db@<H!bJ;``Y>H50Lg=+Ka>BIJm*N+fo|)>+`u$%Qz^P2y^DPDDZD24g
z5$*<~5#VkX&BSml=xJ{m&IXB=ofpT;;@cb}@TQ*A?t4{~ii^qJV|n=g1M6S*8}pgk
zImuV(wXi-ZuzOy{D@Ju;bTE1c&fv~4+H-IHdd2Aa1+x0kmf;vAX6+r?tQ`-{OYHnE
zQ%OV5MD#+4x9j(Gs>pe0e1CP^Hr^25q0ULezH#M2MWiv<?i<g!n*Kgtw0+ZYE5-6q
zAm05sipR}omDi=Mh3s|{VZqQ2!YP3KdMF3B^HjR~0em<f7@(D$8}r${<G~b0+gk`S
zg062TL2Vv5QrNk)1bd}?UNi<7J8MZfGxusvc(%mogyl7VR9&1YeUXOlG>-8{w*^8Q
zPLfO_$*w}ls4SpuRma~=a@tHq7<Yet54*g7FuMUSl#r49n3eLulGmfG`bGWyLkH|5
zOE;p$8PA6H#}x^*HfEN*{kG}EQotK&&x^!X>#r%IMqk5<bOsr9O^ye7dy+ax2j|H>
zeQH>2GQv|NAbnETQ_I@7wO`)%xJ-`H6yn;|C(m3!5`(88VPPTbmyoBSHOn#F>pFwZ
z_rt%E<YbCsO(8{DVKO1<jVVhX=I(FU#`Sn<)~_t7nR_-axwXpJd_puDFqgI6T-1T6
zypTEC&dEhB?Te<%V;%429?!z4d3k?b^tuG|UqmVljD%o<%Q7x<7)ykd?iBnK@t~L=
z`@5enA=E{Q${75ZR&>~?qC^#`Gb)eFj}c1e1~x<n2U!ai%3p*Xs)r)5U(7~T<qyAV
z-7CV4+@Nb!Y?9gATyvta+KW(OdX>;tt9k~xRW*@AaY>njOiIDTmDd?!Vf!OHMBXQv
zwH?Q;i{6WnX(>5Z$(sk0^#WHv(?z#k$Sa@SJ#gx*ctW(YzTYsJ==Q8fg1<ue4T<So
zZcuN$M=xWeUD~Goczk@B)E?lb8)VmqrW1nw1gAH<f=Nnvoj7K?6b%`B>hpwz90K=P
z=Fp={p2;NKUs~7g66IB<pNl>n6eU=h#h3et%wBb+AHMzoI>g0w2SN3iY*-f(i>i*`
z;8EQ5<(U+{&k1xf3PxYfv3Z#Qz)jM~h8KuE5ig=i$S<{AvYjsA=<Z{Qu{g*VWj7>x
z6&HT9By9s}BSE-^z58t{-c+sp@4?@U;j=#$G|i=>>3X;8E<*n+jzche(9Ux5QOZre
zIG0lu<b(N!5uU@v#x%O~9zbuY<vMs(WUaoD%6;cAYc~-N2pxeQhNaV&33QBlsY6D%
z%D9<sGrSm5Tr5D(=%_GvB58eFvv@;rns?CEm^Uc>;IFnk(kQtB*?p}43Ew^TPGP{}
zV8hJcT&h~bilss<FOweO6uJM)`H(<tK{AzB09K=`2z+GQ)ph}PVZJ$|y&A-%1U8An
zR;1)|GueJYT=pI>)Oz8fZ*h!W8#k+NT_ma>JKlNJOD8$Si`BN<JHVIGGd0j;_#yke
zfs~uYYsA#$wnl{N)k@@eR9Vj+1!$)3xQOuf6oZg{vNnLC$yUj0T~5Z(ugQZKghP3!
z4DT2(=_;i!UP%Av7Tl=ZAZ8dump!MVoML8Sf%wbFOqMX3OQVStpoxni;DdYPh-?FF
zZ2l{+O{0{Cj-27x;742h_{fz{{gZm!UnK?_EMPy(+e`iV*Ly8$E;L?>4BU6c-8WnX
zI|UW?zJC_yC_&`W>C&|4&6$QNqL_*3c^%)dt$jLCpW&=Qm%u&u!o49GnM%i2;T_1w
zPHD$Qd99m~N!M~O?ETh;7A$L0`@fOvOqiD5yL&aEjwW~DcHklGu}L<QOg~7Vj~jDn
zo$I{xMyYl6y0kV~Qq}Bivx3oD(oZVyt|2qr1+i*3TlPO>G30K2<}Cb_7O{`h4Nu5-
z+Aqel4H;NJ{wWboWhhUY*Pp(_7VQE_?|a)<A~Ug_qs~B3bw%M&_U3Q3FJy^W$jgo+
z4P8`Ydr=H*+0SA|JSbqQiJL4$JHW!$aZ!4a?><v5=KGJzFSq;EJ2+8=cNxg+W~>-t
zOEY841PWORpt334++^my5aM?Lt_x$+<<-nXZk;2s?j(c=%cq3mqd0C_4MLBi3^AA9
zj?eL_x7Dyd#@F|0g_B(|(k!aW|Bt4#4r}s#+c@1Jf^>?Mq|!MQ0bw*sr$|Xlj2sQp
zNav7N>FySgE@{R_Hyd38-hF?^@&2`AfA4wjUDtV@_vhpm{YM!vWt73v%$<4RbT1-A
z+H<`Vt7Cya?@#w-L~2K;Gtz{Xdk!s~e{jlUN%stG24~)TaK}2P2WttCx!?6TU`+<|
z)px71kAHr*?8s)>ee$c&Y=aL$9(dhXVI{HiN!c8Kh&6c~KM&-syX{dM1{BW|lTdm@
z<;&NoZ(xhQc3YE-S)%t^#==a<&^?Me=4boa0dg+|`Pas#+oZe{>EW4Rw@z~MnerZG
ztd0_nd0JhF*4X|oSUSEExLqR9eG3CzF*$8eE3Xa7Oi3u(`|Je>Fdg6an)a<NBYsw;
zpqAHy&Ch<U<pJ*SBk=>F^XrXIM`KRDoNa49<^h=G+;%DZ0=f+H+{!f-nT$6@>y01m
zQU9D_Wr&XSkKGUz!ZW`FGLV_~4)O;0=yiSfcn$7j7YJOazIqyx>7u#;Y6n;i@t@>X
z{N?i^(DyPi0+=NEte^A8e6ShEsR|QRwGpWC!-23$!}5wLN*FVR+ohZK`AYjOybjTi
z*IGhYUxTCb-pTUGE5ukj12eI@gp9F4X?yPp+kEMWJN){3>c+4QBuUS@Lhfu`lj!Np
zpb_H=I^w3}2!{IR(Rq|CA9tFzpbxsHdRN!sbg7GGN?4HIIXRG|3o<vHE0wRkUqs|n
z%6B`(azE7wq-|Aomtx9_h&b^>odO}liW2UlL783kz3WMGYpCWwTLIr$d|z}G4H*HO
zUrkPo^WahtGZ9Uq31(NH`~Imc`sK`w-Ns+Wzaz}4-8m1V0#^kU#@WUNz}rN)j1z_N
zx_Fv3hv|E`utjytfT^B(ChVOJ{Wc6~4SM^nKMj_MI#9h`5W>06oGo+@*#-@c=CI@7
z*Wy&pjkr#PzcfwzTJa`@0fMoNHu~dR!2?#SB$4S*4*hNwIyiwU&V5?*L$?AUA7Ybk
zC0IrB#?9gs(m5F!#a>El%(D~lDrj2u=8v`%<--kMH+*+yN74t`qvh<j&2XUZN>|t8
zJ+Ck6!38Mx=XXW9EAa>s-7@*S+PuB>UB-1?{d4?mMmc7Mq&||I#TmBpr}@8BvyIx3
zm>xQ)BhL5j2t0)fse`k2GpO0k+^IZ`HN~fwBf@F367-#lG2)ukwxhI%k<j9?5x=cO
zyxS8xU2i-s268!dZ{($x$}jOA(VKm6wOqfK5zlz)yQF4*XhDQDh2jT}g*tq{MEOz}
zBnR@7?A{Vrmv7}1S0Y=LBg7F)BF|YUvSq|V7eCY@c-V->OC&G|we!;>Fudw*5IRn?
z+~$;#xOXA`HWGu8srmBW<2cS&v55m=Qyc~Zl^GZe8#(0lb+gyi^xiOBKZhNpWuHS;
zAQZVU2=Ho_fGMpy`Vr*^fXrW;`0EGcEzvzBdp&vfC?;1ar-f=l{O!M>`_(EAKi(_o
z^?g!h_&~Fh>p^Dw{EgjkDQykICv8u7cQB;@czPmk0|CBjTRs}-X(}(5EM4uhRwL{L
zzoC(EViF$hzVi7@w%><E)86xp_6A4EVEtyGVbnU9oEb4h?&!KH*?CSye@qVF9#oa+
z)@hYi-Fe=SQA*TAtfVqQ-|geP-LRgEU22C<+5Ri0I&9EIp7)kZ<Rf^Cl>crWkMqPo
z0IL119y_C(qXJ}eA;vE=!tMmmC59%IT;<BzNXjD<ZKv@#EnQuQLSgBmNYp=Fe-nr<
z=gl8nIkz5Oxe1ktbm|F;IfK9#vGVDs#B~Z~3B+g1(PTWp)6bT}ttBr_3JrMrMo+Q}
zjY4yi8i@?ZMdw@LDUv7;JUxV!rwfwzLOB=pTjS${-t@0}bHwbporz~f^6HjLC3`tX
zNf5C^(&bdd8mUNW`xzyIYaBO0Z2>xzYE>WLyM4RX6sBBFp1}QXT}C*Ui!YhjkufAx
z0k#%fwu#YO<s)ou|NTPQ5G`qB>HHYX-q|FM2R2{*Av`;{D+8T;CaH~xQvrRVMf}k|
zY?WnE=bqGYu}8JzNBUFd2?-6e%Gh7$sOz}iS1IO109@mpC2=*ZJsTL8$r5__>b2Fr
z<e1k8YfhHV&hywi=1#q)pjfcq^zddH`V{?(?dS&sFArwUN#EuuMkdJcAKk9Kzh{I{
z=@4dnPjHfaXT2kJkQb*3s#ZI9b&H95-fiz6oySwT+Z`HQWMZ(eO!5v%37v#(gL6z!
z7NY|wp!N!DZlpg_l#bR#og~|46pj#B_0;f1<cey6-6Aag$iu5@Go;SkpwV7$jv!f8
zT*~kCKEw&#2~_-zQ4Iyx3-m03VgsAfhX_0ccq|<IY{AsGqo~*boF592WDO?*G3j{5
z?VgW!(7@*Pwn|#aO_z+j_Z%S5iz!_Y(oGD`s9xxqI6)PPZqL0bRNt1#%$f`7vBpTf
z{U*7JRg(hrOPGs)OL*g#pa<!NPTRV@8hQMDD&byzTc1l>a)IL|{k$e|;idlGIjG)C
z8Owa(HP?6*3op0zCs9sQS9<ePvQ<C9a&DJVqkf-l<8GgL-4gDW;^4gXoK(gbvEJvx
zGoyhjG=Mc%Ui|q9HzWrcn!q;ZK7DrI?G9Nxaef2z?j0Fk>ht!}j~A}pEec>{bG=;3
zVOfZ%fAH3X5E@$pMBbE5e$4P94oCr9s6p3533J1@4CxW$2TMHXB>1cE@BS7>rk8+*
z`V9Jt?{&p2g9*ncS<EKlt)D=?j<k5rqw^5PAQf@}6cg&qGf5eI<`JaeQ>5OagJ;=8
zC}4fTXD;(67(l_4!gGTIW<iNoJCXV5nmp*qGi)s7QJVyQ>%Xm+d!dNe<7@J!G;HO3
zDeWb%q9Sp5O}nHZWR~aR3&quSvY1j}{aKyicJDhZ*;TgB8*xbb7jR#gFfgwoM+E6M
zln`q%zeGW2IgG85jYnqYvpfovj=|;{iyHX5$q{$Xh#JC%KnA<r{iz{xF$eL4uhHA8
z#PL_{Ej9;<UR<&I6lx8!1ix`<#dIN0y|h0nh#gZ#TNw+c7iwr`?8?acAo55TzHr!n
zFS5h`Ib*y<Kw1x7P(A&;`T$4jNnD-=Jeh%-KZ7FlfMoni2>YB=CEZZSI(!DhDm^HP
zb;m!6#@sjWpU@j^p=Q$Lukw*UZtDJ8$Tg!?u+!x9+80MY%dA%Y@eCw=wsfpFYvB)z
zJN=x>{O`U5f2^}fj{IV_!nhqb$MJ1Gq<K3hMpXy~@<1RXb{?l)_n6Yju3F9(c78T0
zfEli4v6nVY`n;AuTB|w@uKn#4B+u)b*oh3hjUFU@bbS;j_51S4TFK5aPvB!?avy<v
zC3hm>s>DVn!q)8*{?ej?RazU10WaOoAKBfq{kuIHNv8dO9Qi*aJ}>R={C>%sTGcsB
zA!O*jh}qEQ7@P_;ZF*T9o^k>>N)Cq>h}cUXy_@tZ$*^zriOZC-Oc$OI+N;OjDnL6d
z4K|&EWMt$1RC4bh#*JH57I$oTe)6=RgilCFMq6BswhtE3F%y3}%eWc0&8Yn=)@bby
z1n+*1)HieO#~qh6@O(9|6R`nEb5k>$WltlS=5X=+>$khhq#<B=dRa_JV`*P%e%D~m
zP}#cw+sjR_vkv<75Dt)q{j36Yig^uUS`*+>EiGCqYaXz;+GEd>9~C7tw+k`QQyJQ2
z37r^ADr?vCa52eDmpER>P5<TmrcS%IzShr+@SSh7*-0qAngM%nbO&QQaS2~hzA*Ji
zXEoWwo?z{aWVIH{3}<}x(Fjbj?Y$G3UIPrXhs^3>6?uS>t|{*E*()cReW{u}N?a+0
zVW#H-3-nG0`DQU)F@OHb6xVlouVVcG&vg5m%jyfOrMeiwetj(%wmZ@3nW1=}IlH)A
z-$i7&*_&)fYfJKCL02Riv4K_hH(}_eK*EVge;~Eq_*zVcR^kM5qPkvRlUFUV*EB5d
zdXmhnVdc2bCD_&cj}8SiJ54HSE~P5$2E98#L++j3gJ1p2=`L=E?3|RyQQ<>OssARi
z`JARk<&6O2wv10Y{gptz{~KXs@8SncV03aZ8UE?hpKUQM-E5}&7WO2hFm^tX!d`SX
zI7>EOKha?8MwND9(mGD><G4ywsu;?TD6!jx)YidYlO=QFi{<iD##B$`=23OC+ryBy
zt2cOhp^ZH=exX%SNJJ;HculwA=i|)dt$74cfN#t*x25B7jhsQCrCH<`^>eQapsI`X
zX>n}lW@SvoUBKGI%-Qtv3l81A!izoTwpyoP%c^`=GXvXd;5p0j(p9k(gK6GTTNyP6
z)hf;G+v?6-=Y0;0c;^P5#8>>qB(Sxb*f-Su=#Zu`GwEYXS&wh*CtLH41a-a*El<XZ
zQI2g~tSaFvW5QcH;&V`Z_uC&O@65J~qA)fMGKsgjD?(!?E}R@d1OC<m!YuX&n~Qxa
zSHP}wU|&xt{_>~sPxGcC@h%4cw7<m&lW)J3-q8!-j6Kcy%02i`CmxvtQYGAZ&%>HA
z_mytrP$pB5Fo3#|IyvW@`sZ#h{dljOy$*hJT)LgcT2Gb+ZS%hbm-bm^s;g~u$pD#X
z4*CXm2G4t=66r(oTL=8~ki9OqO!IOdMp=~jg`uE0tguSVRph&TkXhL5q}pwfVqmr1
zK?WOySiL6(Kf8L93D)u6%Y@K+2}lpn$m;X-m}X*=#QVjV)2YC7_MbtLI@UVWN~~!F
zH0)dBxHuska-4oo9Ee7rTtQAb7}4!48=)$OXCz`LK6+PZ!Ox}_ku-=-2pK{{q&rc?
znns>KgxdCyas8X1*>*pDbyzl*Kd(!u+NszeT3(#}&AP<^$}WbHeC}r!JCK@r#|?^e
zuUOeS;jQ$wyIC)ROj-+^&u~J{n3o0+eW4C=0&6iS<|c3Ziun`HvZqJPv~F(N^$|32
zmKMn%@0SwE&fm!cZ}m=fT{KMcT##+0q0cgz0#tRa^6!XOSddPzOQ$*ycyh@E!>WX*
zmhC;TBdN(_e=XN!gDGe1!$hI9G!-rGef;<yUx<t`a#Lwo$Tx#>V_z$$+O8tZu=&-6
zCO&24Vv9%oX1^vaZQGS3WeU<Gt4Px@DNsS?K~j+CM_fbTbv)ue;qf{l`*Ie;>6I1D
zu{^)LX@T{R`*87gx9cu|#|Uz#f!DpK^HyYX9&Hmf4}vF<C>EawWFmgxQHox}*=qbr
z>XS}{^Uk@0gq_X(@{3}okf)bd*COL3iRLpYrF*c~bU!V@Qamn|l3(&l)aT#bPgTg_
z#=h}BerKgV@RC1IL^=LC7NV#zTL<RqGb+=}^waM>-|!!gXI1&TFg#{wa<t}yb_1Mj
zC;3N4a#{v+iM*taB6Qx~8D=c@3{`$NrIl_DbC8zdTRJR%`u=dzeF#=}s!P8~sWo2H
z$J-%oy+W<Nn?A5p$j2C`JHV;uz=sjmh9js<ee3h|!PvQ_7&?v^AzR(pJdj`_#%g-2
zoA#sdb|{a7DSzd-`s=ys<mfVdHjhWt-RwG5^oE-lLSsX-%#pIRb5Ru-P}j(AaQs2}
z+KY$IQm&-`S4<<@`@)JiJWtN*Y6%L^vQtw7le7%Q?F*;XqvORIcS<=$k$&P<PB~-@
zs@!v$c~_8$OCwu{adAw)vb#mpG=N*cAZC{#MPy(8y_k8c#rKfErpkieJ;XNNYw=pC
zUI{EHl!O}}Gm;@3?*Z;hyS06L%N-N<+~dWQsDqh$ob14)RR@c1-DJ%ru3gnh7H)e2
z_u&X7U8#n7J(IvMkX%`p0sY-ClAg6`KO5fEX=Dh{8P$BRVO~1hhK>Co+l4d`-q)u#
z9@%H?6b<`Lk4EkUNXwZZX=gH4b-i23|Kzf*Pj3$x$qeN{ulv^$Qu42KJCC-jx{4T$
zi?usTWsmHg91C+eklEx0e0zO+Rl`YgyOLWf0b!Y41_+5pHNzhouHg@Sp??S!7&a6V
z9Lm1NyIn#4mw&8Au+}e3QDgV~+mrDb_>&X-CjnTBL75(`#HATYC}6q1%-Xpa1y^Tw
z#Sya)$DLm=)Y?FK2HZuIk`z&wQg1SXYly8d9VxC#%>8xm?xE?!d?548W$?`|(l!#H
zOH<|Vc)dLFcx7~LF7DqSFhVsKYoDvt6|-~&a<I(q$NIzN!7;$%wzgTr`<2xx9pl;Y
z5|=$Ch!@mx*NFHNcOXDY3ym+tFf;j+K$q8>7Z&h%HCyvU`j&>4_#_KfW9c?H`{+R}
zVI>0)7r1ioE@+QM>-aT!lCOI>J<L-gK4ukK2!uWFc5-wS=^0tEvCNe0_usL7qm)sr
z`#92t8ae9aOn+_ob66UA`dox!)Bow5w%A+pwMdJtUAkmn<?+MGr>CjQ&nGl$p@qgH
z`=n&lgviT_#B@^wVoN>T^+}=}L79bA@V;)I6r+rH_0f`Tr59C?#P!OK9OxR34_N3V
z`|i?<4_KkD^~#CMhx4e0+cF~#cybkA9blg~_D!TnRmbdh4qL{5L*s7@^C3&&bt8}K
zG+oUNES3vor|1iBWn8*Gs${l^G1++50&}~ic~C$=2Vqy3VLgY(V1A9cChX72NtCyD
z-;Pgu#M|iFzZ7v2{=oMXm7~s*Tv{(Wpb9cdw3c*lZ#I}qQWVV~l0M#=HzA>dD6|=@
zsjH*y|5u;Ih?l-btEY60(4PT90<KQ6+_s|;++Yjv&II1*JpOXYH56~wED}z*r8B=_
zF1`5Ks{vcaF^847xLO-FB~{wB86Hw0n)L-gGKpq|m-?y*?G&bpjV>A0+oJlz(YGI*
zcvqyk<OD2z5CN_|skSEr9>X87I!=ZSzL02hZqW7^nMYUWe#HT~I;DavZs3y0Yv{sA
z60lh&bfWI|>-7w7A)KctmN=#0x_JkR*M5V~?DJ<rxRrjVLc!GJi{qT~3`<x`C<)kH
zTq}MmSC%OD-?3es{hfvdD-E>`nfqCf<vN%(_$Bhqa@Q4;hCj6cs=LSOEfP}uK0Tg|
zL`_PoHyQSy|Fe4|zigayLmxKLwBS_J?Mhgr*r?c{I9m1Z$9?nbj{9BZN7w5W^MK0H
zT4$EZ&Rg?;CIGtC(>d{U$;B=gtZT>I$?|e_*M9eF=WIYsOougYHR7+L=ZztM;;>^^
zNl>q8NMPnUOU+qzRsFf=0oZ-(h-a+|iVOsc&iEKIY^X81y9jKMc5c3A_Cc)05R*hS
zbc5d!eu4?f9ImC$9`8L)O&?wYcEjTB(!#m2UO3VP_!?zrH~X|slwRC$o)Kb>cM#!#
zV}Mv?lJ9&0LSbae(*zrGL$)<G%F$0!TZkS~`r@hZUCN0Fbi=sgg~nElnNBqKZ6oJ*
z{{F*^{}((Fo<IPZ7Q#*EQkE!SWc_UBJ|=o(mwjuH@jE&9Ikg*(+%~KnOG4sa$2^2(
zZMFC8_Mpn;=7;hrL@>E_7@uit1ogobl%)%B8p=1c3mG5Z(K*%HH^`_Hs5h{^w`ht#
zjYRjHc~_e(solbpA_}^{_vOr!$`%hT^`Eu7FN0Fl*3yf@VS;4L{=rQt#vS16$UyHB
zl&W7yPbih1chamf@&$RPFNHt12zTrzFQXX|Uo-?8pRC@cvaZN|2X_QNj~)F|8Btlg
z>WaKa4C1B}Pnzt0L-fx4nrP2E-?s!`+4m_}|2idwRhyBRzHKwF%gEk-rS7tsdyFz(
z(J#xG71utSa%ljoR^{C7+x679XZIg<e8`_IHhZyO2JM@x^Xj9?&$m0`CV_@WKCGdq
z+qHyt!Lv{Rvd41tw3vV6cZ@^c!+9w=A^_pHoXbH~d94U@)qs4AtK}YSC+>u6qELRl
zHv@o;B2D;kvd)C!dWz?evKja``&HJu2#EH5p8wj2Fkqrr646tPkf<znfxB6K1O7EM
zH$|6|$6fLTZ_+Gj*2D%r23{WHj@Om_ro7+#y?-oYK|naM_K~0asgGS^i_GT|(kH2d
z@V_Q0^Mg2ANKDL44(zO(Aod>wu~H#@?`eZEo|6!z_mICQ(f4`x=SPscXb1rt4i!$4
z21x%S>;hT}o-KX&1p?^g=6v*gxXZaW@I!dcE&5e-%&*=V=vw4V<^4b+tr&<b74Y&t
zUt4CewOVFfli!B~E}W3|>Gw*XGYpVE#5FsJK4nk=3`llT{goc%3ZvRio`aZ~2!E=p
ztE<3fpX@@03m!slMs2;Wx)Arnn-@)OGvg8+DThqZv15<B;}BwSzVyw%TqrO2V!l<p
z{RGs8sPHsUnp)Q0oFy8a&DuAOKeCV66F2DkvdME7aK)$Y*)4U@nt`@P^s11OMpaKh
zmrs7Tp|Px&f()a!v0fco77LwE46qa)ktNmnkNM${7CGY(<T=?}`}r`q*<TqxE!>?c
zuVr}!w%Iv4@RA0awJ|%acNGS?c8l~-Yv~tNP;Ft&u@+{3aKJpXce^@<E*C@zc5%Of
zpgVH8T)<S`bs9FRvgCbG-3WqlNx7Gvf3tLnI*8$U1fX?81yXK13xoJ=Z2%tgwQhHj
zTTA3%H&)$!kay6*>%sw!6)bzMHSxeRaq0Od5hwo)&5*lg8(CA}>10TKeT>IGZi0vc
zqr%Gan%Sa`+?JQ(Lp2Q3`kQE&q1=V%7F0OADY$@?+7Ot*L_-Ff44-%)Jkdvloh944
zMsN-d!bRObZM~OFHlq*aO*5&!6uN#ILme#&=d5Z^XJfvid*MKc2k>6RXvO?dq`orW
zaqQ{T652yC+zK=dNKPX)*#3mEnxsA)@1G<neo?0%9{fAV|0%P0{Rt!BhW;mdMeej_
zEwH&Dc7N3bZu0%dK`jWW2)_FD%i-Yn*;pCC=4qgWOp6P-K2r(Vtj)Z1r`d65U%y&_
z|5D6VMW?#bno5A7X?T=B6SzBF;2hU&xev~T2&1`;q`}ipb6&$+58mwWAG{0~%ATkV
zPl~BJ`Pm~pqRGcL^dr$fM6fyHdYwibMUf7uURgMO;;_CI6*)VrT|oB(DA(y8>TR>$
zu(KZNEZex%^~-i8%vbFCiz~J>HSy;ugNkcCwfA%+3Za{M=gRxcPX5Y9kfSv4Mb+oF
zR1dz1_rVN2xYIxP2?(I0c!X<8aiXsDZgpHXZwW5Zu=I_#85cJt&%A&8``=XQ0L5Y3
z?#!p~#>-2ESpwo;Gst0_>OJ`V?Vb&Q>vN&jcD*&n<OJ)kOc`HN)}mlfABC;%*0HOZ
z#@5Vd(%uoJse}^^8KOi#<cE{bQRv-MrfJoWF!y2=4RJ&GuFvk!rqRZo-rqLLR^f%Q
zOLrO1J8MnYxKlV!%ay040XURUf5rkJ{dB9u;xU;rPa$8wj=fg3;{3*28FN}JfHROk
zDe*@$ibZb6OqOs1nL!sZSmGL{?Z$m=qd-_?qq)UVDdaqCkMYK6%{Uw$lU>^bTvUVb
z-07XQ$W8OqfoP3*Lz#AF$LOJ{`vVD251ITc#rlCA3VA9t<A2=9+r?$rDut{uPStOR
zzH$i6GmDhJA`vlo_)d!J1c@!pn5*N~7Q8MFCI^RN+b|fGzbLnsJ4t>GEG@sxs_vxC
zv$e-Kh!5K0y{tu=W+_73KrRd>#1PAl6m}nXcNP=3o2qx&X-e3}6=fXmw?_AVKQ6k9
z7c?gr_n99z13&r^ni$srr^?K^;+%opJr<Qz_s}*MBW$&c)z`#7+688H=Q$f(Ri3o$
zd*RBTO+uS|oee)F<{o~prhXCU6?zf<MyI`jKCx$98qr?zVnY%!Dt!vU>^EGt#u&mN
zW&R|(^$%Bg`E;l)Z9sU&3tQ@%kvmzU<!bbGe<!zCu^NQ`?LB;KzVoud>G`*QLR_=O
zT9m5HHT-)irVOmUom7L7@S54_+pjU3iY-6!g|bdO^8n4Y1NI?XwKhqj*yxAtUC{w%
z9@iC~AC_u^`Gppz#b;58oTGeM1N18!$5%gB^GAh^%@t}Yl{!#k?a<S-rr#IFEM*ap
z=X%57gLfCD-?47Vh#(jFVLzEXH5xSv5pl%YgxuB|60<monfrNVQTA7}Egz`ILM#IQ
zSjAGNVr`B;Wi_%j6TC||za4S<5m^8`Gq^r{R{{T?&Jds%q7v*xt4tqVae<&3w=}V&
zz=^b9p{(R)5u0b1$dXkxPHLXUS4KXG9(zaOVQXufj-zmrBp&=sfb)`7Vnz6_$?wcH
zA{FpNlit||b(d&{G1k!o1QL+AK4~3;XHPX8+)1yV#OrZkwZAUq_o#{!ao>V`%96rG
z*qAo=ynw#Nzc$@>e;$0xT?f3Gnv(??%!hIDP4wLk3Od^zOM=~sLjikJj@}5mgvd&_
zVsXmb>PXpgCsrH&&JMN=5&&wA+=YSHi2Nu(JB`~r=xa}y0c|l<hqPHQG)1>_xW9an
zHJE*n4_d7v{9Xq3Ez>Z~tI|<kB6dSgB(uD1hpjvN4ab9m(TB<{T}0W%>iyH$xvooJ
zi_~AIzi4M>h@mrirfGzpfR9WFHC#1)Ks#-Pe-ru7F_S1JP9`<EU7mjlYE$XWRqp1W
zPKOF6b@I#J*o`$9uGc{>$_LyXIzY`~8|ttiF-O_{bhZ9o{LPNuwWOvP+_c7dzftjY
zp<|)ko|n|3s8FkKf?&L|QOJ&TbeBhWP{t7J6V!2dWYdZR?-5O~ZEkJSP$B%gO`XN%
zs=P={IEMdleh%xAXRq?rf{w+9^923?<wJJd_zWz2M$!FBkt8wH)GLgV8kFd|wC2k*
zMN)^jL-BBMCr!D=YJh5!pKZlR`KmdnT1(cU6`Wi4*h#tNj6>uC9C+7-)AUw~-0h+d
z$;^Fo-K~~sKl|Oc!o|4+#nIi+RkOE95U%-Gq91$pq-V)9aL3~aCNI!u>}?gs=}6%l
zgeeVs_2%!DG*twrN^0)v(a4*>!azYLN)@8Ns@*Utb)velG=g^YSOD~~*Mz@ku)2)2
zZxl@xyB9h5jK4(?gT_7?8xI#<*NzA{yk(N<u~t&a7Ljyk2^KlHBOF#KL=9tK({Wow
z+(wAE=kw?czx#DpY<^qZ=z=ld9cd`s%OR{C%@{r|AJp#jB(=7$h3*L~6_nI^7E{f*
z($lsmlXUFeHhk1Uaengs%8pa(X6#V+h5V}ibstHAq13QIS9{gaDfvv)hNQ&icO+ZK
zdZ}aP1^H^xR-iNKF?U<T#OQZ^>_zm6wL#PFFdv}2OGhO%`KNt#>#hcb0a2w33EUkX
zxx*c<9QRsm@{)8o&~>)PrFsF2mi=nb8zvCNrLn&Q-p#9hq3#{i<@aDWp6i_D8gL~F
ztcp7t+x+cT5$w0IC()kkp(}e@`Rm4OKz;0c6NY-V!}6PF!x0q~{ugzHI4QsOmO_it
z{P`12m)!U{)YPcWto4JDzz_um81f6N`s$m`gZ9vd=^n1m`qgw1A8P8HzDothM42<l
z86RJBh$gx>t;<&E&#VmfeHd{vV7|_y?`w6mn<`=8eYAHEjR}Kia(wpb(6a3OvlwP@
zRF^i&s=Y$jBBilOqwUq!`(P^#%(yzdr*_wbRbhN>EgF^CVuhOUXQh9Bi*cGMRnNNV
zmFfia20QGdpi9Azu`{<DK{jet$+qV-6%!WlCO1~Q58^u4yiNPD#&Q*pr**GOR^xwE
zO$@lxTS2&~x~uN0pysL%J;$56<!nqO+@h-z+F!mN4o4|rDu<(=F@!&DnuJ@xV=oUA
z)e1oX9mf0id<S)*{`4fZo5uKz5Dm#%xw?jD;WJC1l42@uC*qaf;tdTV>*3))%dzbj
zrL5Na8crXk->IDNk@Y7faU#kiHK55#GS<}tAAKBuUrk2J)f;lWa`*_&PM}9xXz&BP
zNk4R38e+!aVDs1mB0Uy&18}DQjr|xpgIu+9SZ4AxR}<8^iAO))ev4$jC1aTP(cdj!
z*?+Sp&a2KwO}#|uzDDj*@(wsi-f_0t+}q*qL-Onr-s>>eAY2DJ&uGR&<5Ox^{;K((
zrf6c2tzGI(&P$ej&lK|SEjp2{_8mEiN1^jAjAPr`M~rj)%^SZ%W%{)rkU{=yPX$=6
zdjEXXmw7+5L(__9drC=GDxxxhi}%}r{8}RyZ%w*skA23)W(lvqq}29{tEsuMefbFf
zSh?wjs$pegKL+?>P}katE5fW*)tCl!_vU+@F*kFDJ)&iJVK=5!mCo^F{mD-TPcyzp
zsps!jDO%g`70@zG?QeE<SCSi-Vw@YiuyDZ2U>*b574V0q6VBzBq2Yu5t4~PApPf%8
z>vdoD`SIE%A7dV!-E8E7UvE%lkDwmDg3OSgu=9RayRNyw3JS^_=@7P|5ZssJ;eX2D
zPWNAHCM6ZEScDXIFHUgZQW*i2y@+7zZoyoBj@Q|Yk*(pz<FisJiI8^}Z2Bf{)>o;5
zfeqOm8<3||su+`1ldtw)PURDl9LX%lrB$u%Dn@bo+cgo>P`;V4q4OG>^$)RsNF!&n
zrx~&eW8lWUG_9-h8&P!Glk*<JZ=-pf2DVDCu4!<fQjh}zlwENMaAsoP;k@tHgDEFu
z|ES0HSfslDL0Dim<115}0cIeEFsuwSKiaUyGh=j-$?db3?j2@W+jIY{+3;+v@fWjM
zdRtMh1Ek+JcI8wXPAO!<$-`ybLZz?N6t4~!XNHT$f4Q6zKEQ>6sx`tu7pLa9lFf9F
z)1s9q&8Rvy`hM;^+M>grHac-@RYSDT!yew{Te8X%<Gsw}xcMcKQ$&qPfonvJ8@9S!
zZMN@R@romhuI-vt<lBGUu1$vF)grHEn5>md>V+jXm~4dPnZ+_2$X1=9e4^!Cc)cO%
z41mz3%QA}sm&w25>r_?}Jg<kH$6kR;SdEV{L1|5dnl}v7JrnZ|ti7P;l@d1R(2ZBt
z1g2r(g4;XW%hU9*5Z8xwjcJ6Ups%*E#@V+>KmqCRA}}5R+0yWEn!E}R))iz-<sALN
zK%CmR{Lp*+^`c|-O+epl<)urZ(s7K<=s`(I&WM?a`*RJSkDzv1TdhSWSBLBTF}Sg_
z_g{P5&isDiE#P8Revh8AFshhYPmwC?6j2g~gEYYNY4QO;rlb&7hMWD7?Zj>!eQ_Zk
z-u{1S))w=VNb@9+Z7KW{y%7)7J<^-(6YC^5FHziPLFQCnz^8B;)TP^W<h2=66i9Vv
zM7x<hsD}v_8b#~fq$3&4%1P#~3CFiow9oD@>_rC9P4gW(tZ@ZzpDdsUAmbg)pHn|e
zFvrxs9sAeZRgHHX1Ba&9q(5yDn>QEu9a4676{KMNP?!b$n-!nhO3fynTi^>O)3=45
z^5aCRVrYjvG<PVz3#g|CP1$-hzQ;r*l701N@i<{1!s$)oJZX+XZ`G?@Im8hXaB>6(
zGh4QW?-ZJzaKGXXuH?DBe@YSMC)Y<bh-E}rmb1e7WcYK&BJ#n2%7RitLDRO@+<PtD
ziBi3n3DfZ>iWA~;-!BMu%kJ<HNv6vPBZUoprudVOlmI>))=fCqiKo0AX58wvz^vw+
zWqn@!t-wW>13a7p;)Ap&335cAir`rCrgDM3Bc6({+#AyCpQ^0DKME*|+}{O0A{bnF
zx`Yd?hUD2^TfF@xGNb9*fGzUgN4K)MM|O{_4~h@h_6lW4sgWPf*|EI6b`qaAGO1^L
zyZeXfk|(Orj4A*TA62#J2HqDWB_IpMtn%OD&DPn^eGxCqdH4^(=;JrM5ZZHVeM9H{
z>8)cTMy}P#cT@}Vukg)RL(sx5u^?@C!HW%h8O=R3nARBzT~BNJsp9Ov*aW&H-ZL%$
zCw_{QLjIa}61xed39P>JA1lwcOgqkcdDZ#*jtk&pm>T!Cv@dLe&u4<&?kqg;`jb<L
z-^jGGZEMRueyMRpfZh^&-A&c+xZ%oQJ(KRtUp1P_*6n}%n`fN^n%_?Afp;=CgkkBv
zi?X?jF|Ko>SLL$aDKlzdbT#WI5=u0^Hzs>cq}Y(9i=blPXF<$473B8m$FpA_i@s=w
zAMM3x#H8->8csN;F&ovcX?j0(;_8~RY#+KC`@>YCE+s?AJ+IIt%lE)-Jj?|?Zs|!r
zPa@f9V~J?dghXW{;ae>2#{tQm3aGy?x6v6Wa|Xs|oMTT|y5f3;0QM0ReSX)GTZ`Ne
zuU6-HgkIhnmqIhX7InMjcUPYj?=mck6E{x}7*)u}^xf$eZyL*La<l#mtO(oi%e|0=
zl0W_pw2MRbhCsc`2!B4D!dKpcbc&5T4orUip5>u(hSmg!rWqdudQ$t}m9g;$_AfC$
zq@FU~YXxB>kGp#VRC2@iOxUTy@{+)>$B>e4f%o%)9fTM?rG+K_SZ*e$OVa@Lt6%h5
zNiam`NgR~v;Ue$+?ycUnB;Y*JS{fm)$1^9X1F_qyu^=+@!CztGd4Nl03Eck3WRK)7
zaj6WZsPnQp$`s1BJgJDmerp~ro+u}gHTV*+8s#07WWgHz95dl!ag}Yw#+`GSaxY14
zE6COx;X|;F|CF5hW(@%55jRssrM$9CmPnbv2ph4`p_00Wi*moIAqMZQedS0<35F#m
zxC_Re^Q3#9#Q4n_+Im*b%HJN5;VGJAV_qa}t7!V_03-C@9yDWd9sv9wy7A>E#k7j-
zw2Z7u1j)kZZheJUz5ao4a_a^o5d(Mc&3we&c={{DxXe3d{4boOzc}5Y)6vD{rmGHk
zVBgmG`||c>2b*JopV#x>WA*2PB!ERXM(hy%+a7Y18=Ao5UnUK$X?Xe(D|A;^cLqBD
zu$R}CV19dYEC9NTUq}DJ;yE_>+h2V!tPYNrn@#DW|A366wVRJf=jr4S78|L5*I)aj
ztr@XcS9&4}5heJPK+1W)4$a3?@j9x6F45RFkiXi}WE*qUtc7p7n-?5i7}s4?z$vXO
zs@hO9B_!B>ioPAHo6F{ct6$b4_gal&R{<yMl}vZ8yFw4h2iMqw;psM}Tjox4L6qw#
zbNUu5)%~o6@U>X9)wi?vu%RzL@egIX=oLl3bd2`34&1jiDBg@cbEH+Tj+Vk#kKcmu
zdDxTa2M@yM;%ztfP|f?p2@lPOU!@*gsah-atBM@mqiu#Y2dV^RpUjC^i3B~noR>ZV
zfDoYeV0Pg<%wfaFPF1t=tMZK(RC%Vtjb210tV+MuqLa13R|>0iUfs$4F^T+ugCG5=
z>F2E8{q943M!?tQnJ>2I;`LTK+(F_X*Uq-FBbPe0sk~9d`@9;H>^&)+cY|#AS_?Jt
zIU!aNGouZn7b&@qsJut_E41CyJZ18|eWVEJSN**pIvd0($B1&|qkb<Dm<Y<V73kW`
z^I&i*?q@i4MHH0lQimqzWf~9#tXD)ZAeK+e+yA7>$spts{Z1mR$w~h(-1<?22waiI
z>nHrvmfz0awUMj8rv1#+dG+Kmb$=tkPI~%MXzU8Ls|;v0EXo*~xa*0bdejWOojD?q
z8JVoNCrnT9lV~|fuku?%Z!JK+%9iaV8>riXF|78s=S$lcrB0(iHj3TTf3r&_5kC)N
zjh=$6d7wymRJe~or(hEv(Qw=C6g}(rz>8n_<iCx1^81}x9+1lWg_($BbH7h3DDzpR
zaC~x)He1ypK{JYA>+ZM3*_DPU8;vSBf5J`4?s8f8=x;=z1VO0MoCe^?UejStCf&@}
z{%WG3QHq*(azzM^6!(MPmOt}?E)=3ww(nRYJc;?WTb&y-|9x2w!BW693?_JmGky`I
z^)@zt(>;?H^7T}ufpesMu%K4&`_j^$)no!0Zq3E8RD?@B7DYAt!c#C){y<TheZ__k
zL6}Vm!6s`+^&Pj^!HISX+YPC0pp&B_Z3AS5k3M~U7Qy1C)a0&zlx8C1TbKP%wjDUL
z4VlXQ5Dl%tD{qJT2qK=V#<=8QSwrj8`$zXP`>hHPs}TVy@DhJM@SP9ZufpZD-;@Nq
zMksUJ@VKDFvdxEIGMUUp)V}}pWekVqo5GNC@&)Ubme~~#Xi3$l8(;spWj%a*C^@i`
z!d^!C73pfx8lZf{cdI==ywKCGN?kr?(yq95eQ#1nLXH~dd&7NW$8dY<{u`?}uNQ!`
zx4C0#==Jvdg!C4c8#1K>)5g{|Tk^F79W~Bn&2>l9y;&DKczMm~JIHRY_%BVZlpnZ6
z?cQY;x;@v#<Nj#uSvv0-V+{M<HtUq}i2ksK&THw%$m!eLc%@F8iG06n(Ud~J+bn?P
z4rB<;EN4;M2p`PXHaw(NXErG4bRzW`)Sr}~p1J*0WMpDVBbQ{~P&l{scXF$Wa*aNI
zHoyq$Rqubl96PR<YOtG9U||aVMjTZ$gj>EjT8I)<QSfb}wM0z)_F%&Qx6K5Z*W>Zs
zgVPXWShlL6@U@eYO^KSl!$3&Ur@2pIl%ZIw{A&peJ<GB^OKN-n@$HU#jjo*f_-&TF
zoEpnK=SEbCpKIX_sy5K*%P5y;de(ZS*#Gj}94*G?8JQmmyWkMKDLw|lP;F2}mjK`u
z^a^$C{2;%jztuKmU&?u_?M7@ZsK)ZlagdzQw}9qvn=3-LdSOVdbL(k*n$<MwU#Xnu
z-l311`04Q<h7g%z;unZ~^JBrz>-7?~D2GCMhhJPX_!CMiMO#K2&R4z{{mOHWXZQ{$
zpHAu4qhXVtpHm8PFpE$8I(Pi&!FzChgryvnTFHjmPf~daT_eWy9(-&AyvAey9qFb9
zSpJmI1Vf+fEp5<SWsg_bxu6=Ids2=3R0&q>mzi-&FcO?^8&5gon0l~#)^ZnPPw6JS
zmF|>sX7~e!Ev{Tcb2RX(W39Zr8nb_45=Nh!gVzPy^6vNYj^!R#c02($|4QpIhv$g4
z-G9nt^XQDN>tim73hP=|fv3vByyhGkzcK>?imi@60IvwQWe<P8`Tf%B(NQJ#Zr_vV
zW_C!dlcyL~4i5Zm5-vEmY!P`e&96~RWC~Ae3Vy~pZx`*%{$FbjdNnKjZ!C_}`4TIR
zKFkZ`ap)G&GRUnuHIT6+BK`4x%W3DZ3Ve~*`iKw?Tn;008$%*~8_t{k-PKj@&iK=C
z>^>xpR5UXZf1q9asnvZ)Pt<~=s47uskS*}^+t~i{->4A5QEdlu?{4fRdMlzyBblN^
z!gQMOv$%%sRYi)1!j`iUcYnic+t#q<<Z_k@5plrLwz|p}E}`!E_+Y7*v*vFLV!2+Q
z@c?i+&vjpcZNzxy=Qba%9v@)Ydjca-+`oZ13-GZQ;GygOuBjSSGio#%T<O((HJWoB
zF6eLB2D9)QDc}))*zC2PzZ~7v{T(dCir_|HrzWOF6~&&Y%Iv}2HkX!HFB_>HGn1KC
zi!q}PAS}St|B4L{%UQ(bE#@+lhFEB2tSZ(SUb}981Hck7GZ`3!I+_}pq1?GkX_KBD
zMj4)0kPb@3K+<yyB`JhuhMA1V&fu4CysMK9;(fMqXPFV{fxQM%3l%Vn7iA^x6S7UD
z&Qj}Vzq9xsHvqnYSC{l!=z<9|D70!7cF*Ga*a<6`X_71d;D(x^Pd@!=c=apm2Jj)c
zmx;}|aQ&n2gy^+l=vKu<-HBTGN039|9ksKoMZ~Q9-o2A77-8I1{6jzI{;skjxH#}v
z2XMPBG&Mo$x^arbGeiX%Aim^tF8s|QxyPS(Jr?~0{MWK`!Hpg)547z_Sa@7zb>$-w
zMw%GR)XAfkcw5oDogeCq_4$cJ)t}lsJoCd0G0+8%wc%v4V+=nK3?^WObr7_Cy--W!
zgk$Zz%3gbI{3TTUZL4X;SO)aS8RXDrvwY59=fFSp65!qMd}uR$i8kLd?HeB+LRnB2
zSCoJgN+;KzZcOHWwc(HCG5)i#x9Xg7)RQx71{QLS5@=HVD@1LnC13^tT|mW7PrU8p
zYc>R=QLT$hRq6hLGbsU?RRDB@!qz!D#bZ&YsU8X@3@wp@!_l`!@j;qvV!C8iK$&)%
zhyZ?qv8T#XPXeP`tyY50R>MmVPdz4z89U*)f_Bm!=001gV}jNc@#7}2q!G~hF{rC;
zj9$9-$pp**!_+{Qmf(%8Di?vNenDu@^OD_sNqS!zyyv9d@3TQPGV9$=rmfZO(h{2o
zSl239aM$U`V0^@86ZoKM=?972%jHsPTLd=qLay&A57fczZux?#qoU2lQ1YX^Wv_ID
zWS&NKTFdN1!%+(?v#VU{H+gR5;yJUbcGcL85gpP5sNv(%VOPAh_Pe&YX5$u-ds|KJ
ziMWS7w%uiG2A05te|*wKl6up1YiU05{TsuQ1-Hj$E$31C-wR&Ut;AF9mC&lqXVw1S
zDIZUW`<%e~^CQ~oHqS0*A(A{M#9nVST+K3)_<%Lkc@O(?P=5m}iyx5^(GSrj6rO1-
zY3oA8Bna}l;KW!%7<}xo)Ci?GCaEstifA_&*>Inud~I7;h<a1Fgq>|pj`p2Prrm68
zwgkISr((JX&75`h@rz%#n+=EIb|X@%&7_)e9{Cf?O5l5Ot1l&2dzyj1x+yvK>c95$
z&5f$Bt2fWrky9HKy=O^O9Ylx=AAx6<%PT*eb{WLO86qh5bBQ0#A4UTA1e)O?)Ztjk
z5<ZVA(tFYAS6S6-?r5NeA{$7c#r7!P@SXSEFp&1W%!KxEl3;q5^$Wc0O*(?(lAMn@
zMVdEBY1uN(7GGVvUxj(=E{(RwFqpWu{c)cTHuNiHggae`vu^uY{&vwCVVxh8yR2z^
zF#xjKcvtd|Y7wq#xZW7zw82vE8ooMQArQ>gaZUKYD(<;?AdQ?#z|EH23E*Z(vr#5Q
z`ry`3E2m9jlAUjD<-gl(PDAg-a@3b8b1q|_kQXr?Z&|~)c{F`g{Sr_qi?LGHk7=Uv
z+wX?~5Ac5I9qjSJOu6bw-Py<UV~foB2?gVMwF~Qx#eiRj4P;$!(8)XSjXP>&e)O@{
z#*Tu;<wy(g&h6*jX?o|sfURLc!NNEfMVLnZm4=DD>Ff-Ab`bDza=hGmw7gUsMlJDB
zu8Jwb&<OBxmZ|v*P-#yx?-g6`mEE+QQ0Rmb#K#S2>(L}mAn%~u@m9}VM5RjFZ=7|#
zCKS4czE(UFqXYSyWn&c<bcqEFOn%3j=fxHA!ZFNE!xJuvL2)tM^f76Kr@nQ86gZW9
z42i8P&1QYwP&Y?;DNVB++{{U<>Xlh3Lq-gpcL*OGoiD~PBhYjue1>Wde1aJXjB?|Y
zQp_3Zuqs<UFA?{ol05jHx!$$K?_wXFzLD){7GA~aR*WvFKE1gf1awv`iyL5+ilS_k
zyB=XjrZNN3cFT{3GcQdkz41fbT;tKg&yN?1+N#-!h(J+H;<<Xl78=mcqMqV7+S4Zh
zhn(b+jyBanT#A*i2@_A1!9D}o78!Z<y&H`LlmJ(bsz2et0BqDvu^cK{97v$=TGT#`
z@sQRiu^4378OP`T8PZJk)^=>;1om(QI|d)kgDY+fDUXfwccs@;=(>t|ksOF-{z9i)
z$|YWqUEzv??^@KPwC5+YWBLJIMTygRS>tJ8MhPl1{KPp3e)&5R*5H($2#>t7t>@)>
zmuXvxSE*lBcI2FrcZxz4y6l3cPrmnwbZncVJ~TUnBBkdBe4F;<Zfg?sme3DOS)37f
zW3`r*xv4~>mf+0%hy$UXL#Jz+N2Pof{K^V@0AEA$q68`pdx&p6<DV7R`o8#&w2RUK
z=|u5iw*3ED0ItKD_p`}&|K%B+p2*+v`G+;|JbCCQ_IBr>Jyr2mX&no>+6WlJXO@6(
z2i98}Z2J=rKF_O2lfFoovqldtaw(D`G;+N)#ocR=vxRNpPb~k<8N^hRv?XkyD&eR}
zm$?nJoC)g7QmXTm(7iBBPdO<T49xetdOU+P&LC&<z+iFnK}@Dvuj+?>E|i*}m!t6>
zyiqIn)sS|9y!R(=6ZzaL+D<}q0%3N<wQSw>e-+|c1y*=tWb-!nnGsK4Ft|{e<D}Q~
z278W-gXHcG>#`fVZ+7xv(8tcJgG+N`n~wfKVw(}9BVmKO|57_igHEHkQCB{a_DvM*
zO9<av(hZ+HNb!&~wel^&Z=an$S2rw_d(^=x<9k|tGeXTypKT4I5ZCwV5<QZJgGIm&
z*-%U7p6|6=hBDyA>WT99v0EKTSY1T^Zr8SG)!qXdU){))w`8i^xtJJ#>e2|q4Kb(F
zK!<Bs(#&Q5^@MBKwLLX09%@sCo<TC8bxZKP>Sh&6dLOT$KKQgCs#ma{bO*gP7^f7S
z!R6%HE*~P>;frugnsMqmv1wj#74>e>lD;J<oZ(M39R!Nz2vObtI@Z<q&%aY5^+Nm>
z-s^7=3sj^{vGnl<el!c9lYN#}OD6Fz*JQp)-`Hn{{^#wQ@<Zfv6PSR;tVO%nSA)Sx
zDd*nvi~q%;<YPL>(I6KMOb=p2L|3na4x#dfO4sbGN@gr&Z?aV@+#!6hC)y*Z8Evwz
z56??cX~Dj+k>!%HEc844Z>kuKZ)D&8j-{-I8D|+r7k7&es8lryEG;?Y%DUS#R9hDo
z&b6dIQJA{@JNQt3hg1fbw0WA(ifI=(FADvw?XlbdzOW~ZZ?0z#*-*qp2r?Dld7zFd
z#oF~NN2j$7xW!ZG927Fc2KcC$b5E3d4wDu&L%1RSeFuVvox6PdifLcHE>I?Ge4sG)
z!fvgfbtLTBpvBKq7-OnJo0wNY(o|^jaGe6`<ik2Htf=cNj%JFYaJ^b-1)GgPUTw5(
z+T5*IyO(e7$MN2<*R(P{9Ei<{kCpj(-mGpq<1$>%EM<@C%<HYEj$B^neNBC}cr-xT
zlDl#<lYC6Y(;WA4`;z<1pWgycbNu5Uy}qVW{B$q@=PUG5QiBYdhU%|bg=4*WJyJOL
zrW^lMS(^cdXq+-jN^^(rXaptJ>3`;559_Y3em|ePuH$@C<udp(kbSwB#x!7TwjswU
z5Tq--{?YRu@|8N>GaWT+?qy!?^~EMX-g|Q%l+Ux?ddwz$Oj8oB$_qdXGCU`3J3Fn}
zj0><_&^USHJ(1(}{Aq=G=nphgcvk!(*%AdkfYtG$VQ6qb?>?1$*QBsa>P`{;Kom@G
z1-=y`<AOVHWDjQo(>V0Yu3E0mr?|Xd4&%{9C4g?6M{nYyAL4F~5<Fp-R2hdt+?Q(A
zET@T)9yBJdZ?Z8hL%!Gymzb&tRT)vNd5td^b|DTKDc&u-3au$*BK=1^6^DT`igwbB
zuL)8NMuw54CNjRs1i)Lb$)!KOJ*n<{rlCCu^yX@&Pa;KqV^N%tI+gXi^dP^>kMs(>
z80y-u%w2{-A1<4CgPZ=WuFIkp(w(ut4RZ79up&PGptHv60wMnk_o^$xqz8HDZkBNG
z$w1A0NsAs~ucMnUcEn8D(39$XymhRhuQ#{%Cd<S0>gPxNX8r?<b>|JG=ycCPuP;qg
zwB4bXiPces_e{MB$nd;ma&c6uH&4~ILYLXEGOzM!4B<{5a@~_Ku3TGeGqjVXSOx@s
z0Q^cn-+ks!c;#pqv}lV>yNebCnxua!>}TE<udL*o?+;G&%P^Z>`z7S_&bMLqaG>Sk
z>CJ%X!Ig*KWtCqLMCx@jVAm2xw@`B<8Cweq0A{vw>jSz3x}Lx8TNlJBKjmm|FSFLQ
zt@iHpqrR74r&HtYo01dnV()S=IW-LIuH1j%H9svDEyo*l&^hv8AR{RdX*MOuTq!jn
zUWckpE8D{48vDrL|1)j!qaHr*kjgVNI*Wh%SqHy0S|SwFI><%d(!dJFNL>7umm5bT
z&P^b9*CHyIDC$)OahYeU8!Sd&-N_^G#^g^4#*4;4$vcHxFU?&`%aeUrD%u;Pm0NVg
zDo=}{5R&CD%vP;q+7>>74K4LnhyNQzU*S#*ef9i6?^E|_w94s;HM^)VFkaek2xvB~
zTBpVN?On=EBVHFy`I6l?d2GfuP_^2%r+=oY-J9$K7=7i*&TT%183sj8A3MbJ_X-ji
z7UWUy%?+2sdR4648bp_PA<s<lW7CTBO77}L9O|=^JAeNn_YOQj*JM}cekK+|d*dMc
zwUQxz)u$=tMcs7eVNmB&tP>s^Dn#4LmcY^Ot5E6q=fcF~r5D>aZ=9}eF?zatpy*r5
zQqhMWfy2fr<wydW$8QSukI8_W<VLnEu_5`*|Dqy|ZBi?}quaV>+;ejb#DVBai}`D-
zhOfK~4<g28xx)WP(^-Z!;l6E{5Tpi5h_r}wN=uCpkS+mf5XlkJ-6$dwN_UR#?heV(
z$cWJmgApUgyWjsf-Vc1?BgfeDJokN_=Ot~ScOF9GZ@y&nx_t#&y8(oZY~5-*dVAh2
z!8!qAcqETen}LulNj0qoTwq{s#f9yTtz28PPJIH-heQ5n(o=Lh`>6N9r_aGF_;|J!
z$C71n`skXd3W9@Tsa&;#eG^gZ$>*NAZBI7x%_=7uk3a0Ojwg5BZ%>rqpraZ9lF`@5
zSf4RVYUekOd-zQk)2=4vik+HCa=1puS=$5C^8S+}QZ~nepv20U)jM~j7X{MPCO`u=
z?_)xdBcD+wa~2?8)CT$qBOW7nUU-EOj~YHSzbxl8i5U6lRGM&G(o##Gl6|uC;`Y#7
z$e&Q$nnNwI%Szw>qc@2z(Es?pXMp)h#FB%eU9D$N8?J`qlYOK5DuT}Uu;=qd81(2W
zP1A0mbc&5r_rvKs)L{!$TrY{m{OWjV!JGGbL8Wc+bZubwu}$46Zud)9he3v-fK(ds
zerblCVglWx?*!6E@SNVI0_{Rn2y(n#ekIp5owM=^--U=n{?zkh{9<GZ6i`Pc94`0R
za1zf#muMr6$5IX4lOZ`UMIn-#01K;^m0ljae*2^8eDhDj4whbwys3GUmNW>dgetv^
zVG%s7L{+x7fKM%h>QeS=AbYr}VKLJpN4?mqU@8M_=>(GXRmY7jlS6MZ28&ajuK=76
zX9Grre-t-dLmDc^t|;x}hS#SBn5E%tSD+p;E|zZw75SJVEHSbm#YnB$Zo{E=swdvi
znXGwrrD{cN)TBbk21S*v9XsiKuA9KQ8ha7jW6|S9Hgv3EWq#(W{{7|VLpoT*NaLUX
z78AoFxx62+{o&ZzdFWl5g%IaHn*z-;dy(4&faRah!wn=T5c!|MADt#64Z0pC;>OYR
zIejD5P}TomuWI4_%a7H+=~b}odXe6eKc5G|WR0zCYBi);`5q@p9yE2l>q4^C)&p;n
zgIXN_+4Oac6)P8IT-9reus+3Tmg!yU6Ml5LD15-BO^&Es%>{@Zn?CM{bhXUE)-=?)
zHR(JQb?O0<I^*kib<QDFNW#Mpe+3{CbyGNw1tE_$u>lS`dj|e5-4PQ+#t$g?W1x|2
zIK!n~F6Z)eR0M?h2&j2CN~<?UUG)*W8$xwe0X$c4N7{!q!|oNvugxCDPhf#CqzBMA
zY%7iRnj^k2n1Sp3mS3`T@tVVi{dh()?W6Osqg_%Ku1M6G${gI9V{|@yehl^TI8s2@
zZ^u@x%K+B9TF3A)3;|qz7`uV&skaQ5%yH^daS{bRqE(UpOGnQG8n#y>F_~%h#nieH
z9O>Eg-?txbbE2FbPeY)lUpvJ$r^<MXN2}T!X>w!$5Z#(m*uG~&*%6~eqnFQ{?E#uS
z8|So76g@*Q1nu!U`UMV~AC$Q6A#sANAD_bPZ~E4I)1e#dBGr0^cJl;g+&~lpE&Y}Y
z)@M00nEyQ4LofD4laH#%hkk(J8qi+DGf))M_TD*cxYNMz=4G*(_}BW;I$d9Sc~S?^
zN05r!QtO`Ox&na%kU%tT{~g3f-3a?340s);Hd*ubIU}|0+H0=zSy6nFg0~sJl5N(-
zC<_-5=vAEl-jKvj&5C8v(c2XhxM%Xo8cG`%`UYcvlkX&HC@;7axv)sD^XU&?jEmGz
zJ?Rp*&$Z282J1!lJPfn|A=}SRkMw;%00{Q%+htT=w-y`EF@UMY<-B5%>LM{g<et=J
zKNvR!oj_0OXPj489>)CKZ34U;J@ueL&K&8+FHBTa4jL(Cng(gODNu%tH>69$m5{$H
zm&886cphIr{diQC%so3+?ww)c4SHLrx%<7V;g}dQ-OCl{HimDinvEX~(;tcL!MGFt
zoBkS9c-N~vOAA!Q$jZz#88He(cos^?XXV`kQA3Zm)`L$pMA2jF1+QddS0y9Etsjc*
ztN<zSUOE*D9UPh^53Fk|(LZ3XTnyU&jBnn!sM^_sby$6z22T-GZUHRg#rHhV(w$iE
zu*r!@V*N|WYgC)V6;qJF3yykKp=MQKj)8&4G+Fn7&-Sx(!dB|(x|>QpAV^j~xpQ%&
z4kFbyz87>BW#&)_$JGkFEedJ{^xyxBefamMHa5uzxV%hqqw9fxA0FWM_YHB-&4$d+
zWP|$%bV$;19x9EUM(zGq4$b?wfeYR+)I*TmUAR#&zp9VP8j?-kSd@KGsoTV>LXkE*
zFVu|&Xd%HH(6i3)^I?L&0qFgg%kS(!`?k+z8E_~-^!y>XC!&|}eBW>j0<jT?eC#sM
z^*$iWzK%LsMZPP;viIz4FDi9dIBF}rT!Evf-%C$0TbCbENM~dx8Mcx7cfK;`uOs}N
zJ{R1q%l0!(4<3P)IkoRo;HYcf)rE)iWqpopYKLM_YcI>83I0NO#b2BNqfQp6RS2O_
zv#a?K%6jE%db~4CaxKg6&?F(xte{+f3YYthtrc9!XN9tv$P&!n8$o#*<>&b9w#I?<
zghqsY^Uj81-^eTPyRbDFNu+r1<_v#w9NeiPg?KsmTk-x6%Q`#%W3cm>+Xf4l-5M6u
z1v>}VQKol5p9H&<OM;FjO;|Hy%_;Il&~;lv(5F?4dyAIuc5b}O(wWYb^eXC;zw=16
zNi1k^ZeqW>|M!ehjEG@M`i&KhWj*;{4WcmSba|pjr5L3w$2WiU5MQXYZ_wjkj@`4F
zf3hjKuFQx+#up0fpTF;?)QNPQ<()V(_h-8!RFN7DPDw=OUa+}&%F1xRi=|HT$&$i8
zB9fGw;c}L&$S%V-Cwlfzl2x*-L8(`yukp^vUmeykAe)DTECwCOqW*ApddqXmWrEs_
zvwe2FJT#=>tgWnBUQQxb5WEej$1NOBYytnD^6#bqYWN@y3oLQ>x+(ouX<6wpylPv!
zyMongiwc}d>f{;4Z#|!wXT$RQ51fwd&A2B-PkdVvG&*+SJE+Ls=1Tpn+v|yXC>?O0
zeG<X^3(6K}cEBvz`8M7EY=E*U^~NIQZcFAYpFFKbWEAB2lBbtxd&nj3db}nEH;3)_
zG_KiRqK$<dpLFhWOiIC?j{myIc|iwDH%Q|qSMBs=H@KMJES90Y5AK=*@oZJZu5UZ4
z9fb~h+3u}?#3vKGm?^9;&F;}?fP62C7c}M3ahv;l^lgTBBy%`31FrKa)8wuX10?XO
zj>fG0*wOhg@X9906W=4h4qz>ebJ-c7j!QN=zLZ_%_tT%E1ar=Ru)NJKFvNX(pjv8h
zFmN62qJR&dX@Fa64Kx<3n|z{wzdfwVhN|($g;9M_ZgZ^uMr%PuL>;)W0-svhn$<D?
zvx(?!c>dGA>d<zjsT1wY1^OQ%WUPc$x(@TqvKBcs7%w<_7S8m(O`D;eaIXWG(D$aU
zUF@<kT|GZK!93*}cel)~Lmxq3-oU%t(*NfF%u}9JA9_HyLZd7bl`_2kqN+@YxW{i)
zr+sbcOWOEIZ(DDloh7N+axjwc`*E>1G`T><_=K-wcP(78En0s~xlN!Sll4Q&U4d5@
zeE~{o)2CKKj<(mKz%H&w`e$Qy;yS0<&4@+)8Ok@WyLit`Bb=eG!$K$c-Al&QYE>Cu
zh~MhWm=%6!_{kmmO6yqo2H-+Ht2r?CiTdzbt(E$wKi<?o|G_592YDi6_4<0l>~`RU
zq3Zhc5_y@>G_!F+BQVn1U#f>~&BPZbXMQ_Elyv%bSjA`OJFFnH|82jBkH)K7A8O{1
zI3{j+5@(&H@-`tt)sliYbQ!=&ldFZGxq$m=ARM({8g$}hu$zdEsg&n~6`EtZ1^sz_
znvVUr4PEf#cy>0(G2o((eWwAH?+uS*t=^OCjnGayNo+ii-Qy{z@;jb&z%N6r^D<N}
zIub-IB1U<;+RxMws!|D{8=LRHfD-?W<yJ>_fB7nR-1gvW>nmrqk7b?56Z7~q?+)F^
z^P#rT;UT4x7ZLATJSPmT>pIcQM%PSvu|GDhK*1bO8v6^F0A66F1crn<W;YC>UC8BP
zKA-Ly#TkvzIt44We8XJtS-NxRGTS~MhZe_e{Ot|RZsh|bRn1w8jaDu6_*|w~9sq_R
zO4O?JqRj&WZU>n!Urs*)ZX$uLsJ&Q}Ee#Q0Fqu3MUa?@8+~gO3V}fj1t+QJXwSZ6#
zz}njkF2k-GN*XT2y=E)#bmwmxqfj0-jsrIIT|)8=d%9@-se^yRr&mejT-5&gxZU+d
zdlYV8kHylc&;h8k217%)gAx$~j-T5asPf;nUw+DmGJkXvC!TGn!u@n8UVK|G)({d5
zE&#2v6}dXR(5qiIBvi6`tCNOWHhwrM75>}!O<X<62a+D`$v$+-T^S<?W2Ah<Xg^$<
zryqogI0c_htMEF~L_m|n@?salumPv`w$B5{>by7nih7&vMEci7_Ot&)v8p3Td94Ud
zVG6wwE?~9JQ|7?sYj8%^`ctL!t0V^FKIcCS#?!mX9pQkGu5IDlKz<Af+-kijpsM&P
zgCD^c&9i^If;)WLYYMWM(fL9i<N^5ue#jHYI_3#W;V5fs7#D@i&u3t4f`Xd4<}eGU
zOi`BsHP%-U!O|wt%?m@ayW*4h<bc}nZ=j{}U#IpB@$T*#<^<l!Eg$7}$HPzm8%Czz
z^y@rv3C;7~FBpjK4)yvj8eV~W9lxeFEK(O<#2z>1m#VAta(vPNa=+O1=rCk_AI8w$
zJiiyvdWU+Nx$@Ar`<Oe{4_gh`F0HCe8i*bN!m%GZBK#lEkVl!?#uZ*oF)O~>KLr>Q
z`4w6&1JB2+6r|ybG%_wv5y9d#V=nkzRP9s*zT~_wl;`~h%~G*(iHXFzi)hzbe<~-C
z@3Z(!xP{AX;}acXQPvp-Cdfi^<F60C?EkRBu>O6-Db3!%5RAa11X^vV&xbIeZwz5%
z)@kggCywf@D87JBs!EA)@cpvIjZV?=o;^c5gKP_bSrlP$d}|$XT?7_2>Yxf`FJSlL
zB%ln4_W<o90QQFIS0hVIK0jwP6QnV{KO&5?M96NoiyXMRKnpA7RJ+UUt%g_Cn#vLy
zwHu2%c-!^$%Wh6|T0upgUQbn~vOzO95jXzdN*zZC>4IUx&y0Q^=BCxeC;{u7S^ljm
zUcHy5bU6?7zr)=o`eo8qB@$6-NKr`NR8<_HBrkWkLOGIm(0wnoa&a<Jl<*K1p}b#}
zHbJo(Z7Kv@x#F(&>anhj-Z1;Af5Ub@5kG3U+uf-bFDGPcQJq(1YQNjEnpD;yget>E
z;aBDD#+gaxJ3<u%w(dVY!#p02&$=jhQ3<bG%$Lz{48Qu5h~tmTWZ+;QHNB(6>wIlI
zUs2~TuvfAYzc6ZgRoj>#Z|u$g^*be3;&h{_`;>g9%&YW#`}hmYB$=M!+OX~4wFQ!&
z<(WxF(-L4gMPeIFm+nM7QFN}M635BEI|u9FTrf^!T&+u*5nPfuq<xN4Qt=20&o-cQ
z4qbdd6oFo$1o`ht?6@-J1rKV<?efbyzsdU3U!NLRsdH^y0bu^1mEqtipR+KW(lGP2
zO|TQ4UQ20rja)bxo$EdjtVGSvjXMN6eAUn2h}?%uhMoREg1)b$Nw7>Tr$`CZJ>8%6
zfbP$R70IVuhv<khhD=(Rh|&k5dwl2S=M5nrT5PU<6;<E>zbV;)e-BkS`5Zz8HY!#!
zyN)MT9&gegXCBa7#)tRBA`^;mqJYWaR%7cma=q@@oPJrXigDpwUYzV|jTA)IxU0BN
zpTnJ@ImQPgS-eZB_$Ds`32ca*h59=-<5&J`QpbP%WWg6PH)O15{ckO^Qv+Pq``nDr
z*aD1|g=t&*0h+%3xlel7C4>8zqxs3wQvElb)8n<T8uEMlL;9Ejxry0+eB~MkygVPI
zT;hTH#&#jsU$LzIoje<jnH0JtR+DH;Iog!L7U?>p>6%j4F~51mC8(M<cOdaA!s+^#
zE2*zivZ(P0W~abkipTG5wvxIkgoUNtOJfb2P-l<KDA#uQL1o(x%`sH#WLD@=TeNE~
zQ^G4J;qd|d4C3(=^xj<YxHU*Hw+h7W)}YsQrdmVawO?B)*EKuS^vrjV1o&-rHQgrc
z$*8#!k|WE31wL>NCenbXRd#4Q{ME+}z1*)h{&xhHNbe9%A97DrX(8d6{q89a*qW7=
ze^@j=YlX$b9XwLm-0@8$BeGX^{S?h$6?}m_c9WtoH|U4-C9Te5@z=|%bTuFUC>T0=
z?!94HGkW_H^&dTTR|mYSD-;Eh9fLkk24DV^WG{%&zePP<Ilkc}LF9KXh*-wSFytJy
z)Z2{g8>I9YxFik@16UF+c#V_LkW7pqboexhAHzE#wteLDD1SMlT6h&sPZ7^;#^pxU
zHxQAyFZoL~WGp?mqAC*(HIAUGbpCEuaK4MspTiVRaiyhVWhUqGJfS@DfT4Xpn83!!
zM_zBVJY+@-u-WqXPm%W+l_xCMx-r7*d3R-I(tE+(+P1n~x$*$_>pJoDYM0|&x_0Z!
z2}aVP_-@tL(M=v_1Nzo9|D^=TdHt<;BMN@=*y-otqHs0@kR9il@MUk&Kn3`?tU=O-
z_Yj*bOK5>hrYCefP1biEZv5{_HoC+N6~==J>!8vroc5vY1f|;>3MhLE<U1Fp)}o8!
ztm4l14b)Qxgg;0teZn8R{rB)s^_7m-tDw3@>vFtc9|*n<u?x|dW@9R`-wvFvXoXQ!
zIZ9;g6Q;#Ric4Pgb1|8N7ZOmZvs0!u29e`Jz<Ixg9!OXxBO%e8pvWea4aY3==-O$>
zUr^cflqJX|0#Xti{rHLJpEQTKZqtvj`X0q*J)3#r5ghHn>qhb7Y)qzyXNR@z0v=x(
zTJw$}56cjtRoR)xD;UaXX{h@ZK_{N1URl**A5r-axk%*dSj>0R`>=?pu8yZxzk)WK
zZdy(r+s}_zl%9gT?$x?`KUR^$i#JaS?XlY(lJpZDY*racJ=S1chN>D1f$*ZtsqqCD
ze?qHUr^<U<8G!{}fAPa781~b;W-X<c4@Ftp>79wPAU7pg$~eK?yUQJg#rN3tXOe1=
zwVGS{rMcwkEfKl!eGWf>o$jY^i1NkjQv3CfULJ-<T8A}ssFjhpAw4z`lQ2$k`=IdB
z0M7i@FqEHii|&`2dzxZg+M7-}9Z1X2?g{(W7N2w~F&;LQ9Hq{fE4XO^lM3#WF9B?&
zGXHwusgK{y4hl4Sgtxub1VNr-pUp3Vc4v0j%%DJ;Tze#-IG$m9eMz&@IFb|YHu0{=
zal_V9`<RS*MY;yDkx!I|-xQPp-~~+Ehxv&ikDE}T;84_HaBcgnog`j|Y<%+}n5HjQ
z5=TM)@!yR1*t&u;W_}&jzC0FyVf!y$)upPKcQi+cN<JiE#A!ewDt}KU{<~h7qx<4X
z;ru`Cjd1XJZR&K@{YcfvYw@<x$1C^<QKhG(>5-VT+vt+};|V6oQI-yk^U7hlU!ru*
z%p1~=Eittv8_^dY3{CbC4AV6%mI61{1Z*C-f7rreEC@AH)%a#c+>>aSx%ShgO?m^u
z15m@GM`o($e|W*eYoWVr4%32J<A#$P+;w=c`+1Qi+`!&g&X0?w?}vm&FZoABxU;-0
z2i>*1-`v;aR8M6z2ETrGnz^J2F=d!2E)t|6`h<^HIDe@PMa~CntQIZOV?SA4|3X6{
z;r&<Kh9YO|8tWMkCSwAB6_M}-`YW1kWw=b9{+_KX1mlLo?A2L|^vm^xxv<n4yavRF
z^kl*Nel1f)6xALnusy>SVB+(v)$xPPm1;`?k_WH3ydspzBPpATcjz{tvb=Wkg?gN8
zQ-tGf{x4jAyfbS8cHZZRIVN`x2B`K6qTu2>EAd+)|9Ib`l@E#W^M2h7SyK>m0{!%@
zf{01Zvld)-cCC2qKP$^UCvMJcZW{sUruD<NOTz=(yMs-RM<`BE;jQ?HRCQ(MV^7)v
z%6RU0OHs0Q$tPaypo)Au>#lt!f5xep(zx;Oq82t`^8GOL3QYI0Js&~(KE%d8I$74p
ziZ--|0#+T;6+hWI>mzhn)tM3KM*%Yd;QkBg0vKEzxGjZL`wL@Batp_Q2D$7iH<rAO
z%==9XP@mNeTK|nQxwt%Xd%0>R0y+KY{fWpw%?3$s1M7fie=du*V*aKG_G&?@j|GBA
zZoLc&DqM--11bhB>cUe_k(SOhD(_|{PS9cQ%k*2jJH3=+h;&K)2yVe80C4wxiyasz
zadS6sOf_~fZr>L2)y2dnJHRs?9WtOQ5P22nobszg<F>8qu1$fE?CJEI0EF`g*1+!;
z)3z$w$&tmIn=Xaummh;osB^@Te>f8@4M9%Dln%i93EMmH^DP2_+Oo~BcHWK>1$8dr
z#5mm1W%VaB&vY)>&lMu05ruE(PHBAR-;U84@1ZUoot7TVkdLb;7e*PTy+<&222{{v
z`CS<BKJ0`>==HsL)ng`<wjyTtH5*q<T&zUMG%J00RYJx_4C`1ejq2wKhmNdEPtWd#
zcf|1$EI$$g2?_)%@E?30uZh$l&rK!BsPQD^+$3C1TfLVHdi25t_g%+le=UAY{Y=mM
ziQF-r&HvtY#rSw~klMHR<0__M+9H#!){coSuhZ)D?S(TVjerB4HbS1}9001Ka1XxH
z*3>{~t$*&6IDO}ev&MkMiH-de_lZN}(_RuNFY;P7or3;lq5gcryXHQIaFFPn(G97_
zZT~uyE%vK}=SU6My8OCKaYN~FBjS-AoZ!*!ENsxem5p!<soMf1+3K8mK((=}a9^|a
zjDhAz;3`ik<!A4H6TgYwnvrPQkL8D2W%qulDfsp2C-(RI-(mv;5qqHTuvLN6f==TL
z8P+pQpdN*>dzg4|V*(*uSTsD!CiSPmXvTc?d*nAUqt5afr@AUtxCowdttJ7}KP_cA
zxqR2Sm(J^(@r1mt{5D@-8?WsU=z>M=b#Y!(>gkJoW=Rg<NZw6))i?rm1A1I0zxy^)
zrTh<F1{&j?e`<sA!Vyt%CX!=wdN^xH4~9DlTtJ~&TNN1y>HR$4=*<Ojj&IqBWoX|E
zmuD^De>^}3w%eweWIV@Gdik(YJ9xt|>{QSA-p+GbRou7De-vs2llr!dA!_Q5uNb7<
z+;hPAgjGZz`gFgYGYJdVg?qff|6IfPK@ddk7?)4B%_xy0!4md8j1Yg5#px>xQJ~ds
zXmcby^)k7WfqU%02w9aeZNP7ND$~JDqggb|+5Bh0*9k%Hj0mi27LH%=b#eZ+9^rRO
zV|hFz7E3NVLOvP%T)mkG>rDs?T?DRiAJrsA+P2sD#Kb&{@1vJM5ou-3+&i#VIJQ%?
zi|T78ylNuvKAP1@gkSGwP`jVi`25O+b%SZE$BE9$Q`qCs3IsI^sh0^yCJ)0m>Cf{M
z<vh5^A1W^MOM3U5$OzV$?}N?QBE+1rWyM@&MP`gfXfVS=cYG(@vSt#f1X8YIyPdCz
zva#9^QmGmYm<8?-F2h6CPM+*LKSx!C?v*oGTKzA}%<Mo&fK&9O4#q<;6{TbNWn6+i
z%KEywoO$e5KB&{pZaVPUZ3DJ-ias{Blyzym)ba1#_RZmXMB`%KcskcYbpabZVQJNE
zfU#ZspANk!ct8&G>`Y?=ugF#`Hw5^DpOqNsVr=cfi_XOrekx@G#(ATc^UdBwT{=lb
zX=wl*9v8Qse89J*PPPQoAt$Q)&$?EN;i*>275)%O<TCR%r<IDqme1=D*Ufh{9wlX%
zbQfFMUIkJm3~4P35$|9tui``G;8GI(YHf^CZeI#fCo-#Ud+-1fpC>_Sk%E225>PH|
z{b{9{k#7;DLN$@A*5-=r^9lEI4TtUX#SbmKU8#lwklG-xmIgIbN4=)D2K@(!wX1=6
zB_h9nT)9WK49EIH2*c&?np`B-!9GH=l-F1B`&S4=y5g#PZDAZ_-gvoE_GPK{Ucop2
zMF3|xdJjI}@}R428nh}|^f%jK%Wi6@>>ceVk_e0FoEPj8wA6VW@ORe8*SYTwN#~_+
z-`>+n0pJFRM(XZkg4iflx<kT;?bQyyrEW`agOqD%2|80yA1F488;+Bp+k$W;?jyCC
z2As@#r3hP0;{|DRFMZCHa}o@5CR2qKj6V{U8d}@)$+lV;BAGJ0$wil$vc0VsR3x;`
zm$amrpkh}8xRO}WxUT<B3)gpxv@}E)d`$wUd!JsnVLUF2#NNJj1BG$+KBD{3Kr7=;
zhJ~GrEsee-K}MiQb`OYO4`G|g9MthAiZS$Kg^2_-RBc4CbtoH4PU^ScwGDdSoM{Il
z&mQ<8<eTuW?p{6gtCO&fseqS~z?PQ(x1kgmDOT_3e`f{qzvr8Z-4<^EuO?FVXh_P_
z&TDYrX12ReEfy%#Xcx2h^oR{^^3r|y-5IVf(QL}G2!`|`Yg``ZtuHT+EcG{#?R=NE
z2RL4eAZd52OWWV)A7}rs2fi%f;q$m5Xd93JE0@h=n@WT$S(5+y&rmYjK1Aw`n(ulq
zX)Fyr-Sj|*fqnAb+GsEqaHp0Fr#xDyEHkrs-4gIs27uL$o;*6$nc^xQ9cfUht$xk;
z@yu5Obrx%O7Er8nkmM!o?<~wLK=$qzs{}+T-5(**|J;-~a3s;I0A}g<24IS<)PxP2
zD|1$MAYA+x`)*_7Pis4GyN<HKy0dc?d0=1aOoLp<@a2tlBd-(0(!=eew4}v*=F5Vu
zu#K^+vd&xgGYraT={=j%pO^U8vK8px96-&hWnwM&=hfzLPO|hW3i)}A&p<UzE3to^
zt{y7&BC{qQnzBXj9eC_CNlXBycH)2^PZ7itQE)@Y$M}Mtd>;*^J`NlbP@7EW?}F#C
z+rO{;L^Vf&a4GD~`%vIM>q>tlrv%bv{cndS3Ejm_`Td1lcU_1z%tzhY4)Zd;-iLmL
zECMSa%54KpZSH>vaa6-RWQZmv5$ROM9#&KQ^BtJksvv5yA>U1<Cmh}3-KStgi{^@L
zvG&JH&qMROd>Jiw$0klX=Td_F#=9T3tsVvi0LVYI(&jJM-xo<=|80A^o=PP8+>jRV
z1t389%*IYYW3|o(LJIlTBum&lR-Jm&Z;1~avy8xbP9XJ`j4Pp}d_0f`6(7|mKE7I#
zYTp!mR-_fYar0l|;kd2I<+YA|FXI5KIKO~f41Vi-+ZfbF8bwpKqUzxS6X?lwy#Q!H
zqm5jORK#(bFW@>#e2dU!fH0P*FH{9>F!=5r&*P)an~^TKLF9s`A{wNh1s)f9MR)F&
z-kbY~41^<j*2%MbZw~akTG{UtFz!tGkV1Q}CAJ+S)w{D3g7&_v=T__{9;6`|D?o@$
z!0_)2rFe-y&a03&#OqY|N>YgisjJAJudKex=~F0xW+9&`;R(GOi&on^u8sG(P`fVA
z5RlE-Qb}hjQr~olz_+z`^<2xgM?Ut2c0EY}%67{hQV~F*-`@HnlR58WMVV(sjAOM~
z$Kk<YqvT4S0!0^(Q<-i@*qmutG7L*UO7o3DfyEx>?LnV|<^fHU`;Xd=_X{It?ToN1
zxJsnnQ#-84Y%BJ6XVxzjjnC>2?cniQ>CJXk+F2GE4JQK(p}VucgHE%Jl9ao(neT93
zeyx7`G>Rep7xk0pB4?~;js2Std>M4E;W6A>&5}`roFL_<J}auaa&x{LD;oTrUp98v
z{iwu<T01&O!Q&e^cc*pw6kRowHOi&eHS;xc33|Vyb%=0YTq*aId4!q;R17g14;lV0
zJPsG=1;k*jCk~S6ww=Z<Dz$IqedO0ge=al#i-L)7NVLW_sM`6LBq9kJHVxwVrm5%e
zlAZ5WkVn~mo?fx=R=?CjSmgj!5HaMvQPkM4KhqN#P_7H~cWe4$yT_X6oXBSHuNkqx
zot*ilrS=(dH>iwfpjm9CkH#_>Sugt<uARZiU^{d@pytQY;)hj%aK)brAFz;$gg_)`
zd<1r^qd$?nq3uj?+@lP#7?H<&Ar)S*S(MRdgWMoF7qQ)*4N2_Q=D=<o1CO7eV=lbT
ziu9-rUepGEJ12PMB+&M`Z#c`yFY8Xfz{>vTJM=%WC*S&SygMVC&`4VwL;nv+^b=;I
zQ4PK-@GfE@`SKz4BMr9rUK2`<^3w><L-kz2oZ{mGXaBj-sFWGua?zc&gXLe5mt~hV
z6-9u|H(aLWQX=k~JIW6&_!@r85tr7Fm){MoIP6w#&YZSz4>|!~E&}luHh8}`vMrq`
z4ky;~+Wp)O0P8)B9&R>_An7Ap$K=J^De$;waEbOa{9MRl4-E<b@HlNy2MaMd`;0xy
z_U6%(jnSOTGAhOTekQ?b)5!p#!EDy)vRGV+crhGdmDSl-`%LAI=t~;k=Ukhr(E7S~
zkEIXq21S>m{)=dLtPXujBg?=N`;y-;%_Qkye4e2o0^oTJRuBrEtPaWie4-!S&^(ds
zxS}!85D))&OzRJ|6XBhij&FN-N|K-$TQN`}`mM$zVV?bfQlpf3Z_+<8+UHeIA!AuK
z3tPdPe7`&`jIQR_ufmbqv8U;~>yiRe1b}XV(QhN4qk4y*O!1tqEA3;W<2Im+-7`Dz
zU8VLZr2R0pAs^u`&eZbO=!W-cGx4Soy=R~QNE_qn{~8Zv6Em-l=N+y-i899Fv#pQ%
zx{YwBX={$-zHM?Z&p_5v|9~T6xQ;C<)r0F!R<kfTL%ftC1vba0yUoLeGQJipe^r?B
zTWDLI<B}UAIE&B1`RkT_x7<j}c(cj9FG=Pav||%k=*aP5>goLe6+&mkrg2!C%%jP<
zJ4@0p$y;6Ql}Y!GRZ_v4;|s2y%&%4-UtFk(r&jk1<y&QA@yzTK^0rscX|!m#YuW_I
z645?KOwA5th}LiADP{pC?^oduraB)Ta1EB{PXB$WWUmC9t2bTlKHN!L-Vb{0dm&2t
z4zY%dB)Oz`8`z`Z4JGCAYKo-gYgZik6V-6uicv3{%tzUkz6I;+8;&7s#pf&t)u}CP
zV%;Fa<>ljI%ga)sqqc46ajco-t(k(6oosx2_N1hRv9fbHX<An{y-`a<;ukU)?z>MM
z7jF$?jswL}pQLKKp?O-xMOu=Ad$6_YSy7Duy@*rpPPf=>S^bjl_1g^e6e4H+>s{1%
z5b!aftyu%je7h0Cnu`3xT`_l|`RGugpLt5`oxe!0wiL1~#5Fufo5f{i*Jz~9yF4+2
zf6bkOMh1$ua8^}7vcxyo6{r2^*ZcFk4`}^2=B&8R(1)1}X0&Vgn)^%Y578n)={^74
ze^i6$ctX;;-#v*wB%RzTJ9ld^@fdw`tJZ}~I!T2DnHsr0X4n9$?*3*6eq?|J<b?~b
zs5#!~WcqttcnDWaVepUXn%&tW8=O%(70H#B;mT(RP?q-;1_|;#Rar53(>-64d#3Ya
zdWCxR*n^Y5!wD<cErrQu82w@<<GhXc!(B{*^M-l#?{$PBN~0PQao$Q1RVHJX=_T(%
zI;T&#wi8bIOxj@^3a+3}uU=y@PzlDAer7iqA!GcSdK6edMQ89fS-a@h-bjD^yZ!&x
z1)f>8CP`Dj|3EIKOrXCM7V;jCjvSVbjy9q|O7F+B`kG$Y+7Pk)!|tB?$#bLmd0f=W
zjpGLAG!_$gL=2P9|IKoT#943%2F(v@X})ycU)oOQz3S5mve)ZT)p!vX8{ic}AhRjL
zyrdr*A{3d*LSxM?zWy&}YM4y@nW3$jW1$W9$aNn#gA0*EY6GhSt8=Bw>c-FAhks2Q
z9q<5WvSs!=)tfqYt5c;-=*hF&HLzFvCY|GA1R1RZ)OJ%=z{e9)B!^V&Yy5<G-Eu^t
zTv(@({eu!v+;r#EH)jA2WnGvE;8m(KOcnlcT>017#TSUGc>eObf_y$yeq~Ka7arSk
z8^b{e_6sb97EvFqY6XH7guvQo?0o`rit5Q-SDy{ir#M>97>2IZ7OdidJ`>{!K35+N
zaz5r75VGCLT21mi)MCAvkxw?Gc~QLEym4B^LGkiq&2kRZ$rTfrebU2mFAKnJ#EgcH
z?oOVx9vILH+6D!<D>cr51#wMwN?%`EA0!j$GMDEe6FoPvQX&sRndWi*)8YI--8#>6
z5B;9rOJ_A0^m*3cRGu<rTii8z!B?*&_stA2-zggy6%pYwI{qBx808Pm@cr#f<ZGpV
zb6q6ScY7KY^z+TKHjm@UF7WXNEU{fie^WTHz)Qm={Po~kIvqhJI6qjbV*49&M}oCV
zi8AoGM#UsihS~=-L{@lAwSKLQmTv+(FX-LcLUg42oXC@N-oK&~?x&WBX3w2koOxmg
z$X(R#oo8aW`O9KUWyfq@7~+ie$JD1LyDk-S_qZ2(;2UHUxjwex(4~cf1>G%yR`zx5
z^}tWU_>GQsl&cpP37e9GuHbFFCTN1T4Q^_qfB4?R{#K+(Ow5Otu^4i@0)KAp%A^s)
zD{U;;)bgqOobMN~3mvYbkWvxtDi~_Z+9ci-{Bjw8ZOxXUgc^W6TC&T)_SD5L3(!;g
zRyib{R}081S<;KAhaAXb!^fO<N&w|<Ipu~}sMyB%<V}iPY#APG7Rwc0|Hq3Pg=xqX
zeah_`zbR)2mm4|t{9<@AWdDuDTub#hLA|D-98U^9iu)_Zt5zo8KSH2IZaW5pRqfTi
z`Oxiue_`$wCi=4<C%Gg6wq)dIm{wD&X98JgzdT3!T6VLRR1MbgU=_^LqhZq5KlR)5
z9CHSge@1eOQN3))=-K%a(_ecbLqlz<M$hi%ozNwLUxWLhpyv5&Fh$_mtkE?eK%$&@
za<V((%4l(tu(Lg^8L?QI^Xrx^ebs;?k9e;XWqsKp4j(b)jq#738+yaN&h9ECcC+&-
zitflhr)ahw)yDJ9OOd;(xH{0Z&AZ{`=0@4Cq0UJJk99m@8z-t>O*T=Rk<$@?n04%8
z%+%IhYW#OSSv6kA3Dd*mshi?EPN^ze-q&vRnwP#SSvcD3yr4AO^E7eO7IAY7P(%|_
zl4s0|?<WX_Liw8R%oj2Zl})Ey9~VeI(wJhGh?iAW3BO3`K^Cz}3F-%mb3$Ats*Hyc
z188bHgCCByZcNy#(j0jW>7e>yy*jMARA}45s2zW-U?tnF2jQd>v->uk{5yg&xR2xy
z(v0Tiddf2JC0+Bh^^e`u#ZHrBN7FKA;g_WW?$mfxleNs)?J=jngiIdQxOr_x_{Yp7
z&`!p<`jJC-BJAqN!=Q(%9iX`Q(ng$O1kG;m@lw$dO$yG~O`VO>`M-V=pZOdIa{tSO
zh|P}%S&IxG-Wm8A5bQ~LEWQ`EoaNR;TO^vK>)#H42$^Hcv1rPQuXPqyd5XbylQsVR
z6A3tEo(gY4TI{p@=3}{yWLajLa?TvyL=$nTLX^=`VDVig>8zPH-CxF?JBmUbLBCfi
zN{=xIvy?k3|4PS^hF<UHf;R3dTN<t5s8-xpS91>k<do_6#iQm8^xl8&VZ_E&#1>qe
zGvkKW#^7!|e@ak~G!?|%XWJaYF<#Kbpt@E38f$iXO2T7tFS*nXIl?CEKa$Gw7CyhJ
zfZbM{#M+<k)0n|yVFFBUP^uyio0Y>=v|Z2>t*H=-lIS>OJ4+6&z^b5+Lc7OC8ZQ+z
z>OI1fIODvk|4dS?3%anHu%ue&qV+>di`(I1*BO}33doIG0`DDx>_uU<FMqNG+RreT
zqqDFJe0J}_B5o71Yr0jwx&lb@#A!H@b>03Gb#W8%X{{xN+yqZ>D^aao(~|+%bu`Qt
zn_PHYd^f`l8XN(jchvr%<n=3EhETSl=h-$kVseN0rX6QCR@A!6CW@t;Vz!UxV*1)N
zfg2_>{q^|0qro>SF{^w&mH6fpjVFB4mZ?7wSw_leWr3`ySYAZnNSmeO_e!m+^OLED
zD$tZ6@MHe?)3$&W8rbD`Z4S^@i`4l|nC+*+X*J9fii-mZ`|l<+<}p5Q9es{vkFAd5
zf1UMn_8|I-%OoQUQvudjyEd)y#OfX*kMk!dv8#xw6ZP!u6DQwj%=_ADc6tXDycdbk
z=y}lN1gK4tqzmof335}I`dWH-AY^u^-gP|QrlVf9?L80}QA&i;^g5WF;84sYf2+^`
zA+hS>k)p@X05H32E0aCyasFBBtx!|hQimoP?NfWcf+b!3uBP7gqbLaOwlQ31R3x=l
zk=W>vM(q>yncWQ_JB-(<mlWc5uez$~4=B~T0vJ-i5(IUk&kE0^jz;Ztc)GG#rZA^>
zs2^BjfUXhz)wXB?_;^1rbsGb?!cs|@Kr?4Kc9pezWc6U&+yC68dNEeNg|BlSy#Ip%
zPTBmpFlmKsfmG6!LTefE;=(_Ce|d)gN8&TRSxfOR=#Ilutw$$YXLp+m1*cWN05JYl
zkNha1ieFq@oG<3ze(hj*ctG@7z?0g{#Nn{LEX<Am^dr8mm-so_oYZ;fG?$wm_AJD>
z;h(PlrWmBND#0%^Z}U9um7a^|N!mVYGaqG%?1uZ9BQs%VnWVZAvu5srHg{<@@9e8h
zHT-A!ThAEN$Un2uis=T!T=yL!M$&dqH#KFp(32j{Ygv;*m`!qAa1}r^k|`6DLtxPQ
z)6L@YE^$+r9CACTwe;>UnBkb`W|DbDS3eLeZnto9(gVEC$q_F+@sJ<3Oj{}Z?MLE6
zsU$%@zY)<XZTt)~4%qV*LD%qs44B{_x)q;&zcrC&|FgY4#g<Ii>^yCuUWS@LKnCL1
z*ht=;-M+nVN|h%~!}V3epuSMYdn1LN-^gP~e6?nL2fdQJSMH0|$MYR|woE4P6x*A!
z!#QI~f1YxJ!|N@hGV~~gY|-EpH}%_BF@uKN&69MY4!b(*%|g7NXUHk~Z~&!*(w*0O
zjz}%^{Vg_rFraow*3fxVz8rJSC=9yRAIZ%dj{2H|u>nVxj^;z%ks&q$Q6;>iI0OWN
z-e^k;6xY@C4!Y1dPBEwCTmLM3wLw#HX58!PfwLL1GAgJmeaXX7ptI2$)-ABzRPo%I
zsmt1Fst}jV!#F*-&HvaO`GyI)L{XZ5vamx*LSb_HEwH6ncf@o2$gC}2+zK{v!m^NM
z+$pXyT=kP2U_f2}fGPZm_fyf1S5mGi`#G*w;VAc)le-E`j`os~B6o=Qm|JTZW#`oG
zrCyp;@5sA~S=&nFPd#PqJHvgw51V9lO;N*9GhUfm;3_2ot8a?;I~%F=nnE%G;RC_=
ztT0IxHUi!m7i@nJCMsz{#0leKDmKvFtIBT$H1~$9z@GsMn136xR<ml7rG8_cx!pE}
z3*t4Q5?ru;W5%D89~aMSEbV=2k9{nY2lylG(tKK1ukESw>+jQU+>ns9-${t^S`8y~
ze5I@#I`%483bUZSMc>PD=;-LEmcQxmCgd&nzx|aCBId8N1Nme%saQoSJwcj+D*hm~
ztpHMtG}QfxCssj&?)(8Y1+&Bc<_}?oy$Y8sV2Ya7SXVN>nNV??wL8z;qP|&oK~U@1
zCHtMV4xXX;x_Kl`B5KTV*SD!P&3V9?rsyq2anQ<MQvXb!q|+$q{@oS%56!HYkF0b$
z8(Z|-K-u8tX>eV%`C4iTottf)ueEW!0xhM~3nwO?vztguAA)n~bigrsi^q-&J3EF=
z_3peN_RPK;f_TL~)C_rMqYgE^oP9=(+UicuCBfzuoi4^Bomx%1H_?NuQfKW^?QQ;#
zTF@!Im(rl%dAVmcFU>wp6QRQs=fKDF!dGN#?M7<>SfML8l*W_7<@swm3EOtKrptIg
z5ZLT)2;oi6&dc*iCesu4Nzck5ee0j~px&((Lmw~+A5#QO!5jB!LE;3?E;`QLujh$%
zvjiF@l}X!<v;x?&R0tYFN}BvoP~*b#-t{--tm?Qp3G$-PhMKIO%}Xnd^r<zAF42uu
zFJf=}N2%)|5`hwhW_hnfP5NqoAlr8()vgzUw8;C%=k^RAo*vSL^}FfZrGH*L=J4MJ
z+B^&9Zxtnr<)1m1ggYiFJhfR{G=z@tG$dX-+B`zlDn;u76HS;gp9u@E#%ZfKy4+gl
zM;HFIl%!I#v6~*=!2;JT^L_LVM)Cu_^rXORbzr_III250Y-NJ34LTjNublv|*)9Kr
zS)f@NTKnb`{`|(`>D8B3Zg9w#0eJq1DVn!eMdKZunoO^v2s#BH9C^Wl1j(!%AB`rp
z=Qf`dEE_wf>ZmMN={m9XpK;r6vQ-Mt5mrWWLGv5G^D}(7wMe(n)H1@`%69$Q@Mz~2
z4l+}H!o13-Hr)eYJruJ7#1uTQoS<U++f<>ZawGfn7~}BGLz|Tnbg=@<@CYr!E;$+r
zi9tLWPUqy>hS~>Ownpoiw>+sibGID(kABn334X3e8Nwc2VndU!IhX+{pIT@JoeCap
z*rhm{dc^uI=Z}A}Yxmm|w`W4k%!pI^wVR6mGl{mn{1J7gyk>95b~pEs7}t<*z2DXn
zmN+4XI^f;~k&)|vn^VKLoWn3GGn*R8iSvLZy#m}Mk;X;}t6|b@#qI2CrYf3!5@;rW
zE@3_7QWz5M!7_h6CZx*gmVY{iNR=ZDi{;G5V>|(m*L@8$R1NSrO%PR7$ug|NXN60Z
zv<;Vt>`$~Wr2>jl345P_lJWr~HS&#;Q29mjPCOH>G|7B%YBv*Gshi4()R%l;V(cu7
z=DL418!#Pfp~0wL#}_~I93I}O016S-!(i-n>`MMZ|K20!4Jz0-8;Ldy;AMVs-KXh6
zgN+{Advt_ZIjRmyLM!b4r-*8IjmXr$_}(*G*eGJb0ORGSYcw1)Il>)%Kj1;4fAESs
z=yF6Ajj;W`?-yCi1u%qq8i}n-`&0TaQ6*>rw_KlWJe$QaF;1X_AD~UIHsbib&oi>o
zH3mt0p_KyU3kS(#uNIbyt6Moo-_Fe4QQBL>cDx#%XI|dTFEll5T3<K0HeB8;B{zx&
z7ADNtv3TP&sCF3$jOm&n7jkxGr`&_ad*T0D+S(eITk=+NO;{JC|EgZDm7fcHYI(Bv
z&JYf?%_YK@Eh0^=^a@hhjoCg{HJb=f2%A`+iof89DZ~2ppkdlUBomX;UP|(bQK~C&
z&zwK-vE4pR*=t3jfH5fJqgBP_jc(JoJdF!4+@ErMwwU)<R<^MVH^a#0RA+7Db~uf?
z?~jGM!f$PV9i}n8mqFsBCkE+%qs^c!5NU5|iG^pR&M$GFK4URkyCY}44AGZ>oCG{V
z1?MFx7ZBWzbzq~udm6JBZ2tf60?1QC6|63AWr}UR^C{p-AN%~P6Ng}u{C{SVH8!na
z&Er~A_GnPJJRn_bL+~rLRGhc46{CdCNZO>qiYMSNZ|?>`2PhR-$|&Ne(d`P<r9srk
zXzg}uyvYoXaWl*y^}ab!oettsI83}Bkl>iGUhn_dBy2Q#Ou@jgPUrvS%j&I#G6_y(
zo^#>qpoN(0$O~q?$9-?Gn(sZ8MM$-5Bz|mZjYPpZF?@W_2o6$v1xmn;Mt(Jog>mIX
z|3K~1pmtkFOk&MByUNZT|M_!%inXGKa|wDd+~)g1zqN<qDfc0~NycQ@Z-g}s0h8Io
z99I5l@)97z82cy9i~V?$9@%`g`rl3!+)$9vu{Ew+nGl<<L+`+4KS*?{k%6v+K;*_@
zvs{yZ@Jl&;y3q&RPitVU)?S{|$}m<sT%^Yw8%y1wneFWwdl=|}oMO4i8Chfv0Eo_6
zbHoqgTtP4j*pI8!%h1{{-+8WCT(*3~TX6&0A0czAd|YIcSdRdo$qw*y+*60*YxLP$
zpZVDw|F5e`tUYD>`Sb}fJsa72Jm?&$?NDT2^-|FRt8S&UuaV|H?{)hv<<cMHU7NPR
zA@<@xD9_`;9K)i%zsWaiZ8KH$t?WACwUn9oCS>f0XXcJtaE5h(?aq(@bUH*q$z8Z4
z_ULN^K7|(pkZC1;OW{=r&Q4tAu%adDZc@x-Cqr7`!;*pT^69wVbVWv?f7!o(16tvd
zCL{UpDZ%5@-xJ!0wAMk=omQ}G1;*p|8-7j~WymIaqNSuYgOUIx8V)WiG9dDdg!pC<
z@=z^t!n(<vA>N<r-RV(>{Ese5-`@YvcPotHY^jc2s)#krD~jNP)W_euW_y>lD0LZL
z?{&HSC*L2m6B?vtujauKOtMxS=P|GW*VN*U9O36q8%&0CqQZv+{i#0}R?i>u9``Cs
zb3Fc>9iO9aY}~0)&WSqQ29wsN$-*dAk_$Bs<T2@#Bp2p9VZ2T6e@Qud<VZq)>SUGy
z{77ENx?!h(|E~Wo@)gLdGks8!<x?Wo{*xIy?M8=tOV5=-E9(Vzcl6oHn&u<9J#x}C
zzh(G>R0e}SdE(`o7V@6dhArbqAM9HY``z_Q>q)=OALtvwd;OipdA6TT&M<_5R&&gg
zQ>3;+8NxKm{mmlnd3}P`Pu*2^F=9ptKII@UJn*F#f}TNiaM+aKtw@o?E**(WEgPw)
z4Ex>K^>JHV`#M15)XljImw-v-ne}STXX*|2Y@5UNmGr5mEz3Vuy<w5gr++&d6ZlEA
zvkIaaIA73U8G1_p5W?dxDDlpvq~%iO0-2wEu4~z1&kErmQZqrPoYkg8%#-T<u8!=&
zUX;Ho(6P2qVxY`ACKy9>aK2!)KG12Jm2a8)Yt{7<6nEUTCTWyptvMAeSlqNUt~xi(
zc(A29Zdmua%;`&(k;v`0-uWReHv~J&;w22OX|_reP7lFlGSis42>!lqqyjBnSy*w_
z>8G)?S7<PP@oYds{wE4*`&-4nSkQ`YN?oLwo+jXTlTZuz{agjQ)om5Rkaf}Y^0m8P
z&u6!$0*xkvr=v33J9_zgJE+RQxw~ql$3fk+SMwhj-})Ete4{BR$q%jmiazv(G`9fw
zz!`0qlGhI=^xC@FhyFWBAHw7=>m~^O9I$7t(7YZ73%_o#`G^}7aUHWIW%Yx`I39Aw
zuwtngqBRBdtfzYm!(%re+%2!0urDlCH6J&BE;{Z@N?Q@ePe|!DE9e`}x>J!*Gp3;Q
z)Ly+*T$9@Gc(-q)Gs|xhhx_XH_$4h6rL{D}dT?~pmeI;|qLesbH%CCr=61NEBI$_p
z>uawj21&=hnzOxf1XE3UN5j1OQ}{nYn=k+4PrY2mkJ2;irjp$Lf}^B0K0F61vodE*
zKV2r&pbhki2_?}WtlFe4vOg_3GhT0cLBY^xB{2M)SO;UA=pM8F>8cVme5a;Y0~gQn
z9)HaMK-*AYa`PYL7z2-L30N(uU&jumAE1OkyHD}JvQlZ^V(Jje+o(O@i+7@u`KlDg
zP~g@gvZk~+Bp{6Ie*3|j%Rjl<_%4-?-6}COC_+HJH{z8UHkGi%+#vO}GL<%q-7@yY
z*R6Q4_M&c9m*ACBO}}!w0|WMPixuEMVavhR9}VFF{g*hR*MIaRo5VkPjT!c~`pE{V
zUeC!t=&<snEq+xs+j&?dl*s@$w9Bhwx2sFRMOG~<W0R*^m$gsQZw&ywh$^$vZTMI2
z)`rxxhSv^Z)w6?Z?J6{XYb&`;imiy$S@#M@4lqp&R}N`plHV+A(xp7W;aBLckgMGi
znHOPK0JB^`H04>%Yg-i#Lz(_1w44w)O|UICMiX&k{VwxBTn9oiP5(&X0H+NYf`T5Z
zofi~Ja19$;Zn0GkReA7L(ldq4{T3X!R?uDW9Je&!*_X}*-MDP#@d^JS)sA7clwIh>
z=BL7=rHkR;tS71c{E7T^%HE&p5Aa^}vRfFXD5S=Ld5wy{-yXOI-O)M2D=Qb{<r*-}
zfM(iQ@{ykc6k!EF5AMg<6Pz!@#0CRndV*zAZ5kOv?Jo#<bx<v0So>Sp0xNV~fxgJO
z2NQJhC@zyj5CPFQ7i928FIV<wN=D-%wH%SZU#ydRoxf+-qJQjoq{v9J*tSmPG5f-L
zJ?_cH1UeGvA;Q4ED(jna<fikAlM!_-+a9TKswv@;NthDT1^JIx16!Hwf?}@cQu_Hi
z(>}lrKOhJbEgf=ek5?l*t|y-EBsx1oIy~DWL9LNcckvk|2d2b|-j_#-a5_i=t+X;u
zpWn+pQ}{Q#y11B`7?S^3pL`0qiN+wPkGIZBy%8_eXiI5t!7#PLG!~eQiK!#+k(M9f
z4m=qU0#z*=Ka)uCQI-m9oE>1tEdP>V(sJZJZXc}F=DAu}HeK5Ee*j`Zoxc3zHl*A1
z$GUq17&hhmumAPGwv%3l^mMNtTy;`E%QP$vLIb>k9zYM}Pb<`~!LR%3c-oNilH$4e
zuEc3;(JIAqtqrO*VQns7GwE>&R6Zu=Xwd^@pAye49<8faO3mQ=tD_%VTq5}LZ)Fj|
zDo&-GHmK5H?L>o>V#<zYkMxSvzk-!wZYw{W)KhTrn{K7UPlLLo#3x9dQrzpq_3iqO
z(;mCGE5CLNr0q<+>!XZgp2wa7qoRN8X*}j|Bcgx?(;3*#Om)Nwvr|GXfz`+<g~z63
zOh1A6gnWXNV_;tNq8GIp6#mvevUDsRy!O`J(S5y?^Ka~&>Gs#msEzp!Rhp?<luyy`
zJ4ZbqVWY0aPOu%{QzC^&>-xkakW81NFkb@0T&)-Umv8Ott|`2Z1sY-dQ$S}8@Q9Au
zGiBTCq*+fwI^d75C{fmGJMZ4CtE0ipsG5<a>_<6)4>0-9+GRKh_wX=>W(^P}KwROM
zU3OVpcGB|3lVz@NhL3P2-A6w1k?!;}Zr<Iz6s8gS_lDL(+Io)^N`5Or{PS+5xAq==
z*8=U1!5xaL_9cRycREzAhrA??9h)yM3lJxTm;p-u@{(r;3anQTo?$t{Bhn>p(ug-Z
zJKLRm?zt^2d?%i~Q%3QX!Nz?C6@n}Ihi@nI+Oua*Q${bj<dU{~d}s#mFs^cFkB9gu
zv6O!UvyQo9QIPT*q)_J1)!KHZ{g%quAzL}c2@T=huAq$SVe>`%$ly6Jq*3a@4Q<51
zKkEGBKmOwlhw#uwc%bLtdhcfQ>jJ%E-?uOI)c&e@*aRf)CXyV-sOsm00pPcvsjtlm
z>nOaL`CYa2ZdaW+(BD^RRlO=-{q}tAn^UcLOSh39ai{CJK1cR(T}`16d6c1eQ6c6F
zKN7RB0wjaJbnS5)|0s2%P%ERS2B~A7M|WKc_&$BZX;brUC!yzj$HQ+u;iG%Qpuni;
z-!R_eSc`2!fe1b}pStT^@7e?-Ye5NzmtTH)n}tiGOPL~QF&!mXJJyIfEWtSqve{AN
z{S^MWn$6!=3jV>`xz{piN9mR5Dzt@D@CgyXEJFQMNX4j*p7~_Iv7Z_J8W^a?e|IVO
z*B-32H48h}bi<`Cwc!|lwl4*uSfrC?4oyL4Qp^mGKfr5=gRjiwTW`w**Nh}HWgUFt
zyEpKZG+c^|xbE{D#XvcPXGVJ0&Rr!-mN;7ehM)eSPMfek{661%pYAp_&^1DRmT0p$
z7-Im2BKxREJ*s=;BOkdYd8CWb=AP-bnPvBDL5{zZr}qbj%6ZZYb07E4cfNB|^6^lF
zKsdbblz)gPxKkFm#IwfTpoDvTDoyw@X@+G;dSQ3E)1BHCKk?+BbX-^PC7wa2ANi3V
zS*lCo2~S$M40t51XB+S-=M$drgr?++?>TwmbNdBYhQ&%E{)q=y>cVfkfwHNOb^7+!
z2q-rQoy2pW`nl)FLWpG$e(>C8t*d`&JxnR}Q)Nol6w6{W>(c3+J;#;1Xksp(#<3ZV
z=CKa+O6oe1uE*ehHvez|P_rN8FmG?*vyQ#3{(7#iB3#>@pDK$shCMa|J1F=ycwpdQ
zM{UNpG^*eF`yN+&f68zj+-tWLn<>Xpja&SMDSGw(a2fl%3hQ?n?z1fVjj%Batn>}%
zTjmCG9~b|OaKoX%sOaBt-sM=6Z9;)Gq}Dc4q|D&EY_?}pGJ>z2`I%7Jy?M``Jx!n{
z1P9=t<j<54Mal2Quo4<-=iWK9Bm}eC)#p$%&+EH$CjNa_Y23e8`<L+5O?I!HDq%xI
zK40spYaKudBt#L7{^5o{=|*>ZQsDn*?@j<LJF0^5-|3#21OkM}zR!SxKdX@df`K@O
zB}CyP0b*qP2oN*@1Z7b{1d&DA!WI?*A?#p4AwUoVF&M%sI|NLKWI;gSLt^%s?wSAj
zo%_z~+i$vix_i23x@T^6-&b{OIaPJ4Zq+$er%qKLOM))i&!UoDeP?d%zzQ$stR`(l
zI}z)@`m4XnzISq5#Yc?at}NnzCX@XzU=fFAyfa`*Q?RQ%TZ1609i&Oxw?q2mr7wSJ
zbE{k4Ds}`h`qj^(uO_Ir@9u46g|=WwRM6SsxCI8P{4;QdpTa-e;^9I}F>^Dex%ARY
z^R^E=+?Y8GZMSVJExX#f`at;=+zKah@9e&XPD)0_w=++1-tYX*@8lw<xE`kLf&;ET
z)`efN+2wBso?@wW>(@ouuiNYsV{0-D0am<>8{=p)R{Cs6M)B+BS^e*_zys#=&UXwg
z|M54B7b~ms3$JFwtu{2*-tXGY4=%hQ`!Hq~vo;=*EbN4vzZretS~*tXReKEU7x<|5
zF1wC5h@($9&s*`u(KhPbV%9=(xH0#Gb7MnAaf#d(FYQMkm`x_Xu)bs5wDETwZ+FHp
z<5rEbw((-_h7Q;5O~kWhJjU6;oyB%@bmF5>GmpaObZ<c=aGeqpNf}hGpcOD>;Xf5N
z-93-Yc3G4`nbV3=D{@sM%P{cBg(BbDRGp?=yR5LQDx><A^IIJ(kd`02`#pF4S5+nZ
z!tM|QJ)^%v6zzEt>Vw|%YPPa!WN<|=J3H-R_}N85&?n&-e%%3AoO(|?XS{aGZUyl~
zurueaW<w+ccf<>ee@%ua!n&dlqn{HaVoWS>;yRmFGjBv(PBWa5tIR`><;-=DgkAl^
zB+ac)(f44to)|qwiogC<aUp`6v6J`w=RZHi^PRc3041!_WTgB7--GU&z7UZ8O=lYz
z@pP`XPe#9zE{M=uY~0ZNUOXgEVqZA?oWG7M%}}gC1zU8A<*K+3V(}(RA@8Z;41tCB
zlo3A7V=8EOxU}1*O`Gy+IC=~)!_KTT!m7Q>E}~3$`Hi~0kD6{=aE(0$>&svMa`UD)
zy(yK<*>xvnWHIRB4}W;$Dm7^<ZWj)}SH0?0skcF2ax96TM?4FguDheT|B_2C$@&N*
zHvSL)@DF*mpEQESPNraYj}$1YJ}h0u;a6^F%!!iIZDIkHruPHlh{)lGKcG3`_$M{L
za@_+0Zs3x*Qoz9ztzic|BcMg7{6+xOAH)CpzjFNui_q+!ZC%nGookzW&?9~F8xel3
zeM9qnt4=@#5`^o)d>i$($S1p8_01K~|Ei4=k?{lf)!(>Dn82$eXWCADOg`NTC!BCX
zjvxAvwo<tq3!pIm7**uTcj0iCNNYnesD%zb^D&xZdCkscVC;#W)f#n%1)db}MJH*`
z>5CI%V;dTgq*z$KZ9dVzwuXw{nAqc_e%Nhq%RCdaZ63pZyvFe!`7A82NU)0xJgI|T
z;roHR`1hJ#5f(4?s!%JnCd#%FO_$c}me2QIV{g}gwYm3%IxHv7cZR-N&T{pzdkm~<
z3>bJ2L$#|~gp>gn^WP<2i|{Hx6&J5f(-I9i-wkHBystMVJ}f$BD(T=1<2N<kw+M{)
zh+t<SLxN($_?twxCBfAK&2%#9&5FlBiOm+%qn}a2N3oXk_jZPfJTqO!$-6ZA7SU#m
znjKtHC;Y1avD-cwYzk4+t{L<}UeU$3H|eKeM*m+ESM@~<kN!oUL`$OKxZ0y9+oEHZ
zR7x9jdyspYPvIBN>!=P%42tg@<>BwC_BEDh^YYW)!r0}W4M#^DaYXLMYHJKU81e^?
z=4A>(<t>l4&DSK;f5sVSG#~lMN1FG)|NX<O)M_k~7dWS%etL6eoS}clGoF#zgW+LZ
z@hHq#NAVNACG-T)S>k39aFv~MyKdff6hK~%n9whP>7+ca`Y!mC<~iy_hrke@p@Mue
zq?|q%iwmcI<fF|$pZ1C7fa@I4jM>3A`dK81;;pgs9`<A~?r(YPo13!+8v2#vg4m@W
z3vK4#JNJ9doe#Q;tTK%&BS``<Kl{bcnk%oqvbo+3;@)Xqz^Zn*m4!08mMzIqRhBEp
zD4zvEZF|y5CpE4xtiGj?BqK;$xPGwPF4B?ApnTCS4y1;v?<K#QU-|TEw#<28V!))q
zL!H@QC{mVvYFdIuGOuQx(++@HXg?N~jDOKZ0*{t~;K8o^)B?zK+|BtGpi}W-;5BbJ
zFqsdGaTojF;=0D}>Wyvxt+*@8ZtGx%;L7qVAV(N}^>`n6bM|H)7^rc19#L0J6k|?r
zmIeboqrWtG?4k4+7-FCXRRa~x<g_~zjcxA|zcNMl?PPgMR2Jh^;<821BBtBvZ3wz|
z^cWZt*u@m-XS9veip|tQQZa)yVd|HNbEf><X})pe#*_pok$>@8hA<*7kfKVOGl<wq
zCO1#y9KY++W&RcqQMxr4v$e_;_TFOP*TXp8<KA~WrX(%8?Njk;JP^OqvF)z9O0m)i
zA2oK=jV;<6oJ#LnKR1AoB5-uWx$_MhHslVjv**%b_Vo|f(CJf5^_zl+c`t^q%83yK
zgKhu{?P&g&-lCIZOBi=B)myxhK&kX#gL%$5=cHzSX}bB^Wu`O2EfC7QY15`uig!hb
z@PFkiUu!7g``W*Mqj}7uj}2WEr^GR-eUbrdk2XbDiv_1-$YMwT{r~WRWOUaII;_4K
z-p$AV<>Sp!M;{5{*_Sc-liN0DIlg($H=18w`ajJBLUMpY)8Vy|(Wxgbqz#S<v?)6+
zGRULjv&)^@pCp6@7xPiq(C=~d(MRXviZlEcXB;gt7Fcj0IbmUiq=(&p$Iq%ib>Z~x
z<-<U9&*7qdyO1)KSB|OK$IuP!Viv}kd$C<_G3k_XVaDP_=p~9oHY4OYJ>UA0a;stp
zdU~9pckk#i&|_c(13jZZf{^~T$H2~F08xs;M<}~O>Z@P<YAWKhTVnTwK_DT1X)<=c
zn&`EwuPEgsK3ltI2+xsOFVpuwdoKox>45=m3=-|?t{?NRZ<RfYrAk}r5b+52@~`;$
z9&i?HGD+KMlj-jQ2yfSGIkW2i=dB^uIzAp^ty`~(SnYB^H~OCHE?gIrL;|DBgjc;Q
zo;IpoS80kNF1zB-WEWgaHh7iiDW{y$TzKJy$%JFZ?d%q8XR|TmcGO9TJmL|L$b}@#
zcflxIp5+z}hF$v7P){kT@+(Iv;d}CvpPW0!MEF1X$xkLTDIGXWl|vd=2zu{^QzU7}
z`QU>O&ima-MYM|z+nR~ZasAszKh`|+S<h-d{jV|p26Otr2R<Meh_+9*WgTnocJSSr
zr#$&7k!NI>Td`4(=)2WtZ+z1mH(Nr7$+}Q3nR~FYqdz{jwz=fe|82hf?_Y^M*{v~N
z-#;Y?9EqZQ5^OdptKpgQ`h*g2y<|0a%TM80=7m@7!>ujO3@Ho?er}Di_*1uU=zr$2
zXFvPdd3y)l3yyb4*PC690XUC-rtiixWcaHocw}c5D`RE>cU4nWF*4J6|0r_`#mqqN
zv=Cc%zl}EpySU7T$$LkSfxQ6(J)^%jv~pj^U5WuBxF?=?VuP5bkP0D1>7E@fJ9h|P
z1gxEYaVgdM^rt_a;*`#{9(B}F9cJI^d~bRTOkp4yis(}<)6(v~3oRnP{MqHM)b(D>
zS2tb7Nx%5n66q%vLmd|J6*git>CyBXi)$hYmNw0G;+X#A+9+Qrr`nd6M`HB%4pU5P
z_gF<SOxmYR*AFAh2-kK@wPD3e&ygQ-3TDLziwHIef+s%liFrqQ^$V~_8o5K>b&Ph$
zzxmB?CR1H;x(f^?RZ`iCWB1p1=ZZSvU`kU){;qkXxt<t5Q!vYx1}t%_4pq+&eBcAA
z5&zMTesu0S^KYLf6!EQZjy&@Jg!um1&FfzOx>%_AM05B9?>}Tnom(b`fB5|$&>Vh0
z3jkrFZx-4ERFp-Z#n?x=;amCR9rM06j-x#Cm`64D+jPI?UcY&-W^Eh;n2ht%?O_$2
zJj&vnRM*5u5X)Oa!gu$M=Y>}|h1<e6T<a6Uk2&U;JPIUE#n+C99d=l@n_Yd2G;TEj
zqc^)41JP;qMe9>TqicCjJNj7k!4NEo_6t9s9($oCR}vPQXb|5*!vc)h+htcE*b9WW
zfUNtdJqGr84D^it9^b@$1$QL|?1s6Y+>RKrR)n;j4@5Ol$0A(qkRbGlPg0=fUgOVx
z_OrQTa^Qgn<_=mBxb1AcD{Il`wURNQUob_4vn?ErGF{>j$EA&*_%jI-444OjkGP*H
zjdWWg{Bsvv*qn9I#c?fEQCvdiHs~>Dlo`Zms~H-dMyz>DGro=(>3DPfU&Oa?-Y?>6
z$2h(+wr{g8<IW9xPYTs$#sY0n?O5$v(mYjPaol<S^Pm4bnQ&4-+OpyovsUrDwy?^J
zEe*emFTS`r<d8#}+u#278NJHjMI8MCBWLH<8Gc3_Mz}J8Q{z}Sy*@`DfU5($rs~Be
zK>!9Zd`5bYuyV5*ewC+pivw;bAqB<1(g_Q$yjdHINDq4O;mzyc@S5g7Ln+`mIKk|!
z!TIQ#`NWSrG4;_mhvN4@_`ERdH0_5|@`Mpdu6?cjn%}(Vy&{_kQ9N*In<wudeVEqy
zVIIPnfRE7!4ZM|L;M8@TaHky@3(ng8@WT&J|H8j;ZegS+=#ii!0q%RV>oKq``krxC
z_o^<IrRXR;6d;agS~E>cu%ogsDDre*3rw0lFwzXBjwx8P!5K!JEy5It-qmAZZ^uB-
z=<n@4ps)XuVgM0(+G(fdj=Xb>i0LAfWK@KA5mEe{k#&ysumAe5o7cbo^{Fh0M(W=5
z7?=qIMVMEZ_hQ%)@6J8H^PTTZWoaZvikKJAVw3Z??rlH*_~Y~FNQviyCr<9t1!4dH
zf*Ck(%S5wf8l_md&TM1yi8ZH5g;T_s5u5=xl$=dm70gA*?r*$$pXNGoA9S$z5f<Xn
zaYo<1>E2>$9IzM&=X7;2{NcZtXk*9=U--i2>tFwRGQ8qqpq)|w^FROdWc=r1xWw1~
z7`tMa!L~4D=bnTSQ{$2N79f>j<Hn5{&ms|1WiGdTJ^aO~e29Ky;K5<QtTL9HhJSwQ
z&E^kI_2Wz!KNaS^$_TEqTTCb;Y)x!&PHvmP>o*%V+$zdG*<5z%WwBBaIEbD1G~o`e
zJ)BsJ)*UJH+<w7NJEMnxl!@VoRgZ$U--qN(fO5s$mWeIRy75?HC{z5$z!tr{D$4G(
z;kD|k9NhW`+`2)b@Lui3_E=<M*8&aqimKS|MCvZeTVEO#8p*y~olHlIXEGk6v90};
zyd<O8_4sz{Etbu^)NAV=)XmbFN@nynWVihKW=e<1e*5jW%gP|%nmcL?-({Ki8Lc`D
z^o;(hYqFg$a8AxU52lAO0|s1^b^iJ1=T1It)1$aXQ1L4wtK805KjtxyY0iu@lEnBC
zn24>xe70RqyL*D$u0XxVF2g|eAsJRbL7-Ex_Oh3~EJbjOkrJo+*bQ!QgJj(FQD<;z
z-G1gXpV>U&2~S9g6LE(o8JkFpamT68(-0>uvNC#kxTB{IIxCUs==gCRR4^6MvDbuB
zw6)<{8=}v_>diBU`9*69!9m@t0P{<_OJljR%)$#PAnj8OmiDvgr=95bsC&#Y=)d^I
zFPgvko4-lrXe5Cg37X3w3hm$*<Lw<YhftJ{9oia?*qCkoO1>lqEDkXhr|Z_Yp8MSA
zHv8|t|1b{yB;XncG5zW!?_%P?Iq$sl5}rDP!Q^-sw)gT=4#%IAqw>dW!U08s+>9{Y
zEG#`M!v80J*EHAN|5w5<Bqx$zZ>37c#wq@4B?nx|w{~owh%?0sRZmRYB1FJVH{kIM
zK;?qMM5NEWV_{@6c2w88rJ@oH`h6sdKk$bh;od!_i(8UD+Njz^AG5#%xA3lB)eco!
zZ7b}CH@snnRhw1#s<j7waK)EDx)6=~bxmZ*lC(!fY~4rCG{n*wp^uJjyE1y5Wml3W
z7NHi}2Q3mT0wejcOnpx~f)Kqd%4|_EzDnO%meb7POCGTMRgSP7{mS}<J=ejJ>bs;Y
zzy0lR=i*fL^+=liyQJKV!5HS5`h3M@gDH+N)VK1*FMcr<qPmjuqKhudZz;!vtVcUP
zci+J%Q{STSuBrDbe}#4Z=}w`D)m*MD!o>G&cE3sf!i6bs*HjvQqi%jRW=ku)Ot)O!
zbmk^_FmI6RDk($KS2w@;&2ye9SOue7)`GtnyWlM*?cR{4XY}`mR_^P#%P?Sd7XiI-
z<HpR>Kxpta5E<0GcL&WHDA`1+4T^{_aoa(AYJlBkmFP2C@fgsbs!!=BCj3A9vp>t-
z{DTfUD3v-QTui#I#d5DX?dGn|!Z?vUaa07uNDDXZ>Kis}7*6;}tHDAz7SaQ2r>V73
zok4iS3mCbxAFjMkFENaeLxv*V=1&-A(pCOb-#r2pQB+KLRf|-{>t6S|B$gias7E!2
z9d=kQ1Yn?@nSbwl-`jlYOJB-`0JkkzD4fa2=chmYX&wne)FSru7fdpde9Rsqm(ic9
zTx?~-<XZ@6$D+24xEXO6SG9H1^0VWveD}TYeKQ^cTbwc!yGdV*6qRqmsj(7~`d5GT
zS3`*h@swK`Xx%3bRYbS*^xHy=UT^#EIU%wiJM1^U@l6I<4!)OAuD3Oq^RWTZ+n#ft
zDp!?{Jy*Y9W3g%?q%T6Nq%zBOTa<~S$u&CqFtO736Jz%q_=8(GbMt|A;>S39&p1j}
zi`3O$_>1?0AN(Nuq&`#Wg!N|EVZe2{<MHUb?HX*fpC*jQ*y=kNpNz%REms6ryd?rK
zcp{V?&Uu;kJD*q7?IJB?Ja?$2RZ<+Hkj*v=jbkC@6^n0^A%U|lC2*!PFlT_SMst}O
zOU!p9puYRv@8-Bs-+(d6_~xi2MaXo1P>ftgyn?|lW9+H#!nf3oQ{MzXPj}qx!UL-t
zw_vlo@CttUcWvnlX8DUxhY3l(O`A3ikAPNq#h(ps`71-!v0Ki{6TC9~ECAX#Kx?2a
z;M@WWi7m%gEoeXTk&hfoxJp}a3r=q@9|n3xfBA6RL*5H9fN&zhMbx$i`q>6F;Y@{N
z)~txy<uXCE;X9hn_Ch50Rh~A%Lx+3GF^80#(Y}v(A(C&hHSrSj)+cKRUq8dhSsXa^
z)KfFQFcWk3$LN1WOf6tTKZr9sPC<%gB8+^n@bNR2G-D7y{)!KZIQb{;6dko!w61Tv
zzIzqs?)p)7=1e}Ncz^n*f12X@!aa`p+~XeiNW#=&4aVOQp_z>M^ni$0X2=*v8{1iT
z(?{)eiC^YJANo-4)FafCZ<J`If&Fohdt8Q9INQT)*s!6qU^zwjV;1#2qW8|(ck}F~
z2Sd5^Yk3#ku0LELPMOjPF&W#MpZw=fMf~yRz_`9JyIyp{5F7~x$rDU>yMY*R;Xacu
z_-9S<CXPj8;~9)eI^#7(ro*qjRx<2D>mu{Y43_8zz7g@RP^>mB{0ck$^wXQiKK8LW
zespmzjx_p{KGO|bqN4R+c7Or>2$Sz=`&4u&eJcB$o)|qYI`mj9<b3IE?`+opB#v$+
zgC7aw8>nu$gKXMhk^}L#P-9FFv8{P0lY{wT^Vqs({MNT>4*I=gLYK?dW;1CS3zHGv
zmSQlUfSeKx;X6z5^y6;-^zNZAsjl;M%nkET3FWI^^{Twi^NnwOW71!~596e<1)NGR
z>w0I?HA3pj=1%3;&D;0|PNm7Bcgj|2>N^z9@VEF|jJNlSE8KOuZ+OES@>>+A(zDgU
zVx2{dyWaJ#S$6*F#DL?yU@9+Dd6k=8VWai0{3?#cWSa%(-@D)a?y1bv0x^FQs&rPl
zo|9cyc1qiuU4wz1(cd+A?sM5oFn}nwD*WzuzdHw6%J^zvg#zvxJh>U~R$yQI+SlgJ
ztFzNJ=w{qOyo}uc?DZI^Nm?H%ri})3V$yal5aA^@XQ%gWce`7PS08c25h)^m?|a`n
z8Al6B76@oZHyJy(f5$uCu_Xcmw&@=|l{D2nv2K@XT`)j)^j)Pl7JYj>wmCz&e)Rna
zj0cZ}cq67c8T6pDq`G*!yE@N|R(Z4{vb1Hjrz#aus6DiWg&Pd9aiG$;x{hRqFjvjF
zZ+<pLUsw^0UE-|VC9YjWZK2h|fLlTSzhIO-6~D^CrkIUggl3IrZ15Ov;f%N*yTNWG
z5$F2Xzy5HntMuZ^kDd||QV8;{9tq(Li50|arE@fdhWvBCcW(283x3et_~tihHXXJp
zc@BLG1C$Itl>-Atf6NyAVodheL0X+TD5*8;!gHT+vm<CSdU-NlUVKbU1no`(|E^7Z
z>QkSZcqQe*uW$(O7W?44{FGN4Qr1U`g5=q>D<OAtopvWacYWUNnA)=J@jWH(<1Oq6
zi8hN8>5uU(T=n@cG}jJ=s>WUmPupT;Dw**47P?(Ym~ZUq@fI*?_(sMi$D7UT_Gzy8
z<v~sJ;3MN>4ec!<En|*QrMBEHqn{tNItuX2V)S4AS^qAxY15`Ws^(pvwrHg<pK!tn
z4SguaUE^wnchiFB9n4~yjbD^kG8MPRyUJf&X}al*$pu3?Y1kg=jExovDqg`V*pAL9
zxA@?eKf7RoQ84+5SMVzi__fGebr8mg&q%|s>YtvSAO7lc^wCE*PkG8ya`D3$0xmkl
zqM<8Q?e<%=c8#Y6fr8rs)jO9713ja^Oo;8l?wuI8!yWF>Jmet{$(3T0$H^z3+z`FB
zbLBj#0nq@*AYu{Xc1hm$wzs8b?Ts5ZCJ`pQofz6d*L!bz48TB5(%dyk6Td(7&_kQs
z+~zhpAtT_Kn%MO}ldj!;inJUN(HA#t*wDD^9K%T5PXA?!ej4!R={KJrjMLiS@+WQ4
zB(aAWCDI%OTL2CEv-R#!X*BciZKjAO43&03lvTSTz8tx*YlA5+f{0yi_;Szr+u#27
zJX1Xz{+x}hys9r$+{#lpQ}MVl@t8VgD^rA`a(bk%aPN#hR&}a$>~3Dd=?Ey(Ef+Tf
zo&}B~MEO@3Te$MDy=T0!I35t|3wOA~?V7Vc_nBa{vD%2$@C-&`O-M0c^~<Z8mp<W@
z%|2JgZ4aBT(Tq<-ngy>g#aZGIjb6WgT`1v;c3Z!uxoZ5X=E{99Z=UzQ=VbYt@N2hR
z<-fA}x5XVg6{_uK!>cxfW9`K@Rv0H3X{z6(NB^@+FkncXj6vBseT%6B*0=3y)#Z{w
z-_l&`>MNQXUfn9AOh!Myg@qKeW%L%3Vk+*!`uzBsU`4iVY1T}}Dx8m4S)+oXP9y6^
zSByGXIvbJXpUSf1(l_;4c8yK?tiEi{ta;NsNvxdczQuC4j1V!G){!bZ^}ZGH9GfB`
zC(L3ie>B{smtLCV8|K?LQ?M1syRpfF0=M{>V7u*LC|i|JS;cXz%HpOSarsIsU*#9q
z_Y0<cjklDuDihestegy+<eqdM`C_muodrAaD!f}3`4PwGXE8&5-1W_mrZdmH;~nqF
zF$KKBzj%%k(kDRj%$&k@G^k)@Vf&wD#X!&KFDrU`;CmYe5KngGO%QfVtztT>YEUz1
z8)(H5?lFKG7%^UmYa-U90{9sum)j&S76d|XRu2aBjWQ;9M&B~&BM|uMZ`Fs`2ml0#
zJ|&!=I37F47)As|`ArP+SSX{fE9q0j_qPxg4FLSc?6!t$TQK#QeP%2ou8l4I@vs-n
zYqcO&@o$TNZ8jOqVzpP{A3=&3^zf^8u6Qbr;A0SnbD8QcHJ*rX3>szhzyP1=_MPf0
z(rdb{k)FHKi=TZi{PZ#LE4?uJRo<gv!Xt88k$LXi*_98^^p=M-#z{EgE*!N<+xIM+
zu;_C-6S<LX&6Z@bqnAaR7)x<RSA>GTEB0@$d)WcaK9}#;?7QZg&6Z#=^wWhIJB`Y_
zwV4>3Xs%p;MRVC?EUu9j2)$=^_thgVAdEO-#)3(U+J$Aa#IdXY6kfGWwOP0AtIaB2
z;TQgyiaS-jsme`Zl56U1MId0Rz^m3Z8U1@pbSKhKagj-j;y`nD;jl?kxq)Jg_{z4R
zF=Ek`^1dnEDD$hj-o4PzB3nGW!a^1a;wy?WUcGi(Jh2)VV*AP1N9Odq=t4eUg#p#_
z-FVbzE>q*BIQnvT%&7h@jK8Cnx4h*o(_L4Il4e4SQFiJW)%C7-y%amQDdVbBM}|lx
zef6tfO~uO;Sdw@#4_Ds48OM~wb3Bd?4C6h<`TqC6e;&PZ1PcRR-=TTh&iEN;oRK%@
z7~@Zl&7#Ve9s%#4`qZcLSQiC=H7362J@3h*c*?F`j#hbuD~k=M=RD^*&9TQGn>491
zkqQ_!%1^}p`OklTZccseYhTMniP3o8<pX|=%~h7lv(k_X6GuHANs~@K-8rP<_V&_a
zpl9@#9;ZFTy&MC?rA-o>Hf>5#Fy~Qey>?#7KyH8|c527UV;9!BCacQCuML7jW9A&Z
z8a!K5_Au704ij}Zb~0fRJN*Z6V_Kax{Xzqf*fH|AnRco%1&8O-<u?U)?{U@SS~WI_
z-}%mW&ed`YOQl?``jP%-(TR<4&_BB{Oui;uk2p!x2k>k@KQp#t<v!i6zJw$rwjU&B
zM$qrHx~_rYp8`1um=0!9Z`rFN1NJb7sf-p>k3|(=S|X)riCv7IT9;|rS(s@tan+R*
z&F4S&h30zK-#^-gDEfG`(RE_s<%-yqxiUnhqc%Avw9GF!+Umdm`=`mIUE`Y9h&pHO
zGYh%FGaR(-oZfAm2q;B#!Ykd7;2MC*{I~;HH(mxue@ionD>L`Xt8JTSYl|aF^()K<
zxKb-0!wQ}x9ovG~#~3ro7_*bY586M7#3UjkB*(!hSGUa7XDK3ca2|^t%UZFh6pIWA
zF@c7EEQ(#dzPZK~`!;L01m5Cm$B?NA--R$H%D=T)KfX_Ut=`)8S^q1yT;ArC8Nx(-
zQ<;0@nw4&YcgZiD1|9|&_LAAQb4Uax&Ii|WLRtLj3%sj;4Ck9@Gyc&S38U>(>67zF
z_LWG#E-JY>@FGMDtQiXiiz^rMVgV@g03%{Vym;NSj7c@k@>NV0>$XO}it|p{X7Jdy
zPW0tMsr+N7IK~krYT4tqjk9<Q{8>nCrpi#bm``SI!?57AzL|)*$Tt4Acyht7su$qe
z?l%~*64$er%y_^I=d&0W^iy-)sIiD>_gjBQm$557o6ZU%{TOt|+`!zpabto3)5EX6
zmrDDc?|dgk)-ma~y49^x0?URDsXDvgr3@FdZGk{~aLQ0v`h_oiA?Iwz1B8{bKzVp)
zSH7h292EuIZoDIVl$9!<IA}~d1?;BN*JBZbRG5ViaTxJP0nm#A#!FxN(jk*CzpAge
z-89{I?lP6Pqp5JOo~1Z;IDS{;-d;8g^o;(pp|%ISH)8-{N|c)77c0P4S?!qGaW$A4
z0IlR2_)Q=sMpZa7DCVGQhiYIV+0@`#o(r^dJde4goh}15kA=+1JIGIIf!hw`G>@q|
z?EqXKJPQVD0%bGA4JqNHMW5K69Rs@%CfE;u_`|9H0a1Wp=uWUEZ-j>hfbKoO-FBST
zbhDIa=p6(Y(Ne6z+UT%bV+X%w@{y&{-GVS>w`5rKLIj&fq(z!c2}Gs$p$rVy;vpB5
z#n^K7D1iq9gt2emlC28d2UXkr;^+Sx3xm&Y*6kB<;^+g-@&6Oo!(DvQ52O8l+}t9>
zA`^-qCTtqtzw(u@Mw?x`IUqzlF`fe!I&(@UK*Y|mh$Q$684*+^Y+)8~ZxK?j1G{a#
zLD{3vX|jCN|3XB=m0rBEJ8}%tRkIfHTS3VX4B`yOpaeR_5f_s$C#L#3cM-STX#y6Q
zBuUgO<PEY6<J-voM%e-^>EP^mu}g@(cbsH%EH3D$l$$NYD9h%pn?qTcOau0We8a8y
z)(yg$?NMMAD(lCwYYd$Q+bnb_PZTP7!py#HD`nMh>TB%cSBma?p(pDYbt=299(X1d
zOjJlwj9Xyw-A_c7G5cA7a7P&e2cBJr1vS|f1|~eS)r@~Vi!TqnI~uh%P~9#QhGQtk
zAFUi&v~8pR$D(1hgYQV!8?^d@GjH2?e#vI5{f<(nR2L2VYeF0*zS$g4l)006^_n)1
z3Ya;^8Xs!hkv~67?I?q6j>L4E7cs`th$FtoB6%sfb>s+bmXcV5MFVlWMj8tTFM837
zQUM)?8^eqd$LPB0#8J&xzxvgwz|OHcV*gCHyp#k!;t`KX2~itD@{#ZK@3O@Mzhdku
zy`yB1)B%xsMw#TT{*IQu>s{~4MFfixce%@5rokk6V8g&s(}zCvp(#yOiviud6L9>D
z`pK(n94T-l0$pMgLVk>RqiMV8diT;{pl9@#4wpTo9s@aeL4yIzpp4-+xOr+2w}L!6
z;nd(*48FMOXUv>=E?c444l*O^UWEP%E##^%R`r*v$dN}L*?jb)A8kJIiBIG%0+DY<
ze=@<EY;S-2+b7Y$zb07z2#O6GHl(&;{z<rmz1VIyWa6Ulf|1kycEDPKe<?dk<`v~$
zF@~ZyOie1m!id8JP7smP65wmsUbDIEia6hW!9~qvh}oi%9BE+w<3G-6&Oh(m=I#go
zdN5Cesak}nqw(Q`|L}q4R}R>}*>I~3k)~y)U_6x`+8IwOxy=u}HvdRpaWk9k@ip4F
zq)ih^gLP2&e06WUS_OSPfZc(`nLEn(;AFeorVd<K@}Rv=$<{<1)!h;>?SAJDcu4Q&
zk=-nRn^$I7Kb>sKiyeEI1L=UmMH`y%!wH<KD|KEwW@j{*{lzjsg9^b&K5cG;7oH<U
z_c}|_he?;pf#;^N-1j^gSFGiHl8eUO+D2(oJ7K#>lx4FbGJ~0|+;H0-P6$?iO`Ka8
z-;zvz(_AC!9}4~S^TdPiUBfC^WRUsH26cu4>=9l&U*EuDsGFFGMJfzm;FBS$A67!w
z$HSplry|r?$gv2s)$YDUujMu)eSUt~Mftada=J*n7S9|Etv-`Od7&(93#IK>F3te*
z%-uEny%@%_n(KT6Hd@%ltP@M8Anse=`c|HaCmPR4gYlQ%S^1Z}>}9zr<6U~RopABI
zd%6arY$u;T@p*axxW%`q{)8tyA$Qg7+MjmXX?c4I*xf|~%5rJ4?@Hi-fqoRP%2)Nn
z$a`w;XV>Drbl|JEg$1JYH1nJLNdxjnlfuszzxc&GhTz7R+JKT@<<qTa*A0H*r`+JG
zA4P}!Y({`NA9!8Zy?f^{unWHYohx9Uy2rq3!+-(JplIMMV_?Hn)d0sW+`wG%4T?1x
zRJzq_tN)_r{MB0F-B9ktar8BYRLF@Zo|vMChzHC)VnqKk*%Q$ts%P@ANmyT_nCE%V
zdtS3~<Hi*0>rUpYvHj46-8v+4cI!`;D9COkI_k8gT$ORlmdU{XPqTDzO)EA6-QU*S
zX$|G@LDbnbrqFOA6u_;^F=boxxzB&DxxtNZ&>Zw@2gP0VzbbW%Ey2_Ug?!b-CC&d_
z{^Rg*PCa(W5Xi}G1*|9oA?wtYkg4Kl#0*L};;ZC{y{*l4HaE?_an^mDvV~wWwy=rV
zCsWs^&h6vaEssQ**CfVHykG=F#df&&X(p2E4(2cp8au8q8I1Q;W7ll1xWRRs%VLMT
zWnZ={#+)A0$!%C^c>+rI_YDr{T31gr6QSLEGV<OxuJgKjGL|8tE%r2%RZ{tA_wn(N
zUeJ2b&MyoLL)eNrWHY69_6!hVo(8;$ScIA!ziu%6G->}bnm(aNue0Ey1Lz0(K3hx3
zc(kR$t2aBufFtXh1FcsFX+e4=m$U-i`FTqX^-SQqRoG`CtvuVHmG?X74>3lxeNu%i
zP++9ny0m5XGBB#tc`fg<YmV#Ak!Y`{J?&|!8D2c}m+_TkrJZ{7AklVo7*n?0{1P`W
zctzWZQ+a{Ui1Xzye>s)vVQP;$>ZqZ=_!xA@HR)t>BmlGRn$k)Oo(&>5tJKXR85lpc
zNhD4gFy+~0=DEuVt9<3{vCwebamO`i1>*j7r87l<cHiN|&c8YpEO~-8n-N}R5YM{>
z6jv9LVsJ!(gh7>kE?BeW(fcnB15=#H;?no2dJOa!=wP4*FN2xEl?WAW=Jh!DWbnj*
zVa|w0c?|qrP1M6lBX?ms8T8H`1A&2>NDCPzSVRhuKr8YlSd+F1+;w~I)5YW?6l&tu
z$Mm_!KmPFv2FxM|3hqj_;}Z)!tl$z3%ZBL6z3@R{?SZ;7t|f-H1(S=yMc5{>ltfaS
z*tu0Cow{P!CZf%V;@;yo?ir$&U&^p1cEg-)pA5ovb1?Ux|JGT}tKa>i=IXJ~1|3A*
z)`)MJA$L<F=9ry56!oxt({WgY*mPe>?5S;S?)B60=7@{dHa808&SSyEU5SYc`S94U
zO66I0<`|0;DXJ|b@=HmDHL*$%%w7o5Mu}s=0R<8Z7;A&^-Zr_e`EDr1`u4;BPjk_=
z4hU#&j+i1?!JNxrg4)Fabqzd4o_HJ!6!ZXG`-**<TmNYP=JGgOzisU`nk`o%_Jau?
zGNN-qUtM9*O=5v4m_3@zV`n|qGn&gTxjGn&?+3$wjlf9Y8@>WT7M8qZcNnzY{A(Ly
zMl9Z3z3<OLY<xpB<u9A@eI{Zd1<aY2YP4tOH2*C6m>moRzT>-@h<h9-*3b<R?O5MP
zww*VQkesmy7xaVgf}d}7Wt#j#W05k%>Z4yqrNWM066JOjAqT&(_0W73wu51z=rHvy
zw{gu*c1c&72hm|f>M^0_NbjS@xp*)<V0a29rnwAvh117Ge4pMWevC8~-$DVp&jP~v
z=bxW(%)J%QJWWY%7iOg|=9^u9(slExu<|bkTpVs!=~~!uTMuQ?>d)UTP!=iNb&n>s
z;GkUc@yum}SJ~vFUVr`9f1UKmVTT>oyx|RR$UG`u#hnXgAGS0Y=o$T`!DA1l$G|iU
z7%<=a-uLDiEoYvb(Jxa2oWanbXX0^{mTS(eqFM#s(V$wecDo&O7$#!wDPXska*r!O
z^&xf{{mDu^g+X=;5EJ~$aGT8at*(E!uL)11Z&tz;Y-c@Id_f0ofcZ%S|KUi&37(@3
zG2<*)Z~r4W{34=1?II1OcjDNg4R5~@53ye+n7i0X-y96}#JY(f^kV@a2<OZ$*aIoY
zXqivaKQpX^^ROe;*0@$KcHVDr^}6Q9S6Z5An#&zch;!L%f}_au!(q#Zbio9!KkDt;
zMKulK*{DP8>Ia99u+N(rf+<;VvE%CT=Hkoa`l|73G?%Y)^aOdLh9hPMuZjUjwS$S#
zha|Pf=<mCJpO6^9n1<+WY;UiR9eqf5)B%3>B(o;)PNF007L|jEpZxfW=82E{R51Ft
z4o3W%xd4=PCRq_JlM5wt;C3ebWV??2wAuPAXE%o&b>n8u`fJCcg1Jr)t*7uCW6SUx
zm<d|%w<8Q#xY}nTq*1oSdF#owA%)Q{qD^&RJvp9iOzmn8@a-`xcT8U!^;j3z2&@Yw
zgE26)3v|^N%g*u-U$RC!`l)%Y<~WPmk9o{v;=5>>Znq#6bI%WhUCg+5<8CSF%dt2H
znC={E{w;s*%7;XqcUSNUFPPjF$HKe3N!9TyDFJojHkX$X?!j~q=jaHmyel5NyI|0b
z%fHH@T#hNw6F}h~nt|jW&HW?+H*MOK*Y<%YO}B3RW;4pDOc!2wVIGMftpL`GU;N_S
zR1n69*PC63fu7Odb-3<x+DkCt$`xAo&p!L?95mf?N;#Tq^C;gl__|upUD>W=aj&SW
zPMqno!YXY|E~^D)Mar#~@~-A`nk@Ap_9y{R6KHpWugSM-ckAEvuL->-<Zir`>TdC{
z@=Dk8G%ce!oxyRMJa&$T#18S#V8F4>LH}%x?N`@FwIor?+~<pSAEW_cNGc^`9RykM
zQJG4=z$AxaE5`u<#q46UF$$PojMyZiY$mkL=m|^^tid!;oD~HQX3>s)OOylCt*<=L
z?D!426!YyZOPTcuVmp}32|JHb&UMNkk^z&G`vkC%WXQ5lEo8}qQJWb}6u@Z#<!n>6
z@st7Ht_~ue=Koj<3EQ_rJXmET<2}s8dQi3z$p+|}aGX09SGEO{zh(37V`=3&fdz~p
z|CS-7b77(5xV9Ky7mWVI#m(j||1WU5Pdw32wLtWd_{l8VcJ~?u%VsJ6L2z%)3j@A-
z%<#T(0>XLrl#=P9IomxD>^qGGxv{NCe0{(-NRnvf;eAc?lRBRtLMU0>LC+lEWNMsh
z$1&a_>PWMHyEPY@A3WwnbQMX;n)5*Fx-qrgck>~pyI_gm&Noq#fXN?RcE#bp&p!Jk
zgN()llV3NjxM)B}IPK~eEVif1lxM>DQI^W9V09M<Dt~r&{x8`4yKpi3Z+g?4QW4<~
zfB3_c{`=5}K9n~E(9B;FfK~s3Q{|b-7KikzBNFrlktTfM3t!kAbkIT5@S}X<_GZ^$
zpl9@V4W9d4_7V&j*iSm?q}<7;DV5u5sg>Irq`L#06=d4@#e2<bUemn%<u6Z7{YM;e
z#Bg9N)Q#Dh%fZofzqmnC@&FU*!HUT;JZgfdd=_L6VA(xeiA2PL>avWHi-E4mx@_&K
zUwCJ<MfrK>?zT@iT{o<V3E|>{*`3r22Fa4=KKHq)N!Y>?QO3%rYghaXT%kWv%`iG3
z$IRcutW4oD*GXxTp-h4_OF31<C@!`h4~9;4zct1^s}N!1IQB4V)oq9A?u^p@469Dj
z-PMsM>=2KSQ~g2QhYe5w$9!>OgYzx9!3=ES)b9*lf--IV5WL?KGPTYL6SiGD5nq02
z%x-t^qi#`rObABbn1Hv#WMDjk--z;rLaBI^K|EUTw?sK(ld`#Td~@u$$6m0qC4#mu
zjOuRh%THbdWXL-s4thW!VLXoSV;BGcKmbWZK~&C?$Jdlt*oyIqEb=({Ouz~ktJTg`
zmtat${LnBKZM8N0wr-0%ySH6CbmIqeAvAET_a*k?2uO?b`BvIk73Z8|Q6?HVkNZ!C
z#Ksy&GXl8@7m~9C)h+{jzFG9~reVNwm@VU1$G382+!ztdh2tUl;-g~1%4U!8(0uH=
z!*yGt&F#oi;FsTh8+AqfMmOBDZnC*5p3TuS^Y#Yw;jr_zi=J&QjT%JBz3E%Co0=bM
zKJCu2QcWLE+j*CN&7qavJX&$f7S9~Z)SO)K`QJQlO2O<?%7@0oEQ9A^F!-2jv>zq8
z;8r<{9;mqDbBj~?*E`siw#w1nffujhR=k2|<LSQly>BYYv%y4Zp5uP^zW2TJ7=v`m
z!X7mRw_w!*3HtY0&w5tEvY_&JfA@C@OWsw!f>m{?_fdOC;iK{U_t`Np_m?tzx;~`G
zK#zeLFyKtSo%{n2JTT8Xx#GkwzCj<eZ@_1JoPQ!Z<{UNV>lLqfMQV6`?Q35<Wd01~
zy;<EDs7bLHZxdK|GOji*!r{k1{&8yRJ^JXQv*Oa5B-!HB<PMGpT-tK^bF&c=<uA-T
ze`0xd5-z#ql63PIF0MS3i?LJNo*2y6{%64$z{PtINkOrqz|$E&n5$q&bH_VlVZEb$
z5w>Y6?L2HmSf&etZjj%^R@=9cHW_7^Mp!V;c5tzE$>av1EZIUHFk4ODLF57(rY&>8
zigF~Z@E4R!eiGx}rs~ng$C78E6OL%<6AWUjtWD30z8?OwJ()(j&XYNZN9qO$2yJ=d
zH<-;fKX!Yjc`}X)OvJY|nF|gPd5<*6J6K2rr_-WOYGB7CCGujIcPy@aP2CCFcXphQ
zhH3M*pAFhAwZMlC1|vuIJvr-dQ|%prz1ay2_{!--n1}^2OK$XTXodU<hwnQ^^RS~$
zBYZsCHkRaKY;UjijY%ZpPJi+{2@o^cqj7iYJxVlF9>xTM2bWJ=-i)u;XXEAub%;4B
z6GU;d??s5^Bfss#J-0DuGNblPsfw|zc~~5C;#@}Dnv>0)^6?ye@WFY#EwTI$e(-}i
zzk*ZqKF0dYGtbPl&c{n5fBAXPiQRJ4Tn~o1x?9#tD?T?7d@w8`R6J>zio+(|;JBzX
zr<`(1^Om=~C8Y~CZQ7J&uly_Tf`x`CKXE*03wjCc-hbpHA4#!)(gE}yh%5hsryTOE
zGI{UK@?xOh(O+Kt_R#k-4E*e8KWl#Slb<xtdCqf^fj7t*)EIURmW7rIGXc1Q<+iuI
zZR5%pMqIm4?GRlldV95DKpS_70OhZKz&5FuF+pE;*<~qCN2#9o(FqbPM%seytX=u@
zvyg<L2fJW^W5xb^-}_$r(;?wliTEA_hXtyH9Y52>C)2zCSr7(VvbNnZjfDA%Fps_V
zU_NPn$0;M;?ZB-K#tVPco-JQUMlrqw?1|V)-%4~kc57+1j{`ymod6^syV;TR;7YD!
z%rmSVo92^Sv1`uZhrYFDy@838QCrP;2^D+FA1(|f$Qh(~@uXtDu!$R|YC;i`HHi{N
zy0uaHb#ZonZRikKYj-(}^z${q1qLIZSOCkv_Px?)SiF!H$S0Zp);CD(ind4C+JvMC
zOvA#I!31igXuLe<0M8LFZuF7xoQk>wm$u0x8DI?-mC`Ts)7t`Wv<1`ta=|w<<&&@d
z&%7~^cnv(Ri;v9N_8c$T&Yv(6^J`=AcigW3l_6;n%os77><imxlf%IF-s#ga*0TgI
z)LS>3CCy|=1YP5r`!&CuY;WO6$(-mrByLC`W|x(g+v41y`^?sh%x7iHd*02D-MO^F
zyLP2Bx7IUP8q8`jvC?|Pzx&<qo|2sJeCIoxzxkWLNv4_@yA7Wo{pd%z(``fO9{0FM
zbHDrDFN@I48{=%=R)(@gs=1t7eD6%f5ub9w(@#G=6+vGRM?>j+5|=2x{Fy2zk$MsZ
z7=3yL?DEt0-;Go5^o*aPGs+(<8wO~Qx4-@E4M|pYrc96oA$(LlxRt@fuQ$5}15*^{
zuF0>@rN_V?jRAw6LE26|V=&{_9n|W*{Ja|gF=qx*E2dy{2h=?pX?u9(wCyi`@r&kN
z?|Rp;A87mT<g5>Pw7IKjEEXXkXsQ(t!!G=>k9}+s4N%sd<lO}<yxZTr%X=n&ek4e0
zRb9Tf42|a&ni%0zq*8kIKMTe{%S5*1Eru%yn_%uIWBcl3A3e7@>)$SoU8`1Elf{5-
z2s@1P0-2yBm!iEUgIk{1=jY8OYc7mDZWd3d#b0Yj4@881z>j#`?QZPC2rc~qSvirr
z;Zcv^-q#|;!*gqt8*`6#v->8l$1ht>V<LphuA@g}5(0VGdMe%vI0-xQh~nn?<4hUJ
z=4F(wg`POHaN<JwFqwZaI<fZ@=Ybs~h$q^j{Wdbnp7*2D$=tO#Xj`PkMC%`JqV;iv
z%R&q+CEhsaVcPZ;(Lxf%hU){4TOwMrl4M1A#v>n#4&fSbXj3{H!m~FE#6aQOoDffb
zbOFLQ6V}=oN5+D2o(O#xSFa1n!DxbN-`1X<UFrRp?%TdCY8II47}G&U2bgK$z!yuW
z$kr|U#Bb0|w)aWl=rFpc@^kA>9g}{Y_pY<%LEm@HlfrAh>&ET6&6iBg&D>zjX2dT#
zO+4??Vq|Uhkj`_3p8Mn7SI_^Vi!N%u{`IdXBP`DKu6MlzUu9;4!yYXMzlv97_0E+2
z@P|J<cjqa>Lq`;RBKfY;D<l1_Z++|3uuo4E+#GVqA-ULJbrH8)2Je+GH{<;~-}z1+
zKP5##LiNKR{%~H?2Tqj>90~?qS4d}7`L|H|-{r(W&*(2Fc6-=+9R?^?IpBZ;QW41s
zC!CP=HxYCviyGkg)qu)&C7MY9vt{rcrWp+6dmS?Opb8UfbJdK=uG{9^`a`#mSm?33
zF6;#_ctI}u=nv99^q~*UMI`a1<5zvf;*vA>R@AGHL1E>=HkrE-g2J3ln>HokQLt}#
z!y6{XEB+pY{#{;`WTum~!DQL-3}P%9%MfFXP1Mi*+a=9gULOLsTlSCe2t$DfBa9bL
zet5wpj(!yM|JL<CYOa6a|2DVXbi-h2&kMM5og(;gKHjeDnq=yNp-Yi<c7)B_kO+v{
zL|@#ZKL(8GT*;7Q5R)*EY9NcN4j8)>-44&Su|4MofV%Q4Lb7kFzujc%^LRn-xGUE}
z%GSoNMFy~$EED!M91f#IsiVB421pNtfN+#w$~Imm+2elHJTW;$XE-qUxyuVlTF*Xz
z;A|LYzGxW0iuPNxHJJQ7udgjbawQr40GUjizG8tX;12xfn?oP3ivjp<8U0xPjex*w
z7+#!fqEC;<qRW~!*KB^ae!phon)^5FC))G+nD&{PcXnnzMzIBrRDESsR9)OJNJ=9$
zw1|LocPbJJlG5GX2n>x#mq>Svh%^Y&of1Qebi>d&Gy~i-&wJl>*Zs&BSckLE{_p)u
zLcQcJ;icx5j!dcd^A}Xt<8w8YP39{OVGkju=L45<?mtFUXNgTx7-5;?ONsIS%EI_H
z!(M!c$t4a!kKT{{M)?uwu*|REwLMnp$99>lB<5k8;og#a1a%4Y;A;aEOLrPBNli<f
zN(1yk{`PGhQ#nV;3L+^Vr07nXd4{W2LwS=D!x^}Ms`ifdn>&EZ{CIMZp6gpS<2FDm
zJThSY%=cnkmJrIPpIl9(R)?Vru|kE|rgX>r63<E)+<xdH4E&l<r{!e_5{Rb$p~(C3
z7H=mgl?<tbDc?B^Id%edey6szqv=a-15^x({T<>zfr;=LbH#iz(oGU5{C4KTRxbne
zn!7ns5Zcx&A&Dcq3qjO>WHIy0y#om?4(Aa#Q#Q)^h{iI)>F;{SjziQJqDd7?XjfN!
zXFt!EX*u1>dXVy(R)CjsB%+#}`Ocw|%8Kx<^#Iv=e~$qv<cLMkms^iZ7<}ehd}e-`
z)nMcxQ-+&><@V*NLGYl%--DknPdhhiTzLX3^5a5FXggl)nA?2XSiWAI!_Oxz(j%7P
zCH$4`t*!o`E$&5w|5RnUu*=f*pTWRF7~cGK7{87J*<W*`0}k``n#^Qd49stH3Y|)M
zT2Krk{ceLT|AHiT%F|c2GP;V=FAHM_>z+?=y?>>`m#wa{ea5MRu1A+pd)uY_O4dD(
z*iDr`v|f!<OIaC~acD}aLI37F^rtlQ3dyWP3JDiF;nU*jrv-5{o6e4%ALN5@)MD1z
zdXs~W^`B#M=;CV^`TO1E41io3<LT(p+Uw)Fxk0d=m2;Raj3{i(fW!Z54na+)W16sU
zGJE`f?_091cKK{R`+>4cFG8`G{w<+&R4Y{z4hnhYqGO8#i$`7!4vSwbmuYfb_`Clk
z`1CH>eZTrEA^wchBx-pf2P<uqNyxhK<w<0>^zJ)V`&`5(i%q(#RhbAz?T|y#jwYZp
zAR>6=8!Ava1Fxdd&x&lD2%tB1OLo(rMk)vl(0GiDSDlqLN1AwsZ}<z5N1j;RhLK5t
zDcaQSuF0IkSU1)p=ySjQOd*Ifh+#vdvvmj<b>JQR-<5@Su062yx=s#bjN;ON1KL1z
zQ8OkSNbm9hYMicco|ZUrOK?Eu#)j`CYZ>WZ<%AVRO{1GX_X^Au5l$?w^50^IBJYu;
z4q|727(Ie>sj#>6uJ)qrR!y)`JACbh4Bk~(ii<07JSHaiTguLd&Sq~=F@IRG{P1Nv
z$J@BFHR!T7k5;8w5HnIUK-O6Bskp_qZE%Wzp<C}25T<P-KrQS%QVU)m8u-r9=6P$@
zM)8{0uttYD%4<lOn3exCC6D``=^R1!dIhU4%nPNxI?i3)1C^r5R@3n$d)o7m-o9X-
zOnMa}pVur{JutC0iH=O~61h7$?esZR;ha@>ER#jkJ$>l(P%ROfGm*Ete9J~`oEhg*
z?p=a+0QiKiz3_|O75-NE%Id-y?OL9XotfJi3oJn|&AFY~JDxwx3O)W0li8~r|3b$^
zei~PH_4jnIMvP7W3Z9xP7}5VZCS2z1!0RTD#ror**<+()*x_V?5;kmuwGr~O>>nwe
zb(zeXY5mE~ttvfdQd1hK4_e6S%O$h8LyES54w-<(Xr3K79BMVw57*M?i2fWQ@MbTy
zbG2U(BC2L-rjNK0JlmYzvSNGFUrO&`2z7!fdl*5T$ZUjK#Al)z?;YiJc`w%gF6#4K
zPcO1n4oFC3%4bjbEcp*}#-GDn>U}@K*LA?!RLASQ>wH=SS%kJ@k3bod6EexAC`i~y
zK0a?kqIsm5XKQbMhw6u`@h+~{9HrgUz*MT`XQXy=Apnr7;_=aDQ;S|11J|NdR%>y>
zdv`CiLSQnbWgWiXzjd5rRL!m;{GkaqS7d8oDQhVKHB&jK&liZH&CI-#zMV)AD$e!Y
zt0sJ9_PbVyV#dByOP#$#`%~G6+8mp7oB8F&jb>&}VOh)PNtnyLkwsCw<vn;|dYLrZ
z9HdT;R*ygphw5ci5Mm%m*-AA19!qlPHQ1%1J4Q#oWoC1*s-N^|k9yIH<0a8B1YNNM
zvO9ADMsU!H2Cam#aD#oCPwdz7vrM*lO$=}u6&9H3g%FBo2O*SIOhyzd+<Z)~{K4s|
zEhlz#jB@&WI`q1++ZlGlddy|Ar;EA=`U!nU$&${GtfiUI1S9>(rxGMl{pD8LZd}hJ
z1A5&LT38Nr38L{(2EK>3`Gl8M$Q$&NMiBa_lWF}4{7CihaLD1SC4&{!<LHvbucXAW
z3Pe54+{-Q>XZ-KV=<e_JmSgd84mg<i^L!#lp6&eoESqKX*HCSAYd4bP*JiDZ;ns4L
z_r0u1WNE)@HRjr=<iyDY{g<z<L=k=k@{s;eXG+_UJoD~zC2JJnq1tpBa{WTNzOiFZ
zb=Kl%OmshtZM+U&VxCbGN`%KhorQYADdmJm-3%L<qdLZKNw^c~VbaQXS)*i^5KB(R
zQhNLZ+)Y8qp>X*?srB8D>qP5#oat=sdQ!m#w=qk$qo0zRi|QTk`KMa-%u;@=$KLAr
z%+-eJ-2Q4gm0uF<s2w(KuWDS6UUYOXS_V{96ON0Gyi95n7^a?UiZcQ9(1vb#r(K+F
z{;;Qc!=A$uI;~QU14xjb()D}j?ueLV%U#V*^_Iyyg}#q_7A3BBIPy0&`=P8Ix1hbY
zgZI)?caC;!&MwEkc(yv3N=7y7bXqz<WaQn-*uKJh|Apx7hr8oEynK&of#hcpudS%_
z+vL}L&7<F_>hg+5cKmLxE+@@p43@XJya)NOjw*k{AJT$u(nJTPfSxbaEdg?WkVTkH
zfjr%4M;qYHQSmvdF1k<Udk&@n?3$ux`zC_U$!+7Ux7YBlV4=6McZcaGGSyxkvIS%|
zYn<FxZ<L?VuNq1`?T3&}QM9}M91>`;gHw0YT264yzqP|mai+kWZ`OLMZ{1Q+aY8Oj
zOy$hvVupt@Y8(uX2>#dzTXs-p@Yh|dzmz>LfLMVRY-PvHN}Fh-mSQJ=)VWSJpMU+w
zK?-}W=K_rl*%^ttkl7a)@r^<;QKO#x)z6Qn_pV(t!1hk#cYtjrXrXvF00#}yiWYPW
z-od>E^k})yir!uw!WK;5P;-P|{zUltS(yXSr1+r*Jw{e{%(Ix1R~N*(#2eN;7ub&U
z3Y;j(M(E-`DXH|{T|XO*3=n*#K(NH(MD;_!l|}w-c-ip5<P^`|L7mjrMBx1FCl*j3
zj(V%#d4hXw4<pO*l1W)RdQ4xiS9ut9(ylDs*;g1+6b=s|^v@SP?Iod}*-3l?9#k^s
zONRZaH!%&rC9pt!s;R1j&LjgQI`5w?w%<ABUDg%j7wko}?E(hsd?ZLkhX%MbCGbcq
zqH<T|84Z1Ri@@c{5A*|AyW#HH)k0L{vVI5P^o3gczx=n1hr6<!u{BW*o|K0#VsBDf
z(#1~MD+#UOFsN+5NxBa?!5-tYhOF=`)CMWVz|k|GDmg4yw$?Mck~fdd)VykkwhRgj
zcEX2Sj#52FhuY4{&!q!;7@aN>d=8;(1HnQS{NRiGsaVzB!leMAqPXLr$2(0W?!oxJ
z^gZ$Ej-43SL%Kfy_%9-2GA>`jm;zi%6E1Qbd9RPA06O%4pC=_6Rp7M~Uq0^0T@m}w
zoiR2)Cn@qcjfb|jTi$VJrK$&Cj(7N;{xJ0f-;c&<uVd%i5=J{x+g1YaiZle@f&)ZS
zn$D3_1-w+Jt<=q?h!%)kTLz}|;9WI$`}Mlzy1!5CHAt4Qp@jpPEej(9%mGAuUXR%j
zH@R>`FXxk`h`FqkFx1fRSQZ0+SM_jFx=W$lEeF3~fEDpg^cB47LG7^Rm;JvqYa5<?
zC(8Sfg(pO^*(>1Nj_a5XTW9yAEP<dCte~$g$I^&3LVo`O>3-~hl77r%($~*cygzpO
zSjw$bb_8mll4q*Ycl_@wL{h@KO`uo$$xzaOU`)<T5=nCpO^{}tefznzA3EkO?Y#>B
z-u3k0r@(cf;CzpfSgVn?2GD#HY-m$mDLl;O-RsHD{R_vZ&g>x&=a_1q@2RBS(j6Va
z*{9kalYo!F5+1J;vOLBlg`axNc$psA^-AoQ*JdK6n&2zgr%I0;<vj=otN^#n`#2EK
zRy84T>QHm5cnK&_4nk_yL4j1)?12qNXm7ZiJMOYPi{Gmq5=0)?FGyb}UUu$?!`2M%
zkwo<0(JE<<{$Ob_JCb3)Wq9t4#B~>;bNhJ_H&bZP{k}<1<MZF?{<i{>nE$ch|GoCk
zp<WdIU{<Q{%Vl+a3cBaqwYtra$*aqu9^m?!Q1`AONqpNEaXLWg-Tvi<XLeMF;<Z!6
z!(LgH^YX)GgD?<B&yr<wKW*Zq0&oc*t1%Ix+u2iyHTnm$XD6gpO#}1sE>ZAQQk4FL
z@B`$54GMMks+E{IRQa%NzqD;Pfm$&7HzJEWH19}Zv0}h>(OFkG;CLB~w}a)SY{Lig
zaWXEvtrVz%>=h98xe>0d&w;HPdd@G$$p@Y|6Bb|T1>#*VH?Npp-R(Eqjd)wfGY0~d
z$^Uk+$f|H{eth%ZczU&b0VyBT#B{vzT(j=j&`C_4m-olR{#u0DM%y}4{HR~wGd$|^
zHcy`bLimH#kCJcqGaak}@8<lu4kc!)Q?&u6#9Nk1-T-iWuhlCLLiYw!8EM|pjE#&v
z;iVcE&xaLGj6;4SZKRM8+jk#w@|viM>CM=fA93F|&nOrA=o}yA_N6(b@CJGh*jSVq
zX%iRIfOi6xZWbZ*o9)r)@$I@Bv5`u`X*&4VarwIndJQC4#7kbzw%Xh~x%e4$17@#L
zKX#pGTV^xS*`7-Y=`GZ%dM#G;C4rC4Nucrem;7}R^{;*yQ~Pi)`mu1y;YwrXoh$y1
zd$FVHlvT-$W66@(fW<O%>5*~kT;<)izh#~qs9}B~NUUUXZ_B0IZ9P<V`@)B;`0Rp9
zRh3hQOJ@wtyUDZ+Ysx;?8J<B6cS1aP0~W#5GOU5&B{+G)fYjDeZ|B}F#Z6lQJhl*)
zGOc;OG}r0pw9G&81QG~2F2mQ`DFn*c0e<p$6*JKQPyZ8*7x8?bvXVxioX_Jx5JF*N
z4)o$)z*a}Bx4F@4_uxDTIGNca%OKVgQRZhmPUBnz1We8MkW3eIh{={3vyMgE7N?6H
zvOS`;5V|gV)q_q?oFHS<Y_=Sh*0fhzR*zd%y$mcIwbdQ$i2Ctxx&WE1lb*lccLdfZ
z*~n!oTf|Ccvarn|CB&rt8_smVQ2r9{?{$GPP}uu3P&1mQWyRylz*tf8ob{ynX4mi5
z;buZkqZzH6|CrNQkuG2nx!eE0?*`cZQ;;^L<2sBNRIg)K<qaRN2c$>gr`YB;hRB)m
zLs2q|H5?xv+tgD>Qo+sKfqfONWJ#VB1Q>*Eq$XR2Cu#T37V7M#Too#;&N5pA<1{)9
zmd{c%B4|zBaD^HWJXq-gLLFQiCe_yz4BQ`FQt8m*J=NBKIsx=akT8YkEB7zQ_O;!T
zrw88c@Eq{L1_1JORrB=i#1un)Yq!nw+f4-j+NAH%Pd|`H#Bu}Ib5-hQ@$;H6s*N{Y
z7JeKOqVIcs8q#fF^9{tjQj&hdK{_0O`2QcWbExaTT>#^Ax>-y7-e!`jiktt)0;jWE
z46Gq=zd<8p-UZR`@^UIx&6|sQDqMHD+D!J@0ixxtM4>*&V?vI?it7D62La+L=+V2s
z4eI2abn}_fiPFv`5S~>m|Co1wV3VQB8%GwwnOg^viy>U3a|<(j$Y8H(<vV<^+2u9m
z-7ZlLuwPD8*`1?3_8BllFhJGs3+B`M?~$Irp-W$6&`%gK3uVRYiC!Z)_Zp>y@iuhx
zE5BoHy}P8#n}v@1g}<k-ZzF($OXV%~`vj-c{M#bl&=QP@zRii^|8JE4HwP$@2f#!x
z8HD*6uVVxN+^Homb0H#tdGa^<{`m3X7^-A^_RaE^X+U?u&L+F&BEE(zyaL<NW}kSX
z=nMXaQv*RQ_b5I9pjnQ@oTvn!)&}0+_TL0N)YbB1)b4XVF$VQde10>7eWk0r13feC
zOo?<hc9j+jz&#htcI&5vBD)AACe4Jk>3{a6Ig7KY|7P%}7W%ykAkKbtn02*Y$5eqv
zkIflo0>;8nLuZR`ZXY_>8y$teB-2|3FI6|KUYH@*DLN+J*p&1V9q0dBw5EMX_SeQX
zTBrRiWzbN(HkuVadsuk?fDGa?R^6g1UsyWwDiI+hj<ZwvVlfnx&lRph9%*+n`+I>a
z*Yo9*Ym!sz=yU~7IF<s)JEtV2emz>}i@$Gwg2PQaV1c&LKap7VG}BpJHrB_}O%_FZ
zpF#<xM7%FefQy|VGA#=prRrY@?qfHeX#7_`UpU_Y;$T{{o)^lQtooWOepAaS15t`d
zxDsl~`Lf7jf`{p!FnxLZHe-+CfRC(dnZkS~{bNAcS)uBeHSQ`lLhlt$^3Q8hNA_Kv
z-!Aua2TeGk?E}dkAI_Tu7h=bFy<;o+GRClK{VqN3J>H2U_zG(@|J{~rD@~*kvrF#K
zp^3*KKj8f8$W`j49r?k?r)=v~XhFr|ORQrbUFPdbZjk437*I4Joiw5FR{W03nfLw^
zd2XHpjm>?An8h%C7ru~Y;l*Ni`iu5M1U#vpTKmWx!43u1T*p>o=oxo6UACsG<nJ4Z
z^eyDe$|DCMrlO}vrW;bgcGW!7c1Gawj3IZX6($&0Mtwo{BfZIfm`#1bOK>YnM?q}9
zq;(x?#fbCc`Hs#hU8@BygK;;6JP7uge}$)do<+mcX4j0NITs$*e7liY$gouKVND@x
zra@E_sBpghvg9A*h4-l50t6)9j>a)TG~~!qQPzw5Q`tyFFq9U}lC(a9=k<NWpVUoJ
z&ikV{Je~gPGDzxAZt+V&&aAdI^npU~of>(KzD_FvZOmYi>Xdo@*V%ZvBm>!roxit#
zITSPX%QKnY_zpONyd*G7<TCZ^1dg2Wd7j7Zg4X^#beWCdR-Mx|Qe-T#DQW*U?lvdv
zhcNW^Zky&(;Zw`clhB43<Z$_A8w+P;ushZZXg>VXr3S}haPqV>x0e}W(|H}G)33vl
z1rLd#HBP6TeONUgS`{@d)=BHYjUVj_tP8Y~6{OThf3DW2`^^)}MzMw+q8y|aZ%Aqq
zLtS9Jp5!g!+wL>|I;Wj7keOD|8H9vpAi_QnR@s#7SAuuV-$i>tA7E07@J27*GqE~9
zVx31b(m0pDImEW6Z-*l{0Rn<DO>Kp86@&VpfZyHr$akhiDDM$KiE+6apJ#}i=B(Ld
z7gwC`5=&1~JHA?mqb7ex3BrA{+!Pkdt+J(uMrNMp_jrG$TD{a26>&{xhyRfjGn)NF
z*E5)G?Z#q|WmLb_O^Me}FHe<6U~5y@^Y^)1Sjn9^o?B|zFd2FHMQSOGq6PRLJ6Z}p
zwg@ZTcQvlQ!h%4{T&ivO5PfAey?z1k3rQuN)7gs(+~Xf4{U19lDPp5%@Y9a#75O~|
zpZ%2xQnT`qdDq>j)C46t;F#E~^BlJx{C;J83IZKs$v&{Z&pj$C@3OU4dfF248_P^a
zVQwvYwsp@THVaQJNxNF#xSiJ8%50H5Bx?}0##p~<VkHw-{+@r@^azA+Fyc2NR!cAr
z%hGKDqE*Seu{sSLKcoZz(cfVz+=*ftge@M{&;)a@?YGB*EQJFxloeq8Ysd3nkUA3p
zsn%q>LPo5JtdslOr44-)FG-3EMNS1)TEy1~3Cnc3X~<qP%@)X7X#Hyg0>?!Qvz`yP
zC(Co6p<H-4;lGnbquBEnOzbYPSJF9?vgesy{_x_&M{D60y?#it>imoReOZgg9R=wj
z&*a-_u684nlP^<ft5TnS484#pd~im2-hNEe(RlVVLBT&Tmv0NRc)hl(>?(R;VK@Ty
z+M1*CzeDU^K)&5$vZl&l?OmFo`@34;uES}9>V;();C3k*;GJySu&uX}gVDJ1aJWLp
zH4D||S*aN4{h6S4U~86G0@&1pEbui~njtcumRPL(JU@TkWWcvnPo#S~`3_iAfM>sx
zqfzgl*`GakOWHy%;A&Z|#G__7$Te6#X6Z735|uJYR516<vkz9&D94(cY|gt0E|l)a
zs4%OrtMBD!4V?YV$8(sNC*;~&iy`cgd1$VBm3~p1&cJQOv>-zKiD4m{RgW44aAGi9
zxX|PG(LNY8%0N-L{0%Gu1=8kZAi7!5Dm*Hh4<FR|w&sPK20!rqOrI!RVtY`&1lk0%
zhSGL--2GBT>(44~Jb!q`oT6315LJN}Wzf?B`aJ*F@9r`mGlsm+19*Xiv$)oiS4dIg
zH<cQc&(0whzQc}QOYqz)nlnjw*P%cBXyXtBkP0hZ>H&(a0H9)eELMwkI$}C`V}Q|m
zQQ?w<CoLDSq+&>)MR@LDa;HzgvHqTsvpcUoz%Fuu-eZI}#VXu@ww5~b&kA54k}aT5
zPr>qTC||l_;}70CBJk{q^&R+TmOoG3*$w_3<&F2x<YCK%xi(^L;On${JQI8Gdlb@7
zhaX~44_$c2pXAAj4_F&I=-}Nd`+WOx3snU)0<gKUKriDO7UvI_-EnBkbIgd^F)8t?
zL1by1#_JvKpqt@Er)nj$ky9GiPSg_1zlq<Z@28}l3|Hm^Mdd}z$1e=!j@q{?3j6nB
z<^hbL6TwSlD4B>j6-irn;W*%JEn~ryQht1kaE$K@P6#|?hpJXCFFG!<z#><_4@8|+
zDU$F*|B2WO)igB$y)CvvOS?0fjb9t}_=%~lcOuibK`#(fu<xRnx0&KltMn6O2FT!-
zGj5Af0s73xkpY;lM<KY$R#*sB3!U!H7)mfOv}221(a0EzF!Bg$C_e-LtIbWx3J3ag
zHZ)MrHMS%vXwLhnSFZbE_s2m&Y<S+aI-D}If81%PAdf(^xEMIsK4PnL!f<QRpa;l&
z_`dZ?f?h3S?<>feJpjaCDAv19e6$`N3LdCttmeHtYB>rF)?Ur7Z9_Vzfj9*Qv%wzo
zF2LWly=dnVc-Kp!s@z{Sgc8m^doU@=XZk7DB#I(rt<9ly1V{v0uWnge@&x<=ylOsN
zG=ALD<2U=_$Z$vazi{V;1OKBI)X!iK->U`uFx@%5+f+`N?xyHr-P9S4={pQ)@g_6$
z8_F}2Ahz|M#$FKv+i&+V04mVm9UM4)XfT=O%H4K82}3rTWdd1LA{2DeCn1Ca$AMQZ
zS;$NK+I6C{AgGUq@>W~6=8o_3KTyO}9mF&Jqc@P}(4uQRff@~eriVd%r)8GXJLJsU
zJn`!DASYBSO&rqBwO$$H!|w7c-YQbJv9Ywt>tNp{iWVVTcrX(g5)k`6lMF|`Y2mTU
z&jBjump(AkCa93fzZChg@QBnP=rqQ}5eP+@5zKoyFNQ}S1U+mAsq8LtL8PdYs?g(2
z^bKw7;VSc6nX)k}KAmWE<Bj$@Qn`ivrx+$TLptux3KrVB<gS!Kn;RlsAy{;?i@+y1
z`eBfIaTSwQ?i4savw=YxWVQ(jKrdU<X-#b3?&}2PK~RC{GiAd*Y}r>?1h%a3Ww!mK
z8#fCHBQ$Bq83^|0o}MzS9|OhEn{R?s>Joww8*%YRXwUm!<e&obt`-xIw3bCCEc#su
zz#?i-AhweatIfN2G>$=5UvbN|1=LAdH=vlFB#5j>8d4_N;7d(X>!#n%!kGd$CGF)m
zV#BYF#UnzxMI}7^8q4BJ+jxxoTBqOEd59jj|J55<O1m%pLz(2%8q999uN;nzcFdo#
z$1f;P3R#Y0ce-HNNM#qis72<gLwo2htCJtxlK(vE%&^Iwlenx&aLNVdG)q#@?QU`L
zGIn7E()6F%(0)<D+K!#edmIh&)24ejTM@7y9*T<J7+`ul^*?}~BpzDdtP6ew3)tIF
z6HOkZ-jN_SVBY!9$sz0aM%Su>X)m6`-A%tCf6`!j?T9LW-%3s1hlE3mCh2@igB8Oj
zD-ZsfAClqLycTh?cUmQp>rSiujM-e4@bspxCnEZKw30|`utzS54$Z*M_+dieIV20I
zVu4~wUw0)pX5q6eNi;%Jgq)iN47Vhyylqydi-*Fz@+p)blt)8}k8)ZNY=OQU8-%x;
zAAqazSGtf;K7j7+$fJI$>N}bA?Xo6aBZiZ#xFL<=zm2wvu=lqo$O-5|0i0Ww<-}v7
zD@a=xbpJ3iUxa=`Ie~{hDS;r=rMk149=w@2m>z0JF*cXdV?HwytTA+jM<6w_07M*s
z$C|^?(HoZPO&%{yT!5&#ye&OvxwowbePx6??U+0#E%Ygkh~BbBE2RzM<*cnPnKi2O
z$`<kKw4mRbob#SaAI*$9Y&GI=f9<-c%by83J#kp$B-brQfA`jN*>9T^vyMqVNRwk{
zqQ^|5{Hbpp_s5pnvN1&m2?k@^)5H=U=lh64>6J^5TzGct&;$rgsAJ_OXXRbZ+XhLH
z4l+up$TIP1dShW&aFVK&=$iQZmrKe48piBXBT~7XuV;)iv~o*6iVF3z(3;(R;&#|t
z2vZKQ5)8OAO7>jhFQ~mTz{bQ}Sv|;Ge-nJ+X44fc6{9nzd0jd7F78&s0`Aw@TlI8Q
zTIj-pra_0U?s>Sh)t!xVc1A!MibsTY`OTbsbVf&W`r-PYE}s$ceSZ00QSVOP;=9M%
z4G7+h;F<Y89m!LM)-EDO9OSrXXq1L7kxEHf^)o%2oRk^cq<p5Ped`KY0@0Ut0h6t^
z=fm6`5=G#s8*yzVHW~jBhb{IFzR^kpHHsO3M=9jwdyhI}t=flyMfiqo%sLof%8ql^
z8;$`-1}vX30yHn9`>3||K<Hul!poO(2;h%#_jDF#4Wz6N?IR?%LSTN(1E-y;bHI`#
z{PnQ65&r5Sw@JC^uwT>UI&eIcF4^a|S*aA>>{277dK-s2ZlPoBzkAZELd08TgRP$Q
zyYZ;2opSBNSBEEVNQ6-(k(PEslQz}%9_hDTWM#y8`%+Pa_)+>gRQiumfZGtrljqLS
z2F{Afu>9sUb^orFB#bv0vGRBqr1>U4E7k8uf*G^4Ta0MB9nUjh`Si*B9{$OxLxN3g
z?*x%X+~hri$(a}Ic{<<EsF+|b_NjJuGJW|IqlftQntAj*GG_YiiA|sPM1mLmkWyR-
zqt>K1qW%I~xZn25S4Y74A>B{>d?}Ez+~Ht0tM}bKNcdM3o~e?{%){}+(Xsow@V{4R
zeYySYF-gwE$l$W@AsI$E*EQMhzl4QciJ7afiQ1MoX_ad8rTyWq_ATV1N@;yW8Vx3R
z*gYcwATukm(%A{x@D-I+TLXWS;fS)7sAwxzEw82$QB$u}XFd0o;b2U-<pZI7$NdO{
zv~f}fA?t)I^@V$}1T0y5nO}FC(hKe#*=R+r@MF8Y+=2mI&H3^oMe;+cj_NP9iQD<(
zJncBM&DWVBTJXG^S_l8Ti@*N%?Ho)q>27PJP79uQNWsVVLEdGrJKnqG1@l0s>p7I~
z+x<VXF+Mf>gl{NWDmyN<DrM8*ol6;auXj%<&@Y)zI=C5$A^MSS0*UuB5hUtYNG@~4
zpJ!VLXDdb+JU^A2>d6Sgqw2iz7kheeZP0k~QPPRLeUpjX&iLUh&BpMP_~onxe?=Z!
zgWQw4pFYW<8`9&<a9-1ofkq8brB4*$bR{D2khy_5Fo28(Vnv7*)?TDL(ASa);4tBl
zW1`U}D|W!^`vmr@rweW-mhEa}2848Z1S%KquR?SdUmUQvrL-Zft~bQg0yOk8u}Vsd
zwh13+|8?CjANB4~uU-(||Iy<MuqaMHX`?M+|M@UKDQx^D*xN#;-mTH$#f34@N?yf%
zGQ~aQ8I^{^Z~YbT7us`t#t#=PVsFzYyH!hAdg|woe273jb(Hgu|4V{&zV8awGkp<$
z#pxyz<iyi7ySg45cdJ>8eu!9jLn^iNR2)IflUjJP1Y(D1dmXSq-G)}S)**qj=jPK9
z%7NgYAGHF*OVg<%;CdrrJG+@Saum*3OQE7dIvO8`E5Ta6BYge0)nETGVc0zMDe}YY
z<^2g6C1sl~0i{k_gVWq9BStJ%wS}s1Jcbvd4-k9ta8QE;JBHKH^K$|eg0_kXwCW2<
zdUZx+2_*%{)~8P;a8crM#M^@}XMv4jkDh3(yFCnVGN8Nplb3ipR6d44HRyJV#72wx
z=A&foH%4hKp=EMB4wYm&bOdQ%gph_`tpo2k|1GUQASW@FtQ_9YyiFgflEOQl1anK%
zdqnkqL{xGbA1=RJxR2y`=>tOhnxJ-VNUuZTb!?mi;#sBsT1JTPFfmHoGqYSH#%nrR
z=91fj9cF(#;nLbqy+*fzuIcPW=@y0dECyng=S+0!V94h+`+;aIRQJ^jRJ^3B2b)&2
zFBQB$whSKs?+~U#V@E~n-&6zQ(#TG=3SOADd<Zek%<*sm+pzZOhM<T2Vs}V()zA6m
z+;uIlD1H|fVb24$G&^ek>u<9xh?VIrxkv}k4)eylLMCUJ2Em1c__v|XG?PtU-H)59
z4=<;tY4qJ%J@^Hy%v}%9NoodkJAm*ws2sWnpONSpgOx(z=%`dCeRk$dN`~zNuJOs|
zf>9LkM-h?rKUJ_B4Q!Qf>DTQ`KCu#^^=~~CUCbT<v`_Xur$smCOP*Es9&bQ-nk<8S
zAADPgEkSz5HVkwH?5(IDUyI%6Ia0U>ceqAjXblN~x(O60EskY9rf5XlCdO<Kao{!N
zOQB8JQOPQT=wpS{Qrm(amT|rMtkPP%9{vBc&DHX%u~%WrE9+cTI6neERd4?r6<*1C
z6>#+y{qV&{gBeRsuJz7|0zi0Z&{UWJPLns`sZU;mVbw2Tp0c9;er96v)pbbb@zUi{
zhywrcHtAU@;tM-&RKYi@XaAtZXZwf$i~r6BfOu}J@ZCL@HZ!ePI4z*Jv&*Wv!h^~!
z>zeRThpT^mwPThe#AXYEE6?2T*$>sZvf%rlFO8@Nh!3Rd6-e~w_2L=}ah|vfnmW9k
z)xLdtJ!U#$M*bUGVrw5;*))A5!8GI80`qFmdH49+<nA}=U{#YgRk$(ri_=NJ@vT@_
zk0OZ?NUV3ibGyd{0*N1PO`x2POvC0@?w3|RB>HH0_HGU<9;;(+6?9gfef`H6%i1!;
znu}m-6Om(_<=G^cZ(arH7C&daadGVnvb>iyzZO8x_e^ekH`Q+<g|pP9Zif^Z3|gzr
zqL-_8Rxf#)iF(!JbNkcrW0a0pj&~AP>!k2#)d&8uQqrphj6i7Xmf0(Owy-dD5uK$8
zeJzA5etn5QOh4utwV0mmLvzDW?+b1tLmC#2FUvEJ0)Ceq8{V0(N(27+@0WVVqkEzY
z-ayhA??9BMif9qgcf0)U{U_z-c!cRX>#cxjR&@(-1qEVT77+U#4-QBIl`V==;VTho
z{PWXCq{HWJG}!}{OJ~iv>2~L*pzmKh_dom(0qu)dE`c{*-65KOQ8o}tiRbGkL3+gG
zk(cc{o)9X@P_>S!H8GF&kjIrnW@9ca{d3itPn-FS9Mvj8x)A$2@%wEQVcAyc9t(Ag
z%K5*B+Ma^p^ixI(FWDDw{PA|^A5ZC7LiOv?zy|ulj%0_N_`;dFtha5ZPdDm!oI{q6
z1pAx2hlQ%LAjpJ30U$7zBHZJbD^Hi3WNuj4I&~?YR=Y3vL!l{rTj#+_2OK8O6-<!W
zfblf?HXVtjL0N#5?(bNe+)hp(NfEIkJqfPWIlBtAvlHGqRr~$B)L)AkOFFLnI?_g@
ziL@dpJ1>ltA_4Fx4<kdYLS^(L7FWkD(I<^S<t7Dj=KxEx+(aztl=>{yFrD0Pwho;A
zLrDs>>48$0gC@bH7JJwo%L~O}N#57rs1ZaYq%KH7uWFwd^BZ$-kjpkkP8$EaXT{>X
z-;><E3;HoPKA)myKg0MQ@Hkx+_(|nEC}4Yi2Zv!^HJjk|6m61nuNMMo)GIp{Tt**f
zvdyRM{8Qk8<(8UHDjC^^i-s0xs=K@CgqaI5>rrfj40Qf7ml<2QS;p#4o%L%Ldvyif
z0Nv%kb|{bUv<$aUBy({%@KK!|Qq>Dm<~14X9(g0b6DF%uXzC1EC8<%$QzcbR^oZd^
zsITc&p}Go|u#IVvzzd7<8ge|v&lf?k`G9)AcC()FbN-G@1K!5Px9fdXU<!$~NeX}G
zHM|9tHaSlmci#P_y7fqxkk=2Sx1XlSm2bSZ_qgzGfGkpuo{ro~lP<Dw0@pGSM(gOl
z#re(DUzfzs-EKEJl3nVlP0C?qB)*VXDBg9zbq~-om{6_pT-VLl4~(22jMo>uf3ehG
zUl@CRvR&H}-m9#GJn;7&-)c2inh1B`tWdV#`q`|W+u8I7qmsn8)TeLhID7S#2(e}b
z2^+V22ExP(Mum;B&d)?5;7%#FJ2etF6+Ayd_BcM-SByiK3PI-zgCkj%H#syj3wkI#
zH5#!Ir}B8ZPTot!M&9~CA4W%;wK_`dGbNDWNI$*=hoQ1xXm97t1R;fs5<vd$;9%R!
zC+GYLM*cQ?-y0e`@$O7fO}@~Ybec9;)qBBKj^z8?<(kP8KQIr{k*$6M(V+5idJ`9e
zW8WJ8u43DI>7dkab4MwiTGk$Q!fJU4Dl)IL%6>1Hgu6<-$}g8S?4o8f5|9#u!u5k;
z1?yokH)n+Nc{#HtfPC`}fWBGttl%3zsVT#+4fW-q5=?17$Ux~D99?I6`C7I$z`y${
zzCRLiVse8XN>TNmXt-l75>k-3@=tKz-7)*ej`NRoD6Ap)#dhJmqpS6W&iZR+GdHsb
z$mY!gm)hHGwuttBTxcUWZa+{w%CcNu)tB9jy=eh^{UwER8q?a9fz=n^YX*Mj8Rm4_
zHTYI9t`h^HaRmGA{41$5^-!|Nt-A8W>uwHQ2M;MOaQby^>3l21xO=MV^X8};LYBQy
z>_~H%uYA=s7I0>N>TEoOuQJ!&St?OVb1_=!I+LQ=eFa!(DsBJ)e0Ix2JO)Qfi9{OC
zaxP{u;55{X(H`LN6DEU<c}4M>j(gHZJE-wk2{L$Mndrq6L}QLok@nGE;)XVYU2A^u
zM?49k94g+=&150ZQp$5Cz)L@m8YfAqosTmUPPF$RERsa>!Ds%4tShXp&L{bcN%bZ=
z7mXS=$tXXcF1`)eOjLjYam63{$yBAw9=2P}ytifA=dNCiF797xbsst{+1WH7ToU=^
zfz6&{m+=kXz2hdm{n%9M&p%J;R8U<k`@~!vn=yAGOciUQ1JJ3#=H|sShM%#RM8Ba(
zYwMNU|ExLkdm?1Uyty7DM=kqzu!b7N+TkoM!2@xE9tke`u#>gEV~%R>hB~1y*6e$O
z;a;ctE)Ao9O#UCs`2Rt%0{d!OjNvvnQ0p<4giM8)cK%~Nn0a9G886HjPoT#{Y|`s|
zH`%?}&eNnn`(cWc(EG|r`S>i2qK7jtTBpu7WsX3cDHKVX{>MSM9JnQ;XddZ?T&Iw?
zvI%GHcIOU42@fqiUS}c$2_>F=i5EPGoTzfQw!R-zG0S(V$$2uta)gw9@R)W=0J^9D
z=()a<DD@z~MyKTFSZHd6moFGeHIDa1uv&$BRoS^sZyF?HsQbJ4WB}|xBChFlG?M9X
z)G97pbsuB8s!$$p+b-_klH`3sa{uSc5P4qEMa7$JPcm(=4$VU(+HPrrGrsq@=KqsS
zk$pzp?9`2g(%sGH(){$O9#!LAe|q}@gs7<?N1Mk8S$iQZ4-S(g=lthPo4|7L>fR?+
z6a=k)=Z~*K|5h>mmEZbv_GRD0Q4_c=bQq>F28eFl)gKRP$5e`1`Zi}4<GWad#-w)g
zL%&^?-T2phN^C#V`QcmYcu<J883J?7eaKotSktll0|wWmt?JjGw#8S#y0IBU(-#}5
zI<%iw<yHUS9`;-Y0;M1?<IFq?zCVk|IFAPs`a1rv7(Ncfb28UdG)<PIdKyh@m;g5;
zt<pQCrR7*F#rzNoLFI>b+MC?2Oh=-U<dFGaEs|`PT~7BSlp`Zib(T)c>M6#)EB5BP
zOC2jW9H*OBWTu%ERkVMsuNzv!TMnZzPWL|Wx-;8iU-ltl4`?umldRzOK+btT4ose%
zu2tsHgo!@OuoMZGgK&yLQ%Dd;V1KA=KWXu7Ai>UyDglz;QNJ{}B1^`e!NyDpqTT)a
z4hU25<%=Norwx8_qEm?IuF4yos3RaN1y<7p9|H=Zrr)i9$_VgeDDsAR`J&;Bj{f~&
zI(}%ew&M2M1DsMZqzC@9P$L!Xs~Qgp-FY$@ntW1Vf@|EzEmpYPvL?iL@E7ulGE4g`
zwSQx04T)$xoRj}iX!5Of-~r}B3+7k=m)}!NyucCU4aX)M57v8=GbRN@{%R`E;~cr%
z|8w~-2ihJixT@9stIKwY-OXIPd6@$pZGKYJM`tdB^uu?nz}u<3AMR3kE&D+St649L
zPEAjg{#Gy_9=)_nL_u?LuZa46C|+@G+=V)|`3Sgk*b@k|!LSV3BGGBA2og`y%46{K
zo3^F@0r^rwP^0(t56_LlP3H9DK}pg~-{~v61U*?HDP>RCA#_?eTZL+_FF@UmoIJz1
z9;y#@2d%ew#_V%4&Suxm1~h^znbHm_LSx_NA+AS2GMU+iu^*iO9)<iLYpdS|^-8+%
ziJGtXABG4^pCS`wx#UpZ83?yDfJzE}e`?$6Jxv=F+lFAzMI0=~99z@z2stmNj|9m_
z(|=-Z{+uUbsuj3(g~ZvQQve<eL;5z#&Up|*0<Xv))+st>0d)2FU0?U=6vaoL*)~A_
z1n7>=HX6swxVvTWgE7Y<BoEA3qd!<KG&$-NPvok68N4qNZk~wC1e`sBfM#Er?*Vg}
zO3~r@BFu1U_HxMl@<npxZK=KH_Jok*?APssG0CgN#lD&Tbe|+X%<Ku(2jI$NIZH@{
z9UrPT?+t{RGJ78tm4%_$wfYU^#CNHt`BX#lIY#lDS<+ySI92E5)XE3>7n-gVNo_6v
zqp)kCKuf<Koc))}gN3enASr6<Z!E#En;ZK}_H$C#s%pHdKvH@wX;=dI%@;CjMHWZu
z7>})Ftav#v#J}WIkecv6Wm2NY6!VcRBa72}$}t~#uJI0JvPSJmx}Qf`({i`uQ`KlM
z)@{!DDqM>KXOG64C9PI+idQ#GTJJwDF9!7zD{R(ixDCyId>#<ot`>nMv5MNnZgMU&
zx9&l)G;yKh<vE1Jz58Jyw|`WPx<2Ym#=g``iNOrqZQ&`EXls)|(!pC@4uqe$`(*Hq
zF-6f1knk&wNX2|}&qdvo%5d4=^1WyK-RBc;-)HY$ZzPQIYgQU1E~Dtj&KgILO8%Jd
z-7VkgSSPFc;|FGSsGD@=6B3o$VpV;o-k#Qh*lJo(RgWig+upVXV))l%;J#F4XVU}C
ze<<6YDAFXkQ<Cude1A$rwfq4Q+ird4-A7ORWKu<bp4B}+|J>_;uK@=tc(cg^m4w3E
zZmqi$T%*256l170?_4+jj?$NVH0FewWc0@ej;Qi(8+z~8TL+N-8t(O8>UmdUdA+U9
zAdnp9rQyjW)d8`D$-uYbC)36I-2UbAR-IW~Pfb|PEHB!7)AqFRdi_S0ZV3(0`5k;6
z978{~pc5JxV_7LpsuQ&#JZtacbe_l#yxoNk8(;piTs692SrlwB()`rB0}*{kf{YUz
zY3v*Cp%pfmYmPymq=>5l&aBD0eFs6nmt%WyvY_U=Ij$9$j7}eLKB3p}S>3h8YHw*6
zskAboYpt~Xc~MZY^QLmLo6H&iTDi@GWJzl8C&YdH?y{jH#|9z=&luZ`=L}!Duq8+1
zF|Cxy8dlCiz@ld(4!!Vp(3qVw;hZP79wgF1Hv51JvjxW1x^H8658q^wZUO@(3mLd_
zqiqZ{ijivb5!j~o->gC{7=o!8EHt<tqqCyc6&5uXBT#<2f@jyb8SmXMc&$!2ilb<r
zwNeDp{rcZ50RPk~=3%{1GCc8H`I*jh&phWH#8Qt`O>BTgx&mq0elAdVp6>S&Gqq-n
zlc}BO%0X8AUTWcA_}f7LPhBU^{2I2Lu<}N4YfMv@%pdAjq*~`_E}IoMsL68LH1ID$
zNypkDG^#YB*a|<R@RgwJOlb4Ic-ghu12p|#zj(;|GxT>O-m&B)j~VVB0X$8y?X9vK
zTTpDJje@-Y&9}?N#2WYm5S8zuU#D-162bKonVT=#zeOX`DJP67zW_%Pfv?PDl1=;t
z-e$OmE<j~{!sh-KI&{`DYWlGHDV%PTEGKMvSd`eDZ}+Rx&s&&wfp!Rze|I~`2v$!=
z^Yueb5fJ+OS315%ZsvS1?B#f)jUEC|^m+DlEI((CA9)sw7h$g;Y11OifI~lIWlcBX
zLX*_YQY&EVeR!M`ClnEnDSaF}vev{7A@UV`9sLo!OWUhf^e!NgfP3kYy)8iP^;<RK
zx9E&*ikdQ+&nJny#>ES8VPU~C&t(+n&ZJlP$C|{ij@k@<c^@ygTq&O&jPM5pjy3Un
zSdV5;oaohBmwd<a{5*>Fc9}<>VsjYHiGgdtxhtBz@~6##?#IvnVvk~(^`;eE&~XVO
z=mQ(+7)(!?v(IWO+NUL0%ZCULQ{2&MP!;k`h=aj$vqp4o-K{;Bu#4$(%s&L}*pB<7
zQn9s}f3B>#wXE%ED+>R{3H5n(ZQ4M44dU?pG8k)@*rHH2z+yJf3*+D;((2y^h~Iq2
zITePrS%%Z8d0>k`Og!>N*bSlBx)JuYtT{h;RqWcHZojsBT4+>0?w*>L6F>^-%r}cJ
z`T%$Hc6x4VvohBfLAkw16(jQ_j9x!jn`n2Nt!*%gg&n`MnlUv&3Jbytq17h{I6=UK
zJ0>D(uOgweL5Ndd=J-Z(!puyuPo?2KJB&XX7Tt&Kno_-a-;qK)A4Q(T2Q#i><MBLE
zC(d)++fMht_PsXV2-t|}O$y>G{nO<AVD8R)UeWw89x1XtA%|B&BNSvssn6lYpM08L
z0)y$&V$E7Rn`V9Z80MDSgbha_zprZEW3xI>RF<(pIn7`Zsg>@euN>(UhnQH)NVeR{
z2gg6N7y;!lJClH4;7PBa5{0ixfW%9)r|G2w{U;}WFlNbLNDD@ytgN&Pn>%~O(XFi5
zhQQ&GFj*`KYYu{AdGW?m8=?W&2))f=BQ8b7ZT5sk7+qgr^1y2R=jV8#B6c`S06Okc
zFMlTe-@}PKL5r8!^e#FkKgy-cZVM3XW)v2B*TKa-A$=6ePaW0Xl+GKNa<8@f#gX9&
zLx6I*pXGxG&QmH2adT>mY&M7Y(^?Tt3PY_B>S&8#V9QF}+c$e~L*7G3(RGZ_THgf7
z=Wjfh&#OJd0~QR5H4ethS<n1PRRaYjn;2eZoc^J72@jG{F0NS0Q%?K`XL}OVcHVH8
zj6fHJ&p_U>%l7f<_LAfJ$tc<(5j}N~qd38@5x(J_N~u^WfnHve(Gf|Y0I2DF(<}x}
zTV9{Qx!WM@!Mx(sZ3Fwix2GnFUA@A&5rs!0al`!UPqv*Z%k{PPhV0VDk>u~`r0Y^E
zll@8@^bHLR9h_46z-N)We-CyJgJPa{N_4@9(D^%h%bQ@CrG)lCRmV7(%|aTEBOHp0
zx-vm;xH4tH--+$mxE8(^PVJ0cR@q$o?@RntI-kWJwxYPEMPqdoE~AZj=e=Zh&9@_(
zoWRm(#2PjHa6)mIKx*jvKrAzvAkyM^oT%(La-l#sQtqXJKs6#ZH$U&NVa1pXfo!&3
zOEqww*ayr)Co)ET3Uh>RoKkW1-ov=_vb^6^zq7r0W*iADP?0B+3NoN&%zol`sI@Ci
z#nv-fv;ny9=K>_khC6lmmVef~9&nUk%D#NwlOIFPfvMXl#K(Nmkwa{n8(ecrM_n?J
zX6(y4wgRIPdhGS$_aSnX4?6Kza(PFA4I;5npS=3(v^M)nIFtc0WsyclY`f1h<<rPX
zII7{&eghkdTr-(9J5w=?-Rf)UtM$i@Iep2+SO8L*0h`XO76yTcoRt4cyku_<ycolL
zr*f7u{8NVEka4&$lqYwKy~^;S@}m^b!K<FTZRy975pkVOR~0&1{8vl0$EpFYvJ|hj
z?to(;sWb4GPJkl{uWIk;&fvO!b039?1ai!xet6I^23uVN2vyNKqI^*kOG&}sK%g^D
zU2NEv>-PM}1#!UiRAx}rntnH!)lf7R_9DIS8~?hJ9nYsUNN4j1BYm%8)e`E&)Und?
z*(Sw=5e&T6>>8!|@(I4?kLPJNq;{iBduTZWBelQWDYa`M#VZBEPOnDz`7CeoaYKER
zoH+X{Kv%2>l@H3#?NR42!PyfE33UqJgvYM6Q>x#vXh?Fz{;SJxp8FLRPKMt1_Mzd`
zlik*D*t^+Ni(yYWKNNIy%y2y!-I+ORfM;GkL+wqOt%yz1wD@kpJhW(e<K{FdIAchX
z3moQooaO~S^i=L+Qacg>)KfyIWD&XaA?b&MK`}Lpf1F%%Bm-XiX4O8*=kx1{1Tcc2
z;h4IUu}kOB7Y}W`VpU`+E3*t+%`k!57a085Nk+vSaEU#KpRks*B`L&7&k4YuYW&^{
zI_biCkL4fq_R?(gYQd$$b=E9mYp17Ou<qn`zbXQY-e0Y%lRs$E+(&hH9PuZujB4WF
z*nPcCwp+MVZ;{IErrC|2WPv(EyU5z)Ghd+M$ZXF>9Bo%v*n6=!Dz^OwYNq$bMD*E4
z#5YPqkKEd=buSzA7atnzivNlVtQ2qfZQgBn^bJawlpUMI(s;g3TI747#_nb_O3*x5
zr82hs=lc)k40gkGqs!g(1`fP()_h+*z4;-{Xs^?T;xs)9`R(^@UK2OnQa)yhbK<E~
zOo8By<(oyebPVmh3OGqc>hI_4l!-|l^HTE=f+)(KvMQ!&Ek^2%3hkn0+<VUUh2L?(
z{I|dF?^`b=-=Z-0rz3#XC6|)yrf}06MUpa65W4Yt;~ucJRVh>I#IGvoryl{LA62=6
zIfgAUf%KQ%KBk$GS$wLCIx1oo<=ERxAOXaL|NFMtIVO63uYT!xN0<SwS8t6=?Y6$H
z&Dj1Q@9p^8mZN5Nq<5;>=Znw1?|wz`{6&*+np_dmu`)E?^EH7=@cnU11VL<gEV*9w
zXD<z4FC<8E7O-_AH*l;0bXZtljJ)RLQjQFR?k}3YlMhUi$bc6+7GK9T?QDJ8p{ejM
z`kJ%MYP78H>ns=^Jo#F~#s=dV&!T8B_Ib`T>MwdO_DK$KeYvZI+5Q2brc#ktOw4vB
zT8ZXu@@av6T=~$}>5-n1-5Z4pJG5bMi7(Gj@<?te8mJXIiC={S|E@kYw~p6Q!MyKP
zrEb3SX#<re-X3@)W*jK4)3mW7`V&pC*s^gR9&-+yujFw>gp%i9b1Uj4qLP}C<!!F}
z6%FAwc>&tPv+l#37q`zqzN6NRyq!l}BecroGnEB5q?oQpP1~b~bwbgpbNT|&a-7<&
zd3z%_w^`23?>g==C=7`RfMz|KHUxmc;Q^63(R*Hi>5=_>@d4luiO6+LQH#)ah%uMb
z-9T>q*Tl|vEfdd=ztlv&6~V={=#N$B4XpfYR?|6pn?&Kpo2o){YwQqo{W~}W`GBtI
zn2b++-Ew(ao$xEp{>Y;n{r&_#=@!V8u2CwMRenLF>l7ZX;no=7pqvbk7KC<t;J@&S
zwSiXFlY;|jLhRq<TlDLG_<hza^2AEM69h^R_Ff4PR?d@>bgYF6>Ei_Q$O3hDK+yEH
zq02yqb-F)@V<9YPJ&L6pNO}d)fj5cUy78>><%iNa%FfEF+Nt7!nnhPYRl8rwabw~>
z$g~ECNOdOc(whL~haM%Tm%-eP4MWKOxG5q*>eiNQ`-h5H{f({l_`mN^?=Q=_K@VWb
zn2y8x1z5U=twli?IS$Vwuw#WRe|buhOdK(%u#M)H&kxa*3Fh7IO2Cp8g$ec3O7Bc{
zi(sDYKoUwfJ1bj`N=Jff{5jgJ4U2+wv;Jfn7YUx*iy{w8YEtO)VU*5JnUvPTs)V%>
zc=(4Ln?gAI+_ET;;G<{*x4BjZ3hM_g8zs3S{8N*1nsUkVz@el4=y1>L!cq}o&Y?`*
zPn>5sqEDv301IpST(FhvK3ry~(e;Q9b_+-%GgB2nv9o}rya<m>x$$tb=VjMHk8MTU
z<@`{2tx|%J2gqac3=}H245;RgFR)Q&athLqfwUW~nnzsjRyDZDBLbeXwt+ZDz{jfI
zKZ*o>$mQ{WWxX+<l*z;Qf?wt?c;RB^^Rfseq2J(K64VAV`qdH$Y>8zrpT0lW1-+r%
zEGf>Vg5`KG)5Il?rh6~S`yGteKDRnobi!8MSXIac66xqYZGs?iLO8Z-1G<vc1)nN;
zOEDl9Uh8FHs#~8t(P=`YUdZUoyjs7D>uwtesz=MUatttIo=W&CPEj)G_15;YU!^?b
zr&K2U47umws%T=B7_rbnwoxCO3yO%=W;fs$&^=#`4J4E~d-XdZm{)=a)6lM8eEq_j
zs5oLSVi16!J_#&#S05h0*~-cb=db3#!e|>-;*~C~r~iklw+zZE-1>(p>F!SHkdB*F
z8UzHSC2v}~K~h?}yQD)prMqt$>F(~1|K>dByfg0y#}CdRv#-6cwbrke)ZOFiYq?<6
zk{QI=JbetXB!;PdeFbWX8tb79YO;zbK90eO;hw>jfH+#N5}ET-&Cim#pm|qSQs`V^
z9d2rT8}o^-Gk@jXl^^kYRcTQn!g7xgq07%d9h)!Y2nGdf2b~4(E^$>sp`uPhpKwu>
z;$IiOS8Sh5<+I7N)&V|}k3yl@CbL~xAz;?&>BGIZCQXb26Trz7a+)Na8ftk_x<AC~
zS<Q1j5pSk(+*>fth=6L$Kf)IM<q_A_(%E>T?u!}qArh={Ba4~Z`MU^wpV)8!M$=(u
z=0({t15QtP1@<)(^DbI(nM4{?H;G^r_+3r|`J(lX$hmM}ljbt~{^&PRdN)wp$ZqD%
zvMMizm=9r~1RFzSYvO+0IK}_z-0HS6WN_?!M}QjQv<jeh9xoJqsISv@dZG`Z8|?zd
zWrMNcSJblOV^XxeXepWDgDSyc6xxrFqa-P`g-@ti@2Fj7a*((@zjO77XTe@wuL9ow
zl8#f+*Lx+!fGoFhQWSa9Y$Tfic2n|wD2XeMyN_mbEKtb`gnTk{FnBKvDH67U#Cz0#
zOKxlf&|vZk?%VN7ozle?77gkwny=C3-)pkRWEqPRb_?84jH`zY_*}^=Z<l$gTg&u5
zsF}BANV4W<IpC`YyPIr=Xj*<7D)lk<eJQJ|qvCEDHkCwazWN<q5%D6-7)8-0!Z{mu
zJATB&R9&-bi|x~q!G1Rh--F@CzCd35J&sf`&R6q$OxZNzd<45&!SQUsIR^NByDe({
z=+sdI1-Q3;0b^^g0erqZS!f^DG#J+?bIfsLG*Y4Y)OM%U4CITW;MSKGI-knxB2TFx
zloGgr+Ve%|h~z<w{D&o(x`B9!$EUh9Uj!F<bsM_H$^6)`D$zqA;BGWNRC$)FfXu)h
zs}@E&ab^KB%;r<}E5b>v$j+$cn=#SXZ#)G!@VNn_;)P+xP~oJ4_W=|0G4x?5?Dp|v
zT2@vYT>NWv!qP|v@T(q&tgKO@*48up?7Jnqk>u1JU6aVnWtf>j_t;^Cx!M-*XZi)s
z{c@Bq7|6~ph8bA2X;!cZ;O^TJ=Nre5g&HS)ptYl^`Y9t4_Kv|t;*-JM*?N=fWLzW&
zD{?e$hgm|a#N$N&8qqCF`8vydTz%Z4LR6IX*PS+rj=21$)%vzzALjBM1TWnS_ONT4
zM^uQQ%98ZY@o&v95d7NO(Zm^~gAY+hGPQ&Jc<=5F)yy#?V)#jA_I@jeNAOk{m6?3*
zReYw!@iJrSVMnOpNO0Z2nVr#85SlbFxCm7edA2Caj2}+c)g*tJZW?bFJ(O69|E{U2
zwZHRoRIrF(HFxs51~ij<EckGJH*;7y>LfN>RXGKTe0-B$R^G1sD6N0wnbj0`ojz^a
zCCAGOwpei6j7Tt~X5B4C0&9?4?;()8WxH5zrMTEN?)QWj7j|!xnTAD1+9k7Dq58wR
zpLJNpMAE>STB;qk&8!~tLv=YhzTY}&K1Fw3aW}K6D#R5TN8@$}jx=C1N>n7f8yZ||
zWMhs}1B}|mv9ghgM~$12f5pU>u9igXMIVp2mYaO;)`Po?9bU5DF+8PYS3*e=^Zj>L
zoe72_xvmKwLFY;{3p7-4@Ur!nahWhf8LLo5wr|Lti`2D?nfopl)(Ve^L-+V-KdY>J
zY}D@UlFL40XlhPQ7i)H0GtNL6HLcNCpy_zPavbZ^)i*>96-HWHAYL6oXlg|Y&NKnj
zZIW&>dEa1~j#TG!1WoUl^Y&@41lH$wXVt1p8Md^wsYcE)vUV^0%$k|mxUL&n$t&~@
z3QWKtC)}Y5>^sojJ3ZZ&&gOd9{?P5{1P4zFefZwcd@lgd2vZ6R`mDI`GSa-u?O_uo
zo1)M1#0DIlRx@Q$)j}KHdrB#slABL}j}m<qKq?%qsh2K-bIg+BYi4H8v84)w8bt(O
zjv5R4CcD>Y928P);0a(n9V1OEFGU=-Wb)BS;KJ%~J^XCcb)D5(p<${#B<{4{bHXkg
zWbXspu^QGdhCgV~MzhdW`UVCdhe>b2vZ|~|Q+VMTTujhZ&6}nLCa7yO^6`_P#hLLa
zA{Hn8Uk7wR0*&t)2#s%lODD0Sl~yMWvXNN6|3?|_O#==nUmhauO;<zE6DOobZMO0c
z;4OA6<u1~ld*PkmwvCStazPqgpp2_>%g^pP)`VhMrtISm=jh`#EdXcqR<UnzscRGW
z{Zj$+NS~Lt-e@OWJPj&&|0dzH5ZO{w6knNkoy?P6|9e=j$~g7*NSde-<rfoCihR56
z@)#U-ywKus{~>=J3>yH3)>YXYHq7i?ztDQFuSb<{JH7vg4c2*l6Sx6<n@sq#0t%Tz
z0%zj_ogI$e_W-Y;HsM39Ep#}2Mk4G{dvZtPZA(_(D-KODBD8ysRAUE^jT(l;Ah$Ur
zjJjW~tiAfIbWGjRAQ5XkZ;~`~x)L{IHHjJJB$JgrjTQ40?}A9b0>tE5>*6*(-z=)R
zQ^--okgkDbCKtP)?Nd7D-uIc0TVc+gEd(xnKY6Qr9h8z{bBpy<B_S2ExaU<>e?GIT
zz9V$k3&^#NuX$8p-SCTVPT!~h{P$(xQiF(7g|$^y>;w(<xZ4CvJhG{Q7b*7x3jVY}
zLx|xuzS2CXs^K^9CMlSBa|Q4a;{aat_rlnr?L<4yBfZhJi$EB+lNB!#f1lW-(Ij``
ztQ9X1@V`Tx(NG0#d?_!#s-``hTgdA^FNJ6QNekF#`)u2apDKi?G%UX?%$ob*oMCJS
zYsVXC!(kdLJH?h?&!c>{?CKi57^kLgP@&DPegwt!6T=~nhwAM1&r`XmPdGq$T+5=X
zr|tnRfVq2g-W|69pkx)t)Nz6zA^T5%a1gLB<X5o{^_?apVa;B#UJFsf8m=bgOVjQO
z2z()t&<KwS8hR6ffy=I|!kjwTwjlu=GR?)haCh~Ddl(PvKfoTG<65Nn1uSUw3ZF*C
z=vWa*?R@3X?XEW?_*RMpZ&7Q19M>hu#ftr)lR`MY5If9k5UL`n2F11i-%CyE$0R0}
z?B;TDt>|8?n3n#}mV&zLw2QC-5j=wKqZ$_eflHkK))z&5YKzKnx=guCX1FD=vDND5
zXcCAv?mVZ#-Fb*+*Kg~iPBM>^DPhxONae29k3g#2pn!^0(*rEHm{o!dPfnyy+WN@b
zh(2W+`^1=#xj{Ds|9hbK3{KMb){?K~nAy>7Sn>F7pz)OM2vKI_?-M1T#71bwMaNIN
zC$+igTVqR*q8aI=C~erzNR)I5-wZz{6)<6QKPWCdz#Dh`GEGXGBQlNFQWc}OEoXuI
znZU^0G139dEsa;<F9CoGoziSpM#3+Rd**wdd*N<naR?+&Z{BhS@iE<<APinI{tWv4
zQA`SOMp7?5+s5nBgX1!h3X~~B+?{$khYmhu-lBX5oZ~C`ZfM*Sfi`Z;fS-ZHG^-|V
zw4t^iULrS3F@_6cBj3U<Ea?~6sGlo&eo+32mduQcK-<mlKY-1*$KdZ>6=RRi{5y@{
z>!8p$MUb)!J3exa&fLyEyRs6T8EDg1d_S$)iK)LKRTamFo>rAcp2?}Bc({$&a{l()
zgLasutiINJ4|U?Ihfngt{q>c&HWXpM+n+={GNbyBpkX4-;8k3&(9~Q;s62eF1{pYG
zWs<KTlmEoWjy81kM+tK}f0x?4q@V=lOi2Y`aY@ie<1Ux)E};03(S}VxjaF#l4$>uR
zu~asAPZ#re@)&s#@~UcL!;x+ifsBhZm9aQE%u}5&_#HnSKwe>MEF4*POn4JBfej(+
zKrmfqTMgFbw5n3zFk3T=5P-{g<g<co_#r`6?s~GQ?;$|{`zH*sM|L+P_2W(dkmB>m
zvKpcNq}|Y3r)}w<dKKAs2j<~*ql~Je%!Syz+dush#?Hl@P^ozQV7}*Rp~+7vsFizY
zwQUdCR&H;GzZg9JUdaSxYh=)5SQ#cDLQS^?EiC$Dp+Q1X)N@vfC%0Wg1fPawmnr^0
zA3`_wM99#4VY6Z_V=>vaM*GcL(%}YniWM;HufVOx{hoRmV=}I=yWK*zO&!t{+HN^~
z?Y@UwD(1GD35C=Muh;2{aRm^(jKm5*M&s`74Jyqp@gQ9=#4+9v%R@bvuwh_cO5$mu
zEAz(@-t0HWQTFxfur7t_=BO)VjT;4gd(q-pA<wWo;WD;~$THWHxZb57PkcMXg}K^;
zzZnOGIq$1#1qx9VWL+ey2fo9uyOT>YtNUkiGo1kO9ouUAf7%z)uKk1&hVVUqd1iCt
z<oCC6h#O<2WJM~CHlM)InfMVZ@$ty1pu!i{{Y|%~QsR)MC~SYyo01tZ?Rc4_x|yk*
zTCwIe$>wMDzdS6{oA_7Lyvnfgg!a2mFKl<0j_a{twr_li^F8OCCxx|s1Kx;uj<GdQ
z+RAo9lKo1UmE{1^vx{;UUm6}#f$T-JApI@UOl+Pm3!Axo2l13YQ{E**G6ZNDduuO}
z(8#Y?0oc|Xn!_<~P5qmF@3+Q?#W@R+Cd3Vck<Yua0qbVx5%;X0yPLOdZ6>v}FKayA
z=tshvQk0C2GX5IKm_-Fzh+QD~l!|lkdG7~F6njoygj}xtI1Sv46jV%rbL55*hTo%?
zn<_lXlM^VMVcuPj6p-p5dKL(R;LKvfI5ExIYwjkUOzZI1H(5|*ke*vR6N+}7^YOV5
z2cH@Bfjl=R1!^Wg@AeD+o|l20qlVfCBj`C4v8qcncz+IX7Rl&k@jT^)jT-+Bch^Rk
zTY5#}nvgPq<Us<YLu0qv*dG3=SC-N@v;KUAtvzLA{7y`>ZM?^;t(vI%iBBEzEo`QH
zXOj;Dhb@`%{u{nUQfRDV;;$*oG6F;0Kv%UN7>;jzQ3uwZK3txUshi=qhBnY){pK8b
z3P5|Tw&U&VoPh8L;XhMf8yr{NBaa%-N{0O<Nw5kxK{Z*b6G&@P^(p7kvu(YOpO99<
zx#aGF7(QqjNZWITK{GUMo72nLH|4<l;I9H{SID~IqCmKeLjyy;fgQ^g;QL8CK=ei0
zMP&H5dr`?@>`WCA+8nnNEsj3%JryW*-mQsA4*zsrb#>#{2x0id>~Aufpo7S+-r^#(
z(bHhr|B+Jlj@wLn0|6bAI<EYSV=-;{iQ01AlYz(^4&ezm{+n;1w&^F(qh(>DJ<>_z
zV?HmJXFF-OmfAM4zu9cy(SZVqfd1n{(@(^_&%9(r8O(hkZ^x<ioZ3_1gwgnxo#I^7
z^Gw*qh*p8YHMjZH$=r0n-)WAwLsQcdC$h7^6QcZ<yikFT4JbxMx%x!d*rJt=JOzhj
zFCAQuDGgTBh&Ss$Cb;H`FBMafW7Gx@PH4o9M=Ne^oS^0K-p9aQ1d$`3<2(>k?-I|S
z5ec8}RgS9b1kY&b>e4g&V9|R0sUG0|H_$|@9~s9V;aBy)3b`QdTQ&80oem9b13Gb_
zffOm{a)YRMMp~0$MS#%3^-1{;GmF6Ds@EsGRj=E?V%0(>ny_iQaQ=~7)`lf)5dhvh
z1JHZ!e#|&g=f<_4C|!W1wiH!hgl|*8=y0j5$MXW{%+LX3rm`;|hW&wsp<}5AM^Sre
zsz71N!SMGR@1fQeJnKd1auDMFeEl!s-VE%^n{AL9e8G@ZD|lQRN3J2Jd<wL+6?}>o
z8%G$|J5NIEZ`h9-D8YqLkFneW<Ouvgd{Rx#IRJQb?(FDVCMN_C0}8^W)@fuKvHswy
zNb`-J(Dll0^eM$e#b`UwCsebeina&K>s<d$Kdjv)u8d0n9Q*nxU{MBJ+u5`6UFw|=
zGEoZ<=tSKDw0a<!Xh^ca!9d)w{*sL)33S_sID}mB+%%OxKk-9QtU1p<Vcao#g}UXU
zw#MS(Qv~V9KZTY;m$xb8DGlJJ8YW2(p&eJ(eK<w=P-`&fn$_Cv{dhuin#xTO_1;~o
z3ruAHk>$gP{(rt6I{^D(lI-Y{i(4>W>WRS3AFZ%j=@&YBzFqy7$O51_#K8m`IJe<k
zkS{?V0LY<n{B+kj2QCacPjxsBp=+L7I+75w={0N%MK5?m9fvv`Y1yQC=e5kL{4S1;
zK+c=I4EXq8Du=5C4<IvR1&+SFF_ErA0in`Z6k(W01+VR|x367*@BwQV6+4OKM>J_Z
z8*oOxmdBzSAv9l0Iu630fqGXAf!QJ3`q*mK_EUH61lRu!@vLO{nRg&bpc@~jDYMzw
zeDTKdx{tK>K!550F@v;0S`g-%)L%JZ+Vn__7zU8Bx=<5(L}Vo7bVfrT4fvk2>KrBg
z!qDLI8X!?>GXOP)P@x#47~cyAa)gH~mod!8=L<uKS0W|+<nq9yv|9sO*ZfZIsZnlW
zJYI{0=-m{Uj*YhQh!f@a1Zd2@c!W45+V8X9Tu|6cf4yKl{qXzoAY<{MkW`D`a*NF<
z5tjO+Y`s7#tFkA4&DWww!T%$I`C)+h^OB)Yd&gObuumDBN2Gxi9P<VbIyYqAF9BbC
z<|1_PM6aD1=f*kvXn1>Gjig)GAcbgn#C*E4^j3z$mzREs$WV%GFig9ehAt1O_uV?^
z6ihy(40VwcIoI>J=8w_!=E$wQgb03L+4uCjF%0=`V-xy`6FSYq?lEqkB;*nv*r6i=
zpjTe}fLH5^RM^@Ks}uEjO%v)e0vP0J%IX&^Q-gq|v%RdfF`8oc<b0%d89B?GGrE7f
zJfB*h-X6U7XW@P(fNP!ru`W4z&SKg3?IR1=N+N_(#{k+VRU+?=&S;v*9sw3Bbyq$B
zN>h+0HwEAl3zNy?)?@{DbvAdf0n8tAr@a>N#lP^G;n!(G7|-wOjZLO~ZLJgGUYA{T
zL|&LS+(s5Ot`tO%@bLtUrDN}Ds?X&juvv`r9tho%3_;A)lX_`KPkaujzU(MNMoyY=
z@u){3J8b^}-QUCoa+osxN;|4T;aMTQTuC&f1;i7#Z`h3FS;L$W%8aR!W~wQGvrZ~d
zJ#hqdVlGNyoU$*f)A2gW=E2R`ivsFSXS2nfw4P<-I0eAo1U&Hkzpe&Fe<J6$I)6f8
z=kcDzWwH+k{~d?tN`&m7?AY$R)2{;cd|%7!J-SW%vy1Lhb|&EtBPre*K12CdetBX=
zoe~ZH6BVjS5YWCiv7u+TLy1oQC31VaPws5jc#0@toP?YvojwsB_MGKT<*EWXCuG;8
zUrQL?#n2P_ly_`G)OzkBCK@8AtuEg9-8!y9*gDR$e$C)fOHQ+*B9zh+{4sU&P-X;#
zEn73w(i)7a<*zg_o*Jk|9=Xy{vy#-f7Y5<z*R=PrtRLyeVVr)pAui7lXq%6Z!sb9x
zp>bme!(&N*B{7jYAbT~Iz?M&n4$q~xZB5b~Rzg1B8rhy@%u^zNx~X2(&$`%`HDc}e
zVHNf(>9^+u9oRpSON~6^Kg94(mN{-;S9+6MW73FeXfR8KG-3_17TcZ_^84zhzeMIy
z!)gX9>PLb&PKc+51u-^@7*lE_81lq_I*3lR*luzfhQ|@&L)l$M2S;XcvVmFjmv|vK
zkyrH6FwuyToYP&!80#SufWP$#6X;Qg7cWRfy<2u<J;}Lvm1=LfM=t2Y-2zf5?o&Ai
z)yfGZw4JvTG-fFy-!D1;(=CZZeix{&o;rI`dlQSfkC2D2DY4D~142Ze!o(1r)X~&-
z?OMLV0Ql5fjB>S%(&vGazp*Ce>2)cQ?{-2Fc2-bNfn76Y52wI&KU9Qcn(=<$V7h+U
zMF(%w=(<`n`1-oSM_2F+;KOA#oRkZyyO5c;Kufibd2y0jNXQ0XJU?mSN2Y;jy=v5E
zj)}#6ZIO;yfb|(3edrHH(ZO`bKM1~KUm>AU!XsLLa;(Psz=i2fsk>BTIsg&+Ky5mQ
zt~=}wLW<A%`ee-6#q-&NfMfA7L*t)d0R}cJhXR^{A)&v+h6Wow&*r`o4#9wt!1F}V
zOWBV<+oKI-WZ{o#7na9|?p`!7b>altQ%e{&21G}smi!LR#BQn8O(_u-;dLR$j)7Gf
zuF@nzlW8n%-EK8i;vm%XW2(q|f(874F0+5W5$+IinQrZPZzT}-U2}l@_FAAupA###
z`vo<iwV$V&3{#_;koqx26~vpiOrm;s@uA<tCl^-F*G|eYTYzj3)@1g9uI10~+q@bf
znPI_xrA{{1xQGw&*bc&1>N?i>7SV1pXko-#Ce~CH4k@2JhZ9Z>2I*hrQ4`hO1xfv2
z;L22U;?_CnRzYMd6K-<50YP3*9J3bM2=8j7(lJDzz6c6GxwE|1&1y3P`*(aM)D&ue
z|E3JQlzSYs<RBB=o)h4JsK)#|Tkqx>xyb$o+2<mRuV8+!!3XOH#SND`X5jG#<ua-k
z1OTN&SX#9A08U|E%z_)yOGdq`K6m&d{OLSv&i18}_@hSF5F)B7Mr#PG;L|yX_OB})
zVaV-A*$g7Da~7Kbw&2gNLfo>#OaIjY@xufnvX^|_MC{ZY{KiZsIPa`ylAzWFNHmIH
z!mE;<R`SVHvPKX2b@YdNI0Sy3-~@9E@mBPhVs-R6Y;5-XQ(S{Z%q`CQ*K9Yg)RoW}
z&V%prXxX;4n%(&twiWQ>BO;-AQ#0K*LMwq$a_=X?LveCx_H>l4JC{1bf$e7zo2O2q
zz`zl3F&8$@i0*meXMAi;QUk>>8|DoTC8>qZ1JmusWl`XMUlnciC|3)DB2!#o+?1;Q
zRVp)*og9Uy82?KXK|<OzgVvh`{w%T!C*0X(ONrkwjI~~~5nzIxx9<QDVnQ*xREyuW
zy%U~f3*<71vs(MCBmFtkDcLFB`<jShrp>Tn4amy~nnZEEwrRbdU0`T)=7^h2`16vB
zrD@D-0@gFfe&<Qu{W@n0L9$j7A|_sx@N7preyq-2<!7!PoPVBW|5lwi7~Uh^Bg(YN
ztgda={=xX-`~pN0)aT6mrPCL}vD6RnI&URl1&e_v2#uR9JdL-|DAVMWw?5U{QM~lS
z%Pd`&Rl`b`YMSRC)JuBS&zvLN4NOE+VG5pIDa;N$6qD&Po<>%NKkYW=N#iT0M}Op0
z-Oa`OQ;7}_4}G4>@yb$Qy+Zn;w9yftOBfx2c+s^s=HEHCV$qP$JMX<SX};8t8Elt^
zQ(E}6z5~5e5Af42nl??_Ba6r0Z&Q~y{9H{1I^3RJnRo$gUiZcWwnTl|<tO7%{x~`X
zl?ltkys|i2@_)gVoZ<XQ`8APIiP{Lk5h1)8JAjxYH}y&ROZ8!Sa|yn>L?KJ-wvq?)
z)J2zz>LX$$a@#2HB7+x4$S1S<p;fiRY#Q_UW=8>L{!^jc_`xJ`TFwIqdak6C58M2V
zE&h&Z?Eu_w1()N5329%S<er3&+TRq;9!s3tOD92H1%n1pFC+VHxvS8j&HwEm;rs?*
zhLTl$;a>v1-zhWQSY8UJWpS)lH{pAmjzqa`B+G6l>*r+bSYHZQxs=AtNx%wgKlVqE
z6uRq>HPT@m-yAtO;*-Zg5Q+rd3$Gbj1ih4o``W*PhLiaBGCt*s_RBYkq~_+apL{QG
zDzK58`xei=L#NNo{D&e<li`d@G)!E;!0xp5_JD34;O4WBjnTIX9Nt<IfwERW&*U44
za(qAMq&S<REO|n>>~*{BSXT#$X^awmX>@^v_!>^4&n(Rj#(enw@@C$LN!6L(HnU#`
zw)HBTW_9|uu=-4Dk#CU+Dv-7`6|2D=r2%C1U%y~i^I~SOMK&}(5~lO!XELc+6or8k
zU-mO`9qsO=nr%`*gUa-GY}vzl!zNnh&)=vITR0e3^2Mob^ggEbLsU>X3U36&_&b=Z
zQL>hg1X#iYScDRY?1+PN(}VFBr?^!<!i1tkf*ku0-te+}oS?-W{bEtWL}?8$kG&fU
zy3Gh6ouR<jGLGrc@+z+e_62wsfS?d*Ab_MqZJ#XG+v-IV)HRxSO(o>jz+bFDdeAxV
z&6{h`!jV|cj<JKc`Y_wj1)!lsY^_%+y3q#qKbhf?YHAZ#R(Ql-@?Mry0y$V1hYz|H
z)Mbz<(S87#AK!&uRkeVwq=IEdZvl!*3_*fkX1DufuOQ8)gbeSLunUXPC@>*1w$tw;
z)p1ADukDXbuDHsN*L+LSnTk`qwZroaJ$VvX#09b^Ec7~VY=e{|JPvGl^JWBDc++pm
zy6Zq96jF#$O|^WX<+d`$!B>7ti@xfxMK+_kEfkdg)lXAw&arIjt?SpwXcp%X;g-0B
zG;eWV=m~P<&QUQS3gH)?m46Xh`naLfa1ueAZsaCP#y|u9vFGTgLau;yTViT^N}>BR
zH0&U45#ERUS*xptLp9G{xx=G>ik}L@ARUcLqcmUCt9Sah`>+Tp;T&pN(EGf$8fv=<
z{#wP^`xX|XDcc~{we)CPd4I%e0$mg09MFahaaLMnYN=U7y?~$M35Apygohi$ega4L
zfulI>p6rE};5|#mris8IDJq{MM;eW`re$1aKdQ|}6qz6|DWJ#p=tkFoaExsV328r1
zSYlj8WzyF^JLZjjP9?a0|DiOim(f%Qv{6&sa}2{U3wF&!quq&PKi>=O(##|!F@RkM
z@;$%$9CTH6af>>F35*Iuw=ga>F1D(gSe#|sp$!uW_7PoGiQWzl6{0r;MPopQIkN^)
zsm<IBu0ju199S~d>6a1$S@fKlHPi>(6Qt`0(P34@=7iXg2ku<ym`naU9NcaNi++WN
z70?lugIwEw*m(LraB_r1Pg+6d9hbgo<d>riGWy#J4-&8XgK|XpDS9N6NPC-LafR)C
zQNo`wyiFnxZ(2MNs>_NDDR3~C`Yz1GeZSe5;ShTxxT58~HC;;u{L%)EKz_oQ8uj%N
zu&^m^#eCFq2`|pp{&Q%Qp&YqfRM$B3uM2yUfH~8KL5O$vI3<3T>K-~ZRw~ZOrZI+G
z_-Uf6tBW`Vyx{@7@J?FfDdaTV<5OTyoO)<8jgYABt*w@NRZsX!9-IOLyVI7?uB4NG
z=-*I7%Atn1s5vGHjV;b#7;cb9*H7OTjTDFBW}z7Vi!P6cuG>C|g{aj`9_A2Al|26U
zK1&A~O|o2f5p{pVSy_+hamQpxK$MA9MF$YM%D}zahN@MWDq>eWbr$b7iei(;k5^Td
z(ai05$y!uryeTAKJXD*HdVc><%fxSrFxiPFGL$Jbk4x&{RSy|9XQ+saLB_v6#pSq;
z#{Df;U*{!7xWe%3bdN2s#9G`~WQtu7HNht$+zj+t2KovEY62FgpSX|YeiY>oHMuRA
z!@F2^mB`GxCeTSYg`4dB$gBPN2IWmZD`)>Z;1;+q{#gwW1iCJ%`iyY&2Yj8U*p3zP
zl93=g;{wK1q9F?AIM)lweX`$oOV}nshdFPu;vX~IZ=^XnnOHC`iRO=^s$iz!8X1eW
zyM(St-V_c#3suzOAKI^7Ev&7J+tNT4y8s^us8hi>vCqn;EP}{N%Q=;aE^WQ@s%GQ}
z9dK`CF&O=0omx;uK#euMOs^(Oc-|srBZIY^imdxZ;+hipsC3`Xn$G|tEf7$uSrd7t
z%17yh`^Cl>v-*?WuO%{8#j{S%bd8+{cd?>-H!*!(`nW6X+pE#i`m9D-JD3Nz7(Vf*
z72N-hgtR)9t(M^j)I}FTWd$WiPq8smTLO=I5_Z02K4?LMT0N@O!v5aj(7uYiY6z1d
z50<v(X!py_>PJIEwV*(aLTyanFND31ri}`{E(g}U**F-6uhuN%=RW3$C}n&gZZ>)n
zB(7l7@0j85X6D6t2Y4!J(j>n>Y8E%{MO)kAAD=szl(AVN8e7y^Kr5wdn1~+1Xin*x
zBwON+Rn-XFEH(a2&d?_^i>a1?!4Z}(fopK&zW(TgzH!RnqPY7DpG?Xd&~i<YIi8dT
zr<it!JcLt-in!)y$=+2#ccgf^tSTQ^ZHitpEzQ+NuuB{X5F6Mt;P5rykjk6bufMTe
z&Bum~|1`l7Eo3emoc&NZlb_Rg6$p^5GOeulN?WH@<cqzjaWRanM2^F~^<$3RJpjTj
zGNHfB5N_|QRb9NPW?^VnQx)sTlUsu#z9^{EMP2uNH_JwS5Q>ckYu#D}y<5OP8vSs~
zA~*i1nr~qKfU5c(ujNTocm3);I$r3AEKBOfg~HAl%qOspqq&j1zJqCuA9U*<<pXMa
zO}OaPK-1zVbw<&op*AG=>Ef?(OgbqGtY{t^@F6u2kUR<WF%^0UHW^<p*7%_RcovCt
zHZ`Ify#`itYcx9C0<n(dvz)x~o#B}Y^*94xId@2(S^>$i_?9-;JlZS7J`JHMT$1md
zQ$sdQ*ny)-d#+2$N-eKFIQ$zH{jwWQnFvnCmTSQ!`FVEI>6|pgR%J+{3pdv;@;vTd
zPa}wffU99-Y#h^PFDLlmsp2a!J_crY!+kykqeh4Y?bq&?q0cKq?8|eBL7>jH&Hj|<
zDMdkrZnqC3W0NtfZ<VaY?xM!!EgG#8GMXle%4>5`*&;A>2IPAO0eEH*pp;XrJJTUj
ze{EY}39zajq-(M=$0=Ca-a@vMrU!27Q@GF{(<m^En^`VFWssFOxzO<dAd}4|!~Pdt
z2_s)9X;$VR1s!-)@?_@5iLd{6iJzYqHvX!b8mZ%+9i$Wb!3J`P6Vq85kHy%U%ypYL
zEHhW+5<Yq$^356c_rryLp9~qL4npcY(cXo;pO*N`_<LB6U!GPSqIYZl855^FBrfNw
z2Xy%e{{gAKj3RcdA>XdB%ky677_USttUD7^3TK7C9xd=yV^VUE1o004gc@OW@SU0G
zlAAF()hiE9Kte?ok;X2vpi!|@-yI+^y?oEh6JRk@xl!g_r3>zx%p^^bPcd8?!BHS>
zjpk%2AEm8Q1@BjJY=P|!oj7>wb@b*EAseX*o+w<VWbM;)F{1LY_(w~H_c&QsfJD52
zDNAY+YV^K`v7ySrqOfULH)!`C4=|#;MnnapQ^S>sZ~tlmB1OecIks7FUQFqfGD%$)
ztsBlZ!agv?1&Wf<k#R4`Hc=n{y!<fEJmz%xE!jochRsY(GKIoydm6`O^&)RGUixL~
z&n`&C-HZ?iU;k%_YM1doV&Hnz<BgUl(fEC^CbZ%G!&EgtZ8d9J18<2{le-iV%)ak{
z0LVPLGqqnq!q;6zbVQz1??3QU2|n~yp`9*M4RrD*f6V}kyNO=TkL3tX^#Q@E_?e3F
zcroPDg!M+EW8ZlSI%f@gujSsTaH=U9>++X`2yAOKk=_(f<OVW_QpC;T`|TYA{N(w4
zbVuSs+&J4o){K0PMMCQpd#<s7eZ|X~+AWiF)>ZDzd#x2-GuFgRfu9M|pp}Sw`j>Db
zuC91S)vq7d-Y~&4K&Dp=BjTii6<OtQl^^9(gknu)TWV{l^Qaj>e+mzs#p41SjesEn
zm+5&iXm?!qFUWaTOIlC@L<J(R&8`MVxjX~Qy+u0=>_x&NF94Ptkp*VSt{ctpfj~d#
zu%ej?m4t2K!sHCgD|a$<*nAqKlWYx48L^J0MUrM6qwpFJN}wjG{j0_KnK5q7320UC
z^T-564Yg9)#`J$dIRrK$OjDc9Js+bq(`R<)pbD7xp(4KQh%UG%35<D+L3)n(Vefzj
zXp0WIcR$a2Vsg+w+Utg^RG+eC8|bdt_psIm^x_yctSwXIBhNX;q8Wu8k==e2kO)NT
zJ#i_E39vH6`iqGh7;m|i;&+!ne@Xd(hC=N{snN?zy)DC&53x2+ChOu9D{HR4nWy%Z
z8h3HFksXrZG$r<*pShDIyyQqW2j?~avdt%kjj}e`|E9E}@T())W%%Dgx1e*c=0cZ;
zrp}qt=7Jz^-vw_0xv2bi$8SR>1X=EEyN?r+zO_smV!62E7u42G;WJ^?9J4Mp!)lSc
zs>A@hBM;uB@<)X}G}rTab{38U43UD{>csoujhmUv-tmd)z=6Dt+2at5f-pfbTcFmT
z0m)O;Pur08fJK{Do}-7=W~R;2UruqTn1(wVT}Q(m)BLK@U>PIKNE)otvq^<83bTX!
z?qPl$0Xb=m4?e#LuWb?zr--|Sx}^fwNgLw=19*uLNpx66WffhBT%&%9@EC3Cn}dGQ
z{g&7}V+da;bxmN|Q~UK0W_6`u$zJD&k>)qQ0WS16DZaW6E~;xp%rX0T<5-#Ozp&H5
z64d^1c=E5$^>gO;{f4Yhv48RSU#Ae@s#gv6vy3P%Zt@smLbm$9J6&s#`<!8a%t^8c
zhHwUKQcz#Q72rRK2<E|;#vQ1^s1Rp=7Ev21nP+g!a?8T}<G*gEl?~^b<r^QF6rLp%
zo@Pp~4CEAHJ1+OqbJ&N+1XmSUk>o0`S@m6TE>3EHvnmB^M@?t7OkE|qrY6n26YYlP
zyZ_QYkt==3Ge)Xx!gVP4o7r;N`5&!ZJJErav!O`D;)*<;oY!bHdEA3m^5QZLpp(5N
zbPxaN>+8(M9Vn=vWufbRzg=hIL>|H=3|ycKhFJjMQj!ekbC~@F@>51MRW?fSMzfX4
zB<b{+2^OQfyz;tw^yLXpBM6JjY~TO{iv=4m)V0@bm&+2xZ0zBfB#;0ef04tj^Avjh
zWM~6?FPYy1z6sE1OoYHeo1NA&$IqM<arq*N%~Hwj=c7K;?>l6ejpTp5Du3qrK$LBq
zlaomIq{~<Glq2P1VB=&`KSAh1tjOxdmv(0Z!VIzV4OMZJ7RxLpXZM%7dp41JOntF*
zEyrbK$u(hJMy^m<EMcvjvz7A>Dbd7RRG`4Z^sEydE@J0Q@*0_na|Po3p751B7JqgL
zH?d*#5k3nwNqX(+yl~i><BidI?MArE+@d-Z!Z2_7ok1@*o`i|X$Cvu#S#Jci?tX}R
ziQiW9?=55ogH3am)re7QxNr1^6gQoFTeO9020u9K03gDRo;i*vYlUXxwgvCG;R*fc
zWCK0Csy{ZkiEI*1YK_-Zd3El}8t1}g+`xsO)Ym5tj|DSJHEC$49tMGp*y;1Nzed0?
z{H7D@@_DCk#XL8nYoJLgNJGy%g(&xGI%Ia(K(Lh7({7m6uqtJ$Ce%3H&C$D^XrGHH
zT*t&$3T!imHsYz8%<EKsc99&|1+(bG_movO01=Bv>r3s7WHOFw+>jqbIZMV9>rlUE
z95Ns5IXudkamL-S!;4-n{k~tBqm#Ri{B=I;KY&vQRuKp;Ad~>Uz0SUVt|rfgvvZ))
zQ*D6R5lR@J;Sw8iUK2OgUc&oX$H^XMUS<L3d$@jCQypcSq^@LMXqk^%!zlAVdbse;
z+a{$f1Bv>%2paY~EMKka1PRD$uD~nB>z(u*ps%buXV+xEXE$K~763Nr(L|+W%|Kf}
zyLCjmWZkqDTSYq@F?5mFtTa*AwCmgDJduOrI4Z9x5s|>3zFrU{N^$soO`O{-8Y?DM
zN5z@_f!Cw+teBL-JY6`7^fl=zpDq4(ccb0Y(<WJ<XGv&#bD4095#k)>JG4(qODT`D
zI<|p;S&eDQ=k?j66%Ctqoandr_-^X=aw2s8HQ&nZM4g?z*}Y05yxX7uMU6@j{RXan
z$B^f$IPoVO&UET$DCcBIMYAWH8HI5OwyBNkNAmnCc%RW-^xjxS*_tDc<Wr3#l-Hls
zg&&d+u+WQee)FzVd~im#6AeN_bu?eD5j%>y^QPR(0NuXfdLRM%ZkCU?8&I*XmaxNL
z-av|xS?h-7S=mhyGzl2r@3&w2Zb#KL;?7mo$MSJLHIawAnkUrUTXO|YWHO4!&Rx{<
zLXd0Hwqo$ymWQQf^)a&G4KIS_UUJ_LVrog4C;sId9Qye*L*@nlLt=-hdGohO>9Y!j
z)~Kth7x0?^^!247z?Nb#msNh^2c!dCZ@5fsK<@i-diaROJ^wkN8|f}7Eho0Y^TL!#
zV_&<^*6>=L`M&iZ`8qA8L5r<T1~cmSG@ba1h-hXH$?m@+DA6ZiZ|GOx9|+mI{~>)c
z)%YFT$|g?O#9Le>gmzJ><rkD|B`f|7o6b5sj<kFRP`6wR;D4@=zksU?^yPZozSnEZ
z)h8zvP9heAGNq6TU7q|?zaUpJC-}y+ZZe?G<OGwKgcIB*cDq$kvYU<&DqI4=hJk=v
zm;amRd2f!a%ggO+7HqFR@gG>sTt<wI89;EG%DrHpF9fd@8eHT>+UNlZh;wmCW5GtX
zK%QFMz93@!<kThSQmr?})XKu1>_(zC@e`9PTMWrwl?;*45k1alT3Py?jhE&d><w`L
zfA~Pz&o=;{im2~>pU>2{UJ!h{>Z4z`DRR5yD3`<#yaX`6-N5=ERkDZ+?f;_aUrGI3
zV#uw`Lm%9omdv0vEjwr06xc^NueeC6EPPp?ZY4r>dr5NN$pxmAS3<|8)k{=)0e)-f
zS|*6~?<XU+i;l00^2Zu*Rx8FM-tG9j16;0BKG8k~wt5nj$J~u7Tbjy62w>;y84ZNm
zvx=6BA;Ur1B-C7~2xKMU>#zJkwpCa4Fk5N32pO8jseLc`J*m$UlA*r>FD6~)vLk#J
z|Cdl|Pc6f!)1yAVp{erIqZXV$4MTUv!EFKoMARjss(l^!hId%c(wpT>t8odw{8v@K
zvyOMTFjWY*n~I6y=3?4+QoqO78_c;UYMBz@4F^x`5+ZT;!%V#;IadNAh>}{Q-_Jmv
zwjw|9m=ee*C=8{)$nhvzD8M+!6o1Y8b?3^3rAmBErzm-_ABai^hukdkOlF)JXy{+O
zypm`s1vPXics-#5$z+TD$z?jYWTZ()%5X0y7H6~Dzur!z$d8Me2<;S?bJ}}L((xqa
zZmw^<0XEs(3BgN~q{hcuM%9K(HtPphTV)PDrJ(WfIfesP8%I>#W4f|_iio-t^nhU9
zskdx0!@3!DRjC62LmhHv2+_>deFZ#8<58crL1}mz-;`aA-xS^gX&8>9Lrq5u-&wF2
z#Xa?f5}rSTlc)ce1%Q8k@1opR_>8kz*%L@KM$z~;mrCLN=hl!z`8M`zQoQIi4ceNr
zX+T+#etkqPjl_uqo?nDoX!2gLjK3u8kW(u#-$O8c0aQFXn&1D_ef{~q_1y2!rg8WN
z2qs4<%y7|e1Ppl21QZJaJDQ$GhR9ei*Z=fI!((;5GWtJiOu}JqJJFmJ<x*YZU~E_{
z4RP~m{F(rl8A1f@&8_QZ9jIcSxc<ugz8WuPtf|MR%ZX2?GRl-@%j+ZKE&&HCJ%S#B
zUo0zNR-m<Rqp@jbt7z0X2#5%b^;onuytUFIEGU^;4RPP$2Y>Ch^k43ud>^dMIeWo5
zD?s8yxJIav8{^<`!kIWzq2;HbAD++Gtf{vKCd5NIUBQ^RQKDBuk@u`~Gsk+@p)lFh
zAYVAPBsR9e1+_OJTpVo0{pFj#jnvwa&b-C3l}N%vO0z%uNFjpDjsFbDk_SziNH)B0
zK&n-+Yp19TduEWikWx`#c)~Q#`k=qMq*Y;UXldcB?xAZdml>s1inNq=sS9~+@?Uw^
zuigqi(PN5(KNBk}58}S)u;Rf6Llc<8(oR`qo-R7TC+jNQEV}A>S**ehIj^YdBlajr
zb$kRy<<)Lr_fRgGQaZa-fe0s~!%PI~7Y`ee>J5jj&vPH(b;;a_TCIF1ORn}X1aC>9
z#okvaOI{1)O6d#;ns&O8N7RlU82B|+vIGT?TqZ3f`Y%5FLkV6ptZmLA{#&d}hGYGd
zgle>u*1Vhf<3F?a8w=}tlcSo+T0ci!2IyqhD}0_y^<Yyp#P2h|BzoCo%&o_yq_?|&
z?NIzxA!eZ`f$nE~G1tDD-Oxnf^r;D%fS+62_7XCdYcBgP(~Fs^UC8G!Tnu_CgXU|l
zk^qygAEaKnXR$q<TF$+YC%A9_H|d>xlcj4S;&GS<Wx=~|P0XNslw=QLdPlr{ilJ+#
zYoy&WR9*!)(7!4tt&B~G);ven<Tj#*Nl#?UU(RioQkCtie1+*4foP7$k%Va9r>iL?
z=NH)ARoxfex~Xo7{v`i3#A#TV{SvB?1dc~1w)7t>K%I-9;A05^Q%}w!<y)|1E=G#E
zA|}TQ?;-Gg76})2GVuqwXGc(os#g;ojdzSGU;^3@TS7sXvV$nfKmSu|0S6iQTK{3O
z8wI1CurG+VJ0x*^ee))&o1UgH=!WV~j%39Wn1*r+tEb-AiofU_ul#`~naS23^x8Sv
zk|q#2+CJ}8LsDr)o>@?Qv7@GN4M5PTO6z$0N^oUy*{FnUAQY}Zn~l^Bk&QD&(qWni
zTldCe#Q0B81>^-8o7BLLw{XjuV0XU1)q#cID_YeFpa_?OFeu8`w~$&d*r8(kBL3r3
zj}&{R>cDD?Ii~5VZ72wIO?@TXNddWP8Xfm`pBescmv*`@I>}@hh#JX~!!_z(bO-GS
zT?eN(G!kPOocJ4h`!5oDCjW4`0NxNAv>JDe^Q9~DH}!c`N@3AQxIKow_hsE3ygL~?
zxSXd)<!ykyrR15kzz0~vq!#mCUc(>LEdc=$+8lnzD^7tI>CLOx1z5oT{3Yiv#8{>(
ze+4|~0#88fKQa&u)vi};Hu?VRJrI`v#Uz@P_T10LSL8Rr;eWLIP&DI|$hX91XjrB`
z#&=f<7KTI86aj%6dh9H9dqz2RTDVt(e?$x7-Yt82ESIxC#4=@`6R$n|IFA(j1$9XQ
zzOfd8koNORTeWqQ4lP-DxZeIxMPFhMJaP7*7r^VOa&Eq;mG2(Vx%P(jthZc3Pyl*y
zQZYmk9?P<PQShX3^Q*YeP{<5Fx8Ae26x82OIV-?rO-$~#25oYa!x4WLLC8T3%%1dr
zivnN<{xqy;k8P1;eERVUY@;gFxX4M5%JBZ5wdOh)>c_>XgJQe0x_EM-p;PkpKFbUW
zx6w4B;Z<<R``_QD{KpBR%(J(8M2w!ER7KsHgR9PqHTA-^wz(6SfH&V)A%CdktAOJ|
zfy%Xi5DRL<=xRhgOIf-TZ}O#sln=X)uB{F^O!|)?L!YCn{;vOM2*%&wAr}Ux_hl@4
z_Pux>twctX?3ZOM8+7T7=8CU?b}+BG#y#aRjOGgrG(5ez#navS`Uuw;>f0(MH(-x(
z7a)ZC!$?@0Ezf=M=7#mCBZtnbL6D5eGc)Y?-#h0;JwD6y;7=S_oHb7vgicg2<gva>
zB278V^4TTWMQ{5!VK8{6B6`c4Hl<Q7L9N&!d+$Gg{k4o6{ZlMI)I-n~hfomf97&5F
z*$&ymh&-JuRiDEs(FleAXq?Ol`bD7csa(M+c-8^yXm;MY1#Ep*r^~vWX4hp`A7*Vx
z=X`jxUaldFl)r5zXe;F1cuM!FQ59#w{c6ch2A7p<lL|HZk%R^R80buxhyWXOL%^Fq
zAYTsP46$ZPUV;4)()(PK%IOI)E*Z|VNC0<SK1qcFzrFmS_{prYH|&K)tHL4{hJjhp
z>9M93G~ApW=TAP|DZbT5dvE$|;$R{JUEJJ0bm{pW&d)oqshWlzF2yrP-<?<@t6p7~
z_d+r-#cCthM@B9_>&p|~xk{wI*}Tp=5eNqK%1elxa@XY=rZIBU)<Vcc<_+L>2J7V@
zF<*4Ui>*Nq1tPP8KmNW=e}Ux*YRnI9k0&I6vy7tnf}4^GJp%#&K7h)tn<9wBwncgK
zW$S8cMl`a-K~g&o^{=yU4qufkB4Gp(nSJZZ+pg@X79|_rB0SFfmNdg0u}da{T9zg2
z{}Y7ybw)sG+|=7gl^-n~{xt3D=ek|1TBm|YT8aADfj9mD0%N~P#5q&reM0gLL{x0J
z@RU>$ufKYR^&K!MTiQXZz_nDcF!A>4^Rd*=88xxXzxhoN@S*v2Mu`p^<H-qN2z3`V
zja)YoaVZV1{2MqPcY@KlY{Q@vr&m-LCpAeM$lfH(c5nws(3+1f>*w27Ub#o>!?`1E
z4%ac81iep93Z8SGt78@P4`=ltm<+5lvS&zg;~1r+ui%JbVq>*`%`Jt`Q`O1ZW+n$X
zi1U?Q9aucV_jP0;nFZWenFO?pqu`-<My&ist8$x&3i$(nTrt~OQW~h^v$+&*^In%z
zto~r$9Py1Oz~rhTj6J?JR|W0{@*uKNzThf?HW~qQ@Y}Cgqz`9sGrydq{P8tRipt(B
z4UJQ(a<xKAz{mc+66^e|JyI-#YEAIL`KI%*uBlIk>gB1f!b_npTi@$?MxelAB$=ZG
z4z1u2=%mItzOt+JMXyWw5s%>i%jS{M6DW{(F~M3MYyHPOzH*|7Nch~b<;dof<#jW0
zNIY>~3Ea&kZ3uhtCNWDll@AA{rPTwkK!Uw8+EYs|h0)GPiokJA2C_ZIe~<NJl&`{q
z;l*N;JchU4-H*I`lCR(|J2{_4Z8#uX(xgunG+%-dHM5k16>jPN;lLav5_bcea=heS
zciz?w!e_pDL5Z$l(fP$T(*Z@DM?O(_N`7b1U^{TQu>Z2Hn~&><&S-f5%~6Wd7(Rqf
z>47&Mi<9+*C&XzNk#R7rKd19^WJ6s54l@*e&+|aelf56lCvRfM)=}tX%<dpGlyCXL
zFm1F+-&CUFNdZ>v@9@}9HRZU`-CsdXh5uL8Iw-}&Tn-jV0*iwkr6-D<o{;pTY`69|
zeWMp=DBxIc>E7&K=~QdKA030z3QIWPRAP3k!2LlqQ9_)g9(y^^@W=`eA=0^qK5vc?
zlOG}1wm__mhdRX$va%OsJ2!>m+^c-o-_L(ySD)0y#12oC$EY`_Uexcmkep#F8#7;N
zXo4lm%MIiSYF*@dEOy``;N9&!Jc`zFNJInE_VzEc`=R0CzjFH7!i0~9740HIk4H5_
z9kzf^#1L>JqP?k|8}E10x&eHc;ssbvTv9s-43lKD+TU66wtawvhAu$5i+ci8AiF6A
zzCb|4!WAB$OD{lU_p@jqaX|E=zMd00gSQ;9Q3H~DvS3m<1=(~V3e0$x%u|X4h~b~{
zS4W((s*mCdrp^xiUtSl#!>NdEGo!4`ZYMxW3fo;T66xDRCg<^yI2Oi2#J~*4yB7hR
z4j)w1H`rqJ8}#0vT3r4eEo-N_l9oid3<v-I^o?+Ob(Cc9$uYx<!lES2MM4vxGDzJe
zEvw)`jz7yjjJyI9Q1crn(qSMDP@SDOlC8p$SYHZA;kmikwT~_w3k4Q#=CveAeQu8C
zJ+4=P;z$hG+sK8-#%x%I9@?HR<Xef4%W5FV;C5c#O%sr*$KM(ytc`ry&<v80KWo(s
zWXFRh!=D?%J~`_#r?&wzy9w4<1-5oQ5Nd>?){Z-7DUeLa<$N#=R0WIxlO$c<bhZW_
zN;M-CC_ta4pUiIf^B~-J{q0)1S|Bn{ydz~^UlKu406xKl$vnlpcTxuL6vVmv_vC%=
z(cmVltP<ms?TjWDk_As>B&AlLjE+{GX5uGTNJDAc9lHzp?%kI%DWUwMU?wv$wNHkf
zQtXX8?o%BVQPiOX_I}benMh=-Z$Q$?oWP+!z;Ut)DV!RO?9E40y|!9}nyO;g)z!5L
zg}BEXwn`srPH=@2v536vxr*fZm`R>hVElimN21@nKPk7#K(>4Pasm0v9E%ku&tTdH
zqLY-bt~>a~urdgpjMxQ`yxQF^y9NX6JS2$RT{Hy5M`56h9*?@(USA%Ed?QE&R4SGp
zyD45Uc^AxcA@3TMf_y)pJK&%qyo1Z8$31KX#yLYk>W{%(-e`nQxMvBxedu%JTj+vE
z+2@_n2pL%HkLgpp+A{d*f4O*DPmEd%n^~A}$}iPQtyjaQkJ4`*(2XN@vz?-AbIyd3
z913Z^e1scP2;I`49&53wlSoIdUkU@#%l!PrJ6!)~%a|kbY7QfXN9M9w;#c#%iOYI=
zF9alXuIDG|A-@9O!`0{O5G)$*^zYZ0`MiB$fiQf!G*If1Dm@{RH0JPh8Kp$QK(l)q
z@F~h=lqO7BCV|k4F(?*&ffg<7-RS73F0B4;4(Qt1v>j61k#_plLB0ix@xuT)4sBB(
zmnnS_>xhyWRwB-KTdv+rq+SM%R#Z?0VU0GVegcwE#QvGt91>?(?OA}V$|05}y*oHG
zc-F3_()hE83ANTw2x57lllIXaGECCi#QDkpC!u~TR-}IDaNnHTVNqPs9N@y;l~Rhj
z=(?@2Fcy=a;+wk06n^@4tS7&Bi?^drZ3h&d#hm>>PQjSK!>9DDP0E0pGbh09y8p>{
zf^rh-O@+mDAihO$?#)Mr$bal?u>Sx1uT<bXr9~yXm;Ldf!H6ma85!!~_w(pJ$T0g>
zh8-Kl$4}nKaUuJHl<vN`E!kge)2!>23!V!_pNsYK`G5&xk0)|3h#&BbsA;WkC+U-m
zJnZFG0;c5sa?yV$_V3>Rr5>m4{OOzK3AbU(y*jq~UvTpODxNDq#VRp$B!9F5!^9aZ
z+iwb++=XNHP)NlDbJZ1#HapadOD}v1ioNAl3RB7?1peH75q+{w0`*<B=VL<YKhmuQ
zHDfkkr4_EyJc}HazqyAE@5T+++YnBYJQ=W#*PmELYvf<1d3ahN#XX!BB(zmU9&w%%
z<BR>dNLqSmzxNwN7qaS`6PMp=#+&+r^8vVDb=&~8Tl`BlD?ZfCs?Y12us>LVcmDl1
zpwg}bnCpZ{hz=#cT=jlIT2O||g6!DAKTh=RvLCpM!BF=+Tdz`Oq<fxcBn#w^d<!Md
zu+O^f+{<>6&j?<o^5%cM<7+I%nP#&Nk}2EUWkmh6<R0n*=lNld4GB>m0-ua-UjF@8
zNEr-h?6RQ#A6su3P-VMq4O0@*BCzNbBt*JX8VLmvY3T;(F6r)0B}7WPOS)USySwwf
zmiyV~?0vrP*ZRBey5^i?jxgv?bIw5{wbYwPddd9zHw}6y&tXlFo)KV#bgYwP;e^h$
zK$09+J_)YrSgt&iP-B=}7;`Hew8WQeX?pT)sndrg^*m=uA(+b&O=V9mnVl{)-AF6#
zJy&<NPIOV{M$2&#*`s2lLe>rTQkkJeMHOH!_`QAI$PV=-s&mDZYn^IadR^8{sRITw
zbg1Nt8>E+g5EWWZUDpO`-L@Xu3}Qg1t<}g-nt1AB=!51RY%k`6#$_eHi2Y?|m(M@l
z=kI5;4n5N07{i>qW+8SoJ7Awyb<jDG%)tO{>|EsHgGSW&N11sZft=!l>n+gOe@A<v
zTc7&NfYK_)!(5NlItJskm*JN$*P@#e?w8+3N)x~-<r&s#$EW)ex=FbqEat+N=@KMS
z=7ZiTt}^cI7r){^7VpM9f%If)o>1d2j&R1#F^R#{`B)CuK@GD4pkXA3)o5wQnOHUh
z9Fre%+|M{rUsb&O1V$1LzEPrkyp6-df4=#D{|E;I*o}@bC`G!O#TN*kP@#}u&06$Z
zJETO{aUf(1o^w_W={q<AEJ%DB*+eX;2iOtll1;<Gviviki*U38S$8guGtC}n=?^s0
z3d{1@E@zEIcZcwxE~cfc<JC3*Il4D0q|`uK=gx>>ZzYy4fmCWYobD(3kS@>kN{Ege
zDd?Zvgj4Hu+BH<4eZ$dT?!Y?egxWgYZ=kU8dS8Sxm0FAhi3{LeB5$&6Ea0IAuOE8+
z`x5?jh=Gyz@0F}n1y!;*A}j)${_@4F73N$42zt(7h_S$NTuHeF{e&|1EFjWyf}+a1
z+)WjMw%0ok+^(P=ckesY16QD2&1C-y&TS)X@A|n~7c<bhDMe{f!!rTt*q7rD`~WB+
z<1a7?b_O5)y`-C25TD2$O+G{R;=ueqX3X1VM_Z>rJUYd-al55#oE2FT7$Q<@+tRvu
zDmM>hZ`*Zcs|&T1k`*syVg6>U-z!^&nqkYY1USj-`EA|Q|9!dseR0ARU}J?(NqN}m
zrN_4IPv<+r2VAxPD%j{gmxW!otwIq<U=L>C1R94k!dDI#wNFF119t#VIDQ$t${(a*
z9Ckmk8(UgVG$f8wQNXZ42VZ}vdwH#919+-jg>^gmb;B*r3()p;Eb`a>*>$@%#r8$S
za(i_$zjN7+Vu_U|V`)b<cc(tV>TBQcJu@?6lJFtO(?Z`eb*F2!NvJ-%Kasftag^HI
zx;W7kQ0uKWsOeQ_I7vaXk;g6YWs?X8ceHB3W%mdS4Qi`XUxi6Tx8)rFH7WeF_1c03
z*Gfx1O-<N|L6aQB&36oi`|P>W*e=_bb!fT69$Qgg`%02Ji*X!4ztnt^*;bx}5bj6t
z{P3sFJ;B2{4;2!}*#xgAhDK?Js>XBc<$w(jG&Gpd7P=p)&!DZ<{J!>Q?<aP!zaj6#
z67MM7e=>i!$Box#dztpx$Cy2IvXjWF5be1~4w_v{^^&X4ERF&{wmko_*u6xy8NLwi
zJU&rw=+(DUy0V*`@>j|l){pYLh#TIT@1#V3yB-rGO(pQE5h2#l<Gx(j3R%3u^f>Z?
zGK~={jznK4XgVZ4T<}!+j1L2QBwX$tpEXSOE@fB8IqQJYKhB$J_?=-ylkfGsOC#Z~
z?ud6_m#bg{YEo9k^YOH@lY50x&&A)9P`{f~p2l1(AAL$!=`_%{jQT3uf6?1M<lYuX
zL`^NN*7f3e7bf;#OWk#ix8--r7V-yDZ<h@dm#<Z9H{$QUs`_BB@k6&Id7C@a%By$$
zF%E(BFBoao{`xY3FSW?XOy!e~Ujz4AH7LZ}ZP#C(-%>OwknS--XBqmkhzkn{i~)JM
zja0}>AwGA912PG(ijVlopZmpj6lN;YP3|ZMicw!5eKJfkh@A#U?qnM{<Wge_Uq12D
zdCR?sia0<M>LN~3DDwuNcO+Iw@dE{sR(@dg!&(#{VPr?^Yr|CBOwxGq2X>X+q{AHS
zo>rRX_%dyaryrEES@GBd_qWG>m_7dfPTJ)XdGu|u2J~}ndJzJ6Om$n7amcy>p5QK{
zK$C6ZO+G;y?oRqRNKxHy_p7yy@ua`aRpW4re26EB`>paA$`~qPYt7*M`rO+&B7^^O
z8C9lBi1j*ZB*V6DI96YUI;3F~W$5%}p5j2$Xr64(OSxE;{qmcF#?i_1!KtqbT-&kl
zvJ__)))%o;7&Bc--~C>)eD3Mmg~<i<;FLS$tP?vnoOGhffrFjW_v@F*61}QFc)A&^
z(9(R)s7>&Bhn8NZiMqenFd^n|2lbI2?AvZoMmv%9Y%x+a?HtZ8frhaJ2qJcc4Jm6l
z^K^z%d6beI2UmXqX`Qod%{q#0x2k%geZ@|UA@BmCFCIFESt(tOkWRAgO#y&bD|9GV
zw`$BeF2cYY>IHZtX^Io%&e^G}nN%(qcQYN}R$9v4UEfhB<8j&i8BDBGU`0vYbcwB{
zh#}RzDV9cf&dR!!sjMD;ht5&=IgH4{RK$JPvx^V8)yA12dVp)FG~>N31ll8I>`ic>
zlR+)Glv|{K4?|@5NBX>*Fzx*Is%9;9v~<q+l48-w`_CGc^e_MU*8ll|>>*Dn-hQ?+
zikLC#Czrhj8<yF>Tux{lBb2Q2j)nl+Qtyh=2Osyi^d+-F@AI@oXiCCZnc@Z(m+aEB
zw~XI(+Wkhl$o6lbsKyPHMCW{-jYjHlU2BwL12mhDVl-@e{6&J)>|C6b%`Z<^MQ()-
zf#kZW2^vi49D?HPhp~chF!+QBa{oR{f-ZbZZfnr%2g`BbJs3%M>Wtei^^SebE|?oc
z`9LT924N~KgWV)r<|NY=Ud`8GLd(^n(IqBD;38(;#d5j@QuIcF$h^dx^-c;hg7j}N
z<bN)^4%+bJ_<kN|PW>jiO<Ld7MdsO{l_d!wEDcMdNtc_@fdk&-sA)A3#5}BBVmOm0
z`cW`6*ghK`Y_fAY7c6+`xd6xd434soG|_1yt;cOk8{QOB_nh-~N(PMVig=iggk7yW
zzpJdCd?a><uBC7t59nSSTsdU7Rf&@(x;YsBj%uLP?0=g7@x6nhy$HG9-P`ruhjTx#
zk*CeK9eC=pxYlnJAldevp@96*e$Y4)drp*{qx~7y%TID^<R3WmpP~!&2qe%$8=0kN
z1MLDD%-d3Y3C)jg!4QbCgmCLcze0K7WRYyo;5HuR@(i(xv8B1ZqF7PjU1`h28FZ~9
zGDo9vK8_5(V&wevu%X~yNE(kZ@W70`p`bCUS6)BF;^o4+WV8q?`}wG}p0!<}`CBde
zV?3I|t!2@7_l%tH#kTwTVwgms$dGw!)PPrbGl$P-ywkUYYJ#XZB+Ad|h)=h}wZ<wA
z9R91)<XJ%*^(LX1FYMY`XQp5V7%9KT>-Xh1NLI?l<XSnGox<*Uof^4{8DoJah~si6
z)umVOiYK4sd-|F2)nPoIXMw^_l^ryS`iD5|4AR&a!2j{13g`Z+Z?C6VxHoG*dBvXY
zj{3H({@?^vJwk@%8sdz~fj0zTM32?E_}_4p?g$}Ys$jgn%g3Hli0Hcqj4+rq1Dwjs
zSg6`Dn|-^R_utdmKb=4huoe;$@h?B+T5!rTttSO)MfyQK*1_x)6i%5_X4#1hfEN8J
zi#QR02#S4SN&VI}<x5h{6111ChxO+Lx^ua3i?cP_x~)g*Rd#g`FfOLNg|rV!8{v3_
z=pBB<`~I*o;X6%sqFdUM7|L$WT#p{2yO>kp@Z&hh$mxek|4@?%;fAFo48qlbwL1-$
z?H8>@W5hbK>CJGygFHnV2~mk(**VRihB<!NTf}>E{7cn`8eQQ<X#8q2O5fAUVRO$!
z4ei;8iYUW0hF?O2tIiX6aJW0+rl`+<6@i8a4e{>-p{zH2S-9GN2V!pZ3}Ta<h?GKw
zaK03`L3<v!?=@he*M{z#=x3;y2+xD8R_630J~3?%ABF3*V)`h!K~4Zb^r5ZxzycV3
z^|QdXzj!^`MqaXNwCUV4l=x%?H#zqGOSFQ`%Y`Mli;MeG3*EqhuhM!k!~U{Z$%F<g
z42M@{sSim=lcz=n+hA1NI#`++01FOL{C!yyT6DeR@E^m%Z?oM&K?j{kL#Cp{8Be8U
zqSEDMgPKRsQykCx&8g<+O6rhHuJ!ZR6X~v=6iap^+TAWq;liF&nWqO2OxgaY5rp%m
zHL&Z2yfMZOlh{#ywG!=krK;wniF`B*%po<?No`@={Cvl<8?^GNc{&bw_;A#zcgUjp
z`1?0kQAp7+b4g)3BVKo(=}VoZBB+0_^~9V%I6~a-ddO1vQ@i`5pO83v{~~DO<6;Q~
zUPriD77fRpt-{^a(cE`!M<j&nfdOFaAnvuuX0)fkyPXZ?XwswSEO*QeN8dM*d0gpe
zO3i_esttIlC^OdG{<YME!UX@kh|bU<t_8N-11g}hx6^R$It6X!8tFl^;mN@KB;Ys1
z`r9J$d1Q;jlGa<PXuDV55Wr)wd4P%JWw|`@itc9^tZhx`7Tr8yz6rWvAdnbs)v&Sj
zZ1qK{V=j5P+1+$_csYhg@}-_m2+p5{XH~-Nrv9+0K#^?9;v|fk{D9=C&(NiLjMYAv
z5yW6n$Q+JU$;-;G?mXTSWC@6!djjNto8>#yvq?cQ?}OUqf47-m01_ee4R%(X4RHoy
z|DE~RU#H9e+v;x%lg|s&)~m(4z}QpS@n`}}hUmbZz01CA5Wy345R)YC3_cA>cT{B&
zlq~=>Rs40P4rqMmh+d$~a?II{$<?9O%sb4|#RwDjcD+O5y7%#NS7+XE1*o3OGO(DK
zXuXOLZe+b}JqjV&78Sd@{lS}V7Dfp}k|XI{sJ(qqZMHqy)M{KN@$Ja;f5iTO@A?O1
zuwTT<OmdnPOYY&IaaU-I(eFjD+3<kcE3mk-FOY#zpFUc_rMOH~dJz_h80%urRzV;`
z!6l^yI_fTi$nOt+>T_{nd4iV~w(OsQotaCj*Ap0Ol>SeR{hXfFeF0wXRyOU800G$X
zA!-X3ly!&qY!}Fu_K<HnfMd6OinUZdxI)~lGPgIr%4G0~ol?*`D1m7&PF|Uw3wndS
zJw&`77Ca;G;fjoQ{`IyDJ3=uZ7*V~I#80d8i$T5FDP4ER<uxz=@B!pk*^Vsg0qm?;
z06T`kd;Xf?SA$|(px*W?66YdNcA+YzaoYxFSm;(a_{Mf0kMf?8GP5S4!je9~YdOrq
zQl@L*+KtrUQfK8Vz@bToP}5+`*?XUr<<-R*KT?D$F>F&kBZlsD7WR6M$e9bf#84+<
zt$fTA3Z7-1I+EaZqfmjW+A*hnH^&q7ux~$$RE6^6$shJt6lw%bC>_E57j~Z1y&kFk
zXPfZ_A^$mWTz`F{6B@)d`<)mx?cTH;Cd6}rcD9;$3Uqonm#r1X3xWVkIHQZ35c7H!
zp?EweMCNTL8HZIO`R&_&At4TCap(AI)#aN8ozcYA%Br5hf`c7oy|`B};{FV*<Be-=
zrwsG^DQU;K?Xt0%niB`UBW$AAzR6LBeYyx9s)Ts-y){nyQzr4MaTR+SFBBCDl=o-L
z6kdl#J`qkHrC;>A2-fPX>cm^#HtOZV_U8}myd!3ILk?037G_O6<1_gX#8_2wq^4+=
zh?`8<^i=eXEo79JvVy(RZ^B=-Y$q@rnRP>Rt6Gj!b^7-YMgM$9CKfX+oj10_6zA0G
z-LH(Fx^1-Ut9EccZMxwL)qaCtY4c)}*j6$4M-Vb3FsV#x`_Z?*j)mm@y$ZH!#WOg6
z;kw`n14u|B#N(sI%7*gfR6DjOypzPv8i`urnw3u-dg`|v1504*{>{mdh{8_?e)8yP
zEMtwSHb%#BEUTtpoXqOO-}k?t2Q(Q&JPzq){#ah`xePN*+t5mkbzSglaJ|*>L*M^F
z$!z~EKZc!Q2B}rC>c2bf`{N=PEs<EjXc_KZOkv)us%)^*o1_!A_5>cIZ#YxZqy0sJ
z(5qIb=Ccd7<<)H^-);YIB$SxPiF@L?+U$IR^+uuZd?}eITMi-;?+qX~9j-cdZRS17
zkYbu7^BsPz6QkdfGG<uyum=H}4-iQC&u(q{v5xkSw4G|pW#j(Qy$R~7`D;mNFmQf$
zk*m$wOgl_QX-`oZj26--$=|@d`KI9THyc3M0Q-wXbD5K?3B0ct%P#W?WbbwJ9c$*^
z5!7*n&9Hyi4EJ%j`GLW_Zadc{EQ#fw5jpV|jQmO+!2^^Y?gzhy{1zjK-c7&xx_^fG
zj;$c;lPa;!8%=H51V-whJTaePSoUGc&@0(#>*kwchBmfCSv25W`GJ4NzKbN+HIQ{F
zj1lV`jxO;cw~8jGmPNZC&Gk;Q;u_Ng`zG~O{UvLhK9Le_>|RXQyqm!pusgo|h4FN|
zTCHUGIr!8+`Wo#a+mXN1PScRApI`siSN{$Ns-(rq=wX;vL3rx8@k2K0SvW|a|EnDM
zE(URKzk2_b+vCou@%nR;LHv8f#a<fB=hQZ+tw-P5T6s28sY26A?3wW?v9ZwEgc6H*
zxjrjON_;Z?hKt871}Db3O|(A6Z#T@u=xMfU?rhjYh34VIypCS9GIS8G(epBt=h4Ly
zOU??{B-Wo<Ci|@+W!xK#uhk`wy?bq2)jh6y4!%C<?&&Tp_T%F&*Vzm5|JndT7nX_e
z2paLm7{aI2l|f!<)snIq>SzaGC3ghso3HXmRq<aEV(%%rNHe*kh@REZV<QG51;LmP
zp6it}55JVf<EyC=wJm4<cw!q}Qpehq73X$vwZG0DFpP@ELwI4(rNvC?Ti(hq*J>|W
z9dwK{QyzeR%c+{4Mu&=6Zm^sg$NKrqZ@|ioBH?BJ>*ociPc6#2TiRAyKeK^|s{g6J
ze?N!kP~F^0P0b;>XiK6k<N}yyvh!cB#kfd7Yv?lI#~}k3FjqkF<!Xgu*zsB}+sxK0
zVvXBgvzW4b8Be|%>TKyO>3STSb695odf)#;_L01;xPD_?F9nswd(%A5bKge3Nwe0;
zVZ>mwk>I_DPRn|sTNZ!dJXa;B-^RqZosT2#Vh^rYU(lLMEsP<foc!z+f%DvNMkxi!
zC1DAZl{K{S;j@Q7{*(-%HjY|_voH`!)d*~9X%KZCJj6u1(8~W`l<NZ*__$RrFOO&9
zP+;{2d8{6xMs~o1fqAd-azO{k6qNikbg|~U4#7l>u3>fse*B=o6+hsk<X1#|kwuCb
zk{1fTIttzP8=K@_zfoeW*7Xp0^()JzBt!S88>Ej6K1}e{cQn-@J9dX5QcC?&zjAwV
zjLYJhXx?}d4)P_fa1e=So2YJ&1@37Hdfs?5-=2^YkgMM$$Af+K@$Z3j!MYb>4Z?V^
z-)#hS(Ixbx94yrdr0Q*R|Lnxv*NBMD&i@LN%%{8X`}0(?)Fme}`#{%v{udXre)I29
zp^gRp4eNil_PtO?aRa!~@}C~i?Rc6ew3qBpVrv6lwZ*dae~3%n+HY0VEh@hNTj&6Q
z#nUx_mFQnzDBEk_7(v3^Aj+nQR8IM0F7yXg-;+<Q);kq?>2h0}6)*@`D=0qP3_d-n
zdRflmoc7_T5CoKSw}z!|Rrf?|tQ=bdSOl8pxPUbK_WkvwSCh-){u#D+0-oIjbY3)3
zFv9a%K2q2fK@g$It4agD%cDo;D|E`$p#vcWAJ#h`n5}G-sPOu%M!{hR#gF!U*Ce8%
zNkPOljd9b1ag^7F(Q*b%-?YohO?R!RT(RMH#;Y&hF55@`6wZ&fAi$ltZM7nCv$pyU
zp8Gg5Tyf|x3Bg#Eaw}wJ6z}&&l4NZN5hO0=DSOy>j=QsMfPq2;g%kcXc>><7FUaZ@
zAG0d?jhHOhIlx}D8k6=XB8DVv#S?HK5?#5?C%oI=Xv2pHhcVA3O$nMiv$tliDt!3X
z>?_{aT3^aeG(DP-1<Q!{<wK_2*00~iUQ03pNng063BuG~e{+nZ-=fjR{O|b?!j60u
z{*ZYcyAWB7^_iA*gV@j$WPgm6UuYB{=?yY@bjiEVCAC-q#1Ru;<BJG4m{07C9uxJ#
zM|Q8e^&CQ8a>r;Y%M$S!t3GhgkF&s4slLS8(kgH2(lmm2h;2=4ZUX<_wrJGVg)t=k
zeKe1NJVC!e518i%J#9>$S^RRwo09=A5NTygZ2yUn`P+aCH$pAf8{M{ft$vuy8R3%|
zxI$@oHOHMq>mT_c7d7=(_16tl{t<G##Up&nzBud$$6aJ(xpFBqd{wOt-wd=r<<SID
zMXkf?8|CdpjoWpl`kdPQXf&Qydg4^RZrX^4{WAb|lI^;zfYkBW&oowctrPJZMe)q;
z_E_$*&a5=mDBXT)GrqXniP$zQ^~aZfRPcePsP27ja9;m#0RP2JA@;rz0Y?;&%-hc^
z;tV1EH9-MFGKsi7VC`fZ9O9=p%hU`DlI}{roQ+5ZfXSIMlvF^Cz#}MjZ>w*xDbS?4
z2cx<C+Fz}4ntAbfd(I~7DYbB!<lvqpmrkSW)O{B@cIGIo3!Zkd<8!$Obc^ra+7(BA
zqj6hsk?!t6v#q?btdA^dkM{pH_CxNFufP`mF|5&#0IsS)2xw}dE++_ychEkx5Rgm>
z;g8)z^y+UBmbd^PKY;9R35g!1McGYHVhCUS!Y?Y=O+|lWYWuQr4DE=we1+tqbi{aJ
z(h->3Kc(<Z^K4ut%@@%{kdD+Mz{3G{CkoUuGh$?3!<rhey~x-5n;HUVgsg4NVLIG(
z9jPnOZl=SVL!<qS-RpvsSWA((;X1HiQrl`m`uI`Xf;X4I)L4t{b&`718<xMh399Qn
z)jQ=-!B9LjJfqBgdbVUQZAZlSW`mz+!_6I!cX=9Uoj$^4*x{$0e9l7+>$nD>TpQ@B
zzZ0G++i9(Y13Zbb`R?TAlN!irA<IHCV><*};pM!n@TX%x6foNY`rOx^oEaP%rf|9^
z-b0s3d!YCw=a!-vxb?mj&z0NB-aL=1RrDx06y<YM7rAezN@4TD=W7cZNm~iO%uiBV
z^Jx8;l`1J|faDi)@%zJWK+NIu*&w_fhx_oTP7V#-!5(>6{rR4VvXxfY&<3s5fQa}u
z%C8M*k>5U+uIDtdJrD~8_F>g*j2p}J6i>a8(ApO3KTRBaR`7}>EI6OEGH6C9u%LP7
zZBx)TsiJN>FHKJ{cTWxZDP?KhS?^4tP%|T?KsJf-H^PrZdxG&32Fm%bht;$?lTEa#
zu7fP}Dk<l7pY*d%n1q+Xr8Gg_S~-+9^a?L_l-UbOW)5O+u}c=)@;^<{<_K+K(Fej9
z-G-D5WnSe(2}plhha=76yZl$PO)Nqm^hS@_P64SlfNhYQYR>hrQ3pU}wc_N7R<{q#
zOF(t|g?lYhv{mu#tGJMuQPhJxIw6d9#%F*r$nPA2>X+vC$Hxw`0M)uM>$W4yb=4m%
zRrj`jsFD&PRdao?<Jd3@pMRu_7f&a9x?o<W%*aRS*z&U<3k`e0bXC4J#9-N@Q3QsW
z>UL1sb#JHOeI%~E^S~W)_iZ9!`K{1%itV#<>87f#h*v4MCn_@WEr{Qo!z2j^Yi|qt
zu&D~Pc*;)rU3`Q`G0zS+4=fG@sBOX~W*bgn>1B5QgPx4p?6BzzzpWH5^P}J`{50Nc
zXvhJvH-fi*XVMi$=2)UzKFnCVPb)iwBasi{diVLH1l92Y#Zid}IeT%$6zw$h5r{zY
zj8<v~G2T0e?l*}dy&Mx|c%Ten{EiCKeAV%~6~xA1gKZmc4$evpW(4cnUKJ|(a;GrY
zaAaeFQIwDrX2}%Pzj9+mExlLF#KSsXSZjV=B^&wjZ|d7$R>41~4e;nGME)=kF4&KQ
zDA5C<gOo!RAVZ4H;9y(^6xOE&g=I2RZl?or?|puDkr|?4TP6xEPQPYw|3J;w#3svy
zxdGjx2|XM~v;hPud(jniP8d)d1J)hpIjFZF>;22F)%*r9$TL6#dzVYQNR^`8@Zc^$
zFGBo^K!70Zz5j^nPJrjPByMZnDcd1|w!b#qV&sC?L}}RC(p*{U^E|J*4uo~Y@8<ut
z4*$<9@Ft~v#}O)c&ar_?WVi#IWKe(@sp%jO5E<+1U|P~Pm_7FH+Q<dZhDCLtEgpFL
zFlWun!#cOaop&iXNsHbfLwtrDyIM*fp>9=XM@$VzZ<ON$He8!7q20v**J9zy9e}g*
zX97o?0VDZuS{RhT{8x(tb%ueau$vK{WTUCSW9DsJ>}}UHMix!h-x>dTBdl_zsuk5X
zWQSS;LyoGDI%{P=J^be7{r~qu=mUk7S#Icj1WiV>zW~pOypLx<{!0tp11y7t&73!S
zE7zg{80f2=VW>*EvMA8i7ZXfL!C@{ZmY2!Ka{a96z;s$btpu?raCPeQ`~8`!37Qk*
z4_e$0j#KaRxg7RB%^wTxm9(`&9matd&FB}PvI&5;2#yOnM=aSt8o+m^G?v_q7=Pu{
zD!2lTX43)YZG(Rtp(3x#1xIb;IOa7hZp?Y=E?7fP7Cv^2D2BoR(_;T^xS55KKcHt>
zp^<U7eEOZ&2)Lzne6Fo1v|IAUEH{C2I7qJBISC&T(IomE)Rc0qy}q35x_SxUq+LBg
z_CW5<%l>4Jw@Ge#gRepz@oeAKU4l7cixM+b(D~bED<B$qs-kHhM}No%I6l(3cHS23
z7r&9ne4%=Sduf$kti@hXf`;Hj6dLq^sU~XpRsfB&l?CpGmbt^Ds+*|@ZgWbqgx7-~
z(#_JWgE%dFMP-gMp1WGxFooP3{2Y_~T#@76|Avvq(cqNZwtOHsW{C{#7r?l2umq6n
zwv`Qyd5w%XIGC>!fvUCO4d?URs6j|MdvX0KHq5Fii=X}S?Y-y4Abd$M{p}+yN@nJi
zaN)~OB_ZS&{<?=2Ds#$@7r{dFy5cHNCN%qIT(vb_t<2Qlm4+1C?>FMVdykC}eK)k(
z&-Xee(l_=p+PSBe*~;)>?xxX*=F$}eKCqe%UAJKh#s2xIJ`gO!O{@NC(TJouh8fO@
z7qTVF>2z=s=^@WO+65t_e*|euKTY7dlTNvPsYuM>=D?RWT5Y90ZSo`S<oYvFW`6Q~
zuNmSqsdCfgUK7;~a}79=8W*h0H22|1wnc(Hj|T7%tB8F?azMsFPq;XqD)r(+`g2xp
z`yKY$`*2{&u-Z^QSTRpLSJKp%6Y#n{i9hDIZ}MfL+g>q3C#0}YZ9%t=m~cXi<w)^!
z_AXlZXtW<z{awAH*mU>iHv@TXe8@Mc;&H5(-}~bTpTA!a&K$=Iv~pg36;Jp5P&}vY
z9uN?cf*mph-S;rdaBl5<YzEZa1Bv`^=9Uk$hwoQXH4z$$QuWKLpf30Efb{<fx?cME
zwj?o(jLR|={e%&)5TuQ?_%bT^d|<M>l0Ec0*PjR_gg|9^CD$<C8LS4upQp)Mha6lZ
z?OUB2KyF4$tIpmrC_e0-3X{AwGi^o1<2kT|v>MwKE!F-aq{I>ekHBv*NfQgv=?j0r
z9u*s?Xw^`U2^y6P=1VsBn_nk$<wHG{k9R*(AF7*W6i8r3m&sl%+>pp1h3*QIV<98J
zdH?$-GYqR6ZN#xmvT@tVR`_nqc$HjBaCgU}-FqoZVxZ77xnDK=s)rwL!SMO`tRpyP
zt5h8dcgdXBqRN2BbT5cR`CR6;fvCR+97&=;p4f{d0l7cVJk7b$Aj~bTF}X1gU#%NQ
zB#5S?BY?T@X!k8(Vi?UPAA<a;1aEkxT*a{@`ZyNi?JI)owhY=TiRK0xSzKSCsWhFq
zb29zkGw)}k4e<5;Y{2#T02vXRs_g)WK?kl%taJ=bk~<=r0Hv!Wv9<3WfSWlZ*KCw?
z35l6-;Cd$Q?lTM{?IGXY{^vsfUN#AqcR!Z<;Y+@0b+GPU1A2|>^BIU>mgoUO27xl>
z<Mxk?<ljwT1Q%3z-Uz<{J6j<7G3dgEa)84DUm}d~1>l+#Kn*1&TIO~H;raSo+1j8+
zuCo^|-H6AZCLh7Yb)~b3Ml|!KQ?UsWMbHb*ZhT|$8fgZC15HwwEefJfpTGP@LJsU&
zo<T&Tj6@tPR?ZT$Rk2AZ96L}P5jQlw`Z6uy#n~}3Sg{W#>d&k#2k2l)r@aRJayaBD
z^q=liBKiQgKNOzzSS49AgtiFdXUiJ>d$-wK2J89Kt)C+?U}`ngTdq+4;~7IQeFoXP
zpOMBvSG;lC7i^n1PJj|~8qd}Ir$1F_4DSKUj04#G_kl&U$!)4AVp7su9R5~dP7f#e
zSrlb#Eu`pKhRY6JicS}q4Q7wgPsCXUsPZD$#%fJwb##N6N-{hax)}iCo%CN7eW5XK
zLr5I%h7Z%}Q5M7$mbB?k-+M?qNhnTyU!?cZPHGwU>SB!MWrvqQ^idPs5j3=3(pd!S
zbb<OdSRs&D3T1p`8x5s9XaT;PMaed>-DKOR=zahX#*$U~PDZoteX&Vo?oZo1;;Cwx
z0Iz&IEDDYPey_=gCy@0GCCH*<?P5C^6WC^Z(RQ~+vIa^B^JjgbowkJ;N0qi{8eW!r
z-gMbLX?qCscEt?gaQ82;FIhfuuziD$67ZQNr1t>T*b)m-4l^`MWbli}1_9n7#X&lr
ziES8_ChgWn)GY;?^FHV<$^h_`yu-;_3A|=|K+>PpZn-;(G;d=vyUj_iZ}D3d_cT*k
z@o|KnBW3&W6vLW8aR$PDnUodCxDFShgGEcrk3QU?NQzdSELgcn(A=oY3tsEbqO4{5
zU?^To8y<0BeX|H&)GsMCycgQ`Mg6}|_bK$frk6D$WJRPC<!}X5{Ho1s5NE8f{$qKF
zcLtEzM!SWq{m}??Z*yJ)3e%jLM!nR%lcc~pZUVuZmL8|NdpgofW0T>|y_O{=l7(N+
z@lD09Lu{kAo0Fsgw#^>*YoPkX07=~&XaWe81LXF!gcP5L-NIci+}hV>R9T2BJc!bh
zwR9Id*unn;Yoxbs_+%KHN6@PEK$_OBUZGZ^CzDK>qgR*ld{X5$;?pFTDAO0J^=_4<
zmB;A%3(Z^=LX&7W<MtIoL(UE$-8fkQE8bQ3Vgv!G%i(dRMIq*9;*t%9weRQI`A*zN
z_t2kuH7U>&;=TOc6W{b42J5ZdEO`$Po%kz)erH_r0{K)i3oPf?(N*SX(b;K4s8Re>
z76PV5EVsr#u{gi<xKkorv<pUEvzHb{H(QIy<_!10jW#tsPfoI1_QB7}v>#iu_}CoX
znj<Wgk{R_;rs-*{9W~D1gYzwRVx`LLXsjB+P)}^ewl5|Tr|iuoc;Lld5oA`b8Rhr5
zv)@G?m}mThxqRRjM1HJ#pR#55^X*)@-e?5$A;MMQO4bz_|L~uo-#?s86-cYd7+N_+
z^Q3hcpSvG@nTcBlc2d(-9?(|nr?K8SnX+XdzqMOcLi4V~Ns*pXeFKf(&86UDwNRS$
z?_wO)Uvb0&^AfC;@;nzSFWE?*@$X3IeoP*y85w7+RgErO7s)f{PQa}YnPQEahfLMN
zuBLNlW_8%{EcaVt0qMh!pXbnAmRL}jRlpjDfyC*K=jXd(Q{T#mblCK;x+(*@XAp?u
z$se6<&Py|QT4l9o-w!<oUgvats5zxXyx{3l?o+z>u)@y(S9|!g@466HQIKOl>LhiY
zm13#l<F0z>b93RCaho@ve~jB7lu1|`(!Iywm&(r8@J7|(s?`{KVWE1dBMtkfzg1Bs
z++QExQwTf+1Wf1~o{5XB@4qG5GD;on%DWipG+uc*ZA8XaB>@>Xr?~CEu*ps%km~zs
zKla{epsC;tXOQn=o7;L`5P3MBxB|sU!)zv?rugeXO9YYinVs{&HsA8)oj_=T?d#I^
zXRKh9i{UdDt6An4eNlYRGZa;mt<;ubeO`%_J)x?Ju^M`LtZ`xdb|&@^d697MYY8K=
zTzG{okBA5P?UdKne0}w+<C=#XBD*8W1a$1*H!h~Wn5wJVKN14igYApaQ(94;_9!DQ
z*$W@IRns8-H0@$Ygrj1b#rBSDcR&y-!$9DD1t&N^dH^Hbn?z^dx>;_Lx*a$Mys*QP
zl#z-3%eGQ)85xdiXbN$7n=gBvHx~|9=bDhYw{xFY2wY;l4v**l+bJ8iSb>9ksLy+I
zb(<2YJ8sc-Ls;wR@+cPN>rm!1A93H~8bafE${9vy%NE`EfVBQLmVVPp&51ic&~=}7
zXNvf&fmp&Qz_6Uq_b#KtAx0w~xk$C{hM#3CV`v9kbH*_vS(W&o*>Od9=e~@2AAX+S
zw_>aIaeEModsVaUmQ5NzS<(5=v-imxayGUbyYKTNc_k$WB%FiWNiMC}>1}Po-8D4D
zw>d}{N2hJ4LAJM)k(NR-D`F|T75rZopCg%2rN@-p5q68Kk=8&9@fzqTN{jvslZx18
zdp<tU(DM5n2>|v|?%=@QKJZnbM<f4?`Tb4xyMpz|#Wl}Yhc!J+aNRz9Xh#-R(}q?2
z$VpQ`C1L8W4G|Vg4_sY@tf4#C07~2%u&YftY<iThBgZe1i}O2SC6O#L67D_?wcW*9
zibHi$5724H_}>FIJVe+qP!MY@%0E<eCj#VXx&<T~*%i-Y`^$5B$R8xy8&r{%L79?0
z-$R&^Jr58@$1d$wz_S_-m+v&mVh>tTeM@}OvV1JyPJ7J+4<$X_fv|Y4Tg<(+mSl57
zJ80r&d6OrQ$n=1&n8x>%+c?r3fTuPfuBUg$2XPZ64Z9vMzWRlI{|92Cx#3GrB|lR&
z9Su%yDmE~iaadqAG01O7_pFZ_^sdJ;qrsq&)Ri(ew+6Fbcaa&SJ8yDCW0LppxTfaD
zy?n!g4{O$@_`GT3BYP{-|F(hT!?Fmh;(+iai<I0rz*C<^H7H<2tF`c+7HtxXWgLod
zbNqfcAwgsjOY5G!zwMfAJ%|Xmz*+Hn5^OH&Ext|7jZ(myw{<1EBw#DFjnTNLi>^TJ
z1*uHYo|ZDDOuGg`ZKSLwE-4<N0$TH*-M5F$R_Z*@HWWN>VnT0+KG7+0SWF;nI{RyV
zDeBx&PW`tRz+Ckhk9yTR-AegB67kL3-({I+T!@bqjM%j9L+8Z#=w*QX{HZ=cMfq9B
z+7z{DMgRj7oV4q6t_UfC|Lk-XMah-K$d0z8*xn+JcwVn3tOKkf9vD7%gI~+r{POtt
zTYN1i5vUAy*5Qz=`8w>UQZpboR2g0XekxwzG|*hrK!J%Sxlxj{1|y9M2Dl28@baO$
z2teB?1`#5sK6cBaV)Lt0KI0KF%-X<f^m4UdgcrIHC#R|bE$S@0@RTUngg7Pu{Bp$b
zoTbxokY}?S3Otd0zL`Jtpap^9i57=a4J3{WJ)<Q_>VMvdSmNYcEj+n~O|Luecr8>j
z+^-SX$7&;GY+s3y{S&FCVrqSqUdik+N8-Y$qY^QMorcNB_Y$7xjh0z@X?eOSqSp(H
z+(+{;K`;C%Rf3(p!pPX_akWhQ7HZuP7|Q1J)l@=%Gfl&!aJl&y*q6aU-4vMDMH^4m
z9J+X}J;)!~a>_auv%caLmA2b-R86v7$`eZ5kkX!veOhzkR7ZHO)W#OTrcJrf`{kcb
zdms9_c#OW!iHY7*5%{tsOyvr7^}Smqdt?roA_X5!`1IuCZS!LcKui`U#z9zZ6szg@
z%VjsfjpzftH7%eNE}kHixql0v+ZAmIimraOKg6L=;uCUzb6%c?GiohhJ`-WK!iwDP
zU=+!$;WCXl)Dyum1w_u#;NGfmPV3{G`V2z_IC$Os*<`pL6Du+JNY?LX7r_%;gPH*h
z930k+g`=3!P`$X`JBoZ)r0ZzCC5Ow0N&lxbt&h|q#I{x!Rl%Gc1n$nS1^yN{_rgjq
z@jb0^3&|f}S`dq)U19uVg;Kc~i6$??DxlFN!D^cD3c<#lLESHS)b1(5n+slXHpv(8
zpn~Z`$E(Xh&Tf&DcLjz*yEyXPzonXlF4@xdJ!If0F0iWlUN_Gf9^$K;v+w|umi|1v
z@~TMI!$jA)Xp=Pg5ASHjQcHEiXZSl0J0r+4XlLZx34PW9;DpW2Z)_$7vA#QU%J6)s
zjj92x6<#57VJ9gec=$XkC=*g*bkqdP6Hu!vk{&>JE<NUV_0!;dZtsbSaC26{s+7^=
z-u~{QKk#l)fikT{!kg|d)W;x%tonf_&nd_t78MR3Yi+nJW<K!T?%A&Q78J);Yk7*h
zT<$#S;Huryu-yxnN(C4BAE*Be5okp3*|D(PrFcp99`0{9gVF=66{Tn!trnXkj^1DT
zXD<hq1AqKaguchqnh|81<`lw2Bf0WbQ$wly&jCxEBjtsGA=zejAxo=*`i~6T{q|5c
zq-}DBCKX<tnyy5u@S?E|<t$Szrn?67!c9r>N5IFF8LgaF|F9JE3Xa3}YXAc*f<}5n
z+M?^B01sHQt^>_D#{gDB&yRE$86ZZKAG_cNu$<p-u>^+vp3sNb49}{0BKq^$xOi{K
z86B)hASL2|9*nTqQg*UziY&ZC9+@8d2m>LTX=b$M9veJ66ARWxoQXb1SloBJx$7pK
zwfESWX<rtaz!PowxnJI~0>-H%+G(uUn5z~Uf-#8uX_jWi^Vb|@ut|^ai?Zv@BRSz^
zQ?1ul*D2A$*|t{~i_}$88oRVbPhwuvdMuZPdG*0rNT*~uU?v%Hl3lk5!uy0VK28H)
zAq$7cz4D>>tZ?4^33;?`F)ewUvTq*=EjY;#LazeCC9C`>WbprU%tsRYTa+6bPtlim
z#(qRJoKL8q0Q3skHIYpzM5z3yZ<cKs*~bZ0TRA6M=fa|nGT!ns?g-Hzb3qzN=6%sc
z(Ks*Yg>Or=(k-5uY(o9^OU9PUL=x?s4#q6O$}cvG3Z$CbspBjjM$vvKS#(-Op9YHb
zSG;9DsT$7fajnVb7s|9`6Z32@mskU4gvu({qVLV!1s|02n<B(vMv+wrk!#RK4BaFl
zXSIh-x3Wy+cg8prm~L!fq_lh+1P#Hg1^?MXdIW?4F+j|zWnZa;Q*T{Lzxo!1WOi%u
zJnv&>&Ho*GgF}SVv`nqp%oy!yb?h3piP=K@*WE+6(|~)MOkq5`y(ODd?%LBVv&cIT
zX#mlIYVj@b68XvXu*AA4sRxYa8*CRp@$j~lGp-2N*Uj7mik$%;JvWMN^#1xr69w1j
z7MR&*Lr9K+N=5m9ddI^O3+U_!hCNl2jEbGt{5I3M;#>_^ds6HGH~ck~v4$<V&_N_s
z;fd0O0U~2cq~{B}_2IjG8zQN^k7ipzB6cSqf>px&z2bkz;*Dp$7+lGvszci3HJK>w
zOb?GsGwa>6H*Qv>AaZjw-UPfqkp+)ii)oMB{U-#v<jB{nJj*~U=Ba9u6X$z~<}YY%
zJ|gug2G`8ID8}-aN{5$1LOX=IY}9$){CVeI&ah(^Oj#s)P_N9}>izmZ6?4eSo9fEc
zH9H%MIjXltEY}$LbgP4j=3|7-IOJ@6*AnzN11<6}5+7KwjFk{li7HHTU&N{!1~%z!
z_-0^m%<e*c@i_IM1@rCW!@b2XGs6$1HIIPD)(vVMg;`;#KX_vWuLrNeNo10C3h=L9
zWMY#s0!*DRvE|mBU$gh$qztjLB{NX!d|B}Dh#=cXxxO$vkCb_FzK*Mzt(aB%C&}@K
zIn~J1h~r9zp^$pkqA^T1MNlq7lr4HeMosW8`7CYE9TX8(omK?7W)03o8>J9+`Jt$w
zyQJh`n|jlE^WMN_EfGV8d^;GCN9Fi};Oe*vXCXB|w`+bHk9dAMHO}qW#!hMD-KmL-
zGNksSL91tF-ObmO6H1AcM)U7ZnD{`u6GN+jXU<GreiY<@^3V-uQw<Q=9w5}E=^}AV
z-LLc77zQwdz3_WSqhU`bt94%E4`VrfDh=Os?YKSFxX(Xwyc3EMMoqxFkMHyApssa^
z&hrx-GIM5sR}p=htH{HX6YOHP*rWT0>r&2=S5|OINdZ0wrxa6hASK3QeB3O0DWwaR
zp1Zj3&*I_nYO)^Z3rb@o8?D_pAL#;24~V3DFD@&lSrxlkE6HEoX3czc+J1?)v$-VY
zI@xrf@l2)uz#4ik?0XAu1U5P5tNw7GP&MiSf>qPbQp=`m!?rHIg}Kb+AqV5U@nS&q
zyT-T%A~#*@TQJ~Sr&dhM-RUEG+tqV>2iRR#4wbojyZiq+yibkDqRt2>od-tiCYuTK
z{e4nOLwmOJ=^<y@V)78%ud*3Dn)$tkJ8lh4h+=tNacshwCW^UFmHP}09XB-{qAMrV
z)obt3ND#K@H3MgJWDH~F5|Y;I8pQg3_j}YnELS%bY?!2Fe79rReeEcq%<{Td&!G27
zp6lYfhVU(g6y6@miNp>gGl3?ShcxNFmpTL=^l38IDfGI+D}Q^hMv$2tMUs~JuM!d<
z|JR`i>Q)63ToUEY#QVW8v@F&E-f3_r)FeV76u+7u>WZd~`UH5_^7=6d*`@Hb-K>*5
zAFjh%V9D9qKCNs@IgUybC0wvYT$yko7~>%0BJ}pS%#f$|!9U5K=jO*a{*>-*h;%aC
ztSzLfd;EftMB)$5P2Y`k;wx(HQ|}6fYuzRHCc#frD(Eox%Dm~Q3i5Obt0r-AdvouY
zQ2U#`%vtSl=nmDG#`=a4-x+P2xTTf;D(rlhF<8XoIsp^esE<aG@QMDq##{&?l1hKj
zAXwGS@N^Lny6@yMyA%0ZuZ3~cUu%J!RL(6Q@dv$d9<{oG5dkjY9M=33#R+i(wZ|xh
zn*S`_(0RKE8wv(1Znk5RZ)udT6Ck<fyKbg3_qSMIg!3BF2FVj1+jen~*bjBxug-ZO
z57R7!$9{E1G&SzZu&-wwgbbc?W;L3$fmT;to_iUA%AmdLR|*DN5r;lH&;1#j{!ygJ
zsQ95#zULEt`OQ6s-o(mD*)yq+3bd1)_YsNXfx!#Z@^p1^LF}><y_IhB(JF?6y`<9Q
z)y*o?${TqH6zg$gHc>cVRg`mCf9R8)454|}n3FFpBI+X%=<-QctwA$^_LpF{yU3=q
z_OElV1gr^+-2j->pSU`vAe0sTeTbz$;g;q_9raDxsMGR)l=;7-LX`x$QscWok~Cx9
z31aR+`Liwa)rPkH%PVZeXeqa1YtjTZ5|>t2q(!)Pg?cJs^_<x#2N%FNmQ&aBLFeqQ
zez9CK0Q1|Z7SSJq4?I}57n_?+-21r}?@s1?H~%EgRM`N7;=J!oY9lJ&J2eDw_IA9}
zD#U0K?E^Ax*Jvy#&g~yX9*jN=GJ6&~4;D7|<u1_dA^>EoBvt&zG<N7@TsyjWBI!ZQ
zuRc8r!JV|&9s9^idMI7M0H_%_L_?s65$@BZtfKpr<wcxFiBfd3*<;`iGdQZr0cWV|
z6k+vzk>DPm?Ph<rD$4agx;D5p0`LC*t+617)=eYc8M}PIWPb^~yuwsxx`w3dg)R}r
z{$$h;^PhrppJtCqw0R-v#hJfLuEjXkax@QCBo+~k`-weWmhgn6P>9^TVnm`0EN*67
zFzN3eov*NKwX3E0R=S0{gA0lo3kn;OE9+I~Rd_}_&NyK*MMV@R{xEL-Or?&w#wx$j
zNU$N{AN8a$b18lkMi#PzCd`tT9o1`RBCMLQ&6;KEpMNx&9iyR&{N_)B+f^kS9FtGv
z1sM4|n+W!9mKHG|<3HV+xz;nfl&W&tbrKkLM{-QWf_7NV<!oy(FYDE$6k<5P{{QJ)
z;k>Iv$=Sq&<a*ul!c=F0nz+NziNXWY`@L;{$h(Jc=}G{!lGm>;MoDanQSh16k)7(Z
zXMz85(@n48VUkYA^(k=2?3EJe#E;-Vk)pa26s~|m0){D;H1nG~6bah)wHo(Bp1rC@
zXrtXSgP*7~HfQr?e%Upf(!h(ztQ=Pxq4!9xXcr@-+mw7<q-6e*8F4iBjrM$!U~cEB
z^8Dl`wJFN36HNZ?2TF}xtS?C`dyAxKUSR4B{wQyz;_iQHh2r56*4(!M1XdK1B<DtE
zn>LkCbj;3d%WnduzmdQzl+zU0NLd@XzuVeV&mes=pfh{ZZW!~X$BQ+^MSt6x#aC^2
z^H86~fa;aQ^<9~-MQn~cR-n-6zK`n~At)K*lHmSiB##DN46@B`kN~1RH2!iLP0)0f
zfAdIaIQ9J`Ed~<XJ-0%erH#-whRaMdn--skYM!h6{>tvDDpy#Rz_H#0DOH$xBR6=r
z{#Y8_i_RBZel5E_yo}tnYS2n|*&slmU!IL56YwZXX8MCWmU&%=n?BfEZmGRZa5;SL
zqM6K{Q%vWG8_-b@ORAaMdq%IBakvVT-+JhO#pqhzJt<6jZb+QoA=c7NWpZsR<SFFF
znPIckB~Z4t%y0QjOI`I*WtS^Y*3l?$EhW2;Gv5x$|0`dNsE6sOJWk@*O_e1CI(xYu
z)Kn)X9?y)Pt|fYT&ZvCKcWMxP^^-T$GC1CmA*1HiFYU=QJ$+<;!jEI1#}j-%Of%vg
z!*W1=kmH)ypH)qB+4(7P#5wmxln3PcTBKb7WR(`m4uIOH@TNG#MhT5uscvFkW2*2X
zA4QKo2nze~vim+{MF^Hv-tcE>T5DWxk9A4mACDr<F21TNA$+g&=SF%_E;{KZm1Lkm
zta@b3h|KN2DwNy8>gQf=`JS=?Yrr%2Cd}Ngi;SGY$#oyK``+;K>XY9$-M`03Y?k#%
zW0x~Jub09XpwAyqB=Y+htzA56c(48EuFU}6yJC()h5OFUKjRSV!9&RTM1PF_21ww2
z=Y{apj45X4vm`G`O|zr0NiB5a*b-mx@gGIxy!`(=I@v-KF{*GH2~5X#0Hjj2Un~(p
z`U(J4xPgt1b<^@;=hFa>t5Q=H(geH>&rhEwEL?vy6nTf<sfByRFb;OZ?2G|++^@|1
zW*}<Ys@o*3^3#Pb9zfd|16{I%Myp882fp2>;_<TMF<GS2<1N$A_zRpDM*43YYbM<f
z&l68ok{0$=UTi~V-XdZ|t6t|OEE=N5HR}uyjX&B?RF&PLy;m|fEmW|Od)erSdW&`M
z3r8OmTHBH9GlxhK#nr^ejTpV)`RHC=Bjj}SIfiG=pIPYPDyt-H0Vr@^MhILazGBe9
z@;bztxCThH_B0T(<{U=5=Y;es|B=N-GZ)S)=tqACx}^Vw5C2tSEquLfBRPAEkIlC}
zxpv(G8}&#iK^b^`Cb?15eoeLBbH4{t&56R|N2N)9w!V0IDl!%()(Dt=`Ffko+EO6x
z?Ex!yMnGQlCeUkt1na=ci;gSk9gWXe4_JG=*|keLx<j=Ay<GQMl9Ivge$^jUyB8<<
z$nW0&QLg)Fujlc{aFSw2S>gzyp^M%~T7|APHuz>rW}f1Wo+(^1J==xBsIt*<y{T6J
zT7sUtcJ<iXs)@002x&c=$O$9}8VivE=3b$M95NV!SD&F_<okT>w()55fV5l&2DV-w
zOhIFK7<eQMcGP>|-caENAU2yBlOH{$T_nHWz7^DAr@cl@3Z?Mssix<v43GZrJ7kB5
zz{-(@xnx&t4QM8%_ESyrM9P-wA{$A^Mx5wh0|*_#P*H><|HCbuX9L+@RkVcNBXvnh
zt=Vw40qmjWA$vTh{cP(1Hs(afFua*oi+Di-F(qa<j{J-q<^&Q_N*kD$pMdVDT7P(_
zp!c92U14Ig+_CZGRp`1h<b@HlH|&df49r`M{wB4Jo}WLPgAz^EAh%;@%I~%5=^3&^
zo;|QqUGt)4em6(RNOD7e9{cO(R+T2VKGOLObQF8IU3{EbqP!rRyXXQ0CVC+7&v649
zq%yG5Oujqc`P0`V<dy}H2!XMey43-{3oxUF?iQDMlQM|jDf)y~iHgoD)7m@*H>FC<
zE5hab^YQpv39bKuR$~65NJ&l^2BMlN@-<{YXq%{P3`9XEpskTUTxi_)xH;SA-1Qe?
znrJ6PdvPl2I@X!`J|OqLhK$!P56XEf10<qZ@<HJ5aJq)t<_2<Tzhkpi@IsD6F!JdO
z490Yb`d+ALk)vMhWR)NE{<7s|*fS3wt93-xE}l4Lh+9i&n@X8~uf+N*iZMA}8E2&O
z#znzHc4gy41}rDt%Q~Mn6Ref%K0oLxyrs1_lljhhl?h8%Px7I8dn6@VyvNtNR@CF&
zype+kn3F2f_?$YqPzJCOQvsO4RgSU(AE70Un1ZKeti!X^;<FCO)loi}Q`&U{d=cOJ
z2=7hh_@Mh;eI)O%9tT%}{{OT8WDOP$N38VR-ukq<WjFTQu%G}ew>v+Wf|NW=KtSjw
zA+ZW1L2@k?y_r&&M?ys=E4IV(T3dCC^mkZu0X(Dxklfh=tOh6nb)%;8wGZr`?Lwog
zV+;zr0J(NyLDN-q$XfiYWz54{w0D+8<@xLR$-j^i)-o&lPD;t}rV40n68alxeXDyW
zYsbk?mFsw$^WUIXZ!dWhTqtEE;j-mKSHjn=YZ2t%8RLDTGYuZTVUVJ>m-*9o@>80p
zx=(wdt&@Y@>7<DcW@q_OOYj{76Pg0^K;zJL_KA4*3+}SjNUlIZLJ^JI&s3Lr#|mk@
z%Fr1<hEHbS{?z`4VfR@UplfwCiL)8XRtMM0PWn;sRWfNwN8>rvp4rNIpft_tRlY&`
z@UGUxulz33M6tg|Yv8jw{jzzGX8z|KyKQnBpYpp#hYWu+k=wzhFaqCoh5@dhIfF>A
zEHpS0kUimEQq$HirTkFYm=A7Xtc|`oR>LnDwio*1Z$U})q(28CTG5<feMnlh?TKZr
zIajJ7|6dFyQwM>SElXiDLOvM>tf#cKEo+t$uL0-7B26&=1~S`rwE+Ak$N?*&5&ItS
z!jwwBx_46qB<47b9?3hj{RyX4m2ByGo=lW?48E@A6JQLIlFv>^63J+Is#;eH9Ac)I
z`}p>(iUvzmjsp_HJ(Hp>_OGW6oo*cCg({04ZQuT2ziyM?EW5~XI-&M_=Wnwmgf4c;
zdfe8m$4h1X;v<KC*dc0Jd*K66@94ryM|$W#-DJCFN=S7+uqFr;qxmyi!8iUayMrwZ
zyM8BBNFs-@xF+lkb}lnUJ8FWOPVe)d6#{Vf>n-h!&LlDG8h(t5^Zieg9J2p-V+p_8
zV*A&EKGOm^PW_mPM40Ft)$t!Mw~S9x=N*gWoiKU?Au?>r2S#=oc8E{5GOfVI%JkZW
z43&tw+2Tpwr`nvL=aT}i7Zs|mhSw%~cVD`_3U2Gk8r^edWD1pYKd_OzJR&O3-W%Tz
z*AB97+o+1fM^a3z>)o9KElJ65am2((26N#Onnyq<=R(J6t-rJ0P(KF`q7RH_3{&w^
z6^fBf?Jp$AW-)FC({U`mqmNJP1m0BR!aaPiR`!Te`1f!yjE2A}dbaJPPAi8xBs*<d
zP(%Y#o3z80Y$Hrx;ayG3V7|TWKN#)Na+o%-NCDF_sZiQSwvxb1Kq@eLYodfPGk#BQ
zSaZf^b({Tk&)xELv&Z7DN&t?4R{j6c^_BrqXzTtkC4#gljkG8!3?SWtNQy|OlyrBO
zfYK?A(%mU3T|;-bGz{JUHQx8$v(G*E{kXsE&7N7a*7N+*#6_fd3W~@40}MXu5@BR7
z3AOAaPYxCr3wJ88;YzyrCXefx8hU0g)g2-5&uicbj#&PMGkGvhW=iNwPWS}2lx)H3
zguc5MWn};#QUB+<jKl4IGW?K;cL~?osPnsF27VX?7)*I*3nx2ag3$3Qfz-g7z@rZ^
zG~q*!iPY9q`)e3xQ8);yQr!TN^3fRa_zcD2oGhs45kH~=sgs^|<IR{86dx&4skpzs
zh)d`3r)mO<3PljvLXgyWKVaoa101DQYO_KhC)@z|U+}9P4b<@RZ+34EL1N9+S@`WW
zE6u$;mZW1^#V_<{eLVI-@36$5`zeOT)23FVATAs@muUEEw*LdagnGkSM-=RbE`DpH
z=uSe-XAp^tR}l}6{HEAQB?ITfp+nZ>jbRb2yAVxaYOU7v6G>YJlJHW!wvZUT_+@y3
zJv{G`ok!S6Syk@=4^UMa&KA`~KeKzetXTe_XY;M{u#^4ROpa$mMHYA1$|9jo;2;US
z=xCtnC6MDb_e_B9W5aZq#g9~w(GW$9(h+^ZM6T{I{&9U!lAHz@f{ntQ%gNFeu{gGc
zeNE&18J{6lO0nlEHt<ug{}WhvhX)s@?39}1ghC;-2jF>(0=iu3DLxMn=u{=gCGG_h
zK}G3r1m|lKr!~9DAvWOO9S6jyWCUSf()FK#EBjD{>|F>G|IJ|=mme=MMv<QU5U~32
z?TTn)7O)qFK<;7-9$O4hBp5)U<_wbZ*1-fD2a&&5J6mw0czKZ6jj;7po6Ma-BNzw}
z@X|zoU5%V1YedDzj#K-FW-F}@N3FqZOqH`X8`-^l`j;;@(%<-p92MzV2OsolInC3-
z<0l5DDzY+gnAoqwQG0wbSUyAUF@>2+7OB$5Ub^32<Tk9MHQ_v0l?b?ELtM(~q*{#g
z3RR)6&kuc5zml?DA}og>!~X;YRru>>ln*N@LZ3$F3vJ7QcRW~9SM>^8P7C(72fA$~
zpEd)Yn?Cz{nE9w9h<~PadPCa)6aEr>WCh9KvY)u@3=iW;v6>kAgn)qrm)E)go_G!X
zL152SrAd*|j3+{>q~DL07i7D}jJN@umom`(P{U&?;fY$dfa;kM9s=F;2JJz3<v>@Q
z_(QPzQv(<e0<?(Ws-tP=RoTpY_S?q;xJkn$6d`erN-y&x8D+VM-zpUOXkP9N<^?n*
zhOb7+x57+ZUdP(v&yK$7Xy|~Bj68|?7(+5$Q$1KkJn4zUvj~!=48XS$589Lat3jpw
zLsZIP%HO{QJFntE6a&r8Qce;-Ew*o36mnN>$HfFbUb4Wi#IBfT!TUs9^1Vkqa<mEG
z)bOOX&3vNzu&<@(UG8nMFQr>-(vVV*>pmw2Yo*YhBN@F9$!wT9bKS6fl3|K^{q@U7
zHioESj;A9hi;7L+Da!+WpUuR}{@xDVH1Ll!BFn#D?9V+&8kr*TFfzYH{%uZTfCCdU
z8KDLAzP@=8lg8n4SN#moCvHscZkiDhCP;c$2Wr+~f{M0<>R}B)XsiQiL`(pI`diPN
zBgxSzc&oZhmg}Ivc?{SICX9{+_CdfZC6T1t+7C&YXJLZ-FQUoUa`dPpT%3P;zY%Uh
zcTSu;p0`n>l|~cj6y$kK{O-|;&2fj~5j(n{P{fz=J6Q44`h>?7sh0{ic>lcH;GiWj
zQo{69bQniqLH#FPtPHpQ<YiUAf!!Gpzoe67!nd89(XT*Yj3y{i2CPIzdZQ_}$3bZ_
zG)oL)m86qQ)4IW(fIu{vA?rplJ9$5CEb;WC2UDh4yQX$7WjPE?vf2#wO5I@X$}l^|
zPk^03utc8}%i<j#gyOK&g;2`#>+41u=bo&7A8_(LSR|D9d*vcJ*dN!c6Q!NtS;KxD
zZ1KUJrsAJh0Fuwy8#uJa+rQ9y2ZlRyR6>1~i%}b+Mka=t+_Ca!6haeY*kE8w9mdPa
z*e8db1#sC~F(6eY;4^oRf;Ou{@@Y9dFM6Fh{h%@+46Np&xc4`p8_lld<05s13WU2K
z-aX8hD|+Pb%FSkG>O!r#Dbiv%D8<Jq*$Hk(!b>nfShVzfJVTF;`*079=rpLpI|6zH
zsiz;Vc@o}uIBfJAq^BQKl5k@E;~AmE?M6aMP@K<cli4JOb#kWc>3H3a8y(kqf*cwP
zBPWp9RWYsko4pNVD+hl+leJrmK)k@j(3?9;ieU1y)vF~xJL=Bq*@R6@1iT5u&QXx4
zv(?PdBR^Kk8T3V;pV2-SYv>{`z#E5_$HbU2GK&T%L2_SBb#u9@j_WwhI}98?v)x!E
z6ar(PY;n_7n(gMiQ;&et0?Tp1S#VZPd7!e?|Bc}0Acxw)CYrsxG;%`g;TDch%Hvz^
zkb=HM3$hdwRHV`%0U%0PGX|bZj3+_~66uy2!`Z4Ff!!6yqFU6|P~M&HcZ@{_$o&4N
zK1d8w>fK0?%ceCk54(1z(O2TjGl+;ZtBI`0{@vIxM__yG1Ie!Xj>3mDn6n3f*D>|g
zLW>L6b)eWR=d;_u)ce(ylj=UAbYs!l6fo4|5xNK{C{!NI#A*MnhNU+H8bQMIuc3|<
z-7XUz6Tg|L)!|x{h=zzJBVU46IE9V?7lGyM{ASt%B{a++?=EcbTUfr}Y!Rqdl`IEW
z%UggF*sc}z^R9tRmApLLYtJcrXd`~A-0Na|lyd*j=sf!bqA4GlMi5+34j=~71Y?4<
zSE>uf6P_T)w#v<bj{F=hf=Z}QCAr3~;H8l9_1-c!v&5y8?KcV?Ee*%(_;mb4R!B>T
zF?1#3gm_5lZFNvze0kxShfzI&OLBv_OSt>+?CNh$(IHAbX@TkqF$XUI7DTlAJ?w+i
zUB7sAN!NfkHX2Tx-ofIVr&@m2VED;yd?qv_G8cQ*9y1m6Io!3%DRpX2PQCVrW2X*h
zEk_Isw5%X5ZkqN{gI1*?`gm{3W);68qV!eNyF8<qJN2h~*Xb%9`K(*nbGuY@;)<P0
zm98vOwQ?(`{-$G$D#qQ7@!x7HL~(u%UfK-|_s($yC<3AgDQ>#SB{4PCV`(gcxeigT
z)Fkxy!n2sQ2&vo2qvwkb<xmgaVUsF-;E&dAW8Xi=4}bUYV4>#Cf)#Qdl*dGYg(=~|
z9k6|Wh*SN9J`DsLMbUf9RQDi{?90trO>-5ohbc+ug07n}%%>)P5DJeZlvtwziAmvd
z4wUT%2(}y0)vH?o7rbbexqUOCX(yXEWG&SkF4Uy($f>TNeO3VCQvA|R_Q>?|BO}4W
zbUwPG-N&4;2r51f@w<AvQ^(^eO1?XCJ+Avyw|zFhR25B|089$Oxk?E%cMu~x?Y#<8
zmp}ayrp%#Fu~$spRZ+6fiyW5wYWPg4@$EugrMzYhI#S+&uuVs4UEbu?Cr-?O2WFNt
zRnY~<b<)o!j?A^NtB$|@Qp5ZF$!w6MOu*yO`#&dt6x}B1VE?5G3lM$|xU<?V`_)<?
zKLAh{LG_>;5cODC+-bY*(ZPq804gTaYa-y>_!N+mpv}8%LjBoimA(Rvd-$h!CEvsR
zOSs0RA2+eje=w12q8j|ZDhjYPWkDtsgobSE1cD*@q;MWrc)|KZj$AAp`vSiz#@G~D
zGiEG3wIN`6GT6Qba)x|O*R8CxP`z)1f*x12hJ#trbM03DJ-YvY;-{e|Gy~s1$UaWB
zbzjEx48m^<-bc#{*{w)&2i*c)6AO(*NNL#XqY*uHe3omp%pUGZ)p7Vpt8cOOg}70R
zl(bu9brn91_Y%jB6gaU$x+V9bPb$L2VoELWgju+W_&*KduDZcB55!B-LSkpN#vYy}
z+KmQdx$h-zKw=j?T!dkmntDuWXrXb@&HPVM7I-5*qxi_+dGL;M(>rO7_H0-S{jQ}L
zoaaHLCCk5Geus=m|6Y6Mq_M`Nen3lt-~B2i6m??1pw+{!xJ)Whq<!RVhl3VM1Rk1@
z##UVjoozHGzwLn2gIOvj8WJAcO!b-O8FT*-G8fSx6)oXa!GHU2r)%MLqYUTR3D~T(
zt4mD23hF2nOtGYQr+ko*BwthV^MhO$=|>BweY?ji%5KkJy-F33oW10J`~){EJE31>
zhlNy|dc6*Ojqm208ych^b4mUBp}M@Nh9?}yeW5M9Fg<{*@3uSVt&DCP-7U9goqJXR
zK|#IVZ#>r?N4e1MWvq8pCdyd&pQYvR_eLyEi7$Q+vLs|fa@K?RxJ6)8NA~TFZ3A&;
zn2(kMzXEHOC1W6&ZIGHOJ*FQ5Zrk_a!SqZ{1=^XuS#FOFl1<}cQjN4hJG7*MZsmO5
z>Zn^wt@=Dx5=x>2##B=WOKuSi>6gD_A0e^0xw8U7L{B;ZgoNr=oJ8DJHe4=IEfU9J
z>M^3||JX%i2ayl4Wt9Z)j|_^GN<(e`DT)Cz2rCljM&E218LmC=#I8pZob6ejyjbmu
zGb;>VCpk68EJ*>$qwMBWuOKsB=?E;cn6qW?5}+jIQBNNx{2WMDa=GTTw=A54d#d0q
zE#<>QKgvg;eCxgwrB*3voI=}!+@faIrLgrV7Dh>A#DY6c7fxlUa~wHJ1R{2&|5RRU
z&1i<lC~O(FMYqX=Dn@l}R@UZ+O+YKh8DZa&_ii%$kTvB?m&IN}!%nZdV)#V%9M@5F
zIl(z;jOXNRwsOh~Lc!FRD!Fm}&5Unuu44wp`qG-@KGL)B(ICBvse_^!kMZDsmPT38
zSm2&X_&$6vTT=q0qdcc^F{t+J-Gw9-)<ms8#qNNmsRtww(dl=FM1q@V<ZY_c5KrKa
zCK@F^D3bbWNvdKrd+4$k`2+W%Bn>po+l#SQ(!mR)lvY@MV<m=JC!_b(1X|(2V!D+Q
zOb|752`&Mtlm^>tw=%>Y=VeeC!5I#>QXpxq{f8zM{=1PzVHY`3JNO!V`X$*Ze{}Fj
zU!dr~!UOC<eI#auRJrGp5BWlsHtQCQk}M2b9}sf<6e@uZQPXhYYvjJnfg&5`<h@*c
zbgzv`%r361<uKkbM=5l-cLXsIVpaM2coO+xF$1`>qw<>NqEquN*22^bD+PW(_}7yC
z_q<w!g3C}a_6+LifiQNC`SCU@PQ4fNlK}8svV+xRmipUi#WF8FlHWs|>PG1B&}$^)
z{FD9-LZ?vAP4QnRW}zY6Hs@9$UP6?I+~>wEXra`k+=^cG*U9LR!3F`t;@-NGx;Kz!
z+u><1a-#oa?>UhYR5KP;v$leM9n0~Y=T45a8e!dgBO=2|g9<q}-iEGkcALTokH2VI
z6HQM}kCEx@kRgaOCk1#%JFI;<_!8|3_2gdO<k>k)nL6h82yKW}0m%&Pm)b`moL7ko
z6}?Y<{$1dP<ReNMkwAR=g<%!)*}n2Nv2<a0f1&YLQHh~66Yu2KE37QyJ90wAX#A8`
z);`%HYWm<^`#Ij4Y$H>suuk?wroxQPrP0#d_f)(SvMs0bLA&C4nBe)Hz^C{?Ae4FU
zVSlE5kELdjZeR2^4u>s6XY%tio3{V`v3E#N9*a{TjG&~NVbfdV-T(j|dRFyi?HIG`
zSd@2gbo9J(=P|+!0CgDx@dNCg<slwtm2-m!qwAkv;Dkq?R!rtK*-jMY1QzWa*c`z<
zXp!Zq`Emnep6nMNGKg37r0)yMwZAlEd|!n@d0ZamQiPlBpQuy#`Co4S{F86zOVBRj
zc>3Ti`ziJKyE?~9QvuUe(go7B4#TgzN2}I~2L}Xt?h=b_Hg~`Zv+mS<-Pk!i?Rk5C
z1Cf7cRfQkvq&wWBIkG$UOB+$4u!UIRv3cdwN4^nRqHgNYx`xp=Z*>T1yP)6STHr=I
z89Q=wRC!2ggwLq;vmB`pS}Z+I6Gz)k@Sr|6VLyoj?v27&i3-cN`FZ1J%D+3%G(&tR
zWTR`;Zr?qMN7A$Kl~8|q$)GS2Df9QhGQR<zTv`|kly7z6XamznKdBLJT!KR+J3(UU
z9*)Iz>9~ZFGh}mao)?o^89Rg9q`K)1+Kwx>d&~T{S0l&V$>^K@ejL_Cc{)*|)t-$_
zF})=`e&Ya0q=rZR@!y_|!dAnDdVw080R$B$n1eOplIx6%61yu?Dtax#4=0jm^ZSS5
z<|Cc$_@XO<YK~K-uvE*YHa0U$2F1>k6zsf+r<ED+U@@j~QR6U5C9dS=Z(Ad^H_My0
zv7SyxZAUD&w{T~wTkmz^#qcmB89k<XDf_rKGvG4yYP-}v>ALDuc2pFu$GMrH1i^<S
zTGmf_@z0v$a^`Nz?Nf<*7IPR%Q<+;X<4m5U`g=!Dnd6&tkUxv^#rkZUf{(<PRcsf}
zX)D&g!Tb2-OO++0Nm-Ujdnt_x4^>5B&Wh5p*bJ)aj>T}J=j8C|99%>XkP_&x?Ay0w
z_<l%7TNUo7+4jDK0pPI@NYm(vMoeG!JQRbufP2ZpYS%c`rE}yK%uGxJKpcvpJbw;y
z&$h-3;vHb*PMFA!do$yfa1m(TZh7At9K8JbwY3EV!)bvvBTXIo;TXMu*xNN=k`#su
zu3dm1v9JknPB9>mLKN=29AwZD>Y@vk6;wdiau#^O?%<=aJjD>6LPXS_0D4`#+4vZM
zhL5uxSXX=Ua#Bq@iH^?qhJwC*>57);r?VRv7pvix?)G{k=H<qmbMcZ1sfSW+$8|7(
z`Muo-JDwAf>XVv>#H$O<V#izi*n>M<X$*A};TpotSoRk|8d~pZ1_zkMa?q>}441nR
zdOkXh_{7RlvC0-^_J}Z;yWf4#Z8zkj#*LZ^Taujc5zcu7f9yfCMjWYS?T-bCLnv3~
z_tVlj`DE_*@Je7};2?FVodgr|Gyqay?bCc4r_ZN?;POjK5Zsum2{M&`&v>EHqNFCQ
zZ&pIW4e#L&RN%2zR0Bv4vk?7d418dShG+A`_G%U%p?I_7huhu(Trdhq9O#7JE<yaC
z7z6mNUSRTb9eBe-|3H&we}bl<{U{(_okoS+o#Y55Fe9z-JRhsba<D-8!sUfh21CSv
z!Y5Oqn`IoXXq>RmW+(ge8iE%~$Y$372^|k%1yv|pmm;|II=XT?i#E%=QojZY#`7R3
z_rKrQRtJnM4x9*+oh~mJAEEoLUcYiaZB?GkXZUz9X!1c-6V2<R;VeN@^rnQ$%;K#a
z_8TRkf6Z;YJV*(YS8&BM-rCiSuN-NLndUt991;c-C4BJi02WlkJs!O+|4A$jof(JQ
zAv9Iv6o*WpKCVtX1Dh~4pb5i^E<J$}d=0qh<BwLm`oKA31($$J(K-^?CJU1tXH2S%
z;-}q=?kuVgE;rY~s~_3*%uNOkQiEvhFDJXxqub(Y#4jl4&Fr%i5BrxO`l0t=zhi_$
z*vTGe)7jPV7BP*;lpyQTC|7lA;wzhxHGhha9*4<5LuPTP4@fW$3oN4@;tK>f%=BBp
zhgT-thH}kxzx}?t3{xy>XFOvYR1k|-cppL4UC$RUyYRYLtMi6OhN9VaOmYn7=|K1*
zr(ThK@9n#^EgcKg+dIShp!ffV2Fzgj$l!XQk(<t(@*Jp+)_uXX?FWncNRB*(8o)#h
zfnPt`^Y&_8O3<V-<|c{-CFR0?hR01El*uFtzZibz3@S7XKz4z|i3>JjBZzHY2k=sw
zIcO)a1)4#r@Mi)cxP_G??<AI-0EhQsAh5E$lT1=e@b=_G9iU`UEPR2(Xf#OBL~#)S
z$>XTl-?svnuz^_)4*fCW5Ayj+ukKphG+>9NSJ?pa2d&mc3m$Nw@v{u%s?MhO0x>^5
z$fT%+=g_n~-Q4<7J?LocxrkTVYe24qsdlg!c@byUUpsdsNG<6(sQCEz1z(sD+N)Oz
z%0-Y12B)P~-jAUq#X)GQNhqCf#JQvXF+VB7e|hly@fx**E8$E(EUGg(uF2dnu`AVz
zy}M`eo6HV!+dOW(nFp5@3!Xm$2%^B(IT%zI=#wb~K<J5KW3Dtrze;_J`=wStkX4`^
z2^W{cod=A34sB3`c1MtuR@RCQ+o04L^94|VszwcbQ&hi!f8rm{(c6~S<u=tACUyT9
zB)eLNYN5o5ngim%QJ$#F-?az`FS()zVUI#i%L<os(mbOC&ZgRCdPn*Yb!|c&9YTB}
z3#%EGYBIRJN?dB@sKVhEoauv>i)ZXy03PWD<DVWExAGXGz(>mW@uhNJtbO(cUDh3M
zC|m1mUBtKlxpKsiz<*D;Gk{#FeKVCMRqQmR5fdQ+Uj3u3?pfYhCF6T={c<9EV6vwN
zEG<ue+5yW+EFv;$H!u)uF}n3i-d2OY3YrrDfQDf`F5O606IuCz=iMrxbd@q>yU{Yj
z51GoXfV2qLxpzixp7QhtNau7299e5y{xaF5=oS6-p36$8ghre}OSy#A`Rrg(2MW3c
zG*;l1K|MBI<i>j)BN(PY7=>Zq&rq34>dnxa`+#=5??J0!u&@h{*@#(?-VfcZmpzPo
zwMEVAqd&HrpBEE+T!nnkl>Ri<W>X~8Y3^=k5}Q|b|7pyNtFC6dY_&ImnV9Woa2y&J
zw{v);WU2=^3E5IGAXc72c+zL8smxaI0&Ek#x$pH2YL=VFd)(Dgneq)JPB9yc=N*d}
ztXbF%^NTAgL`S}4O&}GC(YWN`3BM|#FkWj|<=ci{C+oKKG~K?*t6xoa7jniidS`g{
z9{QB@y-hME&8=Hvw|rN8)Xxcp0O(==a({g8ixrizcj7C5zZQFb_*h?oWCjalxb%C|
zb^mveZ3QfcvBc@CF<{2vnAY{fG60IdDk1X-#gKLpCE`Ni6vE9EbfJvj@;PW?9-k4I
za@{QO1BpFg9V`eWk4z+I`1KJGWf4LMJ`qLuppu#7^zEKsh)D1^UE@C#!e%A(RI`qm
z#)&i3WnBh!vn2!H6$N{DP7ke@;5o^}8xgm_Y49!#2;16X@4h%mebZ{DrErbtfcA{i
zXZ(@iAz>ECSr2xKqT{&u!M3(@>~aqGv_W!%8?h^;P_al1I(et}kTcRj+gDw(Jxr>6
zYUNwc&}sC6@1qKPW@GxW&pC0`^6h3Xku1)rOid7Jv;*Hz8kDqgStv1qtN-0&AIhsd
zqm1Z-ePCIL0W{Yvb9<DrY8$*X!h&2N{|wiN4C3jp7f0c%K6!*mEl;xha1c|_6W61g
z4}0Zcw%x@u*`|Mcm69Cj)6G0hyoEMp)Z9^{N{KGSMiBy=<73%{(<|0qM8~Jpze;)(
z+*N-Ef?UjMoo^fFv?*5f@O|i>g6!zv8<4vYoA)7mVx;!1(W&Xo#AuM+rbLS&_N6`o
z)(VuJbU~kty^|}6Xi4)^Fx?u6*Z$Uc&qQn!NcFzjwc%`LXWez4$DoK)Iey>HKAL@3
z?2ph0_J2r{Igvz8NQIM$ohQ10vGw*f!&xp=Y>LO4Uw<QfZ^^p9TkM%NuJXZ}9Tv}0
zYJwx0V3=DFpYh6L=<S}U*P-20#}IG4xVAP|?!n1}X+(q!AAg5Un-R~u!bWJ!ORZ<c
zWyURGk3O@N&bCHSiNQ`!OSK7mv8s2Ky}2InJ(0OI`8tphXEqay>7Cw5fnTsU-kty@
zWFO?~|JBh#ByCDmC<b-z<O?4_>%IYSS8NH!AP<p0Dcg!Qn8s=>CcI`U^G4?5l3&G_
zi00`1UkfAd?UNOgkvdQ;-2+4bx+xyrJhTXX=eVDht`d08PA?X=<fAQatW&cKklY;k
z&y7t+62n%vvR7B)Q-e<qBOagAH-ajY7LJF`VM2of0!A;W+QISdNm)fKR-N?%>B#*E
zRH%c_oe1a-N7m26If#h3fj2=Ge<_>PGETdZ;FYW^n&9%RLdn9nr<;$3TNa5I*idJQ
zoc&H9pZy!yy1TB0?Efh6;X@5445KW#rx)ygW8iayvDCC`i^UdeOcb5XNr<Bh<GsQB
zH?H81FdE9RJZCn(9yE9O7lXig?p_5dSyaEec6HNg>Z{`IZ&$3DlzaTLsE;U7`=Qq%
zRJ6!X9e$@n4-Dp4SLMI=MAIh`PTa7G2TDh4;cObGTK$Z>l^Se|#KwlLY$X69;|0WE
zJ2P62z0FwL@@ue{Ym0Ba&60d0cvl2@fTL;eJCBD^Ij_3$acf7dXXiFahlvi#2(n~^
zJK8nqQPeZNDeO8wk5+_PUY{d)dDR^1>w0f$4fnoR(`Q*kMX}G#Gn>3<ueM^8-2dkb
zVK0aB&hdc0Tf?K6>i+8Y#OCH&WEj7=w;O$u{eoIspR$H|Y?t=IS|Eeaa;>oh?&g+t
zCwyFRG-^Sxl}{vVx_d_Bf%({(A1RvRoM@MZsbT{2u`rYMvx(zFh~dhi?RCP{lJkb%
z01b`M7W)U<ZbI=#lGBZIG4o}u;nhFGo;5erYu0b^ZrPJb)Bd<~RBJw={gHU35yX)g
zqe3m3m%hKed6L1!_>p0BL)$1Lq#GCeQp7N=wG2+t19u@_G`WFQ>_=s`ffP^<Ms0Qc
zdp*S29Yfpm(s1^@zM<AW^0cIzzgJu}^w3HBb&?Qk{Rd<}>MKrDPVwo8TxFFEVf4yp
zgBwpWP3kzAmF!>^FU+;naz_g12gQqj1T8PRqs$bmCFxXig|hKcMzlb0YSLv%Y1nrR
z&<r964f4?6eX8#XCTroDZVM0NJ>UOqe(uz~kt#LF2g}^tx}FZ*WAoeeCl%A`NALuE
zMzt)V%Y^JE*mgK=vC0BVS1EJAU;Xp48Kg>by<IZ9blG=d&lPUMd|@qz!KpWL-Y&hE
zQ__k7+sXW1y0`ZtZ4#nIcwTZKlTh}}n@X$_KbP>LVh51~hd{EXf$@X8ja0qD*LFIS
z=iFtS+IJ%RF46Mcsq`~zZwD}P4?h!BL0Vb2>b4)OSUpR%9JuxlcQ7||QxlHLtq@DP
zB;2Fd32=S%p;Craeri2T0kdTfghIgnWQ6y^M`*YK6G%)JLk{)4@x{U$lHj5UCJ=@6
z+?Z%W9C>NGpd^m9`AQL+&)xszk03o#&cM&p{>i!{Tl>nK%2#Uwd7fgFOKA}|{(ovT
zS(u*pj7N(X&ADjzi5>9yXDIUQF*mN*>6U91Ucu(g;*Js1rrRrU6nDCDr49#H5uTDw
zu5TZlU)T(=B|XNgMCK`Rpel$F`xpRi&Y?3&%A8mnB3Q}5$E@-%Trl8`Zy(1HafU;h
zNz@9D#X-X)^!|IMl%S`-6b_p6#YVD##mKvyz~*TEvylyAgOo6#$=r~Nup5x{QDJd0
z<%{99Vk0bZP`|^nHC>UTyB1!7^HvuF;oA`?Mry!Q8F@Fk+Ciwp;ioIjToq$cAz7cw
zwW;V7#$#MUiu_{t6#}*{@)`H<rH917ZqH_ArIE@5n7m1+?(WW;<3B2Ezqwy{KIL4W
z751TR-6|Dx2u}|Grv4mjbMz>+*8JkS5~P^*!AGM$32qws2v>tJ#Bz8VHy~C8KO7kg
z2e%4VS{OkKW9Gjcuz;`+J4*?jvY$dZviknbt0IJ)aGBx1*x7Dwqy(xf`+8Dau)~|L
zCR^JLR;7&|ZN-S)CGJpyz34J)pgdZeS9Zi#vs^H(3RXBAxQmY(75wfVz#J^_Mbz=a
zsf<{ZbrCL-&E3#9umrQ)0KUq00A2}vr>Tx=YoVy{i%Bt#P(^#_QV7Tl$EAZX*OhMi
zy@NsN<=u71LY|7(mq78MPlBzx9p<X-e$-)vI3F)ni^!SEhF}f12Cv6!Q|%O1R4*G%
z5p7wiHu2wbJF%cdVBLEv*zFo0?u#Bu0=+Rj{Ec~dnb@|c_1tHVBe>54WH+sWBO1B<
z>t4IWjXvYWmBnvdisFWb%SEbhL_+`gB>vLujJSXlFxGK0YTvQL6%qOqV>iIWYG&*-
zrT1_eY$T!16{!<aG%1I1wV=noi}xyG8?4w+U^KplD=w(vDf=`13qXcrE@T0tWbJ+~
zo;KYCC(U{ffKh@OuH3}J)&%UX#G~&P&n?Itd~(H}<RU|^UeDV{gWUqhc1Yym^o$PE
zW(J*iy^tr-Ee*Dd;$$4Ak?IEVy<P%xBIZLfYwzWa%x$1{jRU#%Bj&`IWZVjHqojqn
z10cnC>VOL`fDX6$dVU|)+gH-fkI|o@JowQ_$*CfMI?^Wdr!(^#gs9Z#mv`Dfvi!{1
z($B%snQ0jY3!6UV{Bz=~=<s2|xH)r>6I2dd40N5GRg@p=F*8j&WnQWvyN?$RsD$ZE
zavMwckskf?D8WC#4ax}gpE!~Fp$;C{qB|&QX1ss;8}wi80In$xWOvGd=tY_ctBI1)
zGDl>~)2zww6p&M3?1rqW?=B01>9&tlbih%gFj5Ds?J`yI4wv(tcNG7f1t3`3LOkwj
zgh++N+HmMg=Dg%XK)8aiya<q{^(p;uPN805GS*FH4GTjv-(gf;lNOMQY0%+61^ERW
zpY*+;D=L}(T(@P}*_nr=KqD^A<yw*6z(?&vX^w1TSPWcrR^x+>b_YIKOaE_&m6xDo
z%*6Ei?-J@^P^f9W{J{1mxJ$ZDeO~_1WPWClX_e5>Oc1FXrF<3oa6prQC&R+79Zzyf
zgc}tZO;QtQbCP$>_|8x(ir77E_}pF~jr*UO3vg7!!UqnZixX9yUM1tJA1D9~tCiW9
zv5xB&UD72vps)r)aGvpV%ct&0n>l(i7MA*I0tHsJ*EMhySRy}C9S$sn6ol2^!ACh4
zh3ZQEP}cl&Oi^9KwA%iulDdsVuH((MmQ`nW)Q3O{Wi@Krvi;hjjjgmaSRF}nonmvj
z^>@jhNbtRIZEpDFw==`DC(?iRRdGTc?8XYFCnC-e<gvS-9KlP5XVnj;bJAkV?ui+@
zQmi$@*4!9uuu&eP+}r|bZ&-ww!O(g{gyt&crQoYB57&s6wkd_E><?e+n<Vr-nl{Cf
z|2rVye|~TpRN$mEFZp7Sh$LxcHGY65SQM2y4_uuCQM>Yf)3qhdihbv3%HecYf@jzx
zL4&jmo-i1<l&)GUIO7T7sr!HUf|q_wb|Aw-*3rnGb)owQ+Ia2XCZq|-yiDDh_~!Gp
zn-6lG``)j}OvO4BQEX+*!uZBnVlFuPIm6_;w<LFmoD^|KH~q~4-fFIOC?<Z}rd9FY
z#<fwREr#8LTfg}(yTUhv$;O`<NH=?Nc-?6m($qQTN$1Sfo2|JjOX(|!jHaB5?65OR
zm_ITNV3t^M7-1q-*pA`rt9f$;y>zPkULHyC%iUA0lec$*3+>4rWS(n1O!My#<~tS=
zXMI5h`uBjYh2Ra$mw_nz&G&Z_`_J4DMUq6$Us<SW!%p09cQ0R`6k2lQ4DyiYyu90b
zC0hr5_b|}b+S!9gtawOQ_vOQ#%oOjkTVhXtr_hJn+edZJ1^tlwo60K?#{_R1zHH;i
z5f#~#*3$}_4EvZnL>0Z1YquakVzJwLdqKs@y#_T5cQg3Pe6GswD~n}ryIZ8r4dBMm
zz5MrCM>42HxViZ<2I_Ra92omdJu_PyU$qVkzeZA`5&qICD$;wy>L}RJk~22K#2g}B
zP?_}{BTHie7-TGdy>Ky>Y`L~bArJhrt)nR3Y)SWd4#2&$Ox#~8zMnF`>#Dcxw9&yu
z*^XkSUx!DS_<dvYgBQi*9f{SyGU|49XF8~0@)bd}vI?34T0|>p`Uqr4_QR7^8ik0B
zZR=o4LNMh$_5wk0Q!rcu^e#K>xX3w>7kX9%*Ey~O%o9@A-n{{?g`7UKFX&tXZwnMT
zLwl7nG8KC~vrJu-KE<%Y1?B&I2!}e+3`8B8vmCy3mo<@#x!?+ooo!`i)RV&{<1##Y
zn<08_cr+4!stqOk;M-y65Z9D0a)pX_D9Rb6i*gn8Uzlr<J>L9_=UO!k7iQ`o>)qd$
zO_dP+X*Z7Y+*-n;v*1G^6825cpz}o>h?e7(iUX))#e-G|Y8+fkI=knCNE>&0vNJiD
zDxb<fZEgd0Ml;)8wyvIzcx|SF;><qJv8yH?a*3kc7h7dx)n-B6FaPa-B0U`~hGY04
zI`87{>%n?UvXo;E8;-$?9wwy9*cw9r`I7hFQUTec2?{1{``NnCevzd&w(3H`&|>aL
zJQY@|QljlrWs2%D)pW>8pKPG#pd(P=#2$p3?~?G;*OB3SU#b_Rf#_nD8{WkAyI){d
z`{#S#r*{AVn_k03j@*s1xdPZiw7d`v&doKVnqERyW)ew=Qis>IiaogltNnkt?w%mg
z$(~SI!$WZzz@x>5(X}hcLgkpJ-f3urXKgEwn6hREUJ|3zhX6&$A1R!q01nScS1TRx
zH<Ba%>{R}2u2%~*t3BI(Dk{W7CvRV9QVt;?b%z%0)3{lFq_~8wf+*D(z%TcqMyRF@
zvXHbd^Z0$!FfR*OQ0aRZNX~K(X0G)JKkNugyMB{I+QhHwOX0+~s7p8p!s#b?ElLtM
zKVIHChLHR{DnlZ8FHC-hU%}Z3X7JLd4qW{#KcFr3t5yjtP)?j1b(~wRO$BCK19<Aj
z9=bdzBAh+L)xd<CDf5&dL@jZyxmV{O9856}msPZtSz;j$7WSMPDO+(Se-50KjMXt@
zOLVGGA1vRrdW>c`5j`2TmB<1{E-oQ~ws0HCi31Av!&al$zRpb?L6H1gI7thr)hK$0
zl)HZ7QT(WIrX3;qiZ%qor|2l}$lz`X-?jK!4)LSERL!3$xE3UX;2O+BBqLj#XPROc
z4fs$_Pw;>iH2~Dr76=%pICqQ^dJMlX%aZXqjyfdI3>S0HKxYRB`^%vq44fTY5It}p
z9nNh97%D%lYq8l|`{rExm8ISvxjf5&qC}J6tZ?<F_Q&h(<eesE#^;9nCGl4;L}>Fk
zxnIyqh%$%i#b6Kf+BaYo5swILyAzlUPOOh7PMIVet9d|U(kg_ya4EU~i3^*q3$c+9
zB-Y)o(50wo(#f;2Iorz@hEP%zjcO_we&I3wMMBxa)%wQ5?lbpzK`BvH+v{7X;>0eK
zqy_#=$HQ}?_TJ6p=FhG(c;zfe*q=VBJjjUVE#@UvITkQwO25mR>~e4B5APJ6yzhU=
ze7XP8ggsL!O!rSf&qKmQ!O4x3U&TUIVH-PDEV_*+S_SKvgUJIa^@FAlc0a%VD&{#v
zo@dt0c~<y?dzs?zP0|O~lX!ty6~qBJmZ@KulcaFc+sfT-QSx5Wzk)P@={4+TOWv#8
zp<<$S6$gu4Kp>D23*|E_c5;o&3@a5X=Uxxww@5d)owKu<lkC^A@9(qP(lX6^y5uAA
zN~;-cp9Q=XWM+*?a6|oe(ImWUoO|jXhjbLG7hwWYe_e>_yyB&PU+X!FzG@G2&&O+G
z;CN>===<n>N=SBVlFIP{RPpT&5#jzGZO-p3E@_jt3~}6C#)mF<S?e`Bq7xaGMgcN>
zBiq7glrJjC@pgnyL-XN>Uit4p>~*5o_GMrs5|#GA+wRNMFma9;JYTo(D`W8saD&If
z<5_P!o<qB}m2tytIlg*F7jm?gkor_R#an7LpKW^L)Z`jzK<3?lJkQR--TX<D-kyN%
z5LY(X_tn?u#(MC46-ZLcEJ*4$FTmmKPpi6`!{=hQ>fRd^l4l$E`CubWr`~=ifAB{Q
zuTUQCVjBVZad!)DdT8Yjq)_2>3*2s)WU&Qq(0{$IXejI<@Ue<uysUViW&Y?4%89aQ
zY@J}DZ|#k|nPNi!EmbbLA{1e7!(yQDg@ugUD3rSxfRk#o8DtnYx%&{x_C$9pyWh@a
zD`o4`M_tcKw(kViLn`EI?4wS89$V>Fj>~uU#PvNhPV3pc0aN2FqD*&e-!$Bzmmv|#
zE)<AcM~$h$*K`Y=Ee|2_-~GAxJYPExS@z>ohAt{cY?Y)GWEfDgv;Sig!=wSzP`Imk
zVnf!)veC7auqvSpzfkFGo0Mm`-LLlFSCSzugjJG4XYp6O09(?H^?EKsCA4$snb}XZ
ze4DjM$#;N}fr<C8ufYj}Uk6QH6Q^m88;hYqbSAGYQa%@}+L$sR^?!~9RiJftb!uS4
zS_je<PM0GQ1Ou$6o@;@JL<_4L>x04H7J)c?4G8pNK%|g*ec5r~m-XXzR5v;_*$)~`
z^KA(m;s|s5d$<q&@qC(7!>4`@`%Ee)&{y5Kr#67eLcRqs3;7L|Sk;Lh*PP^1`)VFH
z{#2fxE>_JmyhBG{CXsx~%SuMG%y7Pa?mD2&4DDyF91=ZCb{HYD<Olht=F)GBREa5n
z?gejR^W-biynlL8U`)+;#{mtHXgOrI|7j7C+J_12dc6VcketNgEFLW{Jwi3{Ce{a5
z^(kS)l3iONckJwvla{>dS&Pp(+XMA?8m&9Tyt#4>txKHM-t_*@EDC0Um?!W(7&uKT
z;5{X_z`ocAx@7eXaA>d3_838{T!w{J6#y;AsqGht1D8gZkKGQu<CUcCM}CheN4O{B
z-^2AA@PMI;4(qY^l09HbeF=cdblu9M>?F%55DYpHW-^-H1GszOfd@Had(qR^22u!f
zIwkhXRYb@_xZERjtV!)vA`iCTzg3}-g*z0&fkWZBq5Px8s2I~O#7#x}ZZc^fYj>gA
zP49j?2|=FK8@@Kq!CSYA!pApR4oQ;jKV%*vb0CRCU}6}*4!B`M%ng05Hd383%Bgvj
zNym%Mz=QbeXo%t0gR7OXrj7X*Sx!RCI9od_x?>571C4%I&0kZ{Lvqdj&PE9$fF(;y
zC<oi?G}AuF$=7YS0+tQV@_Qhi?t?e~z&o_zo#YZ1WCFai`wf23MyLK;wd9I#!t>QD
zcOa?<G7v`YI3*GW_<*g6q9nXx_^dD?RRUrHV4yWn42->=7*24XDA-^M=c#nK=?!ty
zX~VDFDEZB#B*)PzvSstpiIvotP)A2FjfiW@-PIRlIc03uQ45r@{JmN4V?I`*NW!dG
zC1?56oT7AUn1Xj{n-;K5U{<hbGn|QadL7y~NUg$xUn%jI_zO57!tc|(=1Clq-PWdr
zQUAE&N|~Debr&R|X=p!>s6$Dn9^+2?mYss>OIJroRUrC&t3$3i`s;g6FIHeJpkj<T
zs+3tXy+b{u+s3rg?$n_B`d<h>*V*+2Sfkf~NSPC35;VfZ-T*26gu{eWADqejOB@51
z{F&P>8^$Wc!X%1fPqiLR*grY>14hs&yly<M-z)C+d{IbfHqG)Jo|YVR2LktKfmAyL
zzUsmSc{Cs8rluCBgW|Fi<eKdxzeW*~_;3Vo*6$0LUU8#Rr_ANFjn_|DudmNkt#|Y6
z>`u(pJN?Jd^^-;cLZ^LFNCHLp8Sdav-KT$+N<1s#hdRx)2ZRPjklS1nz66>qN-L~Y
zwRiHSk#uw#m5iuE++aGG<ATV0P^y#RdMe#U;&AYz;yhyldbRuzBDUYCe$H~5MhSg(
z5Ka>$m3LF(YTv$jM?2Yy7|B>F8AZ@z3Sv_TY62;~kcOiF0Im~@ygxmbxn#)YPTtk6
zy4rKuL<5LFe;bjGi_0g2k=0I~f17QA(gXv}jZq!JAt`v?weND({`{{^<6(u?1z>Y2
z6az|5@uV#<x)`0zit^70;=FzGWw^9-&=xBh2i5wED%)sp!a6y?UgFKmItZ#gDQr6*
zgMcBmS_UjXyH7^1kMPmfaJ@rTxsOx8&tGQ^^cuy2dsYFA_WW>8dkyq!L<1d#_7I1t
z8;UGG8BCIF4$#wk*Qa5<jN8|V=^km{htu->l>aY(Jia)^3idQS+t}#+I4bU$d|)Bl
zXy0xEsg;>oDRuL2nk9p8F;w!Ekhl}#U7yd(l0?|_S6d>_LQb@R(1VNU1e~r!o2I-E
zUS*~J=iC9TE}5#^Xu@M!5*O8_nty-;=v4wKm-%;MpE=84dj`{n@m0(xlsA()UX6P<
zOSU_HVe5wd3&%X;t#krOOa;nP0j6pHnIuf&BXc6U^~5d{J=q~9Hg|dC&H_teO~nAk
zT-GiWkHS`rIZf>rp6(B%^W-$@+K+;Wu`jJ3PE!cDR`!o)1P-*Yua<}*OA&ECHnxGa
z)^#ub;oQnP5E$onvPaSU!d>Bx_H!^qAdw0P%4@J*fpslByB8i(*A2u8G(fT)6(9f-
ze@(GuW?bE4v$o$EpeN{?NuuVBDWhP!eAIVIk)ANpPjAuwHSN!KxI&NnnHTA~a^6z)
zwJ@||ze1_B78B)b5JD8;h>Ey3%@$msgy197|0kz!BgKl|eyc*~+B`9EeRs0IG{B4u
zv8(Jz;eq?M*D56+S)Gvc3pxEHLJH+^6~^FA*MO(W5#!@uJ#n-$ypD@%rZ7#2%??~1
z4Nv;HH9v>0d}A%HxTNp(yxyl<*NQIXwa+B}tlIZ>a@VYueb|r!x0vBHo+W>wWJ^t6
zfr4wauAdX4e(bB^uoLlPPB#z<Vy3P<O=4Br<KL&^^SDYckN=la9H(}0Kb17dn2H1V
zj`oC(mj{rF@6ve0Ybw^;?>#@xiyKBI#foj-fnpn{*`>-{8s;Z^UJ=_azai{X8W(+M
zplW7do2dZ{&OfHHo8Z!lnJq{&Lfccpc;YzVL4EF`z&<P<z=U$TYEi(moehOZW<BF6
zCT)mxGaE!rFI<;x!QRR>^yMC}IQZpZ_DyM;mwV!O?E(|}a12*<T;c10=WZLlojZTl
zSIF0A_U-O2ub<P;;UN+Hn7FHXJILvU!<oj*9_4Y;lFbP%vtUXAo&vM4L^jAi*k6T{
z>=GZhvEf6mI39b^0*eV#-FALKVTLaQGs9JJHCC~G`=qKq*6exWAzA+!U!C#2!x#uA
z{xTRrD=N4H`Uc3~-ESQWMVjY0jfTFSflSQ)5?VG^hQ_cwJq;@A9rE?K>Q=kw*0{6T
z=XtjoYk=ruB>e0|GUQ?h|3*M|2Y2EF*H5EX7oJhWy!9OM5r%l|h^FV(`JNFnd}DlG
zDXytuGm#|;7EV;co!NumS%A~0Y8R~e?Dq}F^p)aekl)3d3yrQltq;j*8&9rnFM`?(
z`~R@9d(7Aq3mC<G`RHnelx>@8mF(Y3uV~z+`sMZaiHM~zbpEQ|wxP&?7iJD(sCLD*
zQR8!CX1(j{1?T3iiAj!oF>gP`J=u1K!WtYO+f2SgNt~J~>d$H!#`x<w@cBmA^H=&g
zq=>SuC&Mh8zkWoyUfAQ9;V!5jF7)s`0@$WJfx&KL(;(H@Ur(W>N9cUH+!x{aWxZ?y
zC;E!=C|AF=Xw;#feKtA6VE(PSe?h)!w$iJN5{vmN-vp|W!X1v3cN3Z_pCxfVW%XW<
zKjp=3uM!jsJA9L+4q44*n5=l-v5oi+uNXq<y-R5x+(JeAv>S=xj8`~UG^0Fwq$R62
ziIp^8nZ~Vqp_uQ07LqL+3$ON&6WUWsfQ=elHq(R+<w7#+-Rs|WC_T;9;q*W)J86=M
zkBr$mb7$}vN3)k3$+thmNWiK4@b-VP#ZSn9g6S5J3aVke+b;@|No#D&wJBUSi8B|^
z>o!uY1?)o|YYV9(Xm~<ge|^Z-x4Y`Bn@c0*;Ebu+M75cEKDgeJG2!+cUHY5HV05cD
zJB>v|;iD||xbg>i(1KdGrqgVqRJw2kq#qW^Rz@u5e<Tf2T$XZVHg+epu9#{|-C;+@
zu)e_$2K#lcxtOM^@ifsQ<kra<NtlZswf5sa9q%(F;mL2=W@q$0IQ4Nt#BrifsYx7|
z{$VhUODFys&KB!rL-YtP<Nt=56o}vtSy{dptcPABl+xa06LQvHIPQVk8o0~1bud%4
z!=yK2(&6Ani4c2WN*pf&Yjjx_lDJBF>@t0ls(V$~pgDekp2L!$Wyn{N9C(@SH70Z(
z<W5B9o%wMKvxroJ#-6z1rp{Klp@DlVW~CmPs-?1yCq9X+$gxaJ?!+s6U%&vUQlrsX
zWg-r|ZbV#J$E356FNHlL^iq6ORt6Cs8+c7FH%DH&4YCI2k=&;^EoSMRe8?}S8+TXK
zlA}69?|ChxVf3R-maDs+n3&G|--KsiXG||eyLj?J?}LSiX@oQVX@nE+BsW@<PEN4k
zWdGeYfm-f9KF+;;Ex{AIpHH43&agRIi>|;F<6+)1TRx@&k8c0$iS^`<e}48*0;<ZY
z#RG{=y;>WcV6@x*s5Snk$6Y0sUg?~e$-5lymwJ8fSI5a-b^(SH^$&Cwo4%<M$$Yp-
zXr#7^U3u{)BPu(O@je!%S>4sy*969{MGr2-m#yB>K$JdGe00O^ce6Fxl3d9tAImB*
zZ(3KjZIgX1wFok2a&)pYfhOefEna9sHz6%WU9}(VFq!5*zDQ>jcD9UzO!A-DF2(|a
zo+iG7jV199P*e3v55T)yb$W^G`FdgV=n;-e=G0@9khi3hFcT#-d1ylO2$ctiR?YJy
z_BRjB7-E%P{`$nz)S(NDe&HHTdVe%kHJBJ@dOe%y2JI~3z$TfBs7v$Q`Z1ROE2>I|
zgOWJ<6;CCU?qj-jku?3M8giume!K6kO-8eg*U9BarJ?Ib^v~kGgshv%Y`gWi4*x8Y
zk=iRF$}~x@*B9$B%l|ny1?|pJUU$dpoMTS84d?rORcrV*Z`;33-F(RT<lD}6;*Sx@
zMg(5hf9!a|=`C>hNUR@#S+c}{ngA;#p>3k2At$JX<Da$)s$B$8M}JeRK|D7I6udgy
zGxI~-67fY^9jA&yPUCXO&fj&(ek<56wJELEteLlcQqs)+{)>fuO@DGza^gV}WO-%a
zWiFdrmYks}3f7~nimPDhn3G>jY;wwTRW;ZMGK!+vB(zT<93dOpvENb5<&?W=v$}qa
zSj@Z-nN392;|p-$kwJY-jQiVL-=!nD9AK)`dJjf*pN6n(@kkEa8Lu(CbX9Tm8rtDR
zEn&fb&TpLAB=Zo*9MpA(KgZ#{E-BaQ6wX>$jsFvO_^dKygT_rN!S=y$jC0Ojv7yxS
zr=}S?jAoI|B6pTEi@pRaa}w4u+FT4?CV_L~XHr5IR}Oavig<=JX@LZc9`h%P65ZIr
z!gz+oVk=K$AEB`O3%mD`T7`f+ZEG~A4={JsAfYhU&mN;dmTNlmNO2RWz%MtDrkjEO
z)&Rh3cp+Cmu!e?Ogrgsa3I|Afa~?^YjbAkl7*IRDvoCZ)yJWMZg*bQ|hxF1JboQ^G
z)(2iq;wDTJe&t8&z#$mmiI$JzHN7^3EKbZ8ziY4;9FH=J@WH_TR=8<R{?Mo1LYkC*
z53{aB>C~Ofi}w5U>&RCWSg}n#mz<95`+f>0>YpD)u<u;S?te@|34LrZfL4$7HctTW
z*0`MT=Oj_(5nP(}FZM6}0otpgv3)S3HbsNJ4^en{6g-~DG&ZL!lB96=0D$sF7WcqH
z$X@03`!zw1@`}x<1lo&ewiT>0y$%RUP$B5K;BxWh5|BO|hxsZ}hQx4gTECyDw0N%b
z(B0OA-sMmutKz(HbaIc#!9|!*t3YS@&G6iB2M!QUWFG4wyAHciH>r8-KH5EWfC^LT
z3E@W}8#eDmH0cWSG;}DBH4xn*=GQH1W|15g0}Z*3pJEg(B)=~UvZMcR3Dy$=_{4Y*
z$XFQY3xjW_C>S?#6Fe1XO;lZ1)4=2MPFd6D4e&P^n>Px^ovRg4u!x}v7b;=#mNnr#
z7~$%`v7!@QzhZj-X2Q)#FyXdlbAU=D0z&;VfgS0=sjq5XCjv*vCNHI~3H{`S%fi(w
z%|Njh!=SS-?Pk{n@xcffd*jnp%pF5!x7n00t2WP>RWpP#`9E05DE-04H{8Yjc~4aO
zyVM}I*Z!%vP>+;XqzHMxt;@jW>qHU$e^(`;p(Fx^k%$fT6dWzmt;v!!xCq!K!%gmj
zRI#J0+38>*yIcq)rWRC_uG!J3542GG5<EqG`_zvMJ>Z>EM(n4Td8~bTBN><d>PFpO
zqP_Uk6KkKQ9|QG8;2Pw!uz66tB3mPuEMlLgox-IQ(-|jEAeyQo8YKn`j)Kh#k8*bu
z3mnK)P>CQrO&CL}DTY%1_jRk{KPJ2%A&mXrYnTH+^9y=iT;yeq5y@+lrPJhIe+oDM
zfJ$>67l;MP)%lN?#TtGG>33t6FM^z&f`ozvJIj#LW3uwW`WpbQtaBogeEY!thdL{p
zT30%Xvo!i#o$OI+#xv`KM*@%A44sd>GxFs0rvx{6Gx80-@<&ZwnK6uJwnZV(>AA*i
zA^2Hr=1j)rr1E7{_(Z5<jkGFQ<FRkZ_1P_<kE=+%|0Hx4dH+&sa-N96L0Te~O6j5?
zDl+%EAELj}h)uR<g;E3~qfu`qPfmA@&S2ga!}I~H;LDIz%yho+a~IOb$L_LFTBLtB
z(xD*9!aaX!YR*P55WDzY#~A3rdKxCX4v5JpF;OVS??GzS<o=PB%>3eLu-z<;g7h0f
zd6wub;!6!W9q2&f2T=;~w{OXuaCseiVwxT^rc8e<@0t^&#YjOkI!-t!>Y4o%GbFDV
zZ!j@q)>?-ua$!24g%pTozm)dvlj@V)i3x081Y>v`OO_VAjpga-7x&1bN4K)YFKHkb
zZg<VIhTAdgF%jn@^z%c=@6AVY2by%p9~lMo#OH6Rs4xuQzcD&<{-``{Uw@+dGwHc-
z;@hO`zH~pj#Rh&Oov-Hp2=fvsE%{cfEa+$x3;{1(kn*H-i$gibGha!;wsu4fre2Qb
zVts<0J39(<e1!hX=X0B6^fOkvOMBftvElYIX-%szRup3O*XCn15`d%q*XCnYcoqAE
z<$);g{c9RJod@rDqy8&@DC_SjQ7Qq&5in5p(+9F$UVuEE<SfBmHLNoEf}_!%3n2er
z2XIR~$b98RyJN~V>}tC^abXP+3Yr<M89&<3c@x0I*4HD$r&)H%5-cuB6JMYD@y%?*
z`*q_pK`Nx+8Npk;ZWT*~7SWgyc>zivW4MJ!D@51CAq9Ip0;<+l-QIQWR1s4cCrRoE
zntv(rdeWn`*T@;JW5=cY#)$ot0=z^5ScJr5a7HV?Wy*d2O4%^}v3a|tRf+uVg+hiH
zv(a~VdOi{BGzRfe_7`~y=p+p)4LY@OgHLmg?y@IAt(p+Qhv;sa#4G1YmUW}28PN~q
zD!1}mT&rc{tr8^I-l1^^4{)_gj4o5Xr1XeWmqXi)sfTJ!bYtNq?fm~&uuRgHMQ#rC
zv-tiMD-4OFG!Ra-FLj`Efrq~b-n;=9Uw|kKK_LiEq*GU6GrmXKKpLC=7k*IB0RsYJ
z`7hYyrV<w;$cWEgL)FC6v$c$w)x^AB7-RA9r6tCg7Downq7Tq{UiIvbeA|>25&q@e
zV{h>RsYc}c7Xz&Ox%BY;5d}KCZEpo9fei)G;_b5{?ze##KW7k$=B1P4k%KYvATpS!
zU33pWFv`VgW*d||jSbFamPp^9{dnHgfiBrDoc<BbpSJo@QxF+=q?kqFa(|x(1QDRX
z==az`IQI(Ltf>_sO=i^Oczu$2C@;uxgMq0;4erh*<}{5_+g^m*CP6#H&HZl5!(`?+
zvG-;RZj?xmQN&?x)I`D}uPA>$mO-N~(CvIc^Wy7;jZNfw^B)rB_EKtUmPe1i)1WH1
zzJ2ArM*Bjlhgj;j8|3~fb>A+>a<CEd`lNBx<Dg_?baecwT*{TQ!y@na|6}j1qoQ8d
zuwhC<=^PqFKtYfWNku`Bl<t;B5TsKCl<tN>K#*>vK@f%pL8QBL=#K9h&)IvQv$t!#
z-#_1a|M<>YvseQ&{N^`L-p_Sk_jOGTlS)wt65#uuGAObcdtV8BUYsK|xLerBud~b|
zhxSDL*CRa%jb(RL6tx^cs`Q(Hi#7+)6TAnWYtLIJrjPh;0m9v3OBZYLB*!f04ubq)
z2c%&F5S9bm^G$1VugVlQeIX3`@KyEla%olqd}M;V!ASefwZ8hKa-9Uoh(CY_gcl{2
z<G{BT=q>smHy$^<9H4dIES&>NM%enNUFt0WlQe-2w7Qs?UIFQ@A#6|5*#+Gz=lTWe
zIZ!w5FuUtb6^%Z5IigG5`qBF~hX%^`(RpMSkr)Auo~Y?0ACbdPeIpcMkDvbQJBg#U
zCyAS4dt&c1K976XiYs(~vw9(H?w0<^V*!vbYq>$DV-O$&UwL5sVRKcZ9uDj>U0pV`
zXDEPVR&oi*S@L6h%$qJsJd#HXkTP%^_M3~%je3HFGWM-V<Gl~&pN}jw7UR`(U6<`&
zk;u`>6#!TX5eZGZVO4d1VB7hipsb6<Fj|fe3zSBEnpy%V<5F^n8@31esl|TR3|+IX
zSlXc+m8WSC<EeW!!|EjoDjE4L(vmQ@@{6p`X$_yPo!N-6jyhhyo(zqzR~ptZP4nnf
zdk^18=`@uaB_rx8_U3<+46i5^TC(At`iTDM-Ie0Sh5pX~zg6FTxk}H|ovP>&;X>)5
zeyJIy<4tM4b;}WqA{*4)T8F+OTG{3Yp|n|4&y-C0bc&7+1SvjzeShdta2=yL3h4?;
z?Q>a9sUcKtR0uKT&h5HQhq54*&K2NgX<gEC1$6ky5LAT;^Ee(FG3dne*7o8xm4!U~
zj&?qKjRoUbUslOS6p}bp4)<<g7b?eO6ut{VK|ZWeGH-MH`N*?4ZHuW8be3c+qMMNF
z=K7}9N1S(PR#?}$$?2F;z=!?)F(8{x2pxH@x(ADKy)!^9DLVIbVP>I{?2i%luW|OT
zpB<R8+C%%Y-{7O?Qy$po8NP6fWldF*2miPz9ZD>tBDus2X@U{S$JOfO{wen+Ke+R;
z>8g4atu*Jq2k&2(_6@@S9P%-opnn-Y`#`690NV?1U+PRGl~(<oap4c2xsF;arr>#*
zt||@P@rH)JQ=><W1>`Dk-}fbbRehCpgfzo{+o&=yTlIzcm+7190vxeg6-r;KEt5q>
zuC-KJ;{3ylO=C++fQ3)7|M)N-g)MZWprY?Z?ANMMGA85iG<*pU2x30Iku$|E)?~#s
zXU9JfZ)bcBTFR6NVv(GUrr{9sP3G>$=LBBIqP>zDrS!ot3%64%NrR%9j<o!Rx2GXg
zx>Ml<3C@>Z*b5g}F=@jabPNt$Z3^$?`iH)COll;NE=i2v&zfo`CA?JetgXlO#5w3a
zdjtH{JI|~Y%E*eJhg#z0pvT_zCr#s<F3hPMV7<jW<v32-`4F>=W*FU3+0MUC4`Vom
z5vwwUc>flvP%<X1if5guLeR)GpQ?rIVATgW-Dt6e&T;+L6dd;cE0o5W&J8UJJL9Kd
z+vwWGa`2@dP97X5k0je$#r=U{J0vsqbFHt1Du(|DAIu*geBz3Vm93;lL@`6wON#2;
zIPs6Za&QF0JdxDY{=2sjT{TI48Dtkp@%Nzw2AAS<6s`lSG3ysvf2@OltQ=rj)FO)b
zIFl6o*J|?br4qa)Jr2!eb);-8tKzR!;Lqitf{9L|iI>bDH(DHrEJKcL&9rhOH@CXX
zn?gX=i^Zw{vn21ZhY^xvL|cG`PD~wF*2<E@wEA4hL3~6|ztS_plVJ-tbWg_jHhm?N
zV!V}EA^me##PUr?C*dvnx!)m?Vm9VG()S--g>3B&*mPzT`lc}q<hK>+45$lhzI;NL
zrGK-8ORkLPrbC@0QZVhup7Yvn<<Uns&Z79L{ecu?t*o=Q1itz_5fn6&p`PjYTnA<=
znHvAyY`><{XA)%cvr0?$d#t~iG3B6j#Dr+ug8pM6{C5+G7h-k{6stN&YyNRjUm1LJ
zJ(f&D7HO=1|H%#|crvpAGlb8#nx(NWSKuLKEb=&7Fy<Q|Xel4o-fq7#mYRNq=rYhp
zpY*mX0<_uAePT$#jrrQsrwg4e3=SHrD$sV-*emd_>^lphg)DsNtbTE&f}OX2PyBMy
z1eDMY>(C}+bYza6lA)BT$}mJg6qkcOb?mX-<5M0&F-Lp#KAU*o-M~^+vH0}Upz@mL
zE%&B_%Zj_AQqp2s%HO8R`o+Z*a)uli%J+;o)m}_rU!cB|i`KB5wk@fW-oQ7WXp>Ay
zN!gsW&h}ifUD(v3D%RIrF;~9Y{5n0YbG(XIeZwVVe9Zpf$>|AOb$D5X?t_&|cBPH^
z710ozpZqfSKcQ#bo)F01(fg@qDCRw$qZoo1O)=8xV=Q--%S=8rd*GKu(uiemq4S=7
zq&6u}AmHynxl<>JCV*jP&v%PgDoL}9EYtBN*@uu7`%>Dh=+xXM;)iW5G6XF2nNmJs
z8FNRgmHC!;P==Y)SrrZ^o>2seH(-9Ndq{EpGrv>vsWwy6Sq+0Q3zbSZJ;B{qX_n7!
z=0kq<kFWi0FxXPGwBL~U8WKSArHle^s4h7^7LS%qReVA5XSXUbgSY+A);J{ceMC4w
zUHE7)c2g6RSRXHqF*oC{>0+=61_W&#le+)YbR85^_GR^=mr;D{Xx4KurQ)ph|5)fn
z@WGrz<uN6vMroLdU@|+3;BY2&ebL~ox#e0FHO5lV@NPL_tuFl(wJ+PlAxST6UhBNU
zL^(|l6SDqtdf_tr33o8@nubU_6goLj;)YkT=TYdlF)e;eI=PJ4JI`kQ!P66sTb=!1
zk8<WUhLn~Nk%B=ZxI5^BJr?I`G{~@=c8yvU&Wl@ysVt^`lfs<NB+N|nxj!-sG`mAk
zbI6^0&hH17y7GzkOPYGJrJ9vRq#JU}2HWty$HF)1qf&s8Lduys&USS?Us0tGP?jmQ
zk5ztSm-owa9^WtP|NM!*T(5hXdoWhE|NE$K(8`8Kp2kKQ!;JMHGwKvjW8XNF26G*m
z;PX8&^>}Rra@l?}^$^qR)zhbjzgy6h5ewS5-3#sCEocOwaf5<d{_2=NSG)9V#DZ2a
zm*qr>LUbQJrl9@@Q{r3DI_Po+VK?SBc~TjA(^X?GA{l1Yy^BYQZ|y8$G#(3VJv9(O
zTXkLcY&9%=0WSLje7Z7)@qF<lbEFLI#=c*WB7Az+<))Y+FyUSDAO9wfz5Nx(v~BtG
za>$Y8{o?Jz=P}R(S=@puIoQ5a#6llW-mDKs?Ab-qqpDHY%!~jxohC{A5##Vfg)c_^
z47QJiDVbYL7KGWd*EUC!ipbp=jFs(f)agomZxsFkP=JK?ktgV?GjAuE+SxW91(A;A
zky{ipn!R%(iP&ggnmU{hdOs7kzte87-m;PAf>e1llac)Z^6m9a&#bdWgQ0MTs`Acz
zAH9z=Gxma--xDTqO459HJJKqY_wsa)E%dmpNPu$D%%nRNaz5r2!J6R&K#Vv!!%(ac
zzK8sbVSBfO)U=Z)Z<6{JVg0ZTnR$c5n)Efa=<~F}M7s8xf3TvEM_hsSSZ|9~oC@Y<
z@w??cN1kYL<l`PRRqT5b7EP{-ZD-QVV1X-TjL;A>?iMHcO<3a_Ne||nNzx0FKcnjZ
z2u!k968$B9)Adi%x6vG>nzq^QUj7*z|1}Kg=u7VIc2i=NM4b(Odfr-Q+HJj%KtfH%
zt^)xyb-{`L8$tDWA_s~IDLrc9`XvUBG(Z3=)O8L}EDS#V@1EZ?o~p5PE#^tnX~US=
zcU9WoiVhcKR1p4mQ1+PsjA?1(DS+h80S{{Iri-NqHr}Ab8NEeK&vj{Mky%U-;SO&I
z1h4P_fn%Zf1QuaN=+ld{4^T^X4-lD`Z2?jkCS(;@e_M=#>)1@#wc=1RMg_Vomgw4j
zy)g&k`&&6FuVtzMKd2SF7Xk`G@^|9C)H5PNe?uV4{G~o-B^p!DOTrw3W~9<Dt5y0%
zt7$}G?|x+=GpWfIXA$%Tr989JJ^AK7*JRZsQG0=$t<UJua@kftoFmwC3`a~;;~AsF
zn#6K58IvUjNBb^AXFeexg-Vhu75<p1iIWkGLPH;2k&KFGVMHOhoo;oy8A>l9Hs=7+
z{()Z_wru9Ygtjr0t;?iGK`OlkoOvmLzi-h_j@acrk6Bb&4hBS}1|FioiF|GPE5yTi
zl-6za<I6>E@|}6IrcrbvP!^--a(zBSdvyX63kdf*$TS3E6}Ga(cE`~ArnBMV7GDe+
zbFQjUqu`QWzKM6BC28k?>7d*R@P?$F2Inq+1Qi5vCgElxi(={c=l0IA43-0CY9H7l
z^m5ZIkgQ?#sbM$U7wmA?9^=iTGlU?!^8{yA|HF-f9kJ_uu{mz&sq6x-&~0FG=({sq
z?8SQl?yXAtEfC-xPeYmQ6Cy$a76svQgx26}H3yuBfxCCfW8;ij0D|H`&wk{zK!JI;
zyU}cf1TGd(2hghqRq?$8Gk^uu65n|6sTr*qlp7fp6R#>=Wz8*;{P)NL6Tlp?aflPN
zD5vek*a7}=f_=Zx;gO5e)w2=Mx|D$Pzr7Q{Ur&z^Ntgoyo7f0}=udM+7XYH6@H!iU
zqBGqH6PSMw{Gf$+w(E<_`yL{Wr4>@w(ya^EkgY48W?=8PVF&2B+BJwqX$5y~V-NfX
zUyg#J=}l>*gNHzjtQCx3%AX+Ftc{99z4X@%`*WeCV@xNRCZxDu^#u&fu@=`WT;cDK
zMaX5bOB?GZ`vqWt*oGgwtC6ms10@Yh=k56WnuJcEQo4Q0ZPD&HH3#jz@#EQF)xXB!
zlZc~b3NYGbg2xtnm>RtBIq1leJwmV_NZL%A0maVJi+|6vtohA%y??9;2K87S0}A7`
znPWI+-aItMlwxy;G9eMZ@W{*eQo=3R_ANER;UD|coHs6|k@~x-y|ItXL^d?%fb0&Q
zy1r*+VS$412C`j<_xZ^?pN(1f7S1dFadU=bIy%o<7VnJ4`f@_Z=DdD6sqS3IJ+<#F
z#*BqGN0jnF{QhU8{Jy!GU?B5cI@nmHu6013+Fl<h81~Ywe0SUKiw>}x?gus18(R>+
z$=1$rS_WEUsKDgR)e*n@K^iY|b*48Mk)zaJYAZ1Mp>8vUT2LS;hEQOc1AS2b0b|gW
z;$j<<K@5Jp&w~sTRr;FNe{~6jSq~N&8jtz1bQf}PDSLwhrK$6vr@O=bZ8v2Vv6eUL
zU-ur;kLM+-<F=*&@o1Mc#-k|Vs_WE3CtQsuwK8%>$%j@!NpQcL`Fu0xwfDrVj;g5X
z^q@u~B~LV)g4OG6zkQLjdSNuvEz?(FzV7F$sEr)7>LbqmZS(kLk#%a-gR+I6I^T3!
zlILyaqaq53G*WbLQ*xc6ODr@C8AXQ_;o=Wl9M%#FzyIc^SaCPHTrglb=(%)3VJ1E6
znm*%)xL@(vYOD|T<+~KhPU#5pUPuqZ^|VF3agU@f`nrBaq+hgjW$lBRsaKe*H;qJ{
zc>zOo|3YLy1?1|4vsP0VdQcy&@A%GmwfsyhOz13kPCj8Ha(m`-+ZjXjP%0tEIn{;B
z(2-iK4APiTZf%Qp?sJFMcQI28-80bwKl*2%Ybs$I`5GTOm9)z?FDk2%y~ACn`WF~^
zg1P~2y+!wYR7f1MU)(s%gJjDOcRo8il`4uUJYUE%*}Rwd5pI|o$C29iy8UW+)o3?q
z1y>&)r5kj_jb%dA6<v}>wBzXB6T?j%MLvs}tUT+O%_n+i;9OQ~8p}HV!~yQuWWiwd
z)8O5YuGj)QC7o$UO>I&Js~dUGO><eb>l&>@?Bh<@gOy(+0;tsp&_pPAn{I_Eh4}XD
z0#^*k)xOyE(Dt`-OVNDz{;DXhL=ui)YQuwhsT9heC-$s`6;Yq&u#othBbyx8UId$k
z`taa+QMutu(e!WFf2*U_wEeL!dJRf1CnHwg&|f~Qv!~EjDWfl!F1nKJk1a@uVM+GU
zo$eAcB;b$L=k8Wh-CLJvK6{j5d&HWA2G-d9$KoRPdL{Xa1P})hCPm}UwtvAe6evHL
z3KTu~*xYqe;Nv|g%{*py{4CgaOR~zFz4+v+o^qz++cQ{XoQG|U11!yWd#}!+CZ;t&
zM-SQHt*;~GN?_hDi+TF+3|Nxd?j=5<qZ0AB-g@ONL}k~E9H(&o!~h>{HGC=2&Tl}G
zODYlxL>W$DMwcoe%J^dmU5!BuCSuel*hbthGLU;2jyz_rfL;Z5FxdlK{fl`rRObVc
z>2f1+v2~?zE{0^-&Xl2J-L4#DucHnyYuyRC&uWGgYJ}sit6?HV1S>T25y8IKt5Ra)
zCT=P4d%>Zds8k!6vv>Qcy{5mfke~o462!OMzAiPvBVQhlh}Q=Hxgs>rVw$MU%`T3}
zmr=GCq<f#t1qSwmRC3k(X_)9yX$f#I=f$>QS#IZbUZp)Bg=tac!xti2B`LgVq0Y-#
z3!O-N2F=Ec#g#v`NWDN{p`~{fi6`ZjhE@%<sPD7Ech94dXyESN!qEB-r%YP%Z5KD6
zYM!2j3NjF}eY0KntlX&YqC@c{RGir$N-z8K#zO9c$b~H_u5KFWBR&%VP_&J`$OQr%
zwNVm|;;#OtcoYg-t0C&Y0!7QoyO@n)Tndv6w5J^<CY@33br+K#nIA}=YMP}E542;H
zYyv%#K06>LFph@oyfCGB%wt<or-vyGy|d%k$Siv3aGx3F62F7=cth&#UXb`JG)K0L
z(B#*DE}S8J;BNenmbTBO#Lkr5WogBNcN)O9UnR6t&kUF39pk7bH!;c*16DZvs8><X
z**T48Frb@b9D6`)eXwM=opM5w#-tDdNdkTk9;g8~IvF2n{OJ-771%4iJ{xJWg|CV&
zkoT7yA`je(U!RA)>#eHMV=_p&+EtWTOnfMC*ws4Dq;KeK870~*wAQ;fROWV=`C19~
zn75WtZdzlj9-gp6e)*Zzx29lpqqd7L=)>H@%tCZUrLMb2m76KF;)W!oYnKVNUmO;`
zKd9N3)$cBINFgexj`eD}cPl;?22ThynIpM`vL>cwUg})4YIUC~vpPK#!gJ_r+ZejI
zmmT%%-1e^>3<^682OORWOwrAf)|Py|LzX&;eqAqY_ra1DFUte&o#m-Z0BV2TK7#|n
zlQ~c-jtc@zsEu&n^)i3c*!DcUVn}}OgMvp!=bq3$2n#$x;KbGf;9DYG;hknX&_>##
z+l$vgfR@+OrQREaUscWro;R*Clp<B0PSX<ttW(3amBrEg22TEC2A!~R%XgGtGE|kJ
z`R1)5zFO<<#seLB8JO?ILov-c8?#DJsHO;{s0_oq90F<lt`I>+og-wpN5eQ}|NRvR
z4k2kr6*;^u8V0XG8BmqNI6~|a5fk&brl`UdgtmC<lCN9H8aXyANA%=^ca?+#t`S9J
zGY!{W>WxO>Yp*JY)v5$$_#1a|SQmJUUFBT9-qJJhs>@vAUj1;j_Zz?lEC5+}C1oPP
z(}vH=Ii|fYVBVv~r(P?mS>H;1X!19WL-rveL`cFd9h|pL5dK_qWKAdjY>QH9z&G{5
zCeR3)lY8YBEr&~oy5fET5dFZ;%XRf83XtPS*-4(cXlEi$Kyx|m)~A^UK^Pk8{5}k4
zi&K!R>(4R$S4bfKSUkWB#TdkZxS-mgmVC(enen3Wx16vAM7=lmn~StK5iTHy6kP&@
zASM=$7Z)zifmREDLG5}TQ-Gr|TXQtB?dHM^VC}Y)ryUm9fha*k_||%4VEz61Kz4x^
z*B8xXllyyYFRuR?p0y_-n&PzPc@wC-)_PBQV*9=v_IU+Y6y2$ojz;ne<;5oFW(^VC
z75}|(e>Z1<2>f)v(R2Mob_PVk#%ohPW>8T<A00<K>Y<ChKbMa`zg!5v@3QvrLT|=e
zUjyYB-B&2ZDP$BfzJ0>iKzRmP1VXnS(FEH>$cy$h9Xn4CcNTO4D0i|<jUW5ee{<Iw
z>0GK|Pu+Cy9QrURla)AIiRn_d=s4csiM8%gnojr=o8{3Hvksj@NlUyPPlnrS?&eyI
z{!SZ(>0zkKxlbwI1m{<+4eb{{Wz05c9bq{$<0)CT`sj4%ceXMnM=iu!SgtY^Gr?|C
z`arflL!_rxr!M3sJgs>rsnUHaAjue@%+1y*{3?|y378?szL8ZHGUiQ!l17X%ipm7j
z_8tnnlhv@<_D}Yt_}8u!Gemt)kotzo6-=vV2;02d77mKU>~pF5l{YSRnj%Hi`)<H!
zNAHdI{DDGy6#64jXyZ*5r45x=XOU(hX_Fdz5o`1|q=U8w!SB8DM0NE78w2`87q>ZP
zIHC{7(oBvWhK6lbCQ#cA6mBU{!ha5F{do9)fT7(9Z4?i;&V=vjxlgEk;*_3XYV<Q`
z%C?`{uX1jpWkC0KoAg~Jc*gZ!YlYZ2r(D!b$9OdTY&u=b@Jy=mnDAu?r`Cr&$^^AH
ziV0vor(&*tzkFC3=eXt9{26znHlGaDkQN)Y++`BO!1QlNl+T#H7gaBxR{R!*23rmS
znjsT4Mc|u%d-z&{*YpdIBKUrT_Y9DMy})x$`Q<<U4eYU`;I(1P@ZBHT0k^(q2OpG;
zAO29$856^y>W@<?3>5b%=IGIRYtM4@m7Z8nlHl&c`=)`a^5i<}w~hY`ie7Kgb;Au4
z*DXr*W1#B<cflYd9&q)ZZ_|2s5Xp@1VQTwRW!5iMkYg^N6bp|xEYH`Tq~AjM`8~;9
z+p^h_-a;>mE1Tw^`k}nL#uU14Up-QR#>Yb=od-6oKYuV3gq^iv@)zat&HqB9oBwa2
z(R51Kul~>i_#ZO;KV<rU$n^h^>Hi_q|M!t;Itejf`eMw3$A4kV4R_Op|GpPS0Q?Wy
z3?YLniryN*>7iivvxe?>;GBQBu}fqa?J+GA=f&S!RM9|?4ktOMSaF2vV{FPL(!4gU
z#XZUj=aDV%(%o>IRjs+-xqh{(ruLj~+|q}-MhV>wuOy|bGLbkb)#t2b(}S25g@oxB
zrpqAV?IarXw_P5wcl&A~_UruRsItG`BS>a)2!TDhd|#>HKR~cQj=w*GK=7%>8z1w#
zi;e-b4w@QnBo!d8`CKv-8$Z8;cmu-{=j(S%m0`*MK=%wb!8>Ws**Ut9KtL9iDvPZZ
zruix5<e_$r()9<8;_NrY$s62rm_{sA0hcC|n0yRk+sw&((Rt2h`KhgyTP*FpXNP)&
zNUzm2Dv^IU8QkU$@ehlSY0uaiYuVCF`Ve&nJzN`wkMcLlL{wR<D;z|Ap+xD$uL<A_
ztcWsY8)C`856G(SWh;qzbv96JVG&<uWgzi#E||ps0jBKp^?{Ly3Bxm2xwzC<OL>Yh
zonut3mTjNz-2VAJNLVI{jT6yk0}P;%MBXY_<q;aZd-P=)gNy+pm2i!;ay5x!3qgx3
zvc0aC|1cdIX-gRAVxvP@UL$Bis(y=8I<BZ*-YuXb8|j5=PofOTk9?zqhpNmBYtf2w
zp|np0%tF3dZ>ZSjJ3g~Y8jxX13}XD!Ra~Nn`OPVqEV@tB>IrMl6yaFqs8%_df6^XZ
z+w+6QucfEPSyoSK8&k*z<0A}TvTM#|R8yHZc&ocx;K7x)_E6ftI9U+0BeHZ$AS13X
zsw|zNfYRm*^H}GKGlkLXH92>-M@>M~#zWrj$!iehH1Y>f=hk%n#EVLww<lyQ#>9^F
zahu0C%)bkUPKpn<=bIbg$skt8=mY9pG$(PNj}>Qn80j%j{^(<aMMMXmGO6A8bqGGd
zyrZxhuD7JZ@K*@98VyF0C^AbOW(cqE$>JL`_*xh*qpFuU+K*#&G&Q3?Q$Usq;IOro
ztVjar1<Vm_+x*vV&-`{7=#0HD#&vlA3`P@@!S!m$@3q~<pIrjSs>ak}3UD55_b<LL
z%qvJjdcQ-Am519xpZ>EBkIBv9RT)uN&Eos+;bZTPkL4ZMyEPp@^}J^c*a)ej5U0ME
zDNy>@J#6Bk<8-*0JF~a&nW=F2sn;Vf!;U_C`5t7)#<j}L!ulJxNO>NkD`{D9JtEe1
z3p;YS?9W>B_7mT9bfQ^K2_DIjeCYbG16%+i8dbE<FHyP^5o%Mi7ob}Ak%Xt<W~iN@
zWj=xJWRRMiXHf<YH@ammJoFJmeh}y@UXb5zk_=^3{j(|lIt&`H19)!bc9zRu4f<zH
z{ws&YYk5DH<u`d*2Qcx)$mrYtdGJG={Q)IH$rw}}TIZ|8tG3s7CK_<2cUG7F;{=!R
z|KSPlcRZm-x1WG->~JaB7u#f0e=AI^MA`9;7?Z^G;W=l>e!pfVTjM*-^LHB2)1{=L
zmQNIH@5_<>K@TGMgC3Nce7M1JY!?O2Wug5^jN^QHsqaE-<Nr+$`uv(yzILsGE$R)v
ziAWyKYWD0+=LXz7!RLgQ<oA}z0@|4uLDsgYKYG6Mi<~M&bcM~l5ay8;#fZo&_v3R;
zqn#WW^?MrMJAR+v$a6Y9ysppm9DJ?l`AV^n<1%yH9gDjiQut&TM>hZAU0FcRkw+sP
zmeyQjL1`d@3>)e?9~#kZ`Wb4FhP$uir~GYERz+}0R4uU9=zr6pj9^pIM|tw$&+h;0
ziEkd_wb-}_n!jBT@v9;VSil&?V=#X=f^LG>R4qQ(|IGaRschv)WeP*aN43z5{i^`U
z`K)*_n}R?P?(;FjSY?{j_2(1=Z~D}2Xia9G)k$R5uM@=w_#p^FE9rvI8ZqBDcOZt}
zYq^Kc{yfcz$=@gMW#q<cYlpa#i}&#~F2`jM-yYya-Z1ZNVB<BYjpJX$4#~w3DGD2+
zQOkc6&u!uud%s98WaE<!Bc;ck*cVNZ&X=m6+TZ$ECyga*23-Bx`~LY|D-jb(t(VdC
zcN578Or-zU+#*FRl-}p2DvufO{m~Wj29fRnn*Gj%uioa&Pa<WO$t9j_K=84TRbsgF
z`>4RSa2?T)UzsgS>GU7nsXy)VO=%+r5jV-f4$|G;I$CZ8>uVB%{M&2G9#`yvKK8<#
z<mz3}phs^+Z+wO(Z@NxpCv6z-Be@a_T|XADlCeM|MD5ZkCDv?YY<p>`g%moA^F7AM
z3s1K2<~nbaiW`fFl_5O75lY&&a(6AzF=+IBo_uv>%`wT=Lrw2~yG++P)Z3@Tg7A6o
z1kO)B3u+_ly3zO1(b3c?y%q3Dn-tyPg+ta2Dw{5-$mm4!2F)e%7bFE0A)&CeyR=O=
zj+IEuHkRstGFs|hOa_+ro=JZA^g~?5gc3$P-965_xuS4R9E8aF`*LCNXx}iEiMvmb
zfQQ02xB#n|N|zL?@Utf6o=4a8F&TM}v@(0Y@8!qkXWY+^jAHsWZ%6J5vmg06mT@1(
zKhDw6xc8XR<N$L(iD#0g!AUjrN%|R@<5I2{rL+XWXEI+Gb*Y6JH{P+&=G{+}#hl+s
z`TMPN-a%RFRmPDvBw_e9RXF59jyPP|D!=G9$9)vz!)Kc%>3J#gnC4NrL^3k}dJm!}
z))8-WyOr;Co8uWuZ6@!t+9;7*zVjPj%kQYh{1pa;P=k44JhGTqf*JEXy{GM>Ygl$r
zGS!FeLSQ{s<?^rPIUNIxs#^n-<`y-tUFdCg2lh!@wW-~;W^ydF{6FpP#Y?3hQw}cn
z&=4;7-B!puX3zKZ)atY*`KkBVq31w%-~!o9xlvH=fs}pWrC!-Rg$HQZZ@A&RUdkDX
zPlFr_XcqbvY|S+qFtl77e|~-QD&0RzPBTbsnC~$Dwnu^g;)=n!vVFnY=e=LUOdou?
zFLdD_x7w@VLna<CFh|%^S1c--Ur}sxdBDp&_3y)O{%qRD;>hd9Xh|DG&-LdBPqumm
zh(!c8{G0>|pJ+^bwbZBtT)i!_Hp;2mm&Z<oDPs>S9n(4=zYXZ+V5=IYV-k-LbP%T2
zHt0#B+kiP7HrV?-e$^pLZZ?8hCxDYTZPDK?ni&_sQ>3K2Umdk3NFk)S>rt3AUiqDQ
zub0V_YoM@5u`)N97!g+b5+LU5K*EK_C+&y$8nuoV43(7gW8vBi)tq)a9%+FGZ53Yi
z)?-{mvrDgM?e)UCli8(3f3u7wNQ20s<qmH)X9ciPHVigZm~8RCwUtl}_U#1DZOMlO
zX)`!ZN5Q0QWsX@eb7w!O028usDCGe=hUU9SJTdoQX`sK>(_f%hLa<K(>>WP(n_9r2
zeoeY@{^K(YlbNl(iE`~*&S|h$h})pMfLBY$N`SEayHZ#34?({}vzrUhNP8)SUPeMj
zCL5Z7wnB>>{}eH>i4L%vu=%1nir@n{0Xw&LUC*W=+}r$JiTKTmfKPEv!wwW>?SPwz
z_GFS(5jo$4b$JQUs3cb!G@dkG&5;xhxCAx1q*oU!Vu1)!g(0AH;KkG}#c2ls8-h1I
zS`x$m4Bc1#LF|N5353Mhs1RItAQZ{tcL``UrBa)md%*HQo4b*7SsBI|ASQW^NOsu)
zsop?fCD_k|#j!r3WmUNIBS)0Cem@nC{lM*eaL);|NPx($HJ<2+fdH@+ZE<egr{+F_
z*R0U6TKVDdVG3=QNC*I#gBGH+op%))r`iMw?`?+6p^w;}4d)pk__I7j2@4nH-K$g<
zNjKJ?p)KDTxY(fb#$7dXHQ=<BH<Y16`GlXol6xHc_Qhi`?`px25&P=@wKw&I_F@*p
z$&7}<y)%l@DR3&(eQ?$T-u`eGD3C}BAe6A%0THCf4#<pZj=%C)pqyhCJN+O>Qvh5C
z3=y_ec2}op$dXr5qI;1(zD>ZAApj5_rqk{U0AJCTct8lU8_{+42aYQ(+dImnOJ`Q7
zt1Zj5)pjG1#`AZ(b=QweTqoj1f39GVeEh+G?VH*H4L`A7jHl(|9LOveRjS>-VVZGK
zLCqE?YHq2Vvv~_YREMY+)-X~;UrMFm7j5+e^RARv(+kCX&PP9zIm(9AauZx$9x)Xx
z*UaB~=Gr?k4Be@|dL~O!EDNN$x_)M6RIBB@R$jYfeRH+)Ti#OU3;RKVt1m3$hxBLY
zs``sewzgSW9gebPl{>eq>FS^6Y5MIwbok)hP<^|sHjH{@?quxjKAX(v=zXpD&)l9k
zW+Ngw<gJ6j25}kl*ZXb<1)o-qJy{>^*h?19lQ4mU&n;ha<3+u9A$Bx?DYal!Z9j5_
zU^EGBRVX98Mq1PXy=MnxtpufqHxGsOA*+jZKq+fW>0BSA?R=$9IZHsEXvSx*U-t?K
z=!znwNDM#4s&Ta@yD*W-3-5lj19_DI30%QDR8FIYFTV}B9*<i+aHp6BnZ(GcG4F2e
zX`}XcO&f5c8&?!a!PiQ7Et;FfF0$1M_&g&fG2tinscDNH_um5cTYhT>g8L?GaAJ@j
zJAOyLofINb)ixU-_5su<8r~kjL9SI+rrlyHA3Tz$tHH%SirH3k3vBor^@g>MTz&3!
z{8LhtbJmYF!qmNY1`MtGdjz2Lrgu~O*n7=%RjLsC!W}fi4Q4EA<p=0o&J)*%`q#tL
zIV%u}luMm$AV`7<BxvC+fJS5xjasiC{;@BhG}KoC;(RF`{57-{V)nwvd7t+B1QrN%
zT7O{McbRC{yOfGBBJ_R(fv4K=_Fi=O2%6nmsvG}oq?{iAo^G51`U&&3tk2ayvr-|%
zU~`goFp4oIHWsdH6-euiq{%!-q!Ph+O=YWvwYalmBvvzvtV#dMsUnmvKOJJd15_M-
z9mnra=Kv!M6O^yk$)RU0(VGDcH!MWjRRGAs+I$qB_snQsipFiCJkO%<NrgoR)Tj0T
zuA~8igWwG~IT{OKS#re<?fmGMxdiKu>dA`OH93Ol);{GFV!;OrOYGG8nQ@{~0B0}a
zLp4}ST0yGNYDxHftiP(nF~j!)K^kifptOZtfvn~jiuawp4=CLc*Am_>oQq?1oc+_G
zUU<*<JO5cJ!SN97Ak1qp&KAGitQ7)Z3=?e^g3h-lEIYWQVUXL}5gm`hM1O|Zi8e{m
z(Cjw>^RP{BkI$thorJ6-`$X+SD8hRR(;&|^iS21ZKLKBl?xUXj#_ijT_AbY?0jcX-
zX_`;UUopYI6KBolgaF*B$fxYwbPc~G2zPn1(B~di&>37B${F@mz7g1lOzjzI7@O3A
zUz^$~tTZ6#Uz(WvO;Dge8Q5IMa&QwNS3njk=*A3?ORAD8$~kO^fBF`*+UXpy8sSY9
zkRM79)^SBEi|00~i!Du8aJ7_<`tRFlP5}3yZ|=f>q}hRMEZk7HZNI4;I2TeH&KX*|
z6+9c+4qq*ImY`~-X`-BjoOj&m>85+&rO?W`H&l1<Zcc}Gez$x#sTsK3$1!$X*9of~
zUw7+Y4HEMgWAr*u?&rzm__uPg$D4ODlP)UQb_~Lap@WByn)~cGtcI9zR~o5#^^?x-
zn%Zf~jeLgFr9zXP<bJqnvah>rJ*Q?Hql#*G+RiELIc?l8pw6+5A#ljL3cGQ4&YTcp
z0Hu^O$cx2P3G^viD*xh-eK~gONF40jw{W=0Ij@|V_DC>HgmtCddy#?G!Z6GDg%2}E
zNB3I~$I*(Rs5D1ebo{hs#_${%JQ`*36r`OEVDIch9IgSMu(t$gJ8CBUZn_{cPmC4J
z?GJA)8#*E+lyO=qP=$EMU)62#aaYw0$<y%f(H?%5(yRqc&U*;KCAm|j@Vb+Fhj7mR
zg=^c3x^>%*b53_Av<QV$VVmY?tgP1lZ;$hZExb<udC*vw7SG>~AFzcC1A){=^8Pi3
z-8E__Jye5A&&m0=U>kZ->=U33CRhMSeJ>;2^+N2h9h*=U^hnPaL@r^%%baRql<S*7
zbRWr!f3v~TcJVw5H5jROX2|?E6MzCleDMQelk_t#3!%N{*R8Zy0~+RdnUu!VnMqP`
zgpO4SP@mD#3VF3%jVP3JBBSN?Ix2@Iehf1V|K1k>RXE<oY6Kz4k2ESeHv0~z(@oBu
zvAVAc52@VJQQeuyJPt4Kt6l%|Yf63qkp!U@(ZU&)g3Du<b!8~UiQ_cF=K`HSV~S1;
z7$f#-m_^nf?0Tf3kGgFXZtc7N;=-QH2UCVm8mAbqg75;0S1^B_vLeh}09Z1ib-T6$
zb@mEwEp{hklQpINS}4E{Lo>weQhUTLXo<3nkxGkM@$-LA1@SG+RqD&f(YvR$puML_
zhg$_20flu_{sj0`QL@*rW|Yhav5xP8ILj*hdo;CQlF%8@WXmUC52SWVXdwEPTOS8~
z!Ri5sYgKipS(>2OK@ROC=&Ij+dcVbK!iKEz`Vt;^49IN1xS|Rk^F*z>&Qqp_2wO0@
zD+CpkYtlzbETwzbc~2xoKp~hGW={e24_o)<-<~5DyEn>VzRN-IxPe=&@vKI}7mMZV
zVwieSjp!3?-u*7a(S7@;xBAP!naS^sUO88d3mOotvG9Jd`B{9=vJsD^VB8~fE&HNi
zY{JRM<?@4WE1~IS7+gJOPF7~qU9&ThRL+wFk0rBUH}*R==ebB?uw>3w@MbkH8E5=X
zwo7vLojV(7q-OCQ0X-*q+FIgQl;d}@l2I3(v>I#|aqn!=5wmqIT89}{&c7hE-Z1a5
zbgYU(d;Y8T<3Jun)xOm27hpgBj$oclWbWL53o}>-EFLVt_I$Na>Vmxonz_!fWPe%+
zrmp8|DzII>(9(2$-V}%+NMz{)2Bfe5$@JOJF9Co85?TeJArQ6<|9%M;W<Dw)@p^=N
z4MfOko;3jh;0Y7dJ(uunA5>gS5bTC_I!H{x7qtzp`WJ-ni^yNnh`G6YnQ(e=nvG3j
z9wAGMqI61mW1n#i(J>XK?muJ<;vTfY7KqxJv|1Q{-_}cCUaj>$F|oWnB86R|;pGUi
zcwUg8a_(C<Rcz&)b(1@n7YyoMKQHYEIges`298VQpC`zh2^wEpI+j|a_Y>m(tXo{;
zT4W{cU<zS26k(o~>?Zc-p*BMoXjvv6jVT^RCS3JoVuPbqJ-?`NuAQkM<WYw_wp@}?
ztG>(rs<wbFII<FD_*2=B*h1Zz$~1xJ9`SAWY7k@B@fm1Vl$))Ib*o_OO(dUqvU^u}
z0_+85&KsO@g3H<mYg6_ut}|B%8uKX7scQiRu#^b1sOPN<XXo%H?@>@Gzx4r$f8bJH
z_%&cw-*T6_c7UGi%<emr>A4tc0HH!QJm(WvHXqor^z&1_DtNSI$QS37cG6huHGyx&
ze2=yiJAdZJyx?PEFV3X1?Rr_pqWb(Ew+RXWg1HKl8DXH{-~wM!A%%9EMHQ_b0k~#e
za;EE_4aTF>M8Y%-=N{4oI=4<@CnPjh=ZUoQg<VW~dT&|jQ|cYpj_{H%bu^cNmrd<9
zPdPWizCW<zi|3KPD=7RR`5-G3yJev5u(ah+0gdH}PoriUmiosa=~x*ePzJnV^eOq-
zUql@258_p?Cn9d<yK&lzXEICjN{aePHu-I%3mV>!l5*oyQ1D=B4M~NxPltLa!0o9R
zpvHVUNk3bKaZFvb9`tjqK@kleLg4FiFNWyXq3&O2JaY*|7oek#hq~e&8Hev1^VfS4
z`nX)=euP_$)%hw=P_5Z}g^0;P<KhFN!vx}2USq}+`F<pN-yV0||1rK01(xP>Ha+T$
zjQr=B5nM8So2=PDpe|A_hH?=5u5zYu=)Ul~(<zsJDTu2s6IYO*GL&k^u76|cl+#^d
z!>|Kf@{~z&<_6A6a!MCGUfjuZOFig6h&s*Y07HXTm{|(vctvHwa=ohr6h>^}PstWU
z>W3T6FTTiRmXM;k_+;!9JebU3T3}n-oC_jN9LL~gdNrTC+M!Y6<2=~y9S4c^MCquc
zFTjXp6LQVW;}laUF1hgH!&)k<n_eknnzy|6`|Le()?*wnnz~n&WShF@{zR=^Wv=_?
z7_Uja7Sgm6Q$h}}#o6g9@ccSW;Vjue9jWqFnG!S`!Sf1_3phDU$dl+BP7{@4;Dqe1
zryfG{)YIqBmYNt`$tq^vb{&5RVu*G71x2DGi`43)KkbBAp^QPeaIHMBh3zOLSj}g7
z$uCiUsAbNytLA?^(;3z>@i<VLp@dec$$_nBT^w{5wU+e5->}8R1Hl&Pj49Lk7bf#3
zy*(Wlyf*SNHY@YL_l#dTu3ytIFg*rfw157?A5Z@Mfw?Bsp0U?aZE6xJg_1PtFRI?8
z4{dK*A*}@8h%I@kh;{p+@p2yj3wIwlB|+)9#1PB93hh@n-cTjTu#JD#)~S?(rg^im
z7<RY7V*8(KlzSTVjy&F4$H4j0|3*#AEVs)*d-GOwsW~2HPdsMVKy5Zxz(@fJ1`ipg
z8{GmSfFyj&utF;2lu-(PF`MWJZgY_N)_5_jbj|Qh-`FP?tgWSuriY(hLfWs`K2|Lq
z(|(PI20<~CuP9VPdO|;~MR`0+_OGBEeVhT!Q-~k=SnS)V*@WwW$E*q6l0KTplg}J)
z&3(iu^U*&i)Oo{#?Vzhgnm@JbX&LU@0;x*mDPo<uqLD|uX3~$@7}_pcn)^g6`9GgI
zd8QE;H`wG2CS=(V%{(g6AFAGVov51$C-4&!*%k*WAo3@L4nd_urz_N3$Cm#?#{Y+m
z{|_1e|0y!=?vs5#Q{wLsxJbVmm$rXp`~VfVCmD}-^H|I)`Jrwl9jm5z>#m_&$P!aM
z?)5V<<}V{}Q|H4@_M<5&)Zo5qX~l2srYOznEtKQt6)NQ}KVi~Z(Vq&AUa3%7J<O{|
ztnobv2IBrxy(Qduq|90i){(BNN0#&j%Ac$wl9`u_=D$4=j&+OSwMw*~<yi=Gg)VXM
z^|!B#eJ&{ev@c6a$YUK~QCf)3iTt)Fabax~#rP+hrdV1&JQS7KP2#9ipe}B-%|h`|
z-d*pAypqS{r`4E;WbGdycd7!3m9EobvfP<Ty0-2|OI@*hzfNqti~!`$#$m4f=OOAz
zI-;l@m|XmB6S4%EWG^{NdCT8ra9IKPqOGI^h@XEDHvW9_15rYe#Fz5JR>I(In@?11
z{P-Q^5|ENh$LcAiOG(Xk$Re)@iV}N`cCij8Nj5@myl6hA$!WmE4nz`BXm}-1pc(}D
zq=-+KF_yiEJ-_o}&DhgRkzjpFh(=+K8C#5@sa`P|^8Twx^~Zuys^OIHdZA0h4PGDi
zh@v}vhpX0Hrg2|SCQ6b}?WDhe>T`FH3cXVKp%}g~f;sQDS$egmw(0tf?Y-H;lLYIO
zoHm|RYV9Fj<5wouH;W9uSlu0u57c`3jbP{`sUv+|J7VHnwW-Ln@AF!`426}8Ap=Dy
zucxZ(>-2O8b|>s<rUkZTT&9Td(tloQ|2Vkery>6_Tm?#HeP-Y1#3#sj0QXW+(IbE;
zp>Vf%FlhDb<qvljzbY<^1+nL2AKKBZ{A7Ev3B1=&e}q|943^z1;Pm}2xsv@&N$<&b
zORH$@uOLh5J07}QS7i8<yZSrHFr_u6BWr^lfn{Jn0z1^y=#IMSdl@^RlNR=p@Gh{a
zOb-IU;TAt$o`%@k9guLU0=S`>HTmcHe`YIhb8vz9BwSWG@AhwA95@gzxBIhG|MSU@
zRG;nDk%k+ents#!cfci?ITlHL^)O8~PQEdwb!iObkll!p`g@o|2u2VQWMQ7V%WSUJ
zR}++-ZaL7@7>bqBdEK^->eucZglCV%R`H`8G#KeGq-^hR_%z%bk|>fL`I>eR8DODS
zW2K8%&l1n5IYm3wW!WjIjZ?zF-z5%5j#IATi+)^Aq^O^esvupO;;`n*FDPgMqiN@4
z7jg2gg~4Inacou{J0(0ct{rmSG7BPjYs;@Kw7S|VqhjM?*H_j&p<S);)Az6%YlZlB
zcb&MhOK$}!Ju2Atv2i@yVrYjaWQXP9NK;U)>LX^5z&5|J29kUK@nDc?X3AZ7@{#i8
zEm9LEOju!vMz}Lo7HgMAwcQ+c>+q0^=pugH^0B~!iSUW7DFG%CO0KDJYbuz)NV4(4
zzFlz0wSfG-OZ~={X=IwfjMi}Le(k0s{1kKlY(e300kZO1m>O23z{32z@cIoRi6vji
zsQX{@J6#fugLKNL&Bt-2JqwQY;7sH`uK=goWzoYm5nOMf`1Vh0LpwElQ4ii&OlLri
zi@99~;yJ2R!UAh1d8QXe>l~U!zP1Lt_iNwphH}w0bnn$?F=98S6(_V0S?c4V1h0S0
zt|0!8BMmr+e5OR^fB2npVFb=nx7%Gx8MBGS98y8GVCJv@jzrV@*}H|1n%$uLDYWIz
z<ZT|gjcH81D_-$yucec-k6i}KNy2q;6_B&IFo6Swg+|3)?lA$UG4`g<4oOg+A75MR
zby_0ZmJPi&Ut3_Lz$SdJhJQ;D^K~PzV<5~VxCSN>*64R7%`H2OR@s{__Fnd!0R>Um
ziQ@uKDxIA%lt*BZA-uQyPD*`8t3f+F8tcT&;VA=IB0JPWPjqc`l8(9WkTHBCw!O~F
zg1qAj0HhXcLvHiTue$E^O1VCb2fb(FUbAU`kJ5y2+b~Vx+zkPW_B%!Y)Fm?an#|kS
zWJod$n)x#3y&ze!oEP`UWNGHWGBD9Ylx8BzeG$c@HVCD3@=M5ZesQCl20-R*C~Yc~
z4BP<-ImRKZs~)0DC0Rw8jVL;=1~s5^qNWIMvN<5|&Yww+>I!liHz}=49=U)l+^aqQ
zX&2*L4vAYww#3Fh066k~wR%0$bj3Rf4($l!or4<KwpEq@j0$}*<CrPg+Nd^%2&?Ez
zAU)+};sK&mJQTw^!V?sVyaP@I1K8UnY8qc1oyRr}m(X2Vr=}DG3DB<rU}sX>O3g==
zDP&Ev`(ElDqTKYpX4yBs#p2R-3Y^mN*r8{46c_<HtJtLcS?+7hYWr|TN?L78NQH}w
z&W9IVe?n0V=6W^DXZSY5J*<I-{ir&mX=Wcbrk6`E;kQ-O{JU?yiB0;(w=qJ@JA*-@
zX4ClHQQ7O~=0R>8H65;&QOHx%OLkG=(a@7B3%Or-l<QdT&D^F!;-@|G>EuRyC&?mG
z`y}R76&+#P4g}?<2?S<D@$X)yHqUPnjzy5eF|%)&P}>KjgrP=g%O7u5cT5xEKK`B|
z+qVo+V`Xn3vgxv*4vmmBvkIbFK`kPxMwIE#0dLrwO($&+T25D>v_Obcl-npZlrnk~
z0pj`t8zkXPfc6;zH*$`Jdew(`b)YV&^-JSn;m!i>wZO&IaTEU-<clcQED;K%2~w%x
z(n8dNE3}D^z%Kj?tH$%B@#5Fq_vo%CX0Du0zK44eU+n~nA<`!!A=Zl-58Aoh!x9rl
z9f#yvK?U!{uS#xf;4;qy6)=em!x5v5rbQI6w&Hp1`0doqT(VCBT>#RstG2phD&u#r
z%6s_?iyMw^B5J-nAFIdf9&jVS1_H@}a8J1VTnKj`CKy7Q>V>7wo(R1I?gK<x*0>zx
zY<U;?3A>dMTffz?&I>17l#33d6QjPe8iGyKC=ax#@GHoHY4#&oEuob+GQ?(C6A-&Q
zb2a=|*$`6x!1)Ifxv0Y_+dZYPqVuxy61ZboKC8EJyvdW79PAf+{mFWY7UN-QP6;o}
zSxY9aFY75mC_5>J;`Uke=8FobG#zD@e9L&U!^;d|lUk8+rVj*($YCXdH~N^`=OYY=
zlAd)7`EC(<Z>MhaJIL5XBGFgAW@ObNj_23?8Q0dK<-7hubG_?l5u9*aJ3hlIZW2dQ
zeIisl<l_4H`U*-u56sV+$n$>*l6}%MjaA)L^4zv`2u!J424*Ane+Z=Ev)Avm?0m%Y
z><IB(tl8ZT*h3V?f&6Xw=pB?(e$dW4lPjVlT^e9|_LeZuAgT^`zBFC3={Sv<{>rY@
zc!TQtmOy~q?f}>shHW|`G{sH7TN>_pKgF&ED;rTdMU)1x-ZAPO07-W*W(pn7e|Y<x
zy(WtuD5UpsPf5eivO_S>4|2q)MXyi$wHqP>?R%9mekHMh#)Ek@24rJ$tqByP%3Xb6
zPBZtI-OE#N$lE>h+1Q#Clsw82J-o-g6N#`{2!M%f+Z%$~%PMIsCSsS@NK?ht^EabG
zNgHe;s~ITBQs{w9YO;qn#n<kEMj8Z|k(ruhnuFGEVg3DH^vCRAiTWsJynwvMo0K%j
zltl5)Ew>S1UV!?ISQQZf<`GsF`~l`c!qYn7-j*h>LkLbQi#>a{wTzo-?7tB>NhQ_I
zT0<fr$tSMDG-8tbz{`YkdOz*>9o+l++z<@bTX6T8bCmFsg}bP*sU0AC&UgX8zA7ef
z|0O&C%wH_p`JHnTiN!HSZjC{fR#F_~Pnt1A@%x1Ldl2FWqtqvev<6f8Nq7!gBbc^H
z9!#uy`$;1SG#$<hWDC@(fIup%m@eczfN)cH4}oH#P}Np5&={wVkWK^JvOq+72U+Bq
z1O#`Coz@Qu+rHjty6WB@1YD0fUo0X$q8*+hB3IYPNXFDvz%x|b1tv6YxF=}ahJKkS
zdjhl{Y$pLCeJvz>xu$@JfbR*I)g=(KXw>|LCR(1n4{h$AY<1p<+L@|9QfrPQzA8yW
zRv==(2w^}jf;msx2a<mhe%+_Ln<KJi;go9orm)?meRuT=A>Q}Ae*q3Tf8{<g(Ie1?
zbX0dH91tGauZ#>m`NrIX`DIt?$|OGCmmp-nnyh~I2HzN>ZaA?N<iwo<L?uPx$HqEM
zyP>bFSNpPjDZ4x5yJstrXm(fRMT-gPpvT7|IPQ*uN2A6zMyRM%elwwtBSS;3A_IQv
z$S!if$}lCafO!Y6#LSS3$QUiSQ@&k9DPr<fiSc8U2G7Us`>vWDW=1bTohUF;@N^Ov
zExsMmH*{OOhz8B2NK^jbQK98iI@&M8GG_l0*UFS_4t+YKtiIg`D$~P<_B*PD24^p}
z>uJkj@7q^(L4HI;tsmv}WCKi*F%2g(+Hq0@hTkUMRzr2saa2K7#}JZ($}y>l7T)>Z
zDNG&1h0s?TnLna4+{^YPBQU-@4M*3tAngXSD{N+=(~pX(_%26oDR2!M({*lnwzBtZ
z<IS^U+4uDuy700}OoIj7RAFi^ja$C3YlJEz`W_u+;^uc>5de~hEQE<EDDLo}JD=F=
zsS6D>jU#A~$=OjEw-q0u?DBv0hOm52RiUs)S$AicZq7>s$C^S+1MISzM<IK*9%V3B
zilUh{-pTOJjQ|c;j)zk_jG>e%Cm&XnTnEpGkkmS;uIqL7Tx+(<6#3TbY0?7iob{N+
zUMf6H6QE0_oOzeC<l5kY(z>q{x3O9_MznmWumOtmm5Wv$(U>*^e{h|EoXU<1Xnb;j
z>lu&il^f#j!kwbxZYx@#?5@-fo=MkyXB@Cp<f^Sy?M0xF>9uIpH0R6h^zGY-RN<}w
zX=|(DNo`~z1$6tELXSu+`WPm5^?iG@&M<gxy=+YEv{Rv~f3UQK#?+mPYv;TaE-f)8
zOy4YXEL_%~vpz1UcD+(0=oT^ZMk+ukK9#1NORSix0Hgt{KGjyIDYy+rZx#t|AkoG1
zY&Kvkz(n83+Ij#*%90$ylly6~9Jkp(9E(#eaTYSoMd_tFuT59=>yv6aS_P4vT=AMS
z!i4QlhlPGT0F=bq*|VK{+}YOiAk5*Gxx676%)@^wSK5omg}LGM@nlNo7X}sfb&b!G
z>Tf$K^HZ;P5_j6e4b<NX+q1J?#kh5hZvU#u7Wd-{@?<;rrTIVAWQ&hTl<j~F9YVRv
ztLU1faOonKL?|^iP;h1b(Bg_%xpV(%vsX?v<SDG{_<?KZ^OmP%y2L;(h1R_|<?8L+
zT2o@%ex$&({V$RU>*#)?!0Tr#&UGSq78<$NvwnD03bTe;gn<sH+banI(}_~dYDZ?h
zNTOdx`Wg`Wq9hxd^>Cq?Q~Bs=c4$Ml)9C~EvF;W`CJT{{;jY$YKH8oE0YfkTHQ12M
zf{CX=Rd6U4{=u$et~JwhL_Tc17Na1g-wo+oD|X-v^<LtDxTrpEHAt(8SO!?%PI&U@
z3K4#23r9-d{3&B6LXo(Xi-XM8u8tVqs*Dl3FBZbP4l3U?=e-yZm)u4D=}y&#SEwAq
zpfg;<2t+O8G>8Szz_^$2(OJ9AfjKo$@uAt=vCM}GYQCWCQ1H6HZd<{%OWB%JczSHb
z_F;IDar(Zf5PaWwasTS!oCj_gd}_Z)urE~aBIbB6zDf835PdiqQvA;~$i4Fmt8Pwy
zfGIB39d#^=c!(;&QoZ@8&_#nUPJ>QJHBOF$)a!2pt%jzOy7|?w=Vu<Uv^U5nI@Eid
zWv3N0pe4+luUw0-5y3F3chlvORR^=KCC~mnk?G2nZ9yUu4_>6P)hm!9tXkW57FKOG
z!LmNdqJz#0?KGpdDnrjyhDM<rJ5q=D0^bn0j9fj}KYMX9iy;O9?irtVHqFE)8jXA|
z5dK>s?xOUVl%hBt^S&NI5zN#YxwNPE_EBi4)j(~-We|VFkN?aH;rr$hJ;Ql?Z0urm
zzl=o`TDHK>-!3f481TS(mGj%Bt_QXX4f#r`1NJaT){S5*FMjG<GPW)^5(zK42A_Hh
zScunnzTF1hRz?0f4Y%ebQFs(Yep~XQk|*G-Fk|A8V>)a0^coQ=#9^EY<JbX(jl^x+
z3S~U#=&Y(g2vhq--2=B}Ra{bXq>45UBf~xJSeJZ7KT5d%B@6@-)B%VfF)q+?@u_Io
z2pPu?A~w{17SVP!NsqUpAZ!<cpa7JV&herwLOu36)g+Vp4{wVoV9|LaLe65Z!+c6p
zcZ&WTPZ2~3fRvE*^Z%k#e!h<!@`zu+6E4X<;|?1I1EZY_{{JxcRzY!wTeog-2o^$c
zhXi+b4-nh~!QI{6-Q8Upg1fuByIUHU#(g(y|NGQlr_RN>;EJN?s_uTjImbK3Gq$$C
zqptIP>isa2zPexb)U+KSr|oP#4H)LDit=df2_k$X3Bmu`{5+{;bMAf3Dd5uic*hG%
zD+jInodLX)kfXR7<^T{h<x*5=BV3VO_UqjF%3iMNp7E@kdmMo?t{`TTZY^8KowjRF
zAAYa<ty?EC@9{!dbd+{UUrTbK=C(<6$U|B8*<2p<Z@j;JPj4K&3a<b#n*1L4=y=0G
zG|%Pg@Hg-VWFse6{g?kNz|+yBx98wGG2leaFn{mq<Jiq<j4%Gb1p-p}Uy$JEmu-Iy
za2Z)PQY`ZBY>&Y4eBQti<jUhA*SBFCDN7hK#7WV`1F9<mg<ac%7@2$_Vy>br@8|=<
zpd-CTw~ruF0+1Mpd}zdZ-O|dpvpkS&30=hbqg@U2d5&VU7IPlzc|opPjMFnwVaSd7
z=Y?dOMtegodKb4YkKWR0mYj|TpdPIE0Awdy_G;OKgc!~gGvS-6|8JV7a>yrGvS4yC
zN-@%*bH%<mTO|ERhuI>xOZ36Lm(!7uym0B17-<GFHdDTF3Z(UECj#DVK9JHYuS^@C
zTWpBzp$yIg*qgvns|9;i8IW>jceJ@DF-G9Z?xGDl^so^J_d3Uz9eMdHGLgzn6E#PI
zWV27x#&Bnd%OZM?nO5=LTtv5ZXkN^SYLx*(pyoMAY2*c6GuJgGOntoG<9XNVkw^Qn
zGKV{nIKOUJ>;F&cA01BY2Q4kv|HS@-*MQs~p?{-ly1SqewfYeC6q<rBBg60?R9RWi
zkG%gW!Abb<LYv!lS7~HZ4pIbzmoTRAxbQt}0gsrWlJ7_XNw#;x#G7-iS0mp)nb{JC
zY$2=8wp_bzkzoNJ`LygVu7>+Bm^+Yqms2@yJ3>V`Tq*&;AM10~_pRtLw7s4pIxqb@
z;+Wnyz~#QUBPUWMpa)nCdg0viz&84p1q5XN<*pJ}xy`WY0Gs(cH3Sb>7Kvg-Y~P@G
z24HCED7$iR4X_<ZWPYOLVrs_h$w?#_wu!B=@dnpTF|4YSMwk{h`B^!{4v^I3{N?z3
zI6qe+jA+E2#B}KRh-y$Zpym>46&G%%-1a+<ZCo7DpxwmDM%?#ue0U1!b@MssKvWYE
zO{NHa>)V$Y@m_^e%Uny9$Pji;JY}yrlua}NBHl*RTvi{0cy{E|$r$)bH%s}R9i1<I
zuc?eLmJX((vuO6(C(LDuN)0F4Vsn!(mCq7s+_%JKM;!GbFNS)NaeJl=dN&@L=8i2}
z_mcxeeD{0$ygS%g*!8sQ!03g;mWcCUsvYd%^YCEvvOU%dF!XCI7@`A=re5JWrY~!j
zbFu`wkY%)E4}Q3B4M2WF|I`mQv&L*+UzhK*qxo(HZE_B`QyH+UnfVVcEBSRGiKCXL
z4hlhhv`JB2Dx9Fld7@~STel*?%IuFlr*5OrX$kt@3(WZ+eKuyt-wd$5rG7fRm10?i
zLI01C!->AF<)m8S@=>d}a(so|R1v8p$FY{}X^nJU;Yv;rdp{!Q&3sxrMqg)iMjdS*
zE2obA&$G`PY)ie66l1CxBLyoq@6B}UA@AUosKQGzmbt^baKLUoB47wyd~BtDzwRKX
z(b9#*HOlf1epORGcoI<mbzXu)a-U#1Y5JeD+z6vCmRcYp@l~5#R>~)Jiu=07TsS&g
zo}&w7wT(FrNQQJ^x_!WCykM}=Z^}6(hXHH*WelcYN$zfg8`$^v9n5pnkwF1@EVQkG
zk~%%X>&ci2br|@}YP@iu{y!{$|N98z`>Na1rUOQs7;oDHy`tC_8Gm6{cVK5@*H~h&
zVh?;cQZ0?V30q9qo|p-qKDaGAn;sWd<t1AYla&d0SJt5)yKrL_hV3{F#OZ66*XT~1
zS8prZ9p5Xsq88_)n4X64u|rz6`FwSX;p1LqOsDAobr<m+no%_eye+H457LO0;8j`2
zl8*py+zAaC?JgrtcngjQ-nn2L<39C0s86}8CQND>=EQv?+dP@{23vm{M^H?56R_sT
zPL7^&djG-uvgLe{&PvG&9svuS%r_@9va^c+rCJXLe}W~SL|%2=LWG`@(~jH|!mTz_
z*G9eTSSPrD(FW#kVgoy|a{)y!+UvP3H~9&m|5%tVP7~{BA=xxTlTS?Eo)g|b4NR5X
zxGK;g7VJmDL8vxlKVDCeh1I;Vko|>&`T4q!Q}5j1$87rL$0!jv;{*o?IK?w-vPk?*
ziW~Nq$`3bM=&$tZR7HN?FHSw|qK|dA8SF6Y1&=jYt^(QObC1q19~?z+J!VHY^#fSa
zFp@IB=c14BGUp_F20J_(qVC7tUS2+?A2L@k7pf8CSY3#4<pcSp_Z`(8<JM`g-vj5+
z4UB7F`Fjsu!ndr9+*965#QDI$*}GLgjJpq7U$EzXb7$g?cgobJlkd=n5LVgwbo`MQ
zSlfHtSJw-|kiP$r7k13#JkmP`x48k#N-GZ=Zs06I%OK`ezyuh`hpToO%c-3p=%RWN
zG6DXDnS1L@4F2rMLJW91dTLPL!Ua(HZ1oPR3bt<_j4e5w?YaMz+Fo`J-cel#V|$Bz
z`V+RUcaCR8%uM+z+)yF8hrfpSC5OLrGw^&`6pwdjB6OTOfhw$>B!jq(sNH7ibnIJ>
zl8yFSe+0{-0^mJ72OeA=<<$&cSkKn3X_-SHCJLUgpHF{-J7-udw?0;XpT`s*GwiHV
zOHdK)1DIK5R0h;>-3hw0e)+&YUfg>S&Vo#NVEHeeyY>h!;6G@CNn9CdQut4=A3EO#
zL-RJ|9Kj>)mrzgq+LdXqiiX8nmHW6U3J9vTTjGlAOGa{3wX_cdX_O2h@6EIbGW#X5
zfaNk{uR6jzm<(Bq@QE`PHl&h%?WCUx@r2+*agh7W{HQQbK=3QZ)6wUEMhEBZZ0n7V
zkxMbr1V-gB=j1m@5_M5H(yJs}NaBBCQT?9Ky4CtAjbhNog3(4fRs?SHP=h|8l)bLw
z_yzGi3+n?z4}%Y2nTf{=42@Ft@Y%!LGKGEoP=Zv|jgrWT-!{3$uE21;B-_g#I1TSe
zRrW~beSKGh5|*0oT~|x+G?+2+x-#XB$$#H6wW0(q)0+Sl`3NpXAYnZ6^?v*Q$$loR
zHa14{Jk~&B3(U(We@|4BMbLTZ#j@<-ZT?1JnVJhmKWgi_EXNHP`r)lt@|4{b2t=lS
z`<Smyx$^{E9__3z`v~_^3|?rtAHZ0pRn=qQWQQCs<4TVbd|?|{)q=!ddBE-YNL5^B
zU3gM3GS!WcXM9^&wELgb=`WT4IzmfGA-dW|2?M^8ecikNP<H==UEJQV8zMN%xRnE+
zQGFjcqnL%~{!e1JV09P&Lyd)wTn78*+3+34neI4W@HQ`DS$s=5z_q4%NJvJJo=86D
zq6VyWY8xLN4q|n9o!EQ4C_dm$-=Gf{{toyqp(W|6RL+a~KgoleM*XLcy*ko!|Dy`=
zE)W9o(F&Z<DMsa%*KyQ`^rQGZ{S4L&^OAbdk?6Nt<Wb%%b_;F}BO(4b`U?Dp=>f8w
zKjYc7*U0$0a*rBA?tGBWRs_I;Q{R6CwY$b991X%Y@fcRV1o(Ey?hPMugzZkS=n?c`
z{ATBpZ`YXm4x4itFec{>LigzK2M~f25~3Cy=#@SVWZ(-XXBqOK|Gr>yhi24elD5On
z!H)fqo6pYgecpgRt6J{65_?CSgLV`dE;skf2B<85Ta@IotGv!EoZCMzpXiX*no5h{
zf9uAQ&LsKqr)%ht!vmMH1htRK_+nRx+Lw?X(%xWqI$PhJN|k$3MSM=Ni$G4gLGLd3
z4K;BE>wy*^sx1mU382$Xjf-Jj$4wCfGWg`l!TE>1R^EFNv&=)5ZL15~0rS41_^!J5
zq3yH<4BK)5aE)N`>pvSB64>hhdd6;M2Yz%1fv-nyk&_R9at%L%mqt@LEaIHDn~xh6
zvaFMd?0WF6OzY9ZxCwt9NzO0CY{_M?kR?G5R`6JI$V&Bo6xRNqfxs6YDajYp7n|A|
zLlqU$`bM93w^<{qXBYs|2F0haM(WST1-Ct%f+v#oye)8eik8S_CoW(nf-H)s^Y;{_
z$2ty>?NcV+4*l5NQ#L63c^#PU_X>WU6WOGf&AWt;w2DK9pU{!%=bWw(U<B@7*13E(
zPP9!BQ8vNuYZ>kYSgw!q;|~357ta)*tKEU*BwKP!+lRP|TOk{E%zXpJz6(%7vTddu
z`8Ke|i*RX39*6+F-*SC}k3PxdaeVa=t-=R0Ya8Wv!L7XK>h)D5;KweTWxawtmlkcr
zLTz#L-SQxkWfkIHzo!y^zzJmB089d%c0X=y7{ijn*B{dKp+)<<Sp9;Ho*SK&h^$cs
z<)vW{sp%tOB#ki_-2=kyVu>6*KhrtO$O5lj9hqQs(Xj@Y+?eu;M4T6KD;)rm@fPO#
z>Zy^fJIo!sNPPKrZq1JdU*|8GCwRtv27ge+67hI1OP4+nYV7-4GoEb3EnSPd)rTOA
zJs3KL$RTANzs{bxqMdm&EX2@73qn3oB64js{~E1#o;n`ab4O9rDd-x2--@+)ymUNZ
z=2`?8bdFHEFy9T8!mPAJmqfC8pupo4ORaKJ;Z+@BfLvYsyIg}gR_l1_sG{tP!5%v7
z812ttUQ}}eCx6CvA>8YB+F4&VPq3ERxZ2L8SGhPQf-HuZ*J{@h?H@b*QzNtJ+Oh{!
zYOha7;_b9ecz*2KIiG7NPrWzOE5Y3*T1+C^mZE*G;L+sR$iQ_!kgw~+YGCR(bfAp%
zZh70FHeZ>2nn7GvOnGO&RfCRiwPA47^{cgvW4`!wrCqv^$lfQ@&yVFHg=tLk`dWAL
zCo$O`xVk?yxixmtSs!WT++ktO(WlYB1%KLEsBN!H**bIh$n-}&Et}^EJF9<hJ#~Qb
zmXU*f!<C%xBK|tM6#yn9EI)f(TL-qGRE7`%9^w0P4;COC_cm^vqwkky0(ds9Z2`4$
zdr!qX!+QvjhAh8?G-vX7ZYc;R4*&_U4;!u&o39x*oh?j4KAhW`ao0jF<kE!KsY9%@
zv6Hvovrf)EVt}_0`&Y|dnqCOf@4JglC=Cni4BX1ya*a3Nf0Q>fU8fbx$w^0)?05Ym
zBAXMy-Lwle2`J=L9<Be!jd7BRj8~Ue@NVqXIHaqQj>=TqQ-Lf(K^SdM6un~?w8(j=
z$%x^j{GiuVo4@}sHXh>z$iR~=re7&oJscu-vey@K<<w408RQ!pV=&5qM?AwFkLkcz
z+Q}ADV~4L1l4RBA6khBdY5sij&1RFfmQ{h7EEyYXk4!d_2LFJs-Bw+#kr;OH;dGY}
zuBbZnour32(RG%2`^Ysy;19OS*zVvbya0Wn`ekzR0-yNCsQA5L^Tr<&k=7!1HOS@q
z-!N|l32+fr^y@@lIkn9jCXq-}{|!nf&#x}VPuSrX6uhYUR9RXwNY?%eJfv8Mb*tHs
z&OVxR>niYZ-x9GokxKNfKE#%wuXE0IwyX&<5UBS|Ja^;X$QW<Wl)c^$Mjhn$vTIq!
zN{(3E7JmAf7!skyK^u{Ut<^y|U$*Bh-cim!k2c8pbTMnn%8xIy`zY%JsG#ySROKdw
z4B5y1nMSG*q}F+h@Pq)AJcX+_o9D^E_mgMvVo1qRVI7q?L7;j1Wa9o(u#PRqQPYVV
zgK*-g@ny=blA+$BJrz25a13*haI%3|QjGp8!I>|w%Vg>MWLuFPWa0_wZ-UCG8`M=x
zvvz*E2E_gRYEyO12KV^p@}kzGx+_Jzs*W0+fnMTYY3h7=pO@6GOMr&+E@sN{2*KTj
z<!C3J?<?O2{Fi~4g9|lSptek!vj;SM3_T9bAvTI^sCfqNN<_3KD`({1%iB+6M&{{L
z%>%py7Y2B^reh*9Zknmt|CXqEl=hgOPmusxt>+%a3f|n}2dDMf26AEO31b6RZj@?|
zGw5#p_k%0D7nrX!{U>%WkZz|rD^5YHE^`;su>J;F9f_y+tX6K7PBmKkyFt#IN^s<1
zyV>3=JIPsNaaPh@wCCk0j-bO5$u|oFIQ%|tSYwTK<US_yh8h*#CEG|YPra51hR$U|
zG8ew}RHuRjKE3^+8laaKe@ckZIXYx^u<HwUA@~YWq#1m5H(~l?-N;q_@Zgau1bHtb
z>4s@L0`b+2|8CgCjt{fcK`R2JMKPNIJG2VDt^lNhA}1ME+V2x~FEk+dH26N;(-LjQ
zwLf4w#N^g3>{e|=+JF5B)+C0<%03mo&gHz#GahOQYysRx`hqKZxM69NLp{NA`mnpQ
z*KJkBzf4hYVSCUn=Mm@5ODba*<@S(W{Bzg^mOpLr==&K3;8)LS$d(WMqhI8$$rxPL
zORUpH`j{WJK>G`)@hlaE^%Y$51RP@FvC-H~lt@OZ7up^H%@3M5#mBzsfn0T54p`_F
zV!#+*yn`}1OR|A|0pnXOgcc?X#`724?kRQV(@|J#yH5l(9n;t*tLchN4NG0uByfYm
zv?H+eHCxPAx(TQ?nm6Hi-0bIn_Mc3w^kGomiMSt<Jrw-4(NwH%zV@~{Yz`f2)WC~K
z3E>#PCL{Uc(E}5x_Cz&nxbBiUP6XIHuR-=Zi{$LJWW*`pI!o@|GeeO^o2<ed*@Ro3
zNtVdJmV#D+G9s)!>O?YQp{>qy_m-gkCpIyXS9iU2lR?^A8E;kPV6Nn!D>c8}59ROY
zmWZE3kEf|KzDSC+U76Gt_$5W$@ekk9wJqvRS1a7Ro<+Q~_=xq7j5kooId6pFDydpJ
zuKJzmi188*SwPx9Ol8|Ts5)b&@VO-ht)-!1G<M=E84%Om{_=O_-KgR|Y0^AN6;;z^
zW;)lZD|aqQC?e6K7oc92*(%`Bz^R$CiyAPUMEdP(vM_O+Nr2KdWhB{+l9Lr-x25}w
z@~o#Yp+Rqbpi4V5%Y^arWpS#uQ%r|s{mCRdICjv9YH6||(;;S+yr!-L1LI`GaQ1t`
z%#5558wyi=Sn;5Z+G!Kwp4i$hZaHOLzy!H$zT=4tInP~bsH;$BeSXaHMXeL6d8`&5
z3;)I~-^qjmk)g@$@Kd$c;`AK65c4xjaR(b6w!6JGYkV3LwSk(dv(Ht84!*3w>y7Qa
zg7^xleYyNaG;8+aP*Z>1^16w@vpP$%dI<{Gx#x?NwrZluOv}d%z5^bEDmHc~HjQe;
z83g_v`Hs=I(O#avevte<q`kny4!!$e_WhdH8(3qe(ID7DZ;=LLHwkR_3!cH_om0Y;
zq2Gx#?rh!FIsaX^3IBE4WdYIau<D?k&0*Z4&8PUBUl9CB_aCtS>d<X)V<A!FmbXX3
zfs?Zq=f4FAkPj#!2Z<ub)BXV*jkg{uBq0R0R-5z)AG?Du%kkv0p|>%h!C#?ieb4K8
z=cAuPa_q{#<oJ76;QNY>nwyKTl%?cXe6I9`F_xPv`(OK^gHhpDf`W-E#s?s1Twr+f
z;9Tm(jqx3c$5TlXg(Ob@75CaJoZGiR;BFyAywQ5$r!F%X@t$^H8Kj$Ky{6Jfx->OM
zBq5VtDpkVt^YG1xuh{Kba~!og)X?I^8V8gbtz$$mpY>WovB5HMRV@D|%%AG_`^g?U
zQEyn?;~P8zpKY8P`DSz&l#HFnM2m+_25^9h68ZKmgo8adPuo~ft!9RD81D2j^m~<a
zv646ACBZu2dC}3%w8vQ4fE4z8o;SYcJ?b1ghTB2CgOcIIBke|l4;1|x-6lra3r`2i
z(fx$`PRTS^Sf?mq5$~W7I=&j$7!{DVd@l`yXtX>UC^eXi5#n6*XDvVNmhZdszilof
zSQld?V;6S?-%^w(a)jCuCfG&i^L;35gC%t)1z2tphLaP+0v(1(DEDkfAsdI&!(F?S
zD|VIIX1}+XANJm21QXLWwY807;YPYr8g>M`W~hqxCN&x<zo)<VuN!wOtVxr0ka-w#
zp15ElA+zuw1{9u5|MZ%<*)d%asHdFuq|_E_x%)|#XlexV!0<{mgY6(NGQvi%v1m&m
ztO^;!v{T}{ODl=kSkb{H)c$q>8oD;rYqvJK6}(%YtQ1A#LPxdkW+L&UAIsOae6jmX
zAC4Bm$&|mhda+@2t~l%)srKj81Ti1qW0Kl_do?|Pv0)=NKwQ;cRa@NXCJEm9ii4rC
zrbW%hK>^LvH1n1xa#Ia1BXIvo{6b_6B`d55sqvJ9r|oTspCC<Rijh2xaM2@wMh0Q!
z!V1V6D|T!tOzF;TpNWb(a6*Kxtchh>P=biBzcxp7dW_@r=i89d7~Qo4%ud=Q0R}p{
zdRBh3n$>d@K<>KjM2<O+&#WX6P&6ge?@i>A)s{N)mj6X<xjDdFdTyYzi<rN{|Lcm*
zVinf1sDsPxk!D^Rl~?=mAkUuid9nem!%}+CwJSpw%$6GVq`=jn$wBkACwO^FBbhM2
z`B0}ma@~DkClQXC-PZ50d%N0;u)gm1kBpw}0LJfSe|zl3cIxi^PdR0M%LAWGc6c^N
z#KAYEIaX%iOkP#B>(y>r(uj!K;p9ObHDAOINzTQJ6371Dbe76}6*x(0yXgj8Nr6(!
zM16fhcWoJVorvfly%XyE(*60>bM##6!u|I$xMH#w{A28{pmScb$9Ng%^?JLs2I<L5
z<%@*m*2%bi=Qs8P^)032)Tpts8*J60^8n-wys&RLDrFE0x@5x>6P)VxC7_i9d_8-o
zvz|s5Qost8n`zsPXrXI#3${H;@6?{m*7N55sof9RdNAi<L=T*WZ$R`FXcT#J{Hja%
zcSa-bWi-tW$K|DWg-LJ5LgU4*R?xe!jS`aUZf_iw3+@F((nrd?!<5}4{tr9wT7pQx
zcSVo~@#3^=qayMb&NOk{)*sj;rtfbLpgKaI<vpeVq%tf8%te_Z{nZlh!5>7F!~p`w
z1G)UTpI5d++(lXVWOHLTVIh?HtdTfJ2KbaDO-y?XD!-`FhYPyQEyEg3QiwTkGlC|t
zBh6;Qo)qC-{?$TYwhl_LJ0QovAz9|A2pGT+SXe+hn56d`ED<rknK~^p@E+iMoY<)U
zz!9C}R}V5^2u*6@;fD3Sl*qbcq#QoW4w2bai-;4j5ODA9r9>x#6WEO#&o;1lqz}&*
zn3l6dTKU;|8m+o0em3Jd=Lf;=G*qoC_x>HJgYodtXXBQ~P@cn`EvATLG>Vxl=z&2j
z-^KY)Eccs{kR1oKxLKwiC8HKcb9$WvmOv5(U3Bv3ZQ~xw+d#YORw`?m9z80js_J4q
zxOqZP``c1X`EN&giRAAd-+97A^_v%KhGNyfIaQ1Ak~tI_FC`p?T|wviEv>C?%=TBC
zI=;T&O+pFA=RdtS9G!ppPNjdx7{NtMS&|D+O&N|umbxADIs7{YjDp}e!fWYwCS6!4
zv2S(Q@<K=wVn0Ug!2FaAtHbyEYxhk}Ih(1|@$=Ku)3dU+6<G{zud>$TqQ8?J5!f`L
zDGVgNx7ijMMBkb=1e09h+L1;Z&8ea8#d5zH`+0)9<r_QUcC6rLQ0x}1Pqyqf9T0gH
zdmaVkCx^5SnWR_gO<K`Nh%X$t3q~<H;bLAU29GOM<8pt!8+PutOweGqO$a1wr~Lu@
zYHjQoQt#)etzQFN{DER=6sCJ>1=CMXcrzG5U;9;#O1S@u%>2fokcP-7lXQ6cdEQ&a
z>7K~orPzyWe3q1W?>3v?K8|s0m=YkY8tK7KA(bacRAe(9R6gY>PjN`nC7q^DROdaY
z6~sL;zy3&R?crhD#R0LIn1lG7Nk*%$!VgYgqqp^w!IQ0DXSre-hoKTh4iNNU^1FZ)
zLMd=<KqwWgt1M;fwnJ2}uV+noACJMAu?*iM$Uz%NimR8C{&^p4rQ{ps&j@MqyI$P?
zjiNRo$GH=D1HO^&?<DL7K#rgb;R~%C`RSN+@CtxJTm-;11QQv4uF$TYsF`Gz2^>a(
z95*>mi|WZ7%kn4!m!I(1x5#YR;XWi*D=6VzwC0Pej!1A3#<KI;P|!O#>KaM)t%T<D
zmuN~V2`F|~g$H+^Z|IhTA^yS6q&D}$ckO$e0$<>Td{9=yQo>V$w;_3!GTRod9zh3p
zyH}+7MTVcieK};a7GPspuuvT9H7VoJ!^1!E4fE!SLX|p2jeX(vyW>z(NTr=ruvDq`
z#WTLF23ym7*F~t|ZF#rxDrDqO0v!^)MTf%}Us~_tOyl2(!zM1xp?2l$YYTyWB(leY
zR~QJiGU(}LXp}nDF%(R8lS_MiD`7M*O6W+^fgdESJtckeWU<%0Q*oIO4Mf9fPHn_C
z;c%mjqq%<9e@e1unS<Ki9FwDo6ItB3EY!<#;6Ek1f0FZVHF-r1?qI}O+H2s2&|_vR
zOv>hME;gJViQ(mdt<KA79B<wsiTxLu@1A%NlOI%B?&$<cmcini51VV@^pmmXBzQet
zCpHn9i+o~F#+`Is*YrpuTU^^ERadV7=p+R&CEsVnz%x{=Ot9XkD|2Zh7eP%8T41U%
z*KqKK*|XAyH*!>+EcEB;RG^va9v)$oPKsGrqn5v>oz;L>{V8U9yo;@l4Fuh>>1pC*
zn3nDZKIPqV^!bt>Adf>)IpDs?o{+IIk+d3{QB-NlYsT<Ea;pRp`e&7$dOk{SZmu`6
zw6{uhJ9GfxwJ5!+_Y+(w>f-<l3Frr~!I5U`dnxxnjV9FqCh|jk*QT>f096VrSknn?
z9>ktO9_Fw8^%4~-pmsiU01O5w=dSd(D$c%@i0ry(Pw(@nmU}?<T8r!|N^rjZ5_O4K
z9e=SW!Xt@56D->avmFmquh}WBmNv)WuS72RH8-1F8wLIg|G_VVB(KNeW;2a%bP9>=
zkee2j@*JXxe0#+ApsUn)Hn9`J`!gIgI>%s`iy(%9o!c7evs7U6{kc9e=S6uWFW~64
z;#efI2R;Ja30#M8sy1h5X2S7IW7J~O$B73}`3}jnXupN_&J^P&t))hRA^^<En}(VG
zXjN&X^_4p%p}o#>qdoZ`wG;_SklQ3es!iCX4gDbSOX1bHn&tHXn>?&5-#qm%z^!EU
za@RY2)%pYBt#K$tklN5Z=gkG`#|nz_#fYPcQ2g42!0DLSDGKNnocGhZDfaA~Dx_+q
zF1SET_R}>pK-R3b$C=S6ROOf}*40iX(Ib)5yaK`$Om$Ddr=N(Jp8j(^Cde);Mp)O7
z+jGA2LH0c;NSV$6s_1Jdzb`kyk}Oa8oq?;AkXxztP*Wi1E_2_f9$(k=#jy``lv##V
zBhyJ|SN9t@kCWr~t78z9`a{Q&xrZ&bY{Xe!nkdHh%wRW2&opvg_12QP!L0S4{OETZ
z`ub6AwK|*}r1SKp8ghm2{idYzb<P1Y^M#6=mZ5#35!=#9ku^2UrH~1{XrR*H6zLC-
zYu>s1mWxKo_DR06Ic@clS%RBSI02<5jI4cw#0Beuxq*ZSx!fK#T#fT;R{7ipe%Lrz
zul5@KJj8gWJ`%oG@o!m~v#~`5-)tSZ^(wnn?_0D&IAaQU>hLMyXvtH66BFpi(mZz7
zf=7i8sxkeH_S!m^KXTS(FuBXC6hgO$_h!5cEd<ow6&-8!99aDAN7mF$$aUAO2Bv-T
zr)w8g2_n&0aY(k><7Zb98w^!{ANFQt7qlFTz!)A4d349z4<>#qyzIC7_A`fw2*1G{
z8D)4i_pU6gel*M_SP>73$sMZ%=|Ct+xWbw^?cEhhFS8m{okx!Lu<#T%xr&-yHamuq
zL3F|AH4x4wM=%*W)Rq?Go<;35uW0W!)!};yP5SQ1OLp;Ml6aWN#Eg{<vI`q)_oEBe
zcbMnNch-NsNoAfh{}lDpQoD6Hdi{bml_HKC+ki($H&4!v+WNPTD>}&H@ZYQ6%jA;z
zT0!HR+Twmj82>k!(MUUtd)xH&L9Yg~D-c&?hVzW-n^ssq{@-ftp56!8teb|H6!e@a
zvU;P<-)Dgl>`-KgX7(_j!+(qoY2V;qLrmyUFM~wTl2Sz4C;gKWP3BaEk9jIPZIloB
z3Zk%y>`E!*O(t#ELO6&C_n7ZB{fg%9{Bg|ff2z>A+*%~2yB9e1wP!d^@(g*P5<Ir`
zJ)}0Nc)+01*sy@uenP@5dhe$-5jTvSWH(jNw=4u~w_DXv0_}E`YW%O$_yWDp+mZ;R
znA;hWU0s40Kn?Ky{VqM(FR$yaI0%JmRq9{6QE9l`+JXrDGTL`a5eiA<VSz8<2P=K6
zL@`ijt`NDN#QcwY1l_29sURvqW(;s*2FBQwVK9K5A1yTL4cv;?)GEX?h8zmVex9{`
z6n=Z(YD?Sbmjb1A54QZzg0T(8kLWQkSGL3gE_*<4rr2!x@wx}z$!8+JyOyW7s<HOG
z*p#eo1_6dI=bWbN%^{7^Lg5Wkn%8_%u-NO_%e>GWWJK`4P8d3XK*7YS2qXJV+AP4u
zU|hPn>mqM|nlV|9rTI{%3~?|gdH!-gEy>2l>ngc_|0ydi>{u#LUpNtdJP^0K-?7^;
zU1i-=o|`s`09ul?dwe$W-{9X<G-I)5h=5YyC2PyA1bV17@${vuAd+#ghX8uyK;Rce
zX%t>RG=gdRJ7ow-y=40uv6Hi;nlr+RKWUJ{Fy9<tY*G>JT?R?Jie-lJoCR6culfu6
z*+rQC!i)G;`{*ZsY1efO?vI*xo%~b0aBErDk5V9nyf~|vHjPZ*YUTQwNHm-m6BQYU
zU=6uY8D+6Es9rTXdx5JWb43hZ>Q7Tn$O{dwRZ3xDxxG<}{i5VtaXnFgLdCt|lRrlh
zMwNh@PC<|bU;i$pLd8GBJ)sUs7i5BnvwHydY!i}{k(9eVaA*qw&D1}ON<_Z@;NnNk
zrP#1t)3DOp)qQZQV5f-Xfda}ZFHB#$OO&hIJk|BNEc9ms_af^bfi1;+=KiL=wH1@5
z6A*EW@2f@rqrfx6liG^-1`kiGK+T@DJJFr5LIH##>61@{ZfS}N2gtbBLu`CG$<Pi5
zrH`RZVfqf^PWr6f^M6nz`@(jQeHn8Yu4f*KC&!yDD>w6-9#p9ls6;AR6*cf0Nq@Qt
z5eZ22uF?41=SKQa>g6)L^B6${!H>?mbJ(=OJ+HzasHyL_z5(srJ#M`76fIbL&@b8d
zF9@x2;S|->F3{yWkqY^^>5}qia}tx?WZpSVEF0p>ZT{xWp!BY7iar$#%;imVs|kA-
zOP5NEK#UUL5(`1RRJ@*fDR%wt5?N01n$1>nX)kyfGHxmG<P)xEVm|Qqp)25Z9_vyq
zz|Cy8p^WQJt&%2Q23|NuHGZ8uONmd=y$Bt}P1OAb)6vN_n2pyk%JhY#A%WMc>TP!E
zFXa7>1{yI}?^rDvUzhzQFJ~_U*VLbAlXhh~lqp{RUJgtTZ||)fJ1tq0xbzpQ6dC^M
z$(WZGa@TC{bARV6{;86-Aw4KwkliXL2!8;g%Q!DJWXMjl<?bc(ZkG*6iTZ#MFtj6)
zi^KrHwyB)Tl0Ce9;*uFK@wOo$5}D2^SY&XP)|S>xQ_Gn01p_#{)&o6>u%~xx@?h!t
zEa7{tvzw_fu5)o6&+XyvQ#NC?wWiuq;mi;QJP3VS#fgsf{^2a9dRhw4ez$nneK03M
z2Ndh+B$Sgg%EZdX2-tWj|4Q$#*#xe-rhb;JN-|euj;!eaZouq7ZUb3{G%gG0x4dtK
zUP>Q{umkCrA57vRv;3C;d@<FzYxH#$0)o3cy>R0AGh&hk0JPdIO17@&HVoq&Ef%;?
z5XTj~;#&g;HEU)c<chpbiMZTE0Z=F{=L=mBk_Rg<ZU^IVssHMG#sCP3)K35)rm`#f
z)_ObA?&3L6uI(+wzO-vb^!#=-427=%IFd$!j57E85Ka`-Ipjv+*H~!n$t{P;a^hS|
zYy=vdCNQz56JL$CzgZFvc|{e<>16#2dN3(3)jWaKRd&i@pU`O6)<f^rMWVP|m)}DQ
zN1Nc|Q$D(#v3I{&(tv$=XuxYPeU(t`YrW$~MK&L0ioo^HWR%yzU-F3c2&~tT%m#R6
z%eGA4(=xsxpqG6i=x{c_Wf4?HFOz(}HOt9YlxD05$h}^D9^fMak;_Yi{91bka9b07
zIj7h4axXxe8aa?Udepx(FL(&K4J8gbCNt;m&GQO6#k<g>B8lp0x=E*rx7Rpfe>{~E
zU?Q6t2;KhLRpZ6rpsk|&#<<tNq&3`6HoS<p)=YJfvKr6CC3nEeQ&SOB%v<_nWUHn9
zV)k33@UQE!FP~9oMwdr;A<lVqVq&zUABU2@s@>w~S)#KF=9xd`-4ycCZMFOIPOVl@
zA*Px^C-r68oE5mBs?FmEb1efMOChQoHHiFdqco;BoJ$4{ib$mz7|4hnAPpiGwl;tL
z<H@J|I?CXKb!zS5quHbUi(p_b`8a}$<VylvS@av@>h+#ib@9v|FXmn>yE*y<Z4bKL
zqVxC>0n7qRh|ueA*XstfO_u9#N`8(P5Sw_f1`fQjxJ#nv9aS5(sdEpl@aQT0VrB+8
zZVwB8$MGcO=}v7bmuod1b0j|@PPMuI*(WQTTtg$|%OV=v$GD~(ymAn5Z{2Okd7jym
zcDiAHmGInTn*xkmVC_A&4Bgg#0U;bbzqKb6q$niml$mG;&uxo$ZN-QWZNv<4rx@jV
z7k2$-4a(i3DMUQDgmj~Dgox3Su*=hnLOKb7=iM79ME?l49SNbj$68IhoVNheb1&4N
zx`j2TP*c3OH2cTwIed17pajkziT8}0uVGBWgCHh=@~=k1v!Ycoly<$t#U4*)+60S_
zq%-V)efFNty@vb{<ReAtMXBU6P92GT)&-rgaSa7k@X0-y%;r|&l24b1;ru?+@JoL$
zpHEF)Ygpx4ZE-5Y{8=U%c<XuOORw90!x9Ab61aQ#zMqR!?Qd?Rj*G2ri3IZfOb}nq
z%gF!trCRn7pG2r(aAH->wP;4=O=)oTuk@6j;|;2DMto|ju6+Ydj{1D?5h}3~S|T{!
zy+i0Uu6y@O)nxl2F)lnd=OXY-x}OE4Hr0J%e;4pMFA9|FoR?=W>7+~)+Z|SG;J9IB
zCi=w)ip<Q>lb~VN;mPC~qLaQ!*`HxN8JbgF8LblS1&#{!M*`#5U3b-!_fH00cVmBO
zLDLv?arFY4Yp)s30KDZufcal+3#fu$Q?LtLXtH9ug9~>!k@=f25psPgki9Hzp&5`N
zV&}xM1P2-!OHZVc6!$~o3ety$%4#L3*Bc&>+{HaiPc}cfMggrCeoeQ|h9bAt`bSl(
zu9~mbx@uFrJfF#l&`p~sp0`nXN|qh-6(hTcVURZ;+vR2%up(MC=$5AZ>f^kh0QV`}
z`vwMZK~z4H)#FKU1bV_=8Qvy3txZKz)L*`ouWlO$E<~Rt7xDXDR<<|5lq8vreL9R}
zFD?Z;J8+=inHnx~SK+3}-sUbyL}F&AhsDxe=HGCAtX6_?ZPm6g<(~d=#r<$L(_FYe
zGAqv9ZiF|CY)D`D`%4ZAYjA_)_P)pIsj%X26r`c%R!7dX8uIIn!)4C5L^+kMI@j@r
zQ6D6u&+p332nD>#*^NSlMhHw#>RLDDyIOTGtuI4~uDB`J3OZ!b+I1v{j01sB_yBT8
zkwUFd_Cay)GdR3^Se!AjIcx(8hwy6SY!VGS2JS_gD%W~>pdWrW>+ju^xw#T^tA>%(
zCt&P9y)q1j3~zQkMVmPPKfx}j5U!q~8-tJ3qxx>U$i+W!Gt_2H42%J1--(A)`2B3B
z1V*tHyvIw^9@6Y)IGC8r-83`)3dQUexE_~WP(|U;1U?2P4l9SVf3ontv3Vk6Yv>k9
zDhs_6=)&=7i}Jmm+nD44WkmX3(+<<SS~h#`S?Rt+Z9eDo&da13Gyi<`0mN5b_|yV9
zf*hqUGUo$P+>VM;R^4-~8dnB2%??w`--XnGp5Sk&l;>1D7=8;};<0OSn7cwKyuNYt
z^mp;Bd^NgC=_Q@teQ<M$W<Q9pld&A<coc<JLL8HMl{cmgZ;q>JSSlRo(<q#=43kr_
ziWeGW#Cl>1!0qcr>p}E$tDwyuoBLw^tiWFfCshGLbJm6DhKH}&H7b#$KP+qaoAImn
zbp6$8htL8lSCwMYkbY>s+&lmLEVJg3`x@@mb5SLLQk-FUr9oC~s3|>9$~4?(@f(3H
zRbS6^9OqGCto<0j2O<dWJUl`d)|K8^EV`E5v-dO8s}1eb!}pa{p$Zthk=fm!77`m@
z?CqE4PElJ7AUi(a$mwy_Awmg|`&lepO16kf9xg}(9H1D!<N#St#GwKO5>cT<edb!I
z*~_0|kQnAn5US#1ySR}fez67x-sas=)j3c1vvAhdbX1`SaFjY#rsx*(iYEb9V;M-h
zQ}bd+o=rBzbZ4y@YeieB2E^(dGkiPD;@ks9#UA&01nN*Qgs8+b|C%d0ieXm-Ltea1
z=X>VUrgYiQ=sWOe6*8F?qzh_AD>=!ywfks`r)rmLFm|tIaLPnVTKzx`Un_e?VZfCs
zgZo{q5xi9N!yVPGfG*^p7b=O^t^9j&GC6SusrU&>$nUaBdly<m6IBz3s2$@Sp_XoI
z?WKs3A)V0!%tO>d6~#qbDKbTS!>_FJPdy;H-2S@zLN8(62N^xClJ?*#Igvh#S^Wle
zb~_#f{zQo~;>Z)qNGl(B1)&T_f0&VweY9FxgE8h?yzS)oBm@_D_IAaRk?C}^Co~yu
zz&9!_)vZQqt6Iaj!?@IZJ5$3EF*8eH9wk)04dpetQ0D`evR#y-1~W2*sJ)iLD&AG(
z?Q(@G_Ri?f;>QwlD-uT5qji1_?k1|mNw_D>feNWVHN=!`%H8!Qo(NO(3~>*<QWhuP
zN;eiG*Nyiw32A&Cyjc-gx~7qnRsy;;<#gS@q?w5h<P#GTwhE={(5<S%V{`=O(`+5m
z=SX?JSf})R;&;fVZN(>d%b;w~R}g6A-=hZUSIQ4x<r<M2tUGflLRZe?yUvlWy4qj5
zB@*0VMU~{_2^zS{6Aj<vs0(PjwneHxqZVO^NPUk*Eu3Iuze<lZlh(sO`yT6YJ^{EF
z^fJ)=Lzh^HSAi!dH4kTlqnTyBfZygi!hQCWL{BHGRwBF;LCI5=zSM22fB>s#!NOW9
zf}Ider+PF|^^RHz`1A2pGCDGheG2<DtK%erDx60Jo|6G6zWc!C#VmY}rfV7CVkc6(
zH_qq3b?<!04|B~h1lDMBCk|6Lxu~+`>>nnxPNR>e+WWr3-M`W`F86X@fjwlCbP>R;
z6$<ArV*`Q3SH>NZmjr-C_X9oE5-mnEIACdRw_FkcY^T-k#F+h;qiPoEgXLgK-<bZ1
z!f;@Wm837QhzZ*>wU=(L=w0(i&t87L^@}@`;#W}spR9SKD%barFk?Pe((~@th`Cyt
z@1AtQX|Slr;Xq$~=Re8D`sHVgP12<t6|zUDV@t(j11Br@H5Q(bIW=|e#|#V_#mpi7
z>rc2|mxIR^{_yHakd505qkHK_G;^(p46;ksc2>Aa9_R3*Xe$fYtGeq^o=FG=gNgj#
zr5OCZt!JEe6rC{9{lkBj2hYkB9w^F}tF<WnvAb?COM({1-u=7%`u71T8InU-!<>Xx
zja~1=0RzF^c1DW^7Fg-CM*V7Yg=}`)O`Q~}OZQU>MU#n#r;6G{v2Zr(pO@S^-80)X
zM1?>}#gWGsfK2Hi#dpidP3_(xBGLO>Ggm{zTPa(ZygyLMqvp<Bl!LaK1nTA(2)6Lw
z^!tM=B`vI#b0(9=;-SbY^n<XpDQ?79M^l*&%~%AJnD!)D$X`ctlI0{qXL%~&JRkjj
zU}t714jq@eNO@sM#V~nsiyy+VR8I!=k3g3z=sZlRER}4~$EHX=&jnTlEVTXH@_#&7
z<$o-}U2_>5vS$d<_&u`eIZ*L*c^U4w$K8>$urVoLdW`P2l>msi9CyiG<f-BhdNNmD
ztdptQ`2KpA(j`?EtV)R*mN)8?)mb``k*{s>nEbAsq!g{xX}x(5V*lbNmh&e{iQw>A
zqu5918Oy0>%<+OWsztHds^6nfyGKsh`D<|Ozr$Z&kKxFzB@3D~)2v%h|FL?uHJC#{
zJ|2KSQfpi;GN0=c?%o9X`3`bu24{B{(9KwG(MHDDQC89Ycn3WyB%s{yLw?l9jI9b3
zGuh1OBP5mmST6Gv%ZU=5OK(t=r(JaYPGT*1{cW`NeZ7@e%jGI(E_@*4FzM&*40s>f
zWHMw;EHl8RP!KhrmnsvT`aIKazA$LEV)YxKVu58gCvdB9JBOgrs-z{Wl9LOEEX5?G
zqXC#G$20?X8Z>nuhb0~P>uAPWZ%DM+N|nH#cuLQ=9E&0DOD2yPsU)#*sqST|df5nL
z0ErwZ!PCsdtj}AAm}Ig}D0-)+oT;BCg1+!yeMX&BP<c%%=I{L~-tU$u(K>(m0aJNa
zq%eR%wPYr<z!}s?5=hz?t?7nr$FG$dNkuwawd}7JwS=p+dC~JOqL|lgG{0AZO(Xk2
z8(%QyC7DE_E;^~$9ZRhNWpVYqZERXJ+J~zaY;d_ZdY0oli;ceu*&j20{(^s=Ci_{w
zIpWlWQj|1S`K#&Ij>iNxAp<-$E*#sNutJ^u0tM}z6)pqj7>#km)B5mV$Ds&zIU4p(
zt_oG2{T7AVO(|E}g}x_ztLeh>zxL@f`}Z2Rd@)K+rz+|l)FIr8$x{F$&GZ?C=-)vq
zkJ@$ZqgUZeb5tQW9n==w2d=N0+Z3qmw4%}a1wVNU;*xkT*B?$*rGv#c1!EM2KL^3{
ziRUg~eF5oQ`ePM-aQe-_q;olnEXVj6k-6!#j<9GhuvNG)=0Zc;^J>5;{nd#6^AW1s
zu*mqt)?5M{q(E=`5Cv41+@G$-1p4=p6_oNKM#p1&b}}>b#Ia(8tgr3lvspK5igxT@
z^|HRoXAax`__kNIo;q5?qwWXwp44@^kREs4`UmHGnbxtTyHGufRPg$rNBOT1GofKb
zlEc}@^2(D)hlQBUOXYtFb8ea_Z|zs*KI2IFw$YA3_a`o=o1hb8Fahl?(&7WW^^MW|
zPc<cjGoKR@CBDotae8~WaYq)eQ5pOxdHqzD(L@WgOXnLqp0#e~qK4Vo@Kh0pA;U8i
zeHg|{)SOhy?s5lp3wY%Maa^ER=Qztr#`O5fup(F%pU^qEgjq+BG?6gLiL~B9R*33~
z>x`-ciq%OrK|Z8A#;@kh_LRcwW=xXj`%2>k3y~Vs0+RRII%QcK1zjy_QZ<_BnMk{s
zBQ+sUBFJxm&W1mWHOY$26eUDS1xD22-Ej`&YS=QN`?w?$pNn^92FwhocK12UK%mA_
z6P5!6FNP|d&kU(<wTp`qszhERoE35LW|k5R#qF+N%m??i$CsC+M_7&G2*khqBxl@5
zI9vFZ6lsHggwk4?(}}HnK~Z1I&&H+|@?*x|SL#IaD{iZ{Ag$8oG5+B83A^K$C?dfL
z;$h0P^OD_pmU7WQlXZ7Z$dcyN$~0lX^*sq)&p@QzOMcQ>vTyWr6GnJu-x{tgaRcL_
zDUnbV`W_^<dMDKHgX23r_*x|l2;P1u#+KeVVXDupqPT4toZ`(r6x%=_TFRT0fw`M3
zOQRN_)+ZJ;A6Zrp{0lzYep;os;70oy)izo7B<M>4)62i7S!-aYY@2UNa(T?k2r|vY
zYf(UMqn?8NwM28Wl5pl&OF?j5c0BE=!AyGK_h<SA`XVD!ss6n=n07axr`bDhkeJGH
z9C>sT<6s)0LRI{YGVa^wM~|rn_oi>EQs1|2KI;@@qyCA6EV+>val|c8vi&VT%XH$`
zR61o=u2g|`jDzKtQK&!moP{yt#+5bOAiu{WaOT-M!89mBt1N2Vv>9F1b{4=RaQ;!5
zeK#Uz_}!ldij#hSy0g@nq<|ICi2()0QMjw}IQ9*R`EZ(k+5`BnlLqBwQwWYA(Yo{T
z-xK1t9cTWl@W}u@;6N_wvb_z0UA8NoePfw0kcwP^$>HBLO_O`Z2pqdGZ+wFmItC^J
z`FON{Di*eAu^a8JaRmhw>)}`Zg*9=C@3#61OYK1Pe5?Jr9bQulG?FRI0V<})g2cGq
zz`NeQ>GJZw6w+|F6DgF^g&*xBN0C!r^k+4|>_#G}Y`jWSzdxvg(AS7wv`pPbn24ob
zD3bSfN{N~D<$=GWm|&U9CDf{~sl$%YW#DnX^jLR8HCtPxiFTsptqLZeK~QNHie2Wf
z@#^Q*td{aN&|-SMvan;rpjcR|%yU=#8oxQCrRC|{gV&!%X=7Ovx<MfG9e(~AO6`lW
zUd5L?p&OA5kNVBR&4<l+g0$(HLi|q}b**_7ly%uSo!WSv&@ZuI^q<FCfVkr&w{&O8
z(jg(c0z-K^;J1bhY<{DWzu*1@2v*AfB*Mo$Z(46<Cb-{3pVeyYba1~d)5&6=m2I0}
z+Pz{{hJJ?rmUKisJ#}Q#JeE#VR%y2dB&1((Z7ZMAm@M`Tu%94QCc?pw#p(xl=54-%
zvYL<f+%E<XC}}*!7tRg-F!!dv_SltH9j;Ph=S~S7=W5S|{lPj<#9k4i+)oW%$&vTL
z>#Gw9PYh$CWBZ~tU4e^M7O<P>e<k$l_9;b=?(EmoX)bkzWg@D@Rw<*0Fg(#%+7-zK
zfpb+r@4U9VxM+UGy(*!rSf24r*ezgSu?wwwf0Ul)6`Z0<dV2wM9VW<z6i-}KoAJw)
zZicQ%5;Gvq{4ffcKn*C@A}ny(H*gF=E;?3cd<D?kR&hiWm*_>^#PT979{toy>yBO3
z1<$;~_X;ebp(wh;cn-p&ix4fWNab_Q!6C7O^-k_fSLV3$spD!xJl)LX-hqD*9px8U
z3ubo$A*yW1r7LKb*Gudsm08_7=ecw69EH=PAv-XzoOW_2`J!3rr62VtbLm2KQ6`Uk
zG5ePP75^dG)ICcqwLB##4KFic&s@sZav$lMr2=H7J8xOt-vkomm=Zd2y*(~xczg<z
z9RPS8_F2qhg@_e8CWr;{D|xz<h0|`0mF2fP>U*hE2B2J{R_&7BKKd!#vUFTfCU1uY
zt4ucU(x`B?6)%l9HRjRXP|H2=Rf;E-orRGE^%5H$5$%zlB`gIgme>WZ2!8T@df1C;
zlBs7}yee&nuM7C(MVO5fV-)D+YH~gUyWM$^Ov!RBKk)mxF31r{$R}DO10`I8zw#P7
z&=!?`8$CpDsCe%QAly8}vg@WP0ESp~@>;`ZqpEsoKCJ}CU<GYYNn8Mt!vG$6T;tf@
z<SfP_JoDuE<Ug&kMc~L1Sz4Pj*1DpBKR_TseeZ0Balr~}(Ykh(@bl8%?2^}l+5L>W
z`S=GXE9M|Kd%_Ux#O(tgVB3}qh0pm-{T@9J0E(4rKS!nXfaU1ZYkke?uQWCO{R)<h
zo1V{_J#oMWrDGY*bKrTxBNuHj^PEnq4yYS47E#~rJh|jmAn`X10t7!pLh?7Iq-Ta>
z!h`+9Wj_6*^yfO<$i~c`Mo6keUE%|PtcR?;EY{)p_t@1<k|#`{BlJlKH4MoX=)P61
zfcB~EN5NSIG*coM-tWPAj^?(vV7ytL7y)qQ`)$^&g^rKzs<b4(>G-pNn{puUbvWFF
zjavoCsp}ELE3!OOcv<c#_dFbS+8LtBtusk3(C@Yaw=tMjk^!XRcM?Ryhh`ATsQ>N2
zP{nv<A)~1HwsH6D6Y@#g4O$|pvx$3AfZzQ}yUYj$LneirnoqDqa){??l~%nAB^3N)
zVL8E(%{;oE%fjD<qkNw?maKPx0KiM9$)|OVuTj<Tux3UGSdZp7!`N!m%eWU`88%oI
zuszYLZF`I48t+kiu;oUfG&<+!Qaa}^kyr1j)qSTVFS3d~$~UOO2e4UFZ1{8M)Dw52
z4SifJsKH3D!BNMf=sj-=0%A7UPUSbeQsm@-_+?lWi5n<>GglZ{!kuFb?KL9Sgc{E9
z?jOicIrw;5b05ts%{t31Q=R!t<5|&p^2;lF%l?)%$<EL#*r(h@#$U*DQcozoeAt)O
z=du6GkNjQr??edhi8<<t9l6O82xU}i(}6v%J~vW6F<l4A2;;M9d6?1)^Rw$R1SGo;
z^&4`!jvP3EhR%%2d)eK3(tHI$u#cs%CerP8q;F}63)$;^)zYNEbS}(;3jTEeS?3`V
z4?O$mkqCUHscNs4MT6vmIKw!ROJ>lquDt&!Pmnnj7j%qD0s_M#c#fVsIPc&f``8XX
zi+|%C=)iUt&tqUd)CyCWT_>rQs4QiyiA0Y?wuDx}y77=!mF70;29VgM)EyqSa$N9-
zoAK|3Vl(GYJd52N#DaJiB-lA`f*S|z0WK;(siee7$LS#Z{s3a?C$u@qER{!>n|eZm
z<cHp@L^<XkUvU2yb8i_I)z|h769WuVgLIdaAT1@0fCv&|(4EpD-6=UDEioXaNP|c-
z)PRJP(%qdy4fXE%U+?`q*L`32hxg<AiGySIUbAQKwa#_!^ZdoN*TGER3|!4gdcwOj
z!nLawVnz&cVafTj^^wGX5&Okr8*2OenB(*!9tS;6a8uDNAb&B<yRM(_0dU;JmLM#!
z+L64CT+IAlIc5kLgcb7de^BKAU?<EVi=^u}|M<_yeD7&Qp?TWX;j1rJI<5hCt`xq8
z)%ylIZ9C;nk-@Smdz9U{|9uqnf5Qjh@LXAyN2^#;<KE#ndv!CvOHn`PeeF>INiIG0
z(OCJfAC+!veMqCSomPP4J_>NzFKU6q))%5*PlNy4RsG-5Nkeq`^q%IcY=F<e7vL-1
z{f2pmn!!=PVj$BKK#d~fo&7rvPfX6L)25UIY;t_8CD)R5MggC&EMkdOh~N5_6x3zm
zKzl_=@PGLNqy!er=8!yg!^eYh|7VNWs+ztpC2x>pV*o)j(gUFS6((uQOzp3%4;3FK
zNFDdq0x0sy#sJM&_>fQdWf(^0fg|9)QpY`gohQkaJp>j|9^P;&9@_5q7K9nN1&;it
zvfA_-*;s$uG-&f~Da}7HK2Qw!;C7htWarxMd&IMEt^OSDZsD!+!7or~tvF;#g+KcZ
zz%~}6x&eUWw{{0D=NeGI*_8ooW{!$hyK?3S|M3K|LRuA>K{u`TpR$;}4!oq}Orkz6
z6cbTe;{4yc_+QU`m=oM(?H@1-(LhX2rf0KGUPT7X%E3uk7PMn5{_CLs_6@?IjxP&@
z84`^Ll-FOhhNJ^qF_{V|U_+$ev>pD}6PQ>bls7UC<Wy%8jjxkGmc95dEr2$z)pM3#
z@-P3_vtZ1S&_I|-c+wKA|HPj6e3q@AVgWkqUWl<l`kz1JpDT2kI~bE$#PZkvdZ6<%
ztn}X}HE@QhJe$_)+?>nX=AXIk(S1YsSY{(IvdLmTa!lFq>u;_x)c<on@RP~~tE78X
z7|6N--Chsl9Cw*K1{TOBJ8<R!`?U6(|G036k6`9uvKXwuqiIG2OfFXi9In)+BB%17
zQ?1gBIhcZ}J#en^y&Wvztg6C-h=imV(5CC&Z%|r@xE&c8A<AFYj`azX1EWrgzJ(6|
zkDtQ~*~A0Q`rdW(2MPuN9Bx(cWq4IpRjuvFm&Vr%bEv4uNL0vft{EoF$MasCS4&^F
zBO@?&ve6uSBw!84ZsgLAvjxx;G`$+fvPS_9ZCfoM&{zmK#^juacy*Te{IM=4{m?2l
z@u;%05A88C3RHq@SEl9;0?%s!p&w1GU>Tpe_Y;L^em}xeW>cQzxmqf3206+1QT8;$
zPj4!WvqcyD-Rbx_l^)JAFW6VKj32jlGVFh>12i`}#`d$zjhaM_05>nkGK+47Ism1O
z1ZdH-%V}F->u6n_%HVZCwr_rpt+q#k<$+<~<=PgS^4o|G6acZl?I~0JP{4i?dd)w*
zp){B4)o*aKLIG>UST6w8OKIR>=4irhnwTTw2IvszNnJ!5<MiDEj(`>ie*sAMgvZaO
zxiYJfu8q0Y_0RUB0K7b7ANanyI3^OmFbr6$wPE1KlZA|ZSEyeBdRHi)3ms}wqi<W6
zQeYm|0f~4&1%uYvu|8+jph4dmxZ~<!n*|FD2i`rqd(J_A97SHn;}w}t`^$LA6qC#J
ze>^0^o*gWuvNRq!pLRam^xL!e`>=@ob4K_Rx`X-S)Ss`ETaFNP!VOSsa}Hr{tZLvk
zH>Di)?)>&@b@#lQS&(_M`cd)W3b`|%Nvjl~)i<Atjx<v{faWKP4e<$3Y3N2!3f7)p
zjY&5E{I7V9P3H|xv!0leFV5XD6jOk&(MQMTBg7Oy4}W6xX<dXNecu0MaH<`TDNFU6
zpj`qs8-Dc^8uVjvm399KUPS-9&o`SYvY)FfhMi%^0!}L-ToJ8blgJ}zkQ5ya%`Y^$
zSX3%=kAV9}HCmc9gWPpWu!mmQ%C$=tu+S0Gi>{#pBu;e@P;iSpM(ef`zpplj(#tbI
zja0;Mye6j`>t^@v_TVlKW8RF3?q-s%?@O{AvkEgWO8WKpr)F~sivTHI&0J~>xDiik
zboT%a6g%|pRd!Kr)jtXaUfGZU<6W?2L;659&Di&wodFXd+UWWH!Y6Gg!zRa0*EE{!
z#QsgC&)aAj7IBoZ4H0QTg>dud?s?n%_|X-{^LmeQJ)1(b15(OJB?_&7Rb!M}HGT)|
zP0xnB5yG^~%P${>OqYfMVz()4Q;@u05xThjX>1`Y0NJ|&y-j^)kFBD1F3G?Z>`d(j
z*a4|dUQIJU28dD8$y+Odf2E<aObOoNXg>57pz!s{$F>sp0+9Z+{j~UQ@(!Snzu15U
z1)>0CrRDgN@Ne;aCy6YQ(;doqBn|HwTI6wW^U{2;hfLv~z)9pJpw2M{%>PRypxZf@
zugjDNJKTA{SsUa@+ct4J^|AlN6~A9fw!F|*4Q*Z+zu*seQ^5*&pTCrP+#Kfg3omo|
zGBWP<pRzUpgWT3O&Gyv}I|S`t#ZeC&-ll;rU8x5L04`Y5i(h<K6ID3uUX1~ZDbPSH
zf?x*|kN#s!oEe>xqn37hx_|X_?!)oUzNZVBjFThhMNtNV6nq!pe^P{<4mDRPcdu*o
z3SFOv4nuQIi0#KqzwzAbJprbUzeHbwJ*Z!-d5K6h^{M+QzjCf))1wDVpRc1e{aK|C
ze1>=^|00_y0$z}vUSg9A0RiA#z1PO4`@HS$rWWAdKh_$=HC=pA5r^LA*?@=Ow6(?`
z<fE7s;TFP5$e!N(YWuZk`-OARm4PoH1Zdlrn4}^3O&(AR+dm=mVJJXbK5k_U`4=@r
zol7*(k^3Oo!hNweZm)pJl!)e(J*db-gTx;Q;=X-EI+Y9m&^Pb|WM0+VY+ucs9&)-I
z%U}22i?CeXU;Vh;Zml<x%WXo0BNhLC8P9kzI0UW^0b(W@l66lVVjAb3M{mXeEi`v&
zXXFv5R2%20IirRanZuu}>m!uLA1ti#d2!2>OGWvld~f4^qenL%3l}GLm;OtlzaH8;
zg$S0$J!CZJOR{(I-L+tv5$Fja@guk8R`MONV6?~#x&Z|3G90T>l0m5Fi6o1(*N2iL
zLNYV`OF($H8g)x*Emxs%{X+nyk&@hhU*Y#!DcY|l9AmPn`m_7|i)071N`<2)>7({%
z`VZd)#Cvx|L%>J^^wyS^3YMH!z`#8%mc6lS+$zc(y}Js!<Cnf@CvlSe+Hu&r{kq~g
zTH-_@e@Hoia9AmxhwdU^$6(>PH(uokk^1iMCkGC36m*vq*nB|_jn(J#elymu3NC>u
zpKV$>bwNU<H){LBYIIuHeJNBUeG4rsx&_3ATJ8tEWIv~!15YyRw8{641=!H?kOS|q
z$*?R!t#F|@)NUl`DLY0IbfM@z_eWnKe*)%u4bV|7*E&`YfDAB|pAvrHzwu<fwRaWS
zP7)}v2l##30VZ<l4p>j+6p64`)fl<Rd|_v`#WV;o4T1F2h1gcHv;&M4N;=Ofe5&$Z
zq{_S|Nt~j?IHIX9VBQR*v`-<n@5YXKM2!aq-vN`SVib_Wu(y~(+rfs@T}W(XS-e>V
zl)`5b&t|(wMO+eZ(Rzt=xu0?b4>t3mbw^KkemVLt%PawMI**ljR@mt<s{Nns6b_<h
z6F#vF6`q=Gat0MctDH-MyXj-;SJ+l0Erepo5kcEx69bZ1*hqQ2=c(2q*)8WY&ayST
zI(S$s67EB2f>77}4`|(q=!P(#;7c+R@N|wu@?d9QNc9x*08K|2<VK2tXj0o+R@wQ#
z@t^0?ufHu_<zj$%S32GIj{Kr(zgWgE1Y6SEXm25%B!t>Hq+XPJwe&NhLDu6vTNc2I
zJ8<ljZl+21)AshDg<J#(E0>m=V)N<XoR<Wvn0A`X3yKCkO+*OAl*lNwOj$yGPZB~&
zGHct}7AeMlA5$LM@Y(#rJrImx^u{EY;2p$NsSJiKq3~ocLQV~&=$2H|d#3)}kAMTx
z)u#She~YsIKx^wW6*+I1Dj<DdDk|IAuT}pdU^-?$S)_|!*x4rp{yyfG$?MU@Jy2m<
zp=pQG-N)5V^N9<Vv<#OsooCJsc|4}knL$0To-~#0Vc$_ZxFNzaF7}iqk>hF?m?a(v
zYYkesqcD@;sOPFanNiU%>9H<j{xG7qB3si4m1#25(e-E5C6@ZotY32?WMp%duAe~L
z>M-MP8$CsOpS2uUxJAcV%Q#{pMa)Bm1@-HuQpob6Wyxw%qDhBDMlSYH+A!~`jp=9!
zGK(}jG4oip?DU`uSB)tt&dQ89&$vM@s7*Ih=KGfOmV?1uue_*DzmfBG6vMF7PDVH@
zogbR5CMgl;uK&h-B_TXU{mu0*S_{#rZJDCtYmWu}YRd*g8a(mW<Jp`l*SVt}CTF^v
zy|%mC4AA{pHmh%rmOa*94)h3wCt4~+7#|~LC5AjZgS;q}(@H*RTuCgBJ9$u-$BaG6
z#ke@0Y>&azsxtB6id9pdI@4COMqGD={~8Zn^`~Q&B#2={B98?l;B8*O@}h!2-4CsH
zJlTARmT*dPnN9}5HpK6X3!MoU3zP=|dq!V@x|1#5bg+*GmYHJfK)lJ-^n^qCNkeWu
zDf0#rhXkpL!U<4{VQtG+{U{dqD@>$^(a+(AhI4?smMC-U`Q%TZXZOJUYJCnf-lP5J
z`Kcb3FF#D2B<8-$JQA!(Ol>HAC`WLBeL1Y6s<H?W7|#Sdl|$VXU!tJoEbj;4_Xm2h
zc?;G1Yt@Tbh%ObLoV1MXs*KiP7R3>#D))U;S0i+Em*)Ppefcpec|W7Iz_ecV*BfYs
zbX2L4zp%{lBI8B1nMI;8Y-dSt(zbFI_KoSFdTc}Gc=Ub<A!C$h%9lco83h^cRtsb%
zdfjCtt4yKUh1U-zE00cUU^fn(f5cV{defEWuCPN~<|b^Z&2E|DD4jie=hU)g{buXO
z!>g~oZdPZp%dd6oev$BiQprS*I#~-srfX-W@h3`|bCfGe?cPmYfC*Bm!;xsQ{iB;#
zB8#>MrxX}tjZA>bbt;uT?#P9?JbW-VJ|qUyp2{QF5uLNxJJso)jeOD36jZ5OVz%--
zbeBa1$Vz4#T1W#n$77=al-ub<RL)}Pr}0$VA|{D<F#Cu`t98II$T?F|o{bTL*2Z?w
z#5QEC+C5Y)j-dZ?Q1<f`u(M~1wU-^5i#T51r&Pvz<I4-vy*PB!Z}j-42Z`=UkL=Eb
zld7@oMfH%`-^PYq>k|CdMJ5+Zv76}y0LhL%ba^k~J(WG>3LR#w@CxO;^6fV<sjt)|
zM_i_atG&{^eY!3*?p3d<^9+B6j!it2-Toj(kvVHdgm&F@oV5bnH9I?A+~|==r(yE*
z%Vcw6cCNEXmD-Y=86BQ^p!{=AB#7xR5-A53E)ZhVwC|;)sCy1IIwgn1p%a<mYW*=H
z17R=Y(s53Jd^}2TeCiMJ!B4btai8@sdP)nd8!!p@7|t_*9gt{$ysGEg2F@Ab--1E?
zSJ`&_HXu|L;TuvL5_RK@J4lLZkulD#iGik<PxW$)HNLa09dAi<pE{lxuZyq0$KNgc
zL}TQj&w~>^g1E&SSQ-eqzdN7+{Yy{Kf@j<F33|?Gd~aaMJ42wSMw|VB_WY>TmJ*IL
zM4QSMD&eh$IRVLbyP&|n!PM83xm*obAyT9J_LKag&WWC+lOGT=K7G&?^)nZC!l&~V
zL4v}&a`{M-gSF<iO~VT(3+nkQDFrw#oF_3IJPctwFbtQ=t$+;2(z8I)#lRmVga=z@
ze+xGlXv8`R5@b;0kM*fx<XDy=^t(A<4u|KvQdwB8*$OkMg>@5_wQyK9L5rvy8x1jE
zJb_X$#*&8E6zBMs_z-`{`({u%`=z2CUfbc%`Zb-&>Rq)M?eM6@7OUw<TCmf2?ZX#O
zC<Ah+9-S48O&_bP0a`KC0Zw`loy1@4LSse78R?!iE*HYKDm;&@<T=jU)Z;Q_^*{Ff
z6Z#rggq2Mbd#XPazEP-74F5s3LvI0rEwVn_>}5QwjLM+UR;Zgxf~5V5pWzLT<7b>j
z4SI)1qzrv}Noj#0`!n6*M<G4dIeTel8syDO{N$YM_@L<rzY%Kq;zX%$Mg?z|GUNig
zhfW)mV5lWm<_M)VXjPo%ci*pHhIU@nZJyYvH-D*27ZO#3;_<#WUySE#M$DqNhx`J%
z?(uke)<8x@?~4vRhfai~QcrS{w2+Y7<?rQYi<ol%JW}s4YAK(G_NqL$?hHZRk8^bD
zMDcbjNVjeb7L#X~#Mr^r`<!{Ae0s3mnfH1&?l#jFAF<3c>hR-NiJF)va+t>VgsM+L
zy}Q_Ur)O{*4{vj2y4rQZ;Q{L!J(1Pd#YBXI7Qxr>_|_sTTB7%%XVpqDp&t(I7lfmS
z`{E4A_@0N1&78<b+-&uZl<w=`!=j-RZUd46v-t?h&7PIWGcJAPw|=zj0b%21I3sM9
zCsobAvLh4YKp}J?|F#z2*rr=#81AP*d;V^zA$6FVahSjGtu)cl;e!!GcO>^|+NX~1
zRi@Bd7~Z3$O3_e$^HS5@Q>GP`9L8GV1JKH&1NEvGQZR8`tQG~}jg6LtZGZ_hd}gnp
zzS|c^;V+5#;<3Hb+;)U=6LqDAnY0-|34d2(OAgC-L?;!_IasqQWBs&JQiPSe7Cyt4
zPO9D<Mct)XrEkek^alEV2i#<HMETMR9@Xu~K5i&wqOg$SHsTb^?NmI<`Vm#y-4)83
zmun`Av26C&VgYkJpA>o;Pw*HBTE_iUR|ZE?DB}%pA?3<3nUy$$qhB~Yk108LZ*|||
z2ksBmZ`e^k0W?$2r9aAA&@ifIjTv`~o_Omp&}>xRt0w+jkzL6#i+w5hraE7ywu)cQ
z$6Z8l-1;VY5mpiySEyU^gebY^VhB-c{%3F{S66miv8MRm_*>qp{i<O*iMbdk4lPu;
zxm(v}sF^LVy|m>IwlnW|U`=55>F#UH3Y_Vr{E3vh?)=#7%?!<|)%*3(LM8$MI;gL|
z-&a{(-VCPh+|O&-SN9#r>^tp*Hw&QhD}Wofqu=l_Qy%GVl>^xaf}7PSrCRFTlR@Xe
zno}Bnn2>Io2CQh;`h?|#>~u+mrXY%PRF2AIr9HFaH8V-`L2lX$;;D*`EoKI4yUW~w
z-)U}#mMkBM2R0SGRi8bpySio}LXHvdmW5fhQRk+J!ZSa*LB|8Byq(GWA0hZ*PhNQU
zk&gveqy~tX;%4iAHp`*$zOcjBk|wg-a@W_GgcBt98`w@Psxh$&7>_6$t~~m6%kFJF
zT<0CTsM;<pAhTQ9`6)H-&x-rPiJmRBa=-?0$n1jUejoG`c1Qxm^h7*enOb%Bwuo#N
zht90^!l%UC*kQJlGv1{;Y>5Xq+=XiOMurfa^e}Tg8P;APpe@I*t|C#1X_`oSwgH~3
zPpoDkcDk-1EpJ$-wV*ZAH686U;W&(r&hJtBRSi)@w(Qq6o)>TWWiR#VVuvMEfAXSa
z32}|llw~cr<edBv^)Ylt$ms+u5y$565HYFDxEA~|LhdDFXWvm#qjjmp&mhFpaBo9Y
zY{(Hv%%M*)ByvWIZ<bN3nsqTXLM|vlPKbe))GaqwFr$j1sD+;TKrc{^LDyJd0#m<z
z9nc5)I><C&;?Ngq9`)jntrvM!eJ$HS-^!N|%XP+Zl$*;^Mjb{FR{fXFBYvJ<oesk@
z?K+4Gr43J}=ZV~{+K*S^tx+x{kI+6VBPuld{SFEOwF<(aZ5#dcQOx>;t5Ma3^pM=C
zUSbx#)ZJr?9g=3e?uO*XC8weLIH`1to_BzZdB-xjST(OTf_^XH>CT^6==R2+*cX?O
z;|yW$s6qd&C*TVQT|Hv(FmE1PaLoySbK7F5+-3ljoxjUY3__KI(-;$SHDCZj;fA2{
zYVmByRo2|<BcbiRT5q1|;u0W9>Vp~fl>CH)@&+D=N18sT&pAn`_6WNjEJ`acIIbrk
zM|?e_2THlO^udfu8cdiROdM{d5&|D|aBvOBgFEQk;a{S?+G$u|?LQxSOQ_G=I>%yi
zB&#-2`yWPPKGdx1_Fi-6z~z0QwATh$QrEVt1$+W@>=R^uiTD{elVJb$RZm|4bQJ#v
zc5igMyLiNU9^EQPo}YbaeLJ^3a$e%Pb!Jq0q?<Km6ZU=#d;q=|xDX+L;7ZP*uM%Pj
zuPO#B+*c^)0mUE%6DV*BmuozS%)+bc8_)0WbY|ipC{RQ@iLoJq$qb#}K}F`x8bdTY
z5M9!0i2mhcf<@go$D*+_u~)T3_4UlR*Bi16;l!YO<1zX0_LAQ6i*6qxD+1lrFoHje
zCdh=v&w<^aoq1>Cu4>8F@>i9>PgQjYlUaTS>?D=;_uT)Tcm?iX+{1dLl9LD%eOF4L
zn!|LSVeuVnNFQgrtAaY03J0?fPrxG0XT0F&edHF~>dHN+d0^HP&&5zT%z^S@V=T}#
z*f{^;s}T}LZ!yTCYI@R<tto%Dry^&%ym0NysO1Uq7^_4k+;|Am{gtf=30KVKjwvX$
z8!khvZ)~=5?}}})B}C!VQL3Cw_F+_a^NJKCoeyY+Xi_yi6XOKeS4Jq)T6fZXwC-9w
zOQ542fK0!=>moji4fB#{F`)aN3KoE$>?jA5F27A-x`$huEx`RcCb;A{Bi#t|WlnhC
zQO0JA;jC@dAq7pBo4?3d{Vd-fNI=E=ew79Lc+H<Vtix;t!cK5q)<v_bjk)g7gAlLG
z^OSSiep$-DJXRS>hZ{y5uqklz&H!%?HXeh=M|`5ezR-PZER`A&;fA65D9Dh_bPOF)
z>WN2QqkkLK#J~$KE0-QTMbW2^aG8E?Ve__|x=Sasa=Gd{MY8rP3a<&DVBe!WgjZ{r
zwwmQDq$X+O=77MDd1X6!Ot(f0>k?TrS0soE!ibmcOmJt^iMz4K)z|q_Gjih;7K@l(
zS_}USIA)d85yP%sCWYG^AqxR??2pfP+;sksj_+7@M`z3?<fkC$6NYIYyvj`rvmWeF
z&6%Sq6>_q@NR7V27VzM@-!?G`Ig1L0Lwop{^*2sN+1j5_cO;&Gk{ND=vf=X4yvu&b
zMfW-4phpmr{sN>h413|hc2LfSp$J~VCJTSUwX;MD|BRDvFz&wGiYtumzEgg#)LxLq
z_*0i0pYYG#y{R>+=?27T$WP1;pD7B8@6_i;aNhNaI8~Mk&oV7ACYO*YNKdVQ9$QK2
zS(VgpDinw!fd7DW<h;*W*}1}p#K(km+?7tbw@0B*ynFww8-oq{U1W%cpsL!v;$1+X
z;GFtzvlyDU)Wv2FY0t&G?u|c0zH^YDoKdzvF&m{Zn9^$_!~FJ|q49?B0NeB2+SruS
zD;!>eaihB%-P5Eue-P~ZWsw`Y*QJ!s*3#6M&t)p+IUO&JiMGs{=n0tq&^)V?|BT>B
zn{OCz>IGA*IOBs|OgYJLW)w&H3D+DcQd^wm&)Gv}m&@ru+jrv|aO_g8S6Wdea(8BF
zzRRO9@y6;HiR{`nD^4!$$utlN#+k_Bndxo!kIYu^Evn$lqn~DbVi(M>G$=m*Equt}
znzo%SrRj$yrzR0h*3WSGAnV0^v7wW!NmJ#%ihw2Wk=)fuoL7uH)=&e7lc14cLNU~W
zSs)Ya@zzl?Gg)~)Bp3o{K{&o(R&wbn`#2bouKOM9P0%dT<*&`Cl^YxLA=_$)i$KOD
zvjfR#l<F)_K%c$Mm($uYiE2dk?mbl*^{(N!<j6c6K!?cHXHEu7hR<j39x8cB*VIn_
zsJ-CTbvyHNU}O_lDum-(1nrjtrG=fe)*<TPLdLuu5l3)DO0t!z3_>ECZ*zsA%hzLE
zbEqy?a|+B4E&@l^_fYka|0Xr3$*j8g@?o)&s(s5m&PI=t(Zp&Rf;*K(E;VK7$cNQ|
z70QeZC9-g?ku$1X7G`30p*ojjjwdKwLxOV#@{-p=WzH*Ku@xNpi04Si6xd{og-Yc~
zIgeI&&imLH1PujZ4J*vt7grfB=5u(dOym73mvH-auqOPaMz*erW=jdykZX%qJN<QE
zNVD$z;yvQyj%w%x){%B81-7nWT6Jk?3uoVCm0_fmR^Xwk;<4Gj4Y%@7?9+T_owgE*
z>?;QZbslNRvY*_Q4;@NErSj1{CtkU9S&!X-L^)Q-loHVNoaz>HgW-UX5*#z#RPD(`
zA6$OfxWmllT$F?pD&vZ>2OSN=5>m4m)>YjRG&btf!Nlu$xRAWh@BIhIw<!8Ee@QIl
zNB&+m#!#>IFQzKOOnU}$3^hnGqrFI|GI*OV=W??`*R1|$4WkpR(}8L;mvG|lP{bOL
zk{#_VfO(9n7D3t{k+0n&n^GViITsa+e2ki(ZN=d$WfQM*O(#HFl~fk+rGzFiIo&qj
z>CVvHHIh>6aD@pL*Z-!&B#spbWx2v57yF^8*Lx8m7Y47b3Ge*!zLd`6aX&Vcug`)=
z!h)gvNJ*a9EW>E(GVqB7L+V7sJzbr42oL|0N@2}pY~Ae|$Fn_;&F&UP=nr#dX)KEo
zUxNL(Jua|T-Qk`G*4{5k*Xffi_ut&s7Jas!NjAn`?;)|*AEF+dJoIUZ!XJWX+wcl6
zHaZ-Y_qi9Hr&Dd6aAj)gvSkTVnLl!-+>GVjf}hetGtf8#BJK91$G1pFkaaU9c(q?h
ze*42rz{n{>^TK)Sxn58YcJ5!=z(dZU<B<(FPDjfrxd9QruM09)yB61j-Bc!7fIe`S
zFJChOwcMnVOouNB@+UU07q0;_BrR2j?$z6iU8wI<j8TqJoW!2UnTnEvCvK*miXG=r
zfT-0MkVRq{{ZEoUwHn#mQ)fs`bw&N6@e1IuEPcM~5qk>9GfZE;Ihz?8-x356e?nZs
zRl-MA4=^Skb38FTxAAj{q%m^~#_XWQR*cMiM9I0|$$ROm&wI$lTUlaC1E+xO$;RMs
z5Bfek=D{l+5x7{y@vPtK9MF;--{?%%GAVVaHw|T=luLhhxK5_GS~U!0nwx$%hs4aU
zOjCfuES{#YJd^$n&Hf)Cv3wp05gmhj$25tnfvGtmjMOTpb6#}9kX1-I<TL3p<b#%Y
zZo28Lf*@SClk=-D4fVz)A>|4q-S50w^Eezb-l|ern^H^RvN!#J29<SbQ|X|r=5y2J
z%H1NU5LF=$7RL#;JC!jI0l*)*zJ15KN4PF0+NP)%XD?wPZN=P+l5`K38{eq03^v+6
zieY2ei*1rtknv#_7xofjG%w^}Zf-IMwNtLu@{8qG<O=k-2m8bXjycN1&U5<|;Twtx
zTnDmeRnRHikk{g_B>K@RvextYM4MFptPuJ1LLt+rx6Cjx4|a-!2m9wHsL?b+ekrFj
zz;{5WAb0l*gX#%jVNk6%;t9Lf%_Nb9m)`=Wkd;3KFTL2bWW8~d@sZrg&$xar!nCf3
zi)}emgJ1d3dWJ<_X5n3sUh9?;tbF&Yy0|IvzU55_k<)|>FH!;ZjWqCwkzXSCHr1xg
zQ`4GZB@B-jt@WusZ`DX<aBQfaCb2&lqis^B{GdLdp)7f4_i3nW)v`MHaRFKW=je=I
zwM$wY0A}6hb8}6hTADSA!;xdhyHgl{DPz3L+PqI#PjxqAR<X>Xk9;Ljb@O)@=#L%5
zJtx|(w~PGoYIg+nssqq-cKpB}$&>V$CCN*XV`=V~(*2Dl4}yoR^J6rrkpI`8vUp5-
z%tC@6Gm>fF({|}vx|t2GvVT-Ptk|I7)T0b=4ls#t6%%s?1srl~`oy$U+@BZ1mrMzN
z*4uAMRCk_J^}=K_om_K#Zx;aR4N-d;1IA$*zI+G8>M`BwZ`Rd2<ZmRbgx(183E`@d
z+8soym=dNi!&JS$L3!c?t66RC7jVQ^RU>WgV+rH0WOy>1I}}O18JQ^2h_ktGD9o+A
z;~_{XOl6yPp&sghpckCLq<!E*^Jaj=boV5g$>**X@gxbWP=n7x(w*r@wa1Wisf?v;
z$bHQ>w2=3j$V!mvS#*+}YUUsvD%qH$`xytj9N@CBNn-lkPgsQ_OzM}A&lC+&0YBdQ
ztLYr3wPb7`;hs^I!tlDDW2xQ(axwjEJTAue53S0XLU&s`WJG&?&Ar!ZcXNRFJzdOd
zMk<Z2iASl#XpqkW@-Zy3`59sAF>`OhwQagu`}615>i0r@X^`>($3oaMfcAW2AdGBU
zGt)ue$cHbYZLocMWX?842(g@mcMfmbZYd;2H5dJ$uuu|wCXuXq`BC-AMnsFB+rM<r
zitCr9R%IR6iEy{nvJ`+IK{173KKlcw{%3*>2k60K)<S;lgK9^vD?ROw-|6wcNUkeq
zh!uZBI=Xb*yP+lWO{}1hH*oe~l_ui@W@WR~BSvgp1)zWos76_v^>jqovv^egVx}bJ
z<$H}O>C&xq^;_Z%t>JCu^T+w5w7NktJZx_smA>iX{HPq4Y}P9l)c{@l;BTRrQ$pD*
z7~4t}P|@gxT<1G@B&+bfs+@YLCs!zfv3P7Fj)ynQoJ|Wse&_UagFmku!B86Et-qxI
z`z>}QYg&)?XmZc<A)68liksKxQcO8ZfrHe2qwIa%Sn)lI!h}+e70?7hm}(bm!egfI
zyqSkb_}$0;O8)DL&6SmnV=}ea#thyV0&hPioRz%8UKS!*+>^^<@SJS#JCxzT9Bn4F
zRg}Vy$$Nd};|EzdDS2*i8t47mhw6lP{k4qqcu`L4w#3k3WNy%5r$w+8zxUsz_F-N*
zPBq2NlL690ih;s`?@G-EDKF;NP;a^`8s8Rt|LY#4>}TFmtSQg*zR1~?iG+S%1r18O
zj7CFWYUV<>?}5^5!QIidtomzZ83knnE-{%_@=R236)LvAveq!!e*N+tb}$I4$tcJs
z<^USfVfqCb+*tENzicEekm~);Df4zaVvQxXp87X>6ocRoR`Lrgey9=;#vW*P*Qp5I
zwyUJBI_IFnipqjqLqsBG1c{ncOvnbcjP9qf`*__ig}c=Rjk53n5K$28up#>t42k<X
zL{0}ZZD#nHJx6ug+DBs}P<D8PL_{60sX<+sY{2?cUFQ3JJV{hYq?0+VPnk+CY^Dxo
zX8<(~6xGpxqPn0uE0j$)ed<w}uJPG;F|r5O@3itQmNC=r-xbUQk~0ptMM#Ba_{W3Z
zE+F@Zp>cGwGmpL!{|~I*v@%9}wiRevlPf#mY^pAVgHE9O=z%e_gH-Q&vqNfcWH=ft
zWg5r^yntUjwdKnEHzXG`#FhZFwaXQ?RXjt5qw!=<<O&7CKaw*giGSO?OmX%%ViH#T
zNSy5@^Y8??i}t3~DTmofQ91b3?>gP^>TkERA|i-o;t^_I!RhAfm4ZB6%$?wTCt`Ft
z`)|^1JHpSwdSJcP;pu?&=k_FzNQ*9F5Q+xC7G5>H53TzjXeLuiQkVQrUP@9u;4Z`I
znTJs`7VdId<TU>ioru*U&)j^^FU&ppJ{U<0jOG{bolNx~NBslk8em0(`B>`{-~R_t
z>!15LY!AFs9XwvvHvZo~7l;LSZJb(@&i4B5L(sm-V?QR0Gb5v-#zqU(#sK5v(PV1R
z>=poRGX{8(d<Gy?SA_s^@ymf%dmD08Z43=mK|>m9|GqIO%*z9I^X?dr)4y*Gq>D@3
zbp)2>4ZseYH}+@+xJOo=JFOEf`&I{<$cnl~<)`;RqFf+l6_qg@D#MS|1p5FQXl7%g
zkHV5o4AJ$yys>Zn3(!YxkJhYD!ES0kDLYHyd0PY+x1FKI3$!GjcRs!}227zvDIFWu
zQqOMABf&<E)J#n1k}&Lkk0+LYH&C6jy?UpSA!FOR6iG)&;EnGc^DoTh01XyxGAjs+
zjvfnbHX)4PGzIh_4|f6V#Or<(8Y<TBIcXN99;bc=U@w4X=a^8Std##qtZ{ODT%09v
zXp}I%mGUUirtjPZuoRYO={-^$S3t#bjKtEZQUv(XJEb+h`xuWuMos#;0zB@a=Xduz
z0T^5jY!KaPv;>sN-0LQc{j8#x0=>`BI3I&1&pq53uP^2yaDN54dxz?Mtqvn!l>Wg#
zJs)*$+N*N_eE4c4UM(08wO$T7zd<Ag!?sQwQzz_!W+d?Xni+@P!{C+GjTR3!40;eD
z?+F-GTfUUB&KeStn<&b3dWY=9Q$L*nKv8AA4^iV~25Ew_0<yAz4M{e)UE;r14h;(c
zWt7mVdfHwS9Xx+;tj^tm6GCDSNH)*>au;0@`3{?~5baNB)@gD6F=NtfG#xKHL9JMP
z7gOsk9dlmf&R6(K5-swF#$H5v?Heqh-k<<m2#X=~{{H!ApT{v&1em(R44^|wzYemT
zk7&2qBH*e1Ovy0EzXJd?4(?ox7+JDcQ$tQ4`mbZU0U6XAXZ>nRRbRl2X7aFkDgr7&
zFTFVES^Q*<8x8o{0(?AFy;@JQZ$@gNGI!1Z`YzS^dt3}-=xe0wP*B-%w9UU3izs@r
zh;}3fF$)lYv%Ht1w;Y}QG8?ohGZOT;9#$I$gQWm8h6cLhHemFYvc8<v1rW$6VuMPJ
z8d||_k;TPSt%nh+Ib!SkC?Mq<4G%|SrqL>EszbixHJ^B+jULGg5dbUij8twoXf!D&
z8ErpY3?OI+43PjVc*hqP+*`k50vMaCzHx95x&uae8MM#Ig}rKLowmx&&lvOv`797^
z0XPQW8wMW4`bit7xEKYAhb48#QWc|J3Afn!eN+t^MDi=jx=6YIiFS-=ASsDl0xMRC
z6B~T`8UX1yuwdvIS(Y~G{hnw&drU@HdBFe(!bP7p`*+f;-vCz14xiMVC9igC#?Ww~
zf;H@|5E6e=u~=v`5KVKjLAua{TunLMiI<Eza34ww3k&=;hK?)1NH`-9@W7fe76eue
z;5=Y73Y3QQ0eIxy?G^wH%d{Cm_SOfl<(O1pSt(dNG2`2`k1GE_h7MlAK!k2(BkhsP
z#m90GLZaP;Zh(rS*ms;%WkVVm>Q8ae7pwgAtU>R}59U**`Ld;-)P8U?#|=9*?WIoY
zet<`YpkZH+e!J(KWEGsA;(u2Vw0&eQc`_h8hK8#lfk(kFy{!#>zHs^5YZSPm^I8DA
zU8EwlOqw+M@fJ|c&KQa74g%1{bua$LT)HL17M=+ROsLq}M_oSLAe|6a#K1fUfJKEr
zBF2j|U&qDg<NzR=42a>6Zz%>nn@`ZV95TA+qs5TqEjDnE6TU2<+#A;(sW&cp;Q`>4
zE$+7i4!*`H0Iq6Q4oHnmrXK_7qHk+efM4j_GXl|#tI@Pp%NsYq^ZXf5?+zG6%|<pm
z1K=Yx0J@tugx|ZJU^gI-Ns<5Zn6I6eO#I#6_XZ@ZI^<wI)7hz)pAY?Heu58;m;ZLA
zyIroO&FF>}$KxSbK>_4g-kSo(f_$zOUklvj+iVBV{dlK*W>*-~s8xQ8uMp}SO9M7W
zFM61Qntd<@(yCmUEXDxAv`+y1T7k#des!#A{*;eS?C`#Mm@Ek|i=1BX^%>Bj|0`N<
z)c_aHIg0B$8W?-;hIF0WvdGDPxQH2I19F!HU_M7%0HEDlr$_*H<t+|Knxb<|tPoE$
z0FNP=^>?Te#!eSH1Xzfg(o0n)cmobR&(ZFfFW}tz7?5#dnG4ez2j;GJ48evZpfyTb
zHxI<#Oo&(1li79<ucNW$6VgsCmTPisg+*1)Y|EzI&Sxy}b~s#}rK?jtB?Y@63V)>N
zT@i#?6V%L_t!u6!b7`sbjNW5dzMFNWSq03jh3>{_qSvmYmmNQ4tZ2XRJP>O*9o<s)
z0SjtW{?n!wz?ACQbT1z;BkzROJ~vfYpBd4~7owfvw=Rc*TFV|QeMQ71yxf{UZ~eKK
zWn{^*m5J^g<F8f#esE><rLJfjX<q3!rw?%;>W>4mW#Wb6#C5l0FxRLrbIF6-i(v0o
z`1;dsdueCfay7?#+KVoJr>u0Nw`?RX=FjbXfA@1gexK9rzoJp3cxw!yQhm~(8*CD(
z;V*(Rd9?sC_SSmmWX=&*g=h9Qv+pxJwj1UOmC~XWiV}=OyDJsJEpOSJlRttGQJMh0
zQIw>0iO5+%9#&1g<8rAYZ$cRN+N+`8$YJ1}7c|%$b?lST`VAC-a02-w^bk);xb)H0
zj6xv2;;izImL>%DMRbHF0!^Q2ZY!%MsxS-Sf#G0g!YWLyngu`)HoaD7|H1Yo7tr`3
z7PkNHIrMl(3RV&JOU_olS7I?3n^u&oqXTei=bV)02uNq;@DRBG@A$vNV<A?PO!WoP
zlH&?6abe|F(O1^q0--~rX}3C7kKLPCc%{wFzYU?#Q9SZ>26!5N2H1N;;^z3RFXwRu
zY<0;StMar3?;>1el5TNirnRR*?x01IfK5ef!7!T0t(BT+Fp4W>vy)NlFqhgqfNMwf
z?gVz1ejj)XlYPH_j>a`UsGnnfWg!vz4bf*zNM+=qz3d(@FoBlrJPrGXqTJG^g!8g3
zitRM%j^&cjUWicL+#$y_HGy58X?q{8q|ZgQLwBkq@z|grFUAVt13bZ6Wq8>kxK4LW
zdz`f1dRDK-I-a|n#%6K>8#BiXcEzY^^0P`J)|CKSZ|-s)M4l}51W@5kYDH*6msdzv
z9;~p03SwGe+{3XoZGIp}_go`6$Gdmwbq~EJeZXTlf-QM(;QQP?4U_NJ@NEFbV(fd_
zZ{lPj(x0l81=q<9hClfj0GX_Lpzu$6H;3+2&nXBUZ>fP?oNXStjD+fO)V})^sokyw
zVX*~X>3<l(t0F&<gc7~+peE&T3b!rI0khYOn$KB|&IjGy3{GqMfM-jkIb=(T)DOYN
zyN3i81w$wrworxK%~q1IP?yM;!QKKd_izolNYBhFO&cLtu^r6>Hsz4ujM!lstg$_l
zWUN~*|HEqdmxyC9NBUtfo)o-Phb!L2KA=x0=Or-P!E(A$W(CeysMm%YRqu>kyD5mA
z4x|=<Fu!XsnH+s(Kca{MB@K5a*8opsBe8kiy|LlyH7rzX^(i@PJYXE85f##G01fKV
zXZU_=T6P6wYxaGDsffdQ0I-e~(x4P97;+TC*74-IQ@N3xir%i%Mw}QgXD82xqeveD
z7Akw`tts!MgMKRPlSi4CW?P8PUmNzxK%Yog?-hnyyjt6k{csp-a`hpf0)B85<Z?;+
zyfR99AQ|Fay@apzMm+ax&`IV(7TU?!yz11w&q5V5$1NLIis_K}X@e%i5=nCouRW^?
z!o^m33SX0y<)4_Cfu@LLuu^Yb(PDB~gN(zW*0MQi5p*DOkoRbf=X`|!!~WGc<{@8O
z5JxG205@pAru^gSC0M#!&RBr1eyyFiJsAR6a~W8?sRteeD?H5LsaXQ#R`UJUj9w^p
z>Z@(6c!5_Sn`W;52myZKggrKaxmzWC@FADyrinl<-|9)6Or*<DQLJuT^oBbB_(NV<
z-k)%?0_sgD(gsE4xQpowNq&mtZ&ucFBHau_3hw*(<P>GI1rQYF{5UNaKlTVDVoNYD
zZ4O*maecZK5wY`8kysFO2>`t(azE(9k{5jO&+DTz#8k?h1Ty~~K~cH_LXae<@NK$7
z8<MNSZNOz9%}(L<0ypNMb!7ww;#K+O>j>-f=-6{wwFpSjKjqHqT58zU?pG2=t}Pm`
zm=IUeHOU;O!NBgzhV;6>5^H&cN~*DW7==NJeA|T3kSOvMsr{ROk!3b<(ZF8Zm_(HY
zR(QgE{lLoSd0u8v+flr;NdM0sc`oaCfdQh0t%yW!y!3n2c&$XcJ9{6XSHhS?GUMK*
z=el}Rrg5i{Rugs{{vg4e2AA4bbmycl{9V(1zMWgUwRgQLGNZg-heP1?oy1Slg+i_H
zhr_9`?nQrzn7Ef9>&Sw>-|NUK;4OuAEIJ}C@Fn5CUB}4EXj>)nLgTbPz+AQXkv49n
zD%(v;pdKMI2t|I6H`O7xfR+FEk%GK@H*WA4X{k2(630#e)wzEt4+Ju;4St0K3I>qg
zGj+2OGJSrLN<{1k50u|)8drW`R#_)tC`cYiy5rqbM6i>G^OAPN9JrSf+@NU!>iF<t
zp{itI#A7lfsRzx(ES+uRpD(?LB#i0wAeyw8_Je$ZC=SGPgaQdKk?9>-b504C`}^WP
z&*Kn2gxzRZ3Xfcj74u@vb#Q&_ZV4cYijB^p(}}T?3e&b$@7rO8$S4IzQ*eZu)WC(8
zDX=>pD^TkPfS}fx2uy;9w0<N7?G&&dy`=$XHV(#-uWMqo<%Vs(!F`U2<!n0xm?AJj
zX|@6w+R{e$Ns9>1pN0zr?$QNoZ75I=RU%V&7MyMgq~|@ZsP=dJQy&TM5S1TXf5x%V
zXklv4t=0r(9+|N})Gvf2Y*{c)oD@Ju$GLt~B90<(Kml(byy*FS;N!<+zQRKd^zt1r
zKFGnJbh(%6uECpJks~UZHrUS}m+I}R%9&L7s`PEGNCJ1#H!7F_waXqlN~<KcLUQio
zWz^97xI>ulM7L7=<(F{FJ*hW-sWB<oE4HpZ*T*NG<usgnd|i{zZEfbQ{U73Pni6)$
zw`O84F(~oHyhwiFuU)7Y%;efBUnA}cD59fNB2pkFGAdx54^@Ci+!a=bR!b1?U}fTz
zF1PMawqTovKRc;TYZ(}yTWXDqm7&8A0O0V#?TuW9E1~g96yG`GFgi$ma$Od)n)SJP
zK93-7vezD7M)3Q$A6w|i@^XUxmg*Mn{g?_^zlnW$LC-iY5PoDGqWQ1`+{wjCm`|>-
zsPa`=^=alYTjg0&08L+rw|`lltxqu%oc^RV<8Mor<|QpZ2DKlr=jsb%-Ga)qk>Ue)
z*p&#c8ng7yVO7?WHv9yQ08iH9#pG(GTB}}0-?epIOpv!!2V_&5uX=$n)owqZix-!7
zMfWZUna?x@4X`)F9DbL@vo4tJ-p4G*Mz>W*JNCig_Ia;Ip93e61+9gQEW%9+`59*!
zm4iLF+;#I$$z&Sc<nm06@g}&3DdQfcg!R+El)`_!__fOtD1?>=m)385yXNfk;!xGQ
z3;i9pKnHzf7}Iqw--k;My9(dG->p{>1Ji-Ygj9wuAq@vLl=KzY$pphak-?<Cu^0+Z
zDBn}RJnVU${P^===s8oe9Pg1S(JkPFNvfkTqKV&S<_@>MiLEQ*&sBHRBt?ZRz@jM_
ztnTw&ls|QWl8Tn{3eRw{=*Gr=yaTd%k)s=X>iw;U)t}8M&w&M!>jxHS6&TO`qnH;j
z2B8e_`pciXdajE%l!fr_mQD_?p~xkHVv<Aa_k^QrC|h42y8d{b<s_9)ABXA9aHvEI
zx}iR33~^W`1*VJ+%kV7Brz9E3sV9R;O@BIv3mR@tVp0*{CcQuEDGsjub5dqHE+HI!
zH62)b2o`5#`sz}wj33-D>I$$D?60`fjv?!i-b$D1-#IJlq0aSD%m}f+RdpR6-IALb
zxso-trkYHh_Y+>vUBxhwcEC<cZKg)=QCrW->LqEyy6$N#u(k9=(V~deWcmVPZ3I%m
zt4K8z=9AeK!O&2<q8kh~CS%TKyw!LGtO)F!dsaK-3(<%&ChK#+f3n!|50yIqBXWRA
z=nR64(<r*k+ySJXY(N3Snl6?a8q;2O?agd<Y9qDCcrxsOiy%cP^;re9LgNE14Z|tN
zJ63WurF}`iKI!x9vnjP1@cMlHA&PQ7jBD|dRWAIWS<)B+luqa51dk^dA;I|YPtd=o
zK+CV{air1`ewq!49Yi-iCM+Rr#`bZg?!(Ltz%6>)gA*C*oX-Yo4}XPI$_~DAi0e5Y
zAw@Q^glne~poY7PLtF~O&3Ft9zHi3(0cruTR~?_?U5HoS8m2`n#k_d1WrI{I)e)%D
z{a)EzQ$-eok*-Jr9^xf3+w-dlc8ijxsYIY_l-8H#FZ}hWol{$<0&Ly#atgVTM#n}a
zh5Sk^jNMp$0-^<>-yGMv0eOykCg3@{hXoVta4}3Ff8(Hp9ygyfU+Gq9OI0UHyR)hJ
zG?H>r4T)@W?T136$oqq}O7eO6o!)IIoEgj?Q_|UA`g0!nk>;jVQAKdw(?V>qGGVm%
zQ$pCKhNcwR|KFTYj@j_MDa3rWN}e@%h6#&+5`TR9D$)x45_|`K)MbWs{yqvk1lE>{
zYG0zs#9oSo28aC}%YcY-@ieWmO`h1vfZS||Ap_R55b`5JcmPwQY5m3Ol=|!so9AN+
znnT4Ux-iW)alkTSusbEb1Tw$nj+CB7k%%v1)Zm6tJ!dbq@s5NKS93GnfFmird{9)+
zIZ+&$@*TY$hK+u*seUL-pOwOD04=Y6A5lERhYUEwfSW5Ho)@Mwd5qm1st1oKON{dl
z%T_t$34B1uQg<kZ`+ax3oTm<!8+@qO$(O>Sd29>^f;~>4<gH!J4&9m|nXB~BEa$O&
zbyGw6=sw-28B!{FeL_;}Zx9A&!V>H07LD@HXM6-Ej@d(vyyhNfdrsRGl}5bT4#f-}
zDs%h&sm*EN(vggk6j;)xVah-_B5L%>cN3P!$cxS%TsSTFq8SU0_=`P$kD6edx+b;-
zPli46bQ#TLz&{6@fQ3urZ-MfN(*lbOz!QvHGObn-f7tkfE&RwwXq^9Q+K%JS3Djra
z@?bwzfUeuEDKThuL;~xdrXh9!;Tk0K4{7BamT-#zh5C)p!Af^u^Am)uFcvDTb!4z$
zg&hjNabK0f3ZZ5!Utr5Z+o3Y9+A>wDl%X;)g-(JgTzs2NT%=l}%n&(e!TQ`G=~CA+
zEXs;6R4=~QAMv91x-g6rQY?}FXf2nO%QqB|ZSAwFS5fZkFee;@V(P8H_>vx+qb4p7
zj2ia%4BRqlnLU+DMlCG?Wk$DNmNPrOwuM+WV`46&CHymhup<mM#b7=p?8fm+W4`{U
zt09+b04Ds9_|A}Pwk(ScCTB`anrq-U!PiJE#`Y_M*o6OWTofFWtsU3ep`a5bBl=)j
z*DA5VuoSn8^U*kVICnbPvJPeWM3=>jz{{ETz2SY=7$i#CwxUGXs#=1!i5*>t`ALNP
zeVJm1apNyoJ>tE@>+iQp#{R|$XY!CpZiErj&$luuZz){%IQd{@9bYt=R(42gIWNr~
z7n88MjA_QE*0Sz(-|FxdQ}<jw``cnp`}@+6G42$wVn|5<EJ{?%FT17s?7o+0j=Tl9
zQD{!NGlZ+PM@HsViecQuuF;uzwhSl3yjz?*7kPjGFtz&>30^nb%7gw0v?H1Uu9Du`
z@Vhv^o=3)SM@-%ijU5u6`y>3wQCY&{H4Y@a-)&E7T?x@|*||FC`aF5nhc;7E)6lcw
z-<G>aGtoPjzt<B7lQhJaKsEG>bsy|K!fzFF$7B?*1Gh}PV-}a(06(V;r9Hqz+H;O9
zuT6{i9Wa(@9`mBb!_&RAs?n+go9QeBfL#uOUn>Laf5Ynv=T!P>t`0}u*hP&V1utE_
zezkmVMSsrVG3&t=%dj16OXL>$^&LM&rj&W1xdg`Z2PRI##@in#(8-J3Iq7DxV6|aW
zU{RU6y&<}Pas3on6_-;^^6#04DY0MxQ`R9Zzz{T*8~@|7v@qet#dIh?>x&Q!HPHM~
zg4g7FVtr3_!6HB#Zff|Vkto6ljPY;%5ul9`R+H{!$$(zoaTqWMi{gNB!Rq`gz~o+U
zp*<i$HFjAH1LPvT#zXGU_%Q%O_x>@kpS0Xk#1Q4iXh;0pP-w<-m2!}=r~!llTpI;%
ze81yhAu9UbV&inV$YE~{QiT;V4z3i11tN+urV^jbY8$LjX#SJj{_|4-(m4)LCO*en
z<6`?KbNlzP|GDT*Qmm!M1-9J$^#AzipAmAYfL_)DL2`ba95Zhnl_&-V?w#sW`B(qj
zXlVL;?*GTJ{(pKt;ZI&p`%8~~CSvul@ULSca+zkI-t7H3*c0q5(adP35Y4HF3BHR8
ztWY_ke*f@`;MGRi^fx_&+ULSo<SO|8J_|rt!g8j}I?Mq@LEQo$L;HD_NBU0?skTPP
z{&t0%CS@}tFkFp<r}Rzna>z0#HrmU77ke0P11IyMc9hi9J3`2FZd#lgPah2%|LQvM
zGLxke=zF~5RWj4rmRVKe(l;Ykxy|vLK(L~u_3s^9kAlOuukQce9-EUlHD#saS2Laa
zCE<n-ntq!5Jnz+`^x6zFn6d%a>8{R2SZOMFv6JFnVJf7%^WSH|Iw{tABaGnn$%k#M
zZ`lIHC&fA00<$M?HHBZ#48qPHdh~J>>-<fzJo(YO_oocp@BeipQn%Qg?!u?|6zTD=
znDBot1<0LntE2tnZ<`;ldpz9mWK|q`J@fu$@ue2bTD-1g!@qpOV_WU>rT)g7J>7`z
ze~ndAw^^I>+NZ1K-!BU-mtpEXxcy^}{kd<Lb@K{b&P&4IO;NMm)x||1J{~%AIXwJv
zZis_pG<g5S-*BTV<ISd3cK8VB^y`TJ438kuwoH8jG`=t~F|q3=TK@*sqG22*lwC^@
z5L!p`GT?RLgYKkhRZry`M7#V@duUxjnca-_Wuklq%2=4T6gn{-=}D{ONBR}}eHOJ&
zlU2#VVea2dfBm!mz^_TKgU3J9*V{KLY|GD(Pe-5SLhsv(IS*MoK5ce_lD?SX=8tPj
z!aAT0_#~I<s?ib|`J+&^Ov8BM&HJOct2_;j{sN1<ppjH$_feI9=>YDlgCgfY1zF3=
z|K~pbvnrWb1S*${QuMrNM&6V<|ELrolX9ExPKYskpTxD10c<))r~dDLS8EKt{^9jr
z0ql|d^?sa}G0d=6$>obv=Y2_z2jn_I<t_Vr7E?`<6tsCEQrl@`u#97Khq?5>X_D^_
zz>MP+E!lXu`rP)Ye2qU;&m*I3^IiYuajceDT}BCEE8+SX`CjRrJIeL&ueWMQu+Q^k
zm!GPjz7ilWjckid)}IsZw3)u#sp!%k6UBeQBoZzAcklO6m@$^hbQII+iF;07ZSMT6
zJG*jCATi|}QTBvg$*~<t$zbXuV}8UJ2Z)*MzcRl6T2gR-MN@Ir9EHVRT;KNJ?;GOq
z1(&`UN!1G$hcyOEWdA<_fC7L07{?+toCfuLT!I$NIKq|*$z88!ck47bFq~U{5q9-2
z%bfOc)l0&@1pM)_aS0k~GZUQDekMnBla6lYRsBi~s9sJs?KL_vYJ$edu#__jSXE8w
zfdry3G=wmT;6-~+hra2gv7$TvWBm>K)rRA$n+v^UM6E{_jik|0)mPSi;!+>H7zmxc
zhrO$lC^P!Xe`rS&<f((+zd>s!)mwd(bH*3yOkYJHw$7?|khZN3)#X&jo$1}`vZx0%
z(>5m1s~%h<0FRBzqekr~{|xJkqwDf^<XyG;R+3a*PG!^?-)z4u+KyoDqxQ8rs~=MT
zvGFlfGj^WGnlWLKK_Cza1OkCTAP@)y0)ape5g1;!d|7T?Y{JNQOp9@a%~(S*q3!db
zQx=VYF+4b8!u%GSZ;{5bv6`iOBCS}t!h{PGLX;DNl-XG!t!A;CvH*|J!#W{7<mBZT
znydbzMQKC={wAAjl1AkUzDU>CU3;xr!N<jri$mvsXQ9V+sv>SciWmqV86HV%)~-o+
zx$|Ap+O=y<z*@O#rGyQHf3=$W3_=!I<icWAZo1bGbrKnLaW$g}@Cf@TDG;JI-)xIC
zT5GSh>(-{*ZoAC{DTGOAm-sumVtr62t5K8pmfM3_F5uIS^ihQP;gR8V)3Td#WmHXx
z0R|}E2DNN~cC8M&-|SI!g4TCwh|18S9Hp6%`}&YPGH}`VKp))l4qKX%29JzoH!YJ=
zPRkWgT&P~w=pMP@*rKe4cx_%xr%p|>9#Gp6a)r}G+UibQC6pJ`m3OVVLZ(TEaXSzQ
n1OkCTAP@)y0)arFFA(_upgi(W(g1se00000NkvXXu0mjf6SPWz

literal 0
HcmV?d00001

diff --git a/src/common/runtime/src/bin.rs b/src/common/runtime/src/bin.rs
new file mode 100644
index 0000000000..913a875c37
--- /dev/null
+++ b/src/common/runtime/src/bin.rs
@@ -0,0 +1,205 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use clap::Parser;
+
+#[derive(Debug, Default, Parser)]
+pub struct Command {
+    #[clap(long)]
+    loop_cnt: usize,
+}
+
+fn main() {
+    common_telemetry::init_default_ut_logging();
+    let cmd = Command::parse();
+
+    test_diff_priority_cpu::test_diff_workload_priority(cmd.loop_cnt);
+}
+
+mod test_diff_priority_cpu {
+    use std::path::PathBuf;
+
+    use common_runtime::runtime::{BuilderBuild, Priority, RuntimeTrait};
+    use common_runtime::{Builder, Runtime};
+    use common_telemetry::debug;
+    use tempfile::TempDir;
+
+    fn compute_pi_str(precision: usize) -> String {
+        let mut pi = 0.0;
+        let mut sign = 1.0;
+
+        for i in 0..precision {
+            pi += sign / (2 * i + 1) as f64;
+            sign *= -1.0;
+        }
+
+        pi *= 4.0;
+        format!("{:.prec$}", pi, prec = precision)
+    }
+
+    macro_rules! def_workload_enum {
+        ($($variant:ident),+) => {
+            #[derive(Debug)]
+            enum WorkloadType {
+                $($variant),+
+            }
+
+            /// array of workloads for iteration
+            const WORKLOADS: &'static [WorkloadType] = &[
+                $( WorkloadType::$variant ),+
+            ];
+        };
+    }
+
+    def_workload_enum!(
+        ComputeHeavily,
+        ComputeHeavily2,
+        WriteFile,
+        SpawnBlockingWriteFile
+    );
+
+    async fn workload_compute_heavily() {
+        let prefix = 10;
+
+        for _ in 0..3000 {
+            let _ = compute_pi_str(prefix);
+            tokio::task::yield_now().await;
+        }
+    }
+    async fn workload_compute_heavily2() {
+        let prefix = 30;
+        for _ in 0..2000 {
+            let _ = compute_pi_str(prefix);
+            tokio::task::yield_now().await;
+        }
+    }
+    async fn workload_write_file(_idx: u64, tempdir: PathBuf) {
+        use tokio::io::AsyncWriteExt;
+        let prefix = 50;
+
+        let mut file = tokio::fs::OpenOptions::new()
+            .write(true)
+            .append(true)
+            .create(true)
+            .open(tempdir.join(format!("pi_{}", prefix)))
+            .await
+            .unwrap();
+        for i in 0..200 {
+            let pi = compute_pi_str(prefix);
+
+            if i % 2 == 0 {
+                file.write_all(pi.as_bytes()).await.unwrap();
+            }
+        }
+    }
+    async fn workload_spawn_blocking_write_file(tempdir: PathBuf) {
+        use std::io::Write;
+        let prefix = 100;
+        let mut file = Some(
+            std::fs::OpenOptions::new()
+                .append(true)
+                .create(true)
+                .open(tempdir.join(format!("pi_{}", prefix)))
+                .unwrap(),
+        );
+        for i in 0..100 {
+            let pi = compute_pi_str(prefix);
+            if i % 2 == 0 {
+                let mut file1 = file.take().unwrap();
+                file = Some(
+                    tokio::task::spawn_blocking(move || {
+                        file1.write_all(pi.as_bytes()).unwrap();
+                        file1
+                    })
+                    .await
+                    .unwrap(),
+                );
+            }
+        }
+    }
+
+    pub fn test_diff_workload_priority(loop_cnt: usize) {
+        let tempdir = tempfile::tempdir().unwrap();
+        let priorities = [
+            Priority::VeryLow,
+            Priority::Low,
+            Priority::Middle,
+            Priority::High,
+            Priority::VeryHigh,
+        ];
+        for wl in WORKLOADS {
+            for p in priorities.iter() {
+                let runtime: Runtime = Builder::default()
+                    .runtime_name("test")
+                    .thread_name("test")
+                    .worker_threads(8)
+                    .priority(*p)
+                    .build()
+                    .expect("Fail to create runtime");
+                let runtime2 = runtime.clone();
+                runtime.block_on(test_spec_priority_and_workload(
+                    *p, runtime2, wl, &tempdir, loop_cnt,
+                ));
+            }
+        }
+    }
+
+    async fn test_spec_priority_and_workload(
+        priority: Priority,
+        runtime: Runtime,
+        workload_id: &WorkloadType,
+        tempdir: &TempDir,
+        loop_cnt: usize,
+    ) {
+        tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await;
+        debug!(
+            "testing cpu usage for priority {:?} workload_id {:?}",
+            priority, workload_id,
+        );
+        // start monitor thread
+        let mut tasks = vec![];
+        let start = std::time::Instant::now();
+        for i in 0..loop_cnt {
+            // persist cpu usage in json: {priority}.{workload_id}
+            match *workload_id {
+                WorkloadType::ComputeHeavily => {
+                    tasks.push(runtime.spawn(workload_compute_heavily()));
+                }
+                WorkloadType::ComputeHeavily2 => {
+                    tasks.push(runtime.spawn(workload_compute_heavily2()));
+                }
+                WorkloadType::SpawnBlockingWriteFile => {
+                    tasks.push(runtime.spawn(workload_spawn_blocking_write_file(
+                        tempdir.path().to_path_buf(),
+                    )));
+                }
+                WorkloadType::WriteFile => {
+                    tasks.push(
+                        runtime.spawn(workload_write_file(i as u64, tempdir.path().to_path_buf())),
+                    );
+                }
+            }
+        }
+        for task in tasks {
+            task.await.unwrap();
+        }
+        let elapsed = start.elapsed();
+        debug!(
+            "test cpu usage for priority {:?} workload_id {:?} elapsed {}ms",
+            priority,
+            workload_id,
+            elapsed.as_millis()
+        );
+    }
+}
diff --git a/src/common/runtime/src/error.rs b/src/common/runtime/src/error.rs
index 4c553bbcd5..9d687edff3 100644
--- a/src/common/runtime/src/error.rs
+++ b/src/common/runtime/src/error.rs
@@ -33,6 +33,14 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Failed to build runtime rate limiter"))]
+    BuildRuntimeRateLimiter {
+        #[snafu(implicit)]
+        location: Location,
+        #[snafu(source)]
+        error: ratelimit::Error,
+    },
+
     #[snafu(display("Repeated task {} is already started", name))]
     IllegalState {
         name: String,
diff --git a/src/common/runtime/src/global.rs b/src/common/runtime/src/global.rs
index b7d78badeb..5cd008fa8c 100644
--- a/src/common/runtime/src/global.rs
+++ b/src/common/runtime/src/global.rs
@@ -21,6 +21,7 @@ use once_cell::sync::Lazy;
 use paste::paste;
 use serde::{Deserialize, Serialize};
 
+use crate::runtime::{BuilderBuild, RuntimeTrait};
 use crate::{Builder, JoinHandle, Runtime};
 
 const GLOBAL_WORKERS: usize = 8;
diff --git a/src/common/runtime/src/lib.rs b/src/common/runtime/src/lib.rs
index 4429f6fa71..d1effcfa4e 100644
--- a/src/common/runtime/src/lib.rs
+++ b/src/common/runtime/src/lib.rs
@@ -17,6 +17,8 @@ pub mod global;
 mod metrics;
 mod repeated_task;
 pub mod runtime;
+pub mod runtime_default;
+pub mod runtime_throttleable;
 
 pub use global::{
     block_on_compact, block_on_global, compact_runtime, create_runtime, global_runtime,
diff --git a/src/common/runtime/src/repeated_task.rs b/src/common/runtime/src/repeated_task.rs
index 2431a2ee17..6cc26e0545 100644
--- a/src/common/runtime/src/repeated_task.rs
+++ b/src/common/runtime/src/repeated_task.rs
@@ -23,6 +23,7 @@ use tokio::task::JoinHandle;
 use tokio_util::sync::CancellationToken;
 
 use crate::error::{IllegalStateSnafu, Result, WaitGcTaskStopSnafu};
+use crate::runtime::RuntimeTrait;
 use crate::Runtime;
 
 /// Task to execute repeatedly.
diff --git a/src/common/runtime/src/runtime.rs b/src/common/runtime/src/runtime.rs
index 0ea041578e..aeba46f24f 100644
--- a/src/common/runtime/src/runtime.rs
+++ b/src/common/runtime/src/runtime.rs
@@ -19,24 +19,20 @@ use std::thread;
 use std::time::Duration;
 
 use snafu::ResultExt;
-use tokio::runtime::{Builder as RuntimeBuilder, Handle};
+use tokio::runtime::Builder as RuntimeBuilder;
 use tokio::sync::oneshot;
 pub use tokio::task::{JoinError, JoinHandle};
 
 use crate::error::*;
 use crate::metrics::*;
+use crate::runtime_default::DefaultRuntime;
+use crate::runtime_throttleable::ThrottleableRuntime;
+
+// configurations
+pub type Runtime = DefaultRuntime;
 
 static RUNTIME_ID: AtomicUsize = AtomicUsize::new(0);
 
-/// A runtime to run future tasks
-#[derive(Clone, Debug)]
-pub struct Runtime {
-    name: String,
-    handle: Handle,
-    // Used to receive a drop signal when dropper is dropped, inspired by databend
-    _dropper: Arc<Dropper>,
-}
-
 /// Dropping the dropper will cause runtime to shutdown.
 #[derive(Debug)]
 pub struct Dropper {
@@ -50,45 +46,42 @@ impl Drop for Dropper {
     }
 }
 
-impl Runtime {
-    pub fn builder() -> Builder {
+pub trait RuntimeTrait {
+    /// Get a runtime builder
+    fn builder() -> Builder {
         Builder::default()
     }
 
     /// Spawn a future and execute it in this thread pool
     ///
     /// Similar to tokio::runtime::Runtime::spawn()
-    pub fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
+    fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
     where
         F: Future + Send + 'static,
-        F::Output: Send + 'static,
-    {
-        self.handle.spawn(future)
-    }
+        F::Output: Send + 'static;
 
     /// Run the provided function on an executor dedicated to blocking
     /// operations.
-    pub fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R>
+    fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R>
     where
         F: FnOnce() -> R + Send + 'static,
-        R: Send + 'static,
-    {
-        self.handle.spawn_blocking(func)
-    }
+        R: Send + 'static;
 
     /// Run a future to complete, this is the runtime's entry point
-    pub fn block_on<F: Future>(&self, future: F) -> F::Output {
-        self.handle.block_on(future)
-    }
+    fn block_on<F: Future>(&self, future: F) -> F::Output;
 
-    pub fn name(&self) -> &str {
-        &self.name
-    }
+    /// Get the name of the runtime
+    fn name(&self) -> &str;
+}
+
+pub trait BuilderBuild<R: RuntimeTrait> {
+    fn build(&mut self) -> Result<R>;
 }
 
 pub struct Builder {
     runtime_name: String,
     thread_name: String,
+    priority: Priority,
     builder: RuntimeBuilder,
 }
 
@@ -98,11 +91,17 @@ impl Default for Builder {
             runtime_name: format!("runtime-{}", RUNTIME_ID.fetch_add(1, Ordering::Relaxed)),
             thread_name: "default-worker".to_string(),
             builder: RuntimeBuilder::new_multi_thread(),
+            priority: Priority::VeryHigh,
         }
     }
 }
 
 impl Builder {
+    pub fn priority(&mut self, priority: Priority) -> &mut Self {
+        self.priority = priority;
+        self
+    }
+
     /// Sets the number of worker threads the Runtime will use.
     ///
     /// This can be any number above 0. The default value is the number of cores available to the system.
@@ -139,8 +138,10 @@ impl Builder {
         self.thread_name = val.into();
         self
     }
+}
 
-    pub fn build(&mut self) -> Result<Runtime> {
+impl BuilderBuild<DefaultRuntime> for Builder {
+    fn build(&mut self) -> Result<DefaultRuntime> {
         let runtime = self
             .builder
             .enable_all()
@@ -163,13 +164,48 @@ impl Builder {
         #[cfg(tokio_unstable)]
         register_collector(name.clone(), &handle);
 
-        Ok(Runtime {
-            name,
+        Ok(DefaultRuntime::new(
+            &name,
             handle,
-            _dropper: Arc::new(Dropper {
+            Arc::new(Dropper {
                 close: Some(send_stop),
             }),
-        })
+        ))
+    }
+}
+
+impl BuilderBuild<ThrottleableRuntime> for Builder {
+    fn build(&mut self) -> Result<ThrottleableRuntime> {
+        let runtime = self
+            .builder
+            .enable_all()
+            .thread_name(self.thread_name.clone())
+            .on_thread_start(on_thread_start(self.thread_name.clone()))
+            .on_thread_stop(on_thread_stop(self.thread_name.clone()))
+            .on_thread_park(on_thread_park(self.thread_name.clone()))
+            .on_thread_unpark(on_thread_unpark(self.thread_name.clone()))
+            .build()
+            .context(BuildRuntimeSnafu)?;
+
+        let name = self.runtime_name.clone();
+        let handle = runtime.handle().clone();
+        let (send_stop, recv_stop) = oneshot::channel();
+        // Block the runtime to shutdown.
+        let _ = thread::Builder::new()
+            .name(format!("{}-blocker", self.thread_name))
+            .spawn(move || runtime.block_on(recv_stop));
+
+        #[cfg(tokio_unstable)]
+        register_collector(name.clone(), &handle);
+
+        ThrottleableRuntime::new(
+            &name,
+            self.priority,
+            handle,
+            Arc::new(Dropper {
+                close: Some(send_stop),
+            }),
+        )
     }
 }
 
@@ -213,8 +249,18 @@ fn on_thread_unpark(thread_name: String) -> impl Fn() + 'static {
     }
 }
 
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
+pub enum Priority {
+    VeryLow = 0,
+    Low = 1,
+    Middle = 2,
+    High = 3,
+    VeryHigh = 4,
+}
+
 #[cfg(test)]
 mod tests {
+
     use std::sync::Arc;
     use std::thread;
     use std::time::Duration;
@@ -235,12 +281,12 @@ mod tests {
 
     #[test]
     fn test_metric() {
-        let runtime = Builder::default()
+        let runtime: Runtime = Builder::default()
             .worker_threads(5)
             .thread_name("test_runtime_metric")
             .build()
             .unwrap();
-        // wait threads created
+        // wait threads create
         thread::sleep(Duration::from_millis(50));
 
         let _handle = runtime.spawn(async {
diff --git a/src/common/runtime/src/runtime_default.rs b/src/common/runtime/src/runtime_default.rs
new file mode 100644
index 0000000000..ea0b5c27e3
--- /dev/null
+++ b/src/common/runtime/src/runtime_default.rs
@@ -0,0 +1,77 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::future::Future;
+use std::sync::Arc;
+
+use tokio::runtime::Handle;
+pub use tokio::task::JoinHandle;
+
+use crate::runtime::{Dropper, RuntimeTrait};
+use crate::Builder;
+
+/// A runtime to run future tasks
+#[derive(Clone, Debug)]
+pub struct DefaultRuntime {
+    name: String,
+    handle: Handle,
+    // Used to receive a drop signal when dropper is dropped, inspired by databend
+    _dropper: Arc<Dropper>,
+}
+
+impl DefaultRuntime {
+    pub(crate) fn new(name: &str, handle: Handle, dropper: Arc<Dropper>) -> Self {
+        Self {
+            name: name.to_string(),
+            handle,
+            _dropper: dropper,
+        }
+    }
+}
+
+impl RuntimeTrait for DefaultRuntime {
+    fn builder() -> Builder {
+        Builder::default()
+    }
+
+    /// Spawn a future and execute it in this thread pool
+    ///
+    /// Similar to tokio::runtime::Runtime::spawn()
+    fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
+    where
+        F: Future + Send + 'static,
+        F::Output: Send + 'static,
+    {
+        self.handle.spawn(future)
+    }
+
+    /// Run the provided function on an executor dedicated to blocking
+    /// operations.
+    fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R>
+    where
+        F: FnOnce() -> R + Send + 'static,
+        R: Send + 'static,
+    {
+        self.handle.spawn_blocking(func)
+    }
+
+    /// Run a future to complete, this is the runtime's entry point
+    fn block_on<F: Future>(&self, future: F) -> F::Output {
+        self.handle.block_on(future)
+    }
+
+    fn name(&self) -> &str {
+        &self.name
+    }
+}
diff --git a/src/common/runtime/src/runtime_throttleable.rs b/src/common/runtime/src/runtime_throttleable.rs
new file mode 100644
index 0000000000..ea51270987
--- /dev/null
+++ b/src/common/runtime/src/runtime_throttleable.rs
@@ -0,0 +1,285 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::Debug;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+use std::time::Duration;
+
+use futures::FutureExt;
+use ratelimit::Ratelimiter;
+use snafu::ResultExt;
+use tokio::runtime::Handle;
+pub use tokio::task::JoinHandle;
+use tokio::time::Sleep;
+
+use crate::error::{BuildRuntimeRateLimiterSnafu, Result};
+use crate::runtime::{Dropper, Priority, RuntimeTrait};
+use crate::Builder;
+
+struct RuntimeRateLimiter {
+    pub ratelimiter: Option<Ratelimiter>,
+}
+
+impl Debug for RuntimeRateLimiter {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RuntimeThrottleShareWithFuture")
+            .field(
+                "ratelimiter_max_tokens",
+                &self.ratelimiter.as_ref().map(|v| v.max_tokens()),
+            )
+            .field(
+                "ratelimiter_refill_amount",
+                &self.ratelimiter.as_ref().map(|v| v.refill_amount()),
+            )
+            .finish()
+    }
+}
+
+/// A runtime to run future tasks
+#[derive(Clone, Debug)]
+pub struct ThrottleableRuntime {
+    name: String,
+    handle: Handle,
+    shared_with_future: Arc<RuntimeRateLimiter>,
+    // Used to receive a drop signal when dropper is dropped, inspired by databend
+    _dropper: Arc<Dropper>,
+}
+
+impl ThrottleableRuntime {
+    pub(crate) fn new(
+        name: &str,
+        priority: Priority,
+        handle: Handle,
+        dropper: Arc<Dropper>,
+    ) -> Result<Self> {
+        Ok(Self {
+            name: name.to_string(),
+            handle,
+            shared_with_future: Arc::new(RuntimeRateLimiter {
+                ratelimiter: priority.ratelimiter_count()?,
+            }),
+            _dropper: dropper,
+        })
+    }
+}
+
+impl RuntimeTrait for ThrottleableRuntime {
+    fn builder() -> Builder {
+        Builder::default()
+    }
+
+    /// Spawn a future and execute it in this thread pool
+    ///
+    /// Similar to tokio::runtime::Runtime::spawn()
+    fn spawn<F>(&self, future: F) -> JoinHandle<F::Output>
+    where
+        F: Future + Send + 'static,
+        F::Output: Send + 'static,
+    {
+        self.handle
+            .spawn(ThrottleFuture::new(self.shared_with_future.clone(), future))
+    }
+
+    /// Run the provided function on an executor dedicated to blocking
+    /// operations.
+    fn spawn_blocking<F, R>(&self, func: F) -> JoinHandle<R>
+    where
+        F: FnOnce() -> R + Send + 'static,
+        R: Send + 'static,
+    {
+        self.handle.spawn_blocking(func)
+    }
+
+    /// Run a future to complete, this is the runtime's entry point
+    fn block_on<F: Future>(&self, future: F) -> F::Output {
+        self.handle.block_on(future)
+    }
+
+    fn name(&self) -> &str {
+        &self.name
+    }
+}
+
+enum State {
+    Pollable,
+    Throttled(Pin<Box<Sleep>>),
+}
+
+impl State {
+    fn unwrap_backoff(&mut self) -> &mut Pin<Box<Sleep>> {
+        match self {
+            State::Throttled(sleep) => sleep,
+            _ => panic!("unwrap_backoff failed"),
+        }
+    }
+}
+
+#[pin_project::pin_project]
+pub struct ThrottleFuture<F: Future + Send + 'static> {
+    #[pin]
+    future: F,
+
+    /// RateLimiter of this future
+    handle: Arc<RuntimeRateLimiter>,
+
+    state: State,
+}
+
+impl<F> ThrottleFuture<F>
+where
+    F: Future + Send + 'static,
+    F::Output: Send + 'static,
+{
+    fn new(handle: Arc<RuntimeRateLimiter>, future: F) -> Self {
+        Self {
+            future,
+            handle,
+            state: State::Pollable,
+        }
+    }
+}
+
+impl<F> Future for ThrottleFuture<F>
+where
+    F: Future + Send + 'static,
+    F::Output: Send + 'static,
+{
+    type Output = F::Output;
+
+    fn poll(self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        let this = self.project();
+
+        match this.state {
+            State::Pollable => {}
+            State::Throttled(ref mut sleep) => match sleep.poll_unpin(cx) {
+                Poll::Ready(_) => {
+                    *this.state = State::Pollable;
+                }
+                Poll::Pending => return Poll::Pending,
+            },
+        };
+
+        if let Some(ratelimiter) = &this.handle.ratelimiter {
+            if let Err(wait) = ratelimiter.try_wait() {
+                *this.state = State::Throttled(Box::pin(tokio::time::sleep(wait)));
+                match this.state.unwrap_backoff().poll_unpin(cx) {
+                    Poll::Ready(_) => {
+                        *this.state = State::Pollable;
+                    }
+                    Poll::Pending => {
+                        return Poll::Pending;
+                    }
+                }
+            }
+        }
+
+        let poll_res = this.future.poll(cx);
+
+        match poll_res {
+            Poll::Ready(r) => Poll::Ready(r),
+            Poll::Pending => Poll::Pending,
+        }
+    }
+}
+
+impl Priority {
+    fn ratelimiter_count(&self) -> Result<Option<Ratelimiter>> {
+        let max = 8000;
+        let gen_per_10ms = match self {
+            Priority::VeryLow => Some(2000),
+            Priority::Low => Some(4000),
+            Priority::Middle => Some(6000),
+            Priority::High => Some(8000),
+            Priority::VeryHigh => None,
+        };
+        if let Some(gen_per_10ms) = gen_per_10ms {
+            Ratelimiter::builder(gen_per_10ms, Duration::from_millis(10)) // generate poll count per 10ms
+                .max_tokens(max) // reserved token for batch request
+                .build()
+                .context(BuildRuntimeRateLimiterSnafu)
+                .map(Some)
+        } else {
+            Ok(None)
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use tokio::fs::File;
+    use tokio::io::AsyncWriteExt;
+    use tokio::time::Duration;
+
+    use super::*;
+    use crate::runtime::BuilderBuild;
+
+    #[tokio::test]
+    async fn test_throttleable_runtime_spawn_simple() {
+        for p in [
+            Priority::VeryLow,
+            Priority::Low,
+            Priority::Middle,
+            Priority::High,
+            Priority::VeryHigh,
+        ] {
+            let runtime: ThrottleableRuntime = Builder::default()
+                .runtime_name("test")
+                .thread_name("test")
+                .worker_threads(8)
+                .priority(p)
+                .build()
+                .expect("Fail to create runtime");
+
+            // Spawn a simple future that returns 42
+            let handle = runtime.spawn(async {
+                tokio::time::sleep(Duration::from_millis(10)).await;
+                42
+            });
+            let result = handle.await.expect("Task panicked");
+            assert_eq!(result, 42);
+        }
+    }
+
+    #[tokio::test]
+    async fn test_throttleable_runtime_spawn_complex() {
+        let tempdir = tempfile::tempdir().unwrap();
+        for p in [
+            Priority::VeryLow,
+            Priority::Low,
+            Priority::Middle,
+            Priority::High,
+            Priority::VeryHigh,
+        ] {
+            let runtime: ThrottleableRuntime = Builder::default()
+                .runtime_name("test")
+                .thread_name("test")
+                .worker_threads(8)
+                .priority(p)
+                .build()
+                .expect("Fail to create runtime");
+            let tempdirpath = tempdir.path().to_path_buf();
+            let handle = runtime.spawn(async move {
+                let mut file = File::create(tempdirpath.join("test.txt")).await.unwrap();
+                file.write_all(b"Hello, world!").await.unwrap();
+                42
+            });
+            let result = handle.await.expect("Task panicked");
+            assert_eq!(result, 42);
+        }
+    }
+}
diff --git a/src/common/telemetry/Cargo.toml b/src/common/telemetry/Cargo.toml
index 2b4023cf7c..da044436c8 100644
--- a/src/common/telemetry/Cargo.toml
+++ b/src/common/telemetry/Cargo.toml
@@ -26,7 +26,7 @@ opentelemetry = { version = "0.21.0", default-features = false, features = [
 opentelemetry-otlp = { version = "0.14.0", features = ["tokio"] }
 opentelemetry-semantic-conventions = "0.13.0"
 opentelemetry_sdk = { version = "0.21.0", features = ["rt-tokio"] }
-parking_lot = { version = "0.12" }
+parking_lot.workspace = true
 prometheus.workspace = true
 serde.workspace = true
 serde_json.workspace = true
diff --git a/src/datanode/src/tests.rs b/src/datanode/src/tests.rs
index 2acc66a592..8b48dd4258 100644
--- a/src/datanode/src/tests.rs
+++ b/src/datanode/src/tests.rs
@@ -23,6 +23,7 @@ use common_function::function::FunctionRef;
 use common_function::scalars::aggregate::AggregateFunctionMetaRef;
 use common_query::prelude::ScalarUdf;
 use common_query::Output;
+use common_runtime::runtime::{BuilderBuild, RuntimeTrait};
 use common_runtime::Runtime;
 use datafusion_expr::LogicalPlan;
 use query::dataframe::DataFrame;
diff --git a/src/meta-srv/Cargo.toml b/src/meta-srv/Cargo.toml
index 2e3216d075..6d051c2eee 100644
--- a/src/meta-srv/Cargo.toml
+++ b/src/meta-srv/Cargo.toml
@@ -42,7 +42,7 @@ humantime-serde.workspace = true
 itertools.workspace = true
 lazy_static.workspace = true
 once_cell.workspace = true
-parking_lot = "0.12"
+parking_lot.workspace = true
 prometheus.workspace = true
 prost.workspace = true
 rand.workspace = true
diff --git a/src/meta-srv/src/procedure/utils.rs b/src/meta-srv/src/procedure/utils.rs
index c4e1688de0..51db562a32 100644
--- a/src/meta-srv/src/procedure/utils.rs
+++ b/src/meta-srv/src/procedure/utils.rs
@@ -24,6 +24,7 @@ pub mod mock {
     use client::Client;
     use common_grpc::channel_manager::ChannelManager;
     use common_meta::peer::Peer;
+    use common_runtime::runtime::BuilderBuild;
     use common_runtime::{Builder as RuntimeBuilder, Runtime};
     use servers::grpc::region_server::{RegionServerHandler, RegionServerRequestHandler};
     use tokio::sync::mpsc;
diff --git a/src/puffin/src/puffin_manager/stager/bounded_stager.rs b/src/puffin/src/puffin_manager/stager/bounded_stager.rs
index c41df95c25..9ab2be00ac 100644
--- a/src/puffin/src/puffin_manager/stager/bounded_stager.rs
+++ b/src/puffin/src/puffin_manager/stager/bounded_stager.rs
@@ -21,6 +21,7 @@ use async_trait::async_trait;
 use async_walkdir::{Filtering, WalkDir};
 use base64::prelude::BASE64_URL_SAFE;
 use base64::Engine;
+use common_runtime::runtime::RuntimeTrait;
 use common_telemetry::{info, warn};
 use futures::{FutureExt, StreamExt};
 use moka::future::Cache;
diff --git a/src/script/src/python/utils.rs b/src/script/src/python/utils.rs
index 4662922f14..8838fe8a61 100644
--- a/src/script/src/python/utils.rs
+++ b/src/script/src/python/utils.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use common_runtime::runtime::RuntimeTrait;
 use common_runtime::JoinHandle;
 use futures::Future;
 use rustpython_vm::builtins::PyBaseExceptionRef;
diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml
index 354bdf642f..a2803ae035 100644
--- a/src/servers/Cargo.toml
+++ b/src/servers/Cargo.toml
@@ -72,7 +72,7 @@ openmetrics-parser = "0.4"
 # opensrv-mysql = "0.7.0"
 opensrv-mysql = { git = "https://github.com/datafuselabs/opensrv", rev = "6bbc3b65e6b19212c4f7fc4f40c20daf6f452deb" }
 opentelemetry-proto.workspace = true
-parking_lot = "0.12"
+parking_lot.workspace = true
 pgwire = { version = "0.25.0", default-features = false, features = ["server-api-ring"] }
 pin-project = "1.0"
 pipeline.workspace = true
diff --git a/src/servers/src/grpc/greptime_handler.rs b/src/servers/src/grpc/greptime_handler.rs
index f6bafde16d..b032ffc847 100644
--- a/src/servers/src/grpc/greptime_handler.rs
+++ b/src/servers/src/grpc/greptime_handler.rs
@@ -25,6 +25,7 @@ use common_catalog::parse_catalog_and_schema_from_db_string;
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use common_query::Output;
+use common_runtime::runtime::RuntimeTrait;
 use common_runtime::Runtime;
 use common_telemetry::tracing_context::{FutureExt, TracingContext};
 use common_telemetry::{debug, error, tracing};
diff --git a/src/servers/src/grpc/region_server.rs b/src/servers/src/grpc/region_server.rs
index 366d90151f..e3eb87467a 100644
--- a/src/servers/src/grpc/region_server.rs
+++ b/src/servers/src/grpc/region_server.rs
@@ -18,6 +18,7 @@ use api::v1::region::region_server::Region as RegionServer;
 use api::v1::region::{region_request, RegionRequest, RegionResponse};
 use async_trait::async_trait;
 use common_error::ext::ErrorExt;
+use common_runtime::runtime::RuntimeTrait;
 use common_runtime::Runtime;
 use common_telemetry::tracing::info_span;
 use common_telemetry::tracing_context::{FutureExt, TracingContext};
diff --git a/src/servers/src/mysql/server.rs b/src/servers/src/mysql/server.rs
index 146295bbaf..dae01b3f1a 100644
--- a/src/servers/src/mysql/server.rs
+++ b/src/servers/src/mysql/server.rs
@@ -18,6 +18,7 @@ use std::sync::Arc;
 
 use async_trait::async_trait;
 use auth::UserProviderRef;
+use common_runtime::runtime::RuntimeTrait;
 use common_runtime::Runtime;
 use common_telemetry::{debug, warn};
 use futures::StreamExt;
diff --git a/src/servers/src/postgres/server.rs b/src/servers/src/postgres/server.rs
index e904845547..70f74a32ec 100644
--- a/src/servers/src/postgres/server.rs
+++ b/src/servers/src/postgres/server.rs
@@ -18,6 +18,7 @@ use std::sync::Arc;
 
 use ::auth::UserProviderRef;
 use async_trait::async_trait;
+use common_runtime::runtime::RuntimeTrait;
 use common_runtime::Runtime;
 use common_telemetry::{debug, warn};
 use futures::StreamExt;
diff --git a/src/servers/tests/grpc/mod.rs b/src/servers/tests/grpc/mod.rs
index 021144745f..30bd168bc1 100644
--- a/src/servers/tests/grpc/mod.rs
+++ b/src/servers/tests/grpc/mod.rs
@@ -22,6 +22,7 @@ use async_trait::async_trait;
 use auth::tests::MockUserProvider;
 use auth::UserProviderRef;
 use client::{Client, Database, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use common_runtime::runtime::BuilderBuild;
 use common_runtime::{Builder as RuntimeBuilder, Runtime};
 use servers::error::{Result, StartGrpcSnafu, TcpBindSnafu};
 use servers::grpc::flight::FlightCraftWrapper;
diff --git a/src/servers/tests/mysql/mysql_server_test.rs b/src/servers/tests/mysql/mysql_server_test.rs
index ba2cdbdab2..a9f7f8309a 100644
--- a/src/servers/tests/mysql/mysql_server_test.rs
+++ b/src/servers/tests/mysql/mysql_server_test.rs
@@ -19,6 +19,7 @@ use std::time::Duration;
 use auth::tests::{DatabaseAuthInfo, MockUserProvider};
 use common_catalog::consts::DEFAULT_SCHEMA_NAME;
 use common_recordbatch::RecordBatch;
+use common_runtime::runtime::BuilderBuild;
 use common_runtime::Builder as RuntimeBuilder;
 use datatypes::prelude::VectorRef;
 use datatypes::schema::{ColumnSchema, Schema};
diff --git a/src/servers/tests/postgres/mod.rs b/src/servers/tests/postgres/mod.rs
index f3ff827db4..6ff659fec9 100644
--- a/src/servers/tests/postgres/mod.rs
+++ b/src/servers/tests/postgres/mod.rs
@@ -19,6 +19,7 @@ use std::time::Duration;
 use auth::tests::{DatabaseAuthInfo, MockUserProvider};
 use auth::UserProviderRef;
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use common_runtime::runtime::BuilderBuild;
 use common_runtime::Builder as RuntimeBuilder;
 use pgwire::api::Type;
 use rand::rngs::StdRng;
diff --git a/tests-integration/src/cluster.rs b/tests-integration/src/cluster.rs
index 769dad4c1e..55daa9681c 100644
--- a/tests-integration/src/cluster.rs
+++ b/tests-integration/src/cluster.rs
@@ -35,6 +35,7 @@ use common_meta::kv_backend::memory::MemoryKvBackend;
 use common_meta::kv_backend::KvBackendRef;
 use common_meta::peer::Peer;
 use common_meta::DatanodeId;
+use common_runtime::runtime::BuilderBuild;
 use common_runtime::Builder as RuntimeBuilder;
 use common_test_util::temp_dir::create_temp_dir;
 use common_wal::config::{DatanodeWalConfig, MetasrvWalConfig};
diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs
index a055527e2b..b3a7269ae0 100644
--- a/tests-integration/src/test_util.rs
+++ b/tests-integration/src/test_util.rs
@@ -25,7 +25,8 @@ use common_base::secrets::ExposeSecret;
 use common_config::Configurable;
 use common_meta::key::catalog_name::CatalogNameKey;
 use common_meta::key::schema_name::SchemaNameKey;
-use common_runtime::Builder as RuntimeBuilder;
+use common_runtime::runtime::BuilderBuild;
+use common_runtime::{Builder as RuntimeBuilder, Runtime};
 use common_telemetry::warn;
 use common_test_util::ports;
 use common_test_util::temp_dir::{create_temp_dir, TempDir};
@@ -494,7 +495,7 @@ pub async fn setup_grpc_server_with(
 ) -> (String, TestGuard, Arc<GrpcServer>) {
     let instance = setup_standalone_instance(name, store_type).await;
 
-    let runtime = RuntimeBuilder::default()
+    let runtime: Runtime = RuntimeBuilder::default()
         .worker_threads(2)
         .thread_name("grpc-handlers")
         .build()
diff --git a/tests-integration/tests/grpc.rs b/tests-integration/tests/grpc.rs
index 55ccdb258d..26c0385c33 100644
--- a/tests-integration/tests/grpc.rs
+++ b/tests-integration/tests/grpc.rs
@@ -25,6 +25,7 @@ use common_catalog::consts::MITO_ENGINE;
 use common_grpc::channel_manager::ClientTlsOption;
 use common_query::Output;
 use common_recordbatch::RecordBatches;
+use common_runtime::runtime::{BuilderBuild, RuntimeTrait};
 use common_runtime::Runtime;
 use common_test_util::find_workspace_path;
 use servers::grpc::builder::GrpcServerBuilder;

From aa9a26598482928b12d3022a531362b6f59667ed Mon Sep 17 00:00:00 2001
From: jeremyhi <jiachun_feng@proton.me>
Date: Thu, 24 Oct 2024 15:44:16 +0800
Subject: [PATCH 102/128] chore: make pusher log easy to understand (#4841)

* chore: make pusher log easy to understand

* Update src/meta-srv/src/service/heartbeat.rs

Co-authored-by: Yingwen <realevenyag@gmail.com>

* Update src/meta-srv/src/service/heartbeat.rs

Co-authored-by: Yingwen <realevenyag@gmail.com>

* chore: by comment

---------

Co-authored-by: Yingwen <realevenyag@gmail.com>
---
 src/meta-srv/src/handler.rs                   | 57 ++++++++++++++-----
 .../procedure/region_migration/test_util.rs   |  2 +-
 src/meta-srv/src/service/heartbeat.rs         | 33 +++++------
 src/meta-srv/src/service/mailbox.rs           |  9 +--
 4 files changed, 63 insertions(+), 38 deletions(-)

diff --git a/src/meta-srv/src/handler.rs b/src/meta-srv/src/handler.rs
index 15eff85a88..5d382f3506 100644
--- a/src/meta-srv/src/handler.rs
+++ b/src/meta-srv/src/handler.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::collections::{BTreeMap, HashSet};
+use std::fmt::{Debug, Display};
 use std::ops::Range;
 use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant};
@@ -113,6 +114,34 @@ impl HeartbeatAccumulator {
     }
 }
 
+#[derive(Copy, Clone)]
+pub struct PusherId {
+    pub role: Role,
+    pub id: u64,
+}
+
+impl Debug for PusherId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}-{}", self.role, self.id)
+    }
+}
+
+impl Display for PusherId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}-{}", self.role, self.id)
+    }
+}
+
+impl PusherId {
+    pub fn new(role: Role, id: u64) -> Self {
+        Self { role, id }
+    }
+
+    pub fn string_key(&self) -> String {
+        format!("{}-{}", self.role as i32, self.id)
+    }
+}
+
 /// The pusher of the heartbeat response.
 pub struct Pusher {
     sender: Sender<std::result::Result<HeartbeatResponse, tonic::Status>>,
@@ -154,10 +183,11 @@ impl Pusher {
 pub struct Pushers(Arc<RwLock<BTreeMap<String, Pusher>>>);
 
 impl Pushers {
-    async fn push(&self, pusher_id: &str, mailbox_message: MailboxMessage) -> Result<()> {
+    async fn push(&self, pusher_id: PusherId, mailbox_message: MailboxMessage) -> Result<()> {
+        let pusher_id = pusher_id.string_key();
         let pushers = self.0.read().await;
         let pusher = pushers
-            .get(pusher_id)
+            .get(&pusher_id)
             .context(error::PusherNotFoundSnafu { pusher_id })?;
         pusher
             .push(HeartbeatResponse {
@@ -234,21 +264,19 @@ pub struct HeartbeatHandlerGroup {
 
 impl HeartbeatHandlerGroup {
     /// Registers the heartbeat response [`Pusher`] with the given key to the group.
-    pub async fn register_pusher(&self, key: impl AsRef<str>, pusher: Pusher) {
-        let key = key.as_ref();
+    pub async fn register_pusher(&self, pusher_id: PusherId, pusher: Pusher) {
         METRIC_META_HEARTBEAT_CONNECTION_NUM.inc();
-        info!("Pusher register: {}", key);
-        let _ = self.pushers.insert(key.to_string(), pusher).await;
+        info!("Pusher register: {}", pusher_id);
+        let _ = self.pushers.insert(pusher_id.string_key(), pusher).await;
     }
 
     /// Deregisters the heartbeat response [`Pusher`] with the given key from the group.
     ///
     /// Returns the [`Pusher`] if it exists.
-    pub async fn deregister_push(&self, key: impl AsRef<str>) -> Option<Pusher> {
-        let key = key.as_ref();
+    pub async fn deregister_push(&self, pusher_id: PusherId) -> Option<Pusher> {
         METRIC_META_HEARTBEAT_CONNECTION_NUM.dec();
-        info!("Pusher unregister: {}", key);
-        self.pushers.remove(key).await
+        info!("Pusher unregister: {}", pusher_id);
+        self.pushers.remove(&pusher_id.string_key()).await
     }
 
     /// Returns the [`Pushers`] of the group.
@@ -417,7 +445,7 @@ impl Mailbox for HeartbeatMailbox {
         let _ = self.timeouts.insert(message_id, deadline);
         self.timeout_notify.notify_one();
 
-        self.pushers.push(&pusher_id, msg).await?;
+        self.pushers.push(pusher_id, msg).await?;
 
         Ok(MailboxReceiver::new(message_id, rx, *ch))
     }
@@ -720,7 +748,7 @@ mod tests {
     use common_meta::sequence::SequenceBuilder;
     use tokio::sync::mpsc;
 
-    use super::{HeartbeatHandlerGroupBuilder, Pushers};
+    use super::{HeartbeatHandlerGroupBuilder, PusherId, Pushers};
     use crate::error;
     use crate::handler::collect_stats_handler::CollectStatsHandler;
     use crate::handler::response_header_handler::ResponseHeaderHandler;
@@ -761,11 +789,10 @@ mod tests {
             protocol_version: PROTOCOL_VERSION,
             ..Default::default()
         };
+        let pusher_id = PusherId::new(Role::Datanode, datanode_id);
         let pusher: Pusher = Pusher::new(pusher_tx, &res_header);
         let handler_group = HeartbeatHandlerGroup::default();
-        handler_group
-            .register_pusher(format!("{}-{}", Role::Datanode as i32, datanode_id), pusher)
-            .await;
+        handler_group.register_pusher(pusher_id, pusher).await;
 
         let kv_backend = Arc::new(MemoryKvBackend::new());
         let seq = SequenceBuilder::new("test_seq", kv_backend).build();
diff --git a/src/meta-srv/src/procedure/region_migration/test_util.rs b/src/meta-srv/src/procedure/region_migration/test_util.rs
index cb3b5a3dc3..2058782396 100644
--- a/src/meta-srv/src/procedure/region_migration/test_util.rs
+++ b/src/meta-srv/src/procedure/region_migration/test_util.rs
@@ -84,7 +84,7 @@ impl MailboxContext {
     ) {
         let pusher_id = channel.pusher_id();
         let pusher = Pusher::new(tx, &RequestHeader::default());
-        let _ = self.pushers.insert(pusher_id, pusher).await;
+        let _ = self.pushers.insert(pusher_id.string_key(), pusher).await;
     }
 
     pub fn mailbox(&self) -> &MailboxRef {
diff --git a/src/meta-srv/src/service/heartbeat.rs b/src/meta-srv/src/service/heartbeat.rs
index 9569d0e431..489afe41e8 100644
--- a/src/meta-srv/src/service/heartbeat.rs
+++ b/src/meta-srv/src/service/heartbeat.rs
@@ -31,7 +31,7 @@ use tonic::{Request, Response, Streaming};
 
 use crate::error;
 use crate::error::Result;
-use crate::handler::{HeartbeatHandlerGroup, Pusher};
+use crate::handler::{HeartbeatHandlerGroup, Pusher, PusherId};
 use crate::metasrv::{Context, Metasrv};
 use crate::metrics::METRIC_META_HEARTBEAT_RECV;
 use crate::service::{GrpcResult, GrpcStream};
@@ -52,7 +52,7 @@ impl heartbeat_server::Heartbeat for Metasrv {
 
         let ctx = self.new_ctx();
         let _handle = common_runtime::spawn_global(async move {
-            let mut pusher_key = None;
+            let mut pusher_id = None;
             while let Some(msg) = in_stream.next().await {
                 let mut is_not_leader = false;
                 match msg {
@@ -67,11 +67,11 @@ impl heartbeat_server::Heartbeat for Metasrv {
                             break;
                         };
 
-                        if pusher_key.is_none() {
-                            pusher_key = register_pusher(&handler_group, header, tx.clone()).await;
+                        if pusher_id.is_none() {
+                            pusher_id = register_pusher(&handler_group, header, tx.clone()).await;
                         }
-                        if let Some(k) = &pusher_key {
-                            METRIC_META_HEARTBEAT_RECV.with_label_values(&[k]);
+                        if let Some(k) = &pusher_id {
+                            METRIC_META_HEARTBEAT_RECV.with_label_values(&[&k.to_string()]);
                         } else {
                             METRIC_META_HEARTBEAT_RECV.with_label_values(&["none"]);
                         }
@@ -111,13 +111,10 @@ impl heartbeat_server::Heartbeat for Metasrv {
                 }
             }
 
-            info!(
-                "Heartbeat stream closed: {:?}",
-                pusher_key.as_ref().unwrap_or(&"unknown".to_string())
-            );
+            info!("Heartbeat stream closed: {pusher_id:?}");
 
-            if let Some(key) = pusher_key {
-                let _ = handler_group.deregister_push(&key).await;
+            if let Some(pusher_id) = pusher_id {
+                let _ = handler_group.deregister_push(pusher_id).await;
             }
         });
 
@@ -176,13 +173,13 @@ async fn register_pusher(
     handler_group: &HeartbeatHandlerGroup,
     header: &RequestHeader,
     sender: Sender<std::result::Result<HeartbeatResponse, tonic::Status>>,
-) -> Option<String> {
-    let role = header.role() as i32;
-    let node_id = get_node_id(header);
-    let key = format!("{}-{}", role, node_id);
+) -> Option<PusherId> {
+    let role = header.role();
+    let id = get_node_id(header);
+    let pusher_id = PusherId::new(role, id);
     let pusher = Pusher::new(sender, header);
-    handler_group.register_pusher(&key, pusher).await;
-    Some(key)
+    handler_group.register_pusher(pusher_id, pusher).await;
+    Some(pusher_id)
 }
 
 #[cfg(test)]
diff --git a/src/meta-srv/src/service/mailbox.rs b/src/meta-srv/src/service/mailbox.rs
index 6d34401d8d..3350de2f63 100644
--- a/src/meta-srv/src/service/mailbox.rs
+++ b/src/meta-srv/src/service/mailbox.rs
@@ -24,6 +24,7 @@ use futures::Future;
 use tokio::sync::oneshot;
 
 use crate::error::{self, Result};
+use crate::handler::PusherId;
 
 pub type MailboxRef = Arc<dyn Mailbox>;
 
@@ -53,11 +54,11 @@ impl Display for Channel {
 }
 
 impl Channel {
-    pub(crate) fn pusher_id(&self) -> String {
+    pub(crate) fn pusher_id(&self) -> PusherId {
         match self {
-            Channel::Datanode(id) => format!("{}-{}", Role::Datanode as i32, id),
-            Channel::Frontend(id) => format!("{}-{}", Role::Frontend as i32, id),
-            Channel::Flownode(id) => format!("{}-{}", Role::Flownode as i32, id),
+            Channel::Datanode(id) => PusherId::new(Role::Datanode, *id),
+            Channel::Frontend(id) => PusherId::new(Role::Frontend, *id),
+            Channel::Flownode(id) => PusherId::new(Role::Flownode, *id),
         }
     }
 }

From ff38abde2e607afcf02468f9cb79659f55669240 Mon Sep 17 00:00:00 2001
From: discord9 <55937128+discord9@users.noreply.github.com>
Date: Thu, 24 Oct 2024 17:43:32 +0800
Subject: [PATCH 103/128] chore: better column schema check for flow (#4855)

* chore: better column schema check for flow

* chore: better msg

* tests: clean up after tests

* chore: better msg

* chore: per review

* tests: sqlness
---
 src/flow/src/adapter.rs                       | 26 +++++-
 .../standalone/common/flow/flow_basic.result  | 85 ++++++++++++++++++-
 .../standalone/common/flow/flow_basic.sql     | 52 +++++++++++-
 3 files changed, 154 insertions(+), 9 deletions(-)

diff --git a/src/flow/src/adapter.rs b/src/flow/src/adapter.rs
index 3a8acd60e7..d564e1dd09 100644
--- a/src/flow/src/adapter.rs
+++ b/src/flow/src/adapter.rs
@@ -271,10 +271,17 @@ impl FlowWorkerManager {
                         let rows_proto: Vec<v1::Row> = insert
                             .into_iter()
                             .map(|(mut row, _ts)| {
-                                // `update_at` col
-                                row.extend([Value::from(common_time::Timestamp::new_millisecond(
-                                    now,
-                                ))]);
+                                // extend `update_at` col if needed
+                                // if schema include a millisecond timestamp here, and result row doesn't have it, add it
+                                if row.len() < proto_schema.len()
+                                    && proto_schema[row.len()].datatype
+                                        == greptime_proto::v1::ColumnDataType::TimestampMillisecond
+                                            as i32
+                                {
+                                    row.extend([Value::from(
+                                        common_time::Timestamp::new_millisecond(now),
+                                    )]);
+                                }
                                 // ts col, if auto create
                                 if is_ts_placeholder {
                                     ensure!(
@@ -291,6 +298,17 @@ impl FlowWorkerManager {
                                         common_time::Timestamp::new_millisecond(0),
                                     )]);
                                 }
+                                if row.len() != proto_schema.len() {
+                                    InternalSnafu {
+                                        reason: format!(
+                                            "Flow output row length mismatch, expect {} got {}, the columns in schema are: {:?}",
+                                            proto_schema.len(),
+                                            row.len(),
+                                            proto_schema.iter().map(|c|&c.column_name).collect_vec()
+                                        ),
+                                    }
+                                    .fail()?;
+                                }
                                 Ok(row.into())
                             })
                             .collect::<Result<Vec<_>, Error>>()?;
diff --git a/tests/cases/standalone/common/flow/flow_basic.result b/tests/cases/standalone/common/flow/flow_basic.result
index 6aaa2c74be..33d4ddacba 100644
--- a/tests/cases/standalone/common/flow/flow_basic.result
+++ b/tests/cases/standalone/common/flow/flow_basic.result
@@ -140,10 +140,11 @@ admin flush_flow('test_distinct_basic');
 INSERT INTO
     distinct_basic
 VALUES
+    (20, "2021-07-01 00:00:00.200"),
     (20, "2021-07-01 00:00:00.200"),
     (22, "2021-07-01 00:00:00.600");
 
-Affected Rows: 2
+Affected Rows: 3
 
 admin flush_flow('test_distinct_basic');
 
@@ -739,7 +740,8 @@ CREATE TABLE ngx_distribution (
     bucket_size INT,
     total_logs BIGINT,
     time_window TIMESTAMP TIME INDEX,
-    update_at TIMESTAMP, -- auto generated column by flow engine
+    -- auto generated column by flow engine
+    update_at TIMESTAMP,
     PRIMARY KEY(stat, bucket_size)
 );
 
@@ -748,7 +750,7 @@ Affected Rows: 0
 CREATE FLOW calc_ngx_distribution SINK TO ngx_distribution AS
 SELECT
     stat,
-    trunc(size, -1)::INT as bucket_size,
+    trunc(size, -1) :: INT as bucket_size,
     count(client) AS total_logs,
     date_bin(INTERVAL '1 minutes', access_time) as time_window,
 FROM
@@ -837,3 +839,80 @@ DROP TABLE ngx_distribution;
 
 Affected Rows: 0
 
+CREATE TABLE requests (
+    service_name STRING,
+    service_ip STRING,
+    val INT,
+    ts TIMESTAMP TIME INDEX
+);
+
+Affected Rows: 0
+
+CREATE TABLE requests_without_ip (
+    service_name STRING,
+    val INT,
+    ts TIMESTAMP TIME INDEX,
+);
+
+Affected Rows: 0
+
+CREATE FLOW requests_long_term SINK TO requests_without_ip AS
+SELECT
+    service_name,
+    val,
+    ts
+FROM
+    requests;
+
+Affected Rows: 0
+
+INSERT INTO
+    requests
+VALUES
+    ("svc1", "10.0.0.1", 100, "2024-10-18 19:00:00"),
+    ("svc1", "10.0.0.2", 100, "2024-10-18 19:00:00"),
+    ("svc1", "10.0.0.1", 200, "2024-10-18 19:00:30"),
+    ("svc1", "10.0.0.2", 200, "2024-10-18 19:00:30"),
+    ("svc1", "10.0.0.1", 300, "2024-10-18 19:01:00"),
+    ("svc1", "10.0.0.2", 100, "2024-10-18 19:01:01"),
+    ("svc1", "10.0.0.1", 400, "2024-10-18 19:01:30"),
+    ("svc1", "10.0.0.2", 200, "2024-10-18 19:01:31");
+
+Affected Rows: 8
+
+admin flush_flow('requests_long_term');
+
++----------------------------------------+
+| ADMIN flush_flow('requests_long_term') |
++----------------------------------------+
+| 1                                      |
++----------------------------------------+
+
+SELECT
+    *
+FROM
+    requests_without_ip;
+
++--------------+-----+---------------------+
+| service_name | val | ts                  |
++--------------+-----+---------------------+
+| svc1         | 100 | 2024-10-18T19:00:00 |
+| svc1         | 200 | 2024-10-18T19:00:30 |
+| svc1         | 300 | 2024-10-18T19:01:00 |
+| svc1         | 100 | 2024-10-18T19:01:01 |
+| svc1         | 400 | 2024-10-18T19:01:30 |
+| svc1         | 200 | 2024-10-18T19:01:31 |
++--------------+-----+---------------------+
+
+DROP FLOW requests_long_term;
+
+Affected Rows: 0
+
+DROP TABLE requests_without_ip;
+
+Affected Rows: 0
+
+DROP TABLE requests;
+
+Affected Rows: 0
+
diff --git a/tests/cases/standalone/common/flow/flow_basic.sql b/tests/cases/standalone/common/flow/flow_basic.sql
index 9356508aba..179a19195d 100644
--- a/tests/cases/standalone/common/flow/flow_basic.sql
+++ b/tests/cases/standalone/common/flow/flow_basic.sql
@@ -79,6 +79,7 @@ admin flush_flow('test_distinct_basic');
 INSERT INTO
     distinct_basic
 VALUES
+    (20, "2021-07-01 00:00:00.200"),
     (20, "2021-07-01 00:00:00.200"),
     (22, "2021-07-01 00:00:00.600");
 
@@ -405,14 +406,15 @@ CREATE TABLE ngx_distribution (
     bucket_size INT,
     total_logs BIGINT,
     time_window TIMESTAMP TIME INDEX,
-    update_at TIMESTAMP, -- auto generated column by flow engine
+    -- auto generated column by flow engine
+    update_at TIMESTAMP,
     PRIMARY KEY(stat, bucket_size)
 );
 
 CREATE FLOW calc_ngx_distribution SINK TO ngx_distribution AS
 SELECT
     stat,
-    trunc(size, -1)::INT as bucket_size,
+    trunc(size, -1) :: INT as bucket_size,
     count(client) AS total_logs,
     date_bin(INTERVAL '1 minutes', access_time) as time_window,
 FROM
@@ -461,3 +463,49 @@ DROP FLOW calc_ngx_distribution;
 DROP TABLE ngx_access_log;
 
 DROP TABLE ngx_distribution;
+
+CREATE TABLE requests (
+    service_name STRING,
+    service_ip STRING,
+    val INT,
+    ts TIMESTAMP TIME INDEX
+);
+
+CREATE TABLE requests_without_ip (
+    service_name STRING,
+    val INT,
+    ts TIMESTAMP TIME INDEX,
+);
+
+CREATE FLOW requests_long_term SINK TO requests_without_ip AS
+SELECT
+    service_name,
+    val,
+    ts
+FROM
+    requests;
+
+INSERT INTO
+    requests
+VALUES
+    ("svc1", "10.0.0.1", 100, "2024-10-18 19:00:00"),
+    ("svc1", "10.0.0.2", 100, "2024-10-18 19:00:00"),
+    ("svc1", "10.0.0.1", 200, "2024-10-18 19:00:30"),
+    ("svc1", "10.0.0.2", 200, "2024-10-18 19:00:30"),
+    ("svc1", "10.0.0.1", 300, "2024-10-18 19:01:00"),
+    ("svc1", "10.0.0.2", 100, "2024-10-18 19:01:01"),
+    ("svc1", "10.0.0.1", 400, "2024-10-18 19:01:30"),
+    ("svc1", "10.0.0.2", 200, "2024-10-18 19:01:31");
+
+admin flush_flow('requests_long_term');
+
+SELECT
+    *
+FROM
+    requests_without_ip;
+
+DROP FLOW requests_long_term;
+
+DROP TABLE requests_without_ip;
+
+DROP TABLE requests;
\ No newline at end of file

From 8b307e4548a1ffd0f2daaa0ac5a8022638686d53 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Thu, 24 Oct 2024 20:02:10 +0800
Subject: [PATCH 104/128] feat: introduce the `PluginOptions` (#4835)

* feat: introduce the `PluginOptions`

* chore: apply suggestions from CR
---
 Cargo.lock                  |  1 +
 src/cmd/src/datanode.rs     |  3 ++-
 src/cmd/src/frontend.rs     |  5 +++--
 src/cmd/src/metasrv.rs      |  3 ++-
 src/cmd/src/options.rs      |  3 +++
 src/cmd/src/standalone.rs   |  7 ++++---
 src/plugins/Cargo.toml      |  1 +
 src/plugins/src/datanode.rs |  3 +++
 src/plugins/src/frontend.rs |  3 +++
 src/plugins/src/lib.rs      |  2 ++
 src/plugins/src/meta_srv.rs |  3 +++
 src/plugins/src/options.rs  | 23 +++++++++++++++++++++++
 12 files changed, 50 insertions(+), 7 deletions(-)
 create mode 100644 src/plugins/src/options.rs

diff --git a/Cargo.lock b/Cargo.lock
index f9e0c1849c..e13a7c6929 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8349,6 +8349,7 @@ dependencies = [
  "datanode",
  "frontend",
  "meta-srv",
+ "serde",
  "snafu 0.8.5",
 ]
 
diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs
index 466514d038..187bdbd7aa 100644
--- a/src/cmd/src/datanode.rs
+++ b/src/cmd/src/datanode.rs
@@ -272,9 +272,10 @@ impl StartCommand {
         info!("Datanode start command: {:#?}", self);
         info!("Datanode options: {:#?}", opts);
 
+        let plugin_opts = opts.plugins;
         let opts = opts.component;
         let mut plugins = Plugins::new();
-        plugins::setup_datanode_plugins(&mut plugins, &opts)
+        plugins::setup_datanode_plugins(&mut plugins, &plugin_opts, &opts)
             .await
             .context(StartDatanodeSnafu)?;
 
diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs
index 38fa75a3a0..728d584b3c 100644
--- a/src/cmd/src/frontend.rs
+++ b/src/cmd/src/frontend.rs
@@ -266,9 +266,10 @@ impl StartCommand {
         info!("Frontend start command: {:#?}", self);
         info!("Frontend options: {:#?}", opts);
 
+        let plugin_opts = opts.plugins;
         let opts = opts.component;
         let mut plugins = Plugins::new();
-        plugins::setup_frontend_plugins(&mut plugins, &opts)
+        plugins::setup_frontend_plugins(&mut plugins, &plugin_opts, &opts)
             .await
             .context(StartFrontendSnafu)?;
 
@@ -474,7 +475,7 @@ mod tests {
         };
 
         let mut plugins = Plugins::new();
-        plugins::setup_frontend_plugins(&mut plugins, &fe_opts)
+        plugins::setup_frontend_plugins(&mut plugins, &[], &fe_opts)
             .await
             .unwrap();
 
diff --git a/src/cmd/src/metasrv.rs b/src/cmd/src/metasrv.rs
index 664066132e..b116790367 100644
--- a/src/cmd/src/metasrv.rs
+++ b/src/cmd/src/metasrv.rs
@@ -273,9 +273,10 @@ impl StartCommand {
         info!("Metasrv start command: {:#?}", self);
         info!("Metasrv options: {:#?}", opts);
 
+        let plugin_opts = opts.plugins;
         let opts = opts.component;
         let mut plugins = Plugins::new();
-        plugins::setup_metasrv_plugins(&mut plugins, &opts)
+        plugins::setup_metasrv_plugins(&mut plugins, &plugin_opts, &opts)
             .await
             .context(StartMetaServerSnafu)?;
 
diff --git a/src/cmd/src/options.rs b/src/cmd/src/options.rs
index 26ac9203a2..ecff735294 100644
--- a/src/cmd/src/options.rs
+++ b/src/cmd/src/options.rs
@@ -15,6 +15,7 @@
 use clap::Parser;
 use common_config::Configurable;
 use common_runtime::global::RuntimeOptions;
+use plugins::PluginOptions;
 use serde::{Deserialize, Serialize};
 
 #[derive(Parser, Default, Debug, Clone)]
@@ -40,6 +41,8 @@ pub struct GlobalOptions {
 pub struct GreptimeOptions<T> {
     /// The runtime options.
     pub runtime: RuntimeOptions,
+    /// The plugin options.
+    pub plugins: Vec<PluginOptions>,
 
     /// The options of each component (like Datanode or Standalone) of GreptimeDB.
     #[serde(flatten)]
diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
index 8e1185e389..a72aea3c9a 100644
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -445,15 +445,16 @@ impl StartCommand {
         info!("Standalone options: {opts:#?}");
 
         let mut plugins = Plugins::new();
+        let plugin_opts = opts.plugins;
         let opts = opts.component;
         let fe_opts = opts.frontend_options();
         let dn_opts = opts.datanode_options();
 
-        plugins::setup_frontend_plugins(&mut plugins, &fe_opts)
+        plugins::setup_frontend_plugins(&mut plugins, &plugin_opts, &fe_opts)
             .await
             .context(StartFrontendSnafu)?;
 
-        plugins::setup_datanode_plugins(&mut plugins, &dn_opts)
+        plugins::setup_datanode_plugins(&mut plugins, &plugin_opts, &dn_opts)
             .await
             .context(StartDatanodeSnafu)?;
 
@@ -762,7 +763,7 @@ mod tests {
         };
 
         let mut plugins = Plugins::new();
-        plugins::setup_frontend_plugins(&mut plugins, &fe_opts)
+        plugins::setup_frontend_plugins(&mut plugins, &[], &fe_opts)
             .await
             .unwrap();
 
diff --git a/src/plugins/Cargo.toml b/src/plugins/Cargo.toml
index c927fb3d33..977b7d9b7c 100644
--- a/src/plugins/Cargo.toml
+++ b/src/plugins/Cargo.toml
@@ -13,4 +13,5 @@ common-base.workspace = true
 datanode.workspace = true
 frontend.workspace = true
 meta-srv.workspace = true
+serde.workspace = true
 snafu.workspace = true
diff --git a/src/plugins/src/datanode.rs b/src/plugins/src/datanode.rs
index f3cd62af55..60640f05f1 100644
--- a/src/plugins/src/datanode.rs
+++ b/src/plugins/src/datanode.rs
@@ -16,10 +16,13 @@ use common_base::Plugins;
 use datanode::config::DatanodeOptions;
 use datanode::error::Result;
 
+use crate::options::PluginOptions;
+
 #[allow(unused_variables)]
 #[allow(unused_mut)]
 pub async fn setup_datanode_plugins(
     plugins: &mut Plugins,
+    plugin_options: &[PluginOptions],
     dn_opts: &DatanodeOptions,
 ) -> Result<()> {
     Ok(())
diff --git a/src/plugins/src/frontend.rs b/src/plugins/src/frontend.rs
index e94b080c89..240e7a1582 100644
--- a/src/plugins/src/frontend.rs
+++ b/src/plugins/src/frontend.rs
@@ -18,9 +18,12 @@ use frontend::error::{IllegalAuthConfigSnafu, Result};
 use frontend::frontend::FrontendOptions;
 use snafu::ResultExt;
 
+use crate::options::PluginOptions;
+
 #[allow(unused_mut)]
 pub async fn setup_frontend_plugins(
     plugins: &mut Plugins,
+    _plugin_options: &[PluginOptions],
     fe_opts: &FrontendOptions,
 ) -> Result<()> {
     if let Some(user_provider) = fe_opts.user_provider.as_ref() {
diff --git a/src/plugins/src/lib.rs b/src/plugins/src/lib.rs
index f0be0bd763..a29ed0e4a8 100644
--- a/src/plugins/src/lib.rs
+++ b/src/plugins/src/lib.rs
@@ -15,7 +15,9 @@
 mod datanode;
 mod frontend;
 mod meta_srv;
+mod options;
 
 pub use datanode::{setup_datanode_plugins, start_datanode_plugins};
 pub use frontend::{setup_frontend_plugins, start_frontend_plugins};
 pub use meta_srv::{setup_metasrv_plugins, start_metasrv_plugins};
+pub use options::PluginOptions;
diff --git a/src/plugins/src/meta_srv.rs b/src/plugins/src/meta_srv.rs
index 97fadfabd5..282ac241c5 100644
--- a/src/plugins/src/meta_srv.rs
+++ b/src/plugins/src/meta_srv.rs
@@ -16,9 +16,12 @@ use common_base::Plugins;
 use meta_srv::error::Result;
 use meta_srv::metasrv::MetasrvOptions;
 
+use crate::options::PluginOptions;
+
 #[allow(unused_variables)]
 pub async fn setup_metasrv_plugins(
     _plugins: &mut Plugins,
+    plugin_options: &[PluginOptions],
     metasrv_opts: &MetasrvOptions,
 ) -> Result<()> {
     Ok(())
diff --git a/src/plugins/src/options.rs b/src/plugins/src/options.rs
new file mode 100644
index 0000000000..7eb0b32903
--- /dev/null
+++ b/src/plugins/src/options.rs
@@ -0,0 +1,23 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct DummyOptions;
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub enum PluginOptions {
+    Dummy(DummyOptions),
+}

From e328c7067ce95f042b273030dde6875b643934e7 Mon Sep 17 00:00:00 2001
From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com>
Date: Fri, 25 Oct 2024 08:23:32 +0800
Subject: [PATCH 105/128] chore: udapte Rust toolchain to 2024-10-19 (#4857)

* update rust toolchain

* change toolchain to 2024-10-17

* fix: clippy

* fix: ut

* bump shadow-rs

* fix: use nightly-2024-10-19

* fix: clippy

* chore/udapte-toolchain-2024-10-17: Update DEV_BUILDER_IMAGE_TAG to 2024-10-19-a5c00e85-20241024184445 in Makefile
---
 Cargo.lock                                    |  8 ++++----
 Cargo.toml                                    |  2 +-
 Makefile                                      |  2 +-
 rust-toolchain.toml                           |  3 +--
 .../src/user_provider/static_user_provider.rs |  4 ++--
 .../information_schema/table_names.rs         |  2 +-
 .../system_schema/pg_catalog/pg_namespace.rs  |  6 +++---
 src/cmd/src/lib.rs                            |  1 +
 src/common/base/src/secrets.rs                |  5 +++--
 src/common/catalog/src/consts.rs              |  5 +++--
 src/common/greptimedb-telemetry/src/lib.rs    |  1 +
 src/common/macro/src/lib.rs                   | 11 ++++++----
 src/common/meta/src/cluster.rs                |  1 +
 src/common/meta/src/key/catalog_name.rs       |  2 +-
 src/common/meta/src/key/datanode_table.rs     |  2 +-
 src/common/meta/src/key/flow/flow_info.rs     |  6 +++---
 src/common/meta/src/key/flow/flow_name.rs     |  8 ++++----
 src/common/meta/src/key/node_address.rs       |  2 +-
 src/common/meta/src/key/schema_name.rs        |  2 +-
 src/common/meta/src/key/table_info.rs         | 11 ++++------
 src/common/meta/src/key/table_route.rs        | 12 +++++------
 src/common/meta/src/key/view_info.rs          | 12 +++++------
 src/common/meta/src/lock_key.rs               |  4 ++--
 src/common/procedure/src/store.rs             |  2 +-
 src/common/recordbatch/src/recordbatch.rs     |  4 ++--
 src/datatypes/src/scalars.rs                  |  8 ++++----
 src/datatypes/src/types/cast.rs               |  4 ++--
 src/datatypes/src/types/primitive_type.rs     |  7 ++++---
 src/datatypes/src/value.rs                    | 20 +++++++++----------
 src/datatypes/src/vectors/decimal.rs          |  2 +-
 src/datatypes/src/vectors/list.rs             |  2 +-
 src/datatypes/src/vectors/primitive.rs        |  2 +-
 src/flow/src/compute/render.rs                |  6 +++---
 src/flow/src/compute/render/map.rs            |  2 +-
 src/flow/src/compute/render/reduce.rs         |  2 +-
 src/flow/src/compute/render/src_sink.rs       |  2 +-
 src/flow/src/df_optimizer.rs                  |  2 +-
 .../search/fst_values_mapper.rs               |  4 +++-
 .../search/index_apply/predicates_apply.rs    |  2 +-
 src/meta-srv/src/lib.rs                       |  1 -
 src/mito2/src/cache/cache_size.rs             |  2 +-
 src/mito2/src/config.rs                       |  2 +-
 src/mito2/src/memtable/key_values.rs          |  2 +-
 .../inverted_index/applier/builder/between.rs |  2 +-
 .../applier/builder/comparison.rs             |  2 +-
 .../inverted_index/applier/builder/eq_list.rs |  2 +-
 .../inverted_index/applier/builder/in_list.rs |  2 +-
 .../applier/builder/regex_match.rs            |  2 +-
 src/mito2/src/sst/index/store.rs              |  8 ++++----
 src/mito2/src/sst/parquet/metadata.rs         |  6 +++++-
 src/mito2/src/sst/parquet/row_group.rs        |  2 +-
 src/mito2/src/sst/parquet/stats.rs            |  2 +-
 src/object-store/src/util.rs                  |  2 +-
 src/pipeline/src/etl/processor.rs             |  3 ++-
 src/pipeline/src/etl/processor/timestamp.rs   |  4 ++--
 .../transform/transformer/greptime/coerce.rs  |  8 +++-----
 src/pipeline/src/etl/value.rs                 |  4 ++--
 src/pipeline/src/manager.rs                   |  1 +
 .../src/extension_plan/histogram_fold.rs      |  4 +++-
 .../src/extension_plan/scalar_calculate.rs    |  3 ++-
 src/promql/src/lib.rs                         |  1 -
 src/query/src/dist_plan/planner.rs            |  1 +
 src/query/src/lib.rs                          |  1 -
 src/query/src/range_select/plan_rewrite.rs    |  4 ++--
 src/script/src/python/rspython/vector_impl.rs |  3 ++-
 src/servers/src/export_metrics.rs             |  4 +++-
 src/servers/src/query_handler.rs              |  2 ++
 src/sql/src/parser.rs                         | 19 ++++++++----------
 src/sql/src/parsers/admin_parser.rs           |  2 +-
 src/sql/src/parsers/alter_parser.rs           |  4 ++--
 src/sql/src/parsers/copy_parser.rs            |  2 +-
 src/sql/src/parsers/create_parser.rs          |  4 ++--
 src/sql/src/parsers/deallocate_parser.rs      |  2 +-
 src/sql/src/parsers/delete_parser.rs          |  2 +-
 src/sql/src/parsers/describe_parser.rs        |  2 +-
 src/sql/src/parsers/drop_parser.rs            |  2 +-
 src/sql/src/parsers/execute_parser.rs         |  2 +-
 src/sql/src/parsers/explain_parser.rs         |  2 +-
 src/sql/src/parsers/insert_parser.rs          |  2 +-
 src/sql/src/parsers/prepare_parser.rs         |  2 +-
 src/sql/src/parsers/query_parser.rs           |  2 +-
 src/sql/src/parsers/set_var_parser.rs         |  2 +-
 src/sql/src/parsers/show_parser.rs            |  2 +-
 src/sql/src/parsers/tql_parser.rs             |  2 +-
 src/sql/src/parsers/truncate_parser.rs        |  2 +-
 src/sql/src/statements/tql.rs                 |  1 +
 src/sql/src/util.rs                           |  2 +-
 src/store-api/src/manifest/action.rs          |  5 +++--
 src/store-api/src/path_utils.rs               |  6 ++++--
 src/store-api/src/region_engine.rs            |  1 +
 src/table/src/metadata.rs                     |  3 ++-
 src/table/src/predicate/stats.rs              |  2 +-
 src/table/src/stats.rs                        |  3 ++-
 src/table/src/table_reference.rs              |  2 +-
 94 files changed, 181 insertions(+), 164 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e13a7c6929..f447ea1148 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "Inflector"
@@ -5269,7 +5269,7 @@ dependencies = [
 [[package]]
 name = "influxdb_line_protocol"
 version = "0.1.0"
-source = "git+https://github.com/evenyag/influxdb_iox?branch=feat/line-protocol#10ef0d0b02705ac7518717390939fa3a9bcfcacc"
+source = "git+https://github.com/evenyag/influxdb_iox?branch=feat%2Fline-protocol#10ef0d0b02705ac7518717390939fa3a9bcfcacc"
 dependencies = [
  "bytes",
  "nom",
@@ -10953,9 +10953,9 @@ dependencies = [
 
 [[package]]
 name = "shadow-rs"
-version = "0.31.1"
+version = "0.35.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02c282402d25101f9c893e9cd7e4cae535fe7db18b81291de973026c219ddf1e"
+checksum = "2311e39772c00391875f40e34d43efef247b23930143a70ca5fbec9505937420"
 dependencies = [
  "const_format",
  "git2",
diff --git a/Cargo.toml b/Cargo.toml
index ab78bbd90f..60d8215634 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -169,7 +169,7 @@ schemars = "0.8"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0", features = ["float_roundtrip"] }
 serde_with = "3"
-shadow-rs = "0.31"
+shadow-rs = "0.35"
 similar-asserts = "1.6.0"
 smallvec = { version = "1", features = ["serde"] }
 snafu = "0.8"
diff --git a/Makefile b/Makefile
index c63acb67b4..dd335c370a 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,7 @@ CARGO_BUILD_OPTS := --locked
 IMAGE_REGISTRY ?= docker.io
 IMAGE_NAMESPACE ?= greptime
 IMAGE_TAG ?= latest
-DEV_BUILDER_IMAGE_TAG ?= 2024-06-06-5674c14f-20240920110415
+DEV_BUILDER_IMAGE_TAG ?= 2024-10-19-a5c00e85-20241024184445
 BUILDX_MULTI_PLATFORM_BUILD ?= false
 BUILDX_BUILDER_NAME ?= gtbuilder
 BASE_IMAGE ?= ubuntu
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index cfa276d72f..c986eedd97 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -1,3 +1,2 @@
 [toolchain]
-channel = "nightly-2024-06-06"
-
+channel = "nightly-2024-10-19"
diff --git a/src/auth/src/user_provider/static_user_provider.rs b/src/auth/src/user_provider/static_user_provider.rs
index 9e05671219..24093d5e37 100644
--- a/src/auth/src/user_provider/static_user_provider.rs
+++ b/src/auth/src/user_provider/static_user_provider.rs
@@ -33,7 +33,7 @@ impl StaticUserProvider {
             value: value.to_string(),
             msg: "StaticUserProviderOption must be in format `<option>:<value>`",
         })?;
-        return match mode {
+        match mode {
             "file" => {
                 let users = load_credential_from_file(content)?
                     .context(InvalidConfigSnafu {
@@ -58,7 +58,7 @@ impl StaticUserProvider {
                 msg: "StaticUserProviderOption must be in format `file:<path>` or `cmd:<values>`",
             }
                 .fail(),
-        };
+        }
     }
 }
 
diff --git a/src/catalog/src/system_schema/information_schema/table_names.rs b/src/catalog/src/system_schema/information_schema/table_names.rs
index 53541bf955..cdff5ebb77 100644
--- a/src/catalog/src/system_schema/information_schema/table_names.rs
+++ b/src/catalog/src/system_schema/information_schema/table_names.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-/// All table names in `information_schema`.
+//! All table names in `information_schema`.
 
 pub const TABLES: &str = "tables";
 pub const COLUMNS: &str = "columns";
diff --git a/src/catalog/src/system_schema/pg_catalog/pg_namespace.rs b/src/catalog/src/system_schema/pg_catalog/pg_namespace.rs
index c6db980de7..108cd92242 100644
--- a/src/catalog/src/system_schema/pg_catalog/pg_namespace.rs
+++ b/src/catalog/src/system_schema/pg_catalog/pg_namespace.rs
@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//! The `pg_catalog.pg_namespace` table implementation.
+//! namespace is a schema in greptime
+
 pub(super) mod oid_map;
 
 use std::sync::{Arc, Weak};
@@ -40,9 +43,6 @@ use crate::system_schema::utils::tables::{string_column, u32_column};
 use crate::system_schema::SystemTable;
 use crate::CatalogManager;
 
-/// The `pg_catalog.pg_namespace` table implementation.
-/// namespace is a schema in greptime
-
 const NSPNAME: &str = "nspname";
 const INIT_CAPACITY: usize = 42;
 
diff --git a/src/cmd/src/lib.rs b/src/cmd/src/lib.rs
index 80e1628bc0..731f527daf 100644
--- a/src/cmd/src/lib.rs
+++ b/src/cmd/src/lib.rs
@@ -84,6 +84,7 @@ pub trait App: Send {
 }
 
 /// Log the versions of the application, and the arguments passed to the cli.
+///
 /// `version` should be the same as the output of cli "--version";
 /// and the `short_version` is the short version of the codes, often consist of git branch and commit.
 pub fn log_versions(version: &str, short_version: &str, app: &str) {
diff --git a/src/common/base/src/secrets.rs b/src/common/base/src/secrets.rs
index 702b6d3ccd..b3f4347578 100644
--- a/src/common/base/src/secrets.rs
+++ b/src/common/base/src/secrets.rs
@@ -46,8 +46,9 @@ impl From<String> for SecretString {
     }
 }
 
-/// Wrapper type for values that contains secrets, which attempts to limit
-/// accidental exposure and ensure secrets are wiped from memory when dropped.
+/// Wrapper type for values that contains secrets.
+///
+/// It attempts to limit accidental exposure and ensure secrets are wiped from memory when dropped.
 /// (e.g. passwords, cryptographic keys, access tokens or other credentials)
 ///
 /// Access to the secret inner value occurs through the [`ExposeSecret`]
diff --git a/src/common/catalog/src/consts.rs b/src/common/catalog/src/consts.rs
index 9cf02b81ee..5a73e8ddae 100644
--- a/src/common/catalog/src/consts.rs
+++ b/src/common/catalog/src/consts.rs
@@ -103,14 +103,15 @@ pub const INFORMATION_SCHEMA_PROCEDURE_INFO_TABLE_ID: u32 = 34;
 /// id for information_schema.region_statistics
 pub const INFORMATION_SCHEMA_REGION_STATISTICS_TABLE_ID: u32 = 35;
 
-/// ----- End of information_schema tables -----
+// ----- End of information_schema tables -----
 
 /// ----- Begin of pg_catalog tables -----
 pub const PG_CATALOG_PG_CLASS_TABLE_ID: u32 = 256;
 pub const PG_CATALOG_PG_TYPE_TABLE_ID: u32 = 257;
 pub const PG_CATALOG_PG_NAMESPACE_TABLE_ID: u32 = 258;
 
-/// ----- End of pg_catalog tables -----
+// ----- End of pg_catalog tables -----
+
 pub const MITO_ENGINE: &str = "mito";
 pub const MITO2_ENGINE: &str = "mito2";
 pub const METRIC_ENGINE: &str = "metric";
diff --git a/src/common/greptimedb-telemetry/src/lib.rs b/src/common/greptimedb-telemetry/src/lib.rs
index d681b20092..faab69f8b1 100644
--- a/src/common/greptimedb-telemetry/src/lib.rs
+++ b/src/common/greptimedb-telemetry/src/lib.rs
@@ -199,6 +199,7 @@ pub fn default_get_uuid(working_home: &Option<String>) -> Option<String> {
 }
 
 /// Report version info to GreptimeDB.
+///
 /// We do not collect any identity-sensitive information.
 /// This task is scheduled to run every 30 minutes.
 /// The task will be disabled default. It can be enabled by setting the build feature `greptimedb-telemetry`
diff --git a/src/common/macro/src/lib.rs b/src/common/macro/src/lib.rs
index 8079be46a8..4ba876ab9d 100644
--- a/src/common/macro/src/lib.rs
+++ b/src/common/macro/src/lib.rs
@@ -35,7 +35,9 @@ pub fn aggr_func_type_store_derive(input: TokenStream) -> TokenStream {
 }
 
 /// A struct can be used as a creator for aggregate function if it has been annotated with this
-/// attribute first. This attribute add a necessary field which is intended to store the input
+/// attribute first.
+///
+/// This attribute add a necessary field which is intended to store the input
 /// data's types to the struct.
 /// This attribute is expected to be used along with derive macro [AggrFuncTypeStore].
 #[proc_macro_attribute]
@@ -44,9 +46,10 @@ pub fn as_aggr_func_creator(args: TokenStream, input: TokenStream) -> TokenStrea
 }
 
 /// Attribute macro to convert an arithimetic function to a range function. The annotated function
-/// should accept servaral arrays as input and return a single value as output. This procedure
-/// macro can works on any number of input parameters. Return type can be either primitive type
-/// or wrapped in `Option`.
+/// should accept servaral arrays as input and return a single value as output.
+///
+/// This procedure macro can works on any number of input parameters. Return type can be either
+/// primitive type or wrapped in `Option`.
 ///
 /// # Example
 /// Take `count_over_time()` in PromQL as an example:
diff --git a/src/common/meta/src/cluster.rs b/src/common/meta/src/cluster.rs
index 785a28f17d..6e9d9c8ef3 100644
--- a/src/common/meta/src/cluster.rs
+++ b/src/common/meta/src/cluster.rs
@@ -55,6 +55,7 @@ pub trait ClusterInfo {
 }
 
 /// The key of [NodeInfo] in the storage. The format is `__meta_cluster_node_info-{cluster_id}-{role}-{node_id}`.
+///
 /// This key cannot be used to describe the `Metasrv` because the `Metasrv` does not have
 /// a `cluster_id`, it serves multiple clusters.
 #[derive(Debug, Clone, Eq, Hash, PartialEq, Serialize, Deserialize)]
diff --git a/src/common/meta/src/key/catalog_name.rs b/src/common/meta/src/key/catalog_name.rs
index 9238f95c82..18ca096412 100644
--- a/src/common/meta/src/key/catalog_name.rs
+++ b/src/common/meta/src/key/catalog_name.rs
@@ -35,7 +35,7 @@ pub struct CatalogNameKey<'a> {
     pub catalog: &'a str,
 }
 
-impl<'a> Default for CatalogNameKey<'a> {
+impl Default for CatalogNameKey<'_> {
     fn default() -> Self {
         Self {
             catalog: DEFAULT_CATALOG_NAME,
diff --git a/src/common/meta/src/key/datanode_table.rs b/src/common/meta/src/key/datanode_table.rs
index b10d536672..e904983cc8 100644
--- a/src/common/meta/src/key/datanode_table.rs
+++ b/src/common/meta/src/key/datanode_table.rs
@@ -77,7 +77,7 @@ impl DatanodeTableKey {
     }
 }
 
-impl<'a> MetadataKey<'a, DatanodeTableKey> for DatanodeTableKey {
+impl MetadataKey<'_, DatanodeTableKey> for DatanodeTableKey {
     fn to_bytes(&self) -> Vec<u8> {
         self.to_string().into_bytes()
     }
diff --git a/src/common/meta/src/key/flow/flow_info.rs b/src/common/meta/src/key/flow/flow_info.rs
index 2057f06950..55f982af93 100644
--- a/src/common/meta/src/key/flow/flow_info.rs
+++ b/src/common/meta/src/key/flow/flow_info.rs
@@ -42,6 +42,8 @@ lazy_static! {
 /// The layout: `__flow/info/{flow_id}`.
 pub struct FlowInfoKey(FlowScoped<FlowInfoKeyInner>);
 
+pub type FlowInfoDecodeResult = Result<Option<DeserializedValueWithBytes<FlowInfoValue>>>;
+
 impl<'a> MetadataKey<'a, FlowInfoKey> for FlowInfoKey {
     fn to_bytes(&self) -> Vec<u8> {
         self.0.to_bytes()
@@ -203,9 +205,7 @@ impl FlowInfoManager {
         flow_value: &FlowInfoValue,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<FlowInfoValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> FlowInfoDecodeResult,
     )> {
         let key = FlowInfoKey::new(flow_id).to_bytes();
         let txn = Txn::put_if_not_exists(key.clone(), flow_value.try_as_raw_value()?);
diff --git a/src/common/meta/src/key/flow/flow_name.rs b/src/common/meta/src/key/flow/flow_name.rs
index 3a331be800..0db86b05fa 100644
--- a/src/common/meta/src/key/flow/flow_name.rs
+++ b/src/common/meta/src/key/flow/flow_name.rs
@@ -46,6 +46,8 @@ lazy_static! {
 /// The layout: `__flow/name/{catalog_name}/{flow_name}`.
 pub struct FlowNameKey<'a>(FlowScoped<FlowNameKeyInner<'a>>);
 
+pub type FlowNameDecodeResult = Result<Option<DeserializedValueWithBytes<FlowNameValue>>>;
+
 #[allow(dead_code)]
 impl<'a> FlowNameKey<'a> {
     /// Returns the [FlowNameKey]
@@ -104,7 +106,7 @@ impl<'a> MetadataKey<'a, FlowNameKeyInner<'a>> for FlowNameKeyInner<'_> {
         .into_bytes()
     }
 
-    fn from_bytes(bytes: &'a [u8]) -> Result<FlowNameKeyInner> {
+    fn from_bytes(bytes: &'a [u8]) -> Result<FlowNameKeyInner<'a>> {
         let key = std::str::from_utf8(bytes).map_err(|e| {
             error::InvalidMetadataSnafu {
                 err_msg: format!(
@@ -223,9 +225,7 @@ impl FlowNameManager {
         flow_id: FlowId,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<FlowNameValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> FlowNameDecodeResult,
     )> {
         let key = FlowNameKey::new(catalog_name, flow_name);
         let raw_key = key.to_bytes();
diff --git a/src/common/meta/src/key/node_address.rs b/src/common/meta/src/key/node_address.rs
index 55085c928f..a1a763940b 100644
--- a/src/common/meta/src/key/node_address.rs
+++ b/src/common/meta/src/key/node_address.rs
@@ -52,7 +52,7 @@ impl NodeAddressValue {
     }
 }
 
-impl<'a> MetadataKey<'a, NodeAddressKey> for NodeAddressKey {
+impl MetadataKey<'_, NodeAddressKey> for NodeAddressKey {
     fn to_bytes(&self) -> Vec<u8> {
         self.to_string().into_bytes()
     }
diff --git a/src/common/meta/src/key/schema_name.rs b/src/common/meta/src/key/schema_name.rs
index f9cbc4a9f2..753b865b53 100644
--- a/src/common/meta/src/key/schema_name.rs
+++ b/src/common/meta/src/key/schema_name.rs
@@ -41,7 +41,7 @@ pub struct SchemaNameKey<'a> {
     pub schema: &'a str,
 }
 
-impl<'a> Default for SchemaNameKey<'a> {
+impl Default for SchemaNameKey<'_> {
     fn default() -> Self {
         Self {
             catalog: DEFAULT_CATALOG_NAME,
diff --git a/src/common/meta/src/key/table_info.rs b/src/common/meta/src/key/table_info.rs
index e5d124fcb8..b3ca42b6ff 100644
--- a/src/common/meta/src/key/table_info.rs
+++ b/src/common/meta/src/key/table_info.rs
@@ -51,7 +51,7 @@ impl Display for TableInfoKey {
     }
 }
 
-impl<'a> MetadataKey<'a, TableInfoKey> for TableInfoKey {
+impl MetadataKey<'_, TableInfoKey> for TableInfoKey {
     fn to_bytes(&self) -> Vec<u8> {
         self.to_string().into_bytes()
     }
@@ -132,6 +132,7 @@ pub type TableInfoManagerRef = Arc<TableInfoManager>;
 pub struct TableInfoManager {
     kv_backend: KvBackendRef,
 }
+pub type TableInfoDecodeResult = Result<Option<DeserializedValueWithBytes<TableInfoValue>>>;
 
 impl TableInfoManager {
     pub fn new(kv_backend: KvBackendRef) -> Self {
@@ -145,9 +146,7 @@ impl TableInfoManager {
         table_info_value: &TableInfoValue,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<TableInfoValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> TableInfoDecodeResult,
     )> {
         let key = TableInfoKey::new(table_id);
         let raw_key = key.to_bytes();
@@ -169,9 +168,7 @@ impl TableInfoManager {
         new_table_info_value: &TableInfoValue,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<TableInfoValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> TableInfoDecodeResult,
     )> {
         let key = TableInfoKey::new(table_id);
         let raw_key = key.to_bytes();
diff --git a/src/common/meta/src/key/table_route.rs b/src/common/meta/src/key/table_route.rs
index c9990ab121..96949d2b9f 100644
--- a/src/common/meta/src/key/table_route.rs
+++ b/src/common/meta/src/key/table_route.rs
@@ -245,7 +245,7 @@ impl LogicalTableRouteValue {
     }
 }
 
-impl<'a> MetadataKey<'a, TableRouteKey> for TableRouteKey {
+impl MetadataKey<'_, TableRouteKey> for TableRouteKey {
     fn to_bytes(&self) -> Vec<u8> {
         self.to_string().into_bytes()
     }
@@ -472,6 +472,8 @@ pub struct TableRouteStorage {
     kv_backend: KvBackendRef,
 }
 
+pub type TableRouteValueDecodeResult = Result<Option<DeserializedValueWithBytes<TableRouteValue>>>;
+
 impl TableRouteStorage {
     pub fn new(kv_backend: KvBackendRef) -> Self {
         Self { kv_backend }
@@ -485,9 +487,7 @@ impl TableRouteStorage {
         table_route_value: &TableRouteValue,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<TableRouteValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> TableRouteValueDecodeResult,
     )> {
         let key = TableRouteKey::new(table_id);
         let raw_key = key.to_bytes();
@@ -510,9 +510,7 @@ impl TableRouteStorage {
         new_table_route_value: &TableRouteValue,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<TableRouteValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> TableRouteValueDecodeResult,
     )> {
         let key = TableRouteKey::new(table_id);
         let raw_key = key.to_bytes();
diff --git a/src/common/meta/src/key/view_info.rs b/src/common/meta/src/key/view_info.rs
index e3843d54e3..e5b5950e52 100644
--- a/src/common/meta/src/key/view_info.rs
+++ b/src/common/meta/src/key/view_info.rs
@@ -53,7 +53,7 @@ impl Display for ViewInfoKey {
     }
 }
 
-impl<'a> MetadataKey<'a, ViewInfoKey> for ViewInfoKey {
+impl MetadataKey<'_, ViewInfoKey> for ViewInfoKey {
     fn to_bytes(&self) -> Vec<u8> {
         self.to_string().into_bytes()
     }
@@ -139,6 +139,8 @@ pub struct ViewInfoManager {
 
 pub type ViewInfoManagerRef = Arc<ViewInfoManager>;
 
+pub type ViewInfoValueDecodeResult = Result<Option<DeserializedValueWithBytes<ViewInfoValue>>>;
+
 impl ViewInfoManager {
     pub fn new(kv_backend: KvBackendRef) -> Self {
         Self { kv_backend }
@@ -151,9 +153,7 @@ impl ViewInfoManager {
         view_info_value: &ViewInfoValue,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<ViewInfoValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> ViewInfoValueDecodeResult,
     )> {
         let key = ViewInfoKey::new(view_id);
         let raw_key = key.to_bytes();
@@ -175,9 +175,7 @@ impl ViewInfoManager {
         new_view_info_value: &ViewInfoValue,
     ) -> Result<(
         Txn,
-        impl FnOnce(
-            &mut TxnOpGetResponseSet,
-        ) -> Result<Option<DeserializedValueWithBytes<ViewInfoValue>>>,
+        impl FnOnce(&mut TxnOpGetResponseSet) -> ViewInfoValueDecodeResult,
     )> {
         let key = ViewInfoKey::new(view_id);
         let raw_key = key.to_bytes();
diff --git a/src/common/meta/src/lock_key.rs b/src/common/meta/src/lock_key.rs
index 324733aae2..4ae2696d83 100644
--- a/src/common/meta/src/lock_key.rs
+++ b/src/common/meta/src/lock_key.rs
@@ -34,7 +34,7 @@ pub enum CatalogLock<'a> {
     Write(&'a str),
 }
 
-impl<'a> Display for CatalogLock<'a> {
+impl Display for CatalogLock<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let key = match self {
             CatalogLock::Read(s) => s,
@@ -44,7 +44,7 @@ impl<'a> Display for CatalogLock<'a> {
     }
 }
 
-impl<'a> From<CatalogLock<'a>> for StringKey {
+impl From<CatalogLock<'_>> for StringKey {
     fn from(value: CatalogLock) -> Self {
         match value {
             CatalogLock::Write(_) => StringKey::Exclusive(value.to_string()),
diff --git a/src/common/procedure/src/store.rs b/src/common/procedure/src/store.rs
index 4bc5f8ca81..cebbe9b009 100644
--- a/src/common/procedure/src/store.rs
+++ b/src/common/procedure/src/store.rs
@@ -297,7 +297,7 @@ struct ParsedKey<'a> {
     key_type: KeyType,
 }
 
-impl<'a> fmt::Display for ParsedKey<'a> {
+impl fmt::Display for ParsedKey<'_> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(
             f,
diff --git a/src/common/recordbatch/src/recordbatch.rs b/src/common/recordbatch/src/recordbatch.rs
index 1459b87fac..71f7f60685 100644
--- a/src/common/recordbatch/src/recordbatch.rs
+++ b/src/common/recordbatch/src/recordbatch.rs
@@ -226,7 +226,7 @@ pub struct RecordBatchRowIterator<'a> {
 }
 
 impl<'a> RecordBatchRowIterator<'a> {
-    fn new(record_batch: &'a RecordBatch) -> RecordBatchRowIterator {
+    fn new(record_batch: &'a RecordBatch) -> RecordBatchRowIterator<'a> {
         RecordBatchRowIterator {
             record_batch,
             rows: record_batch.df_record_batch.num_rows(),
@@ -236,7 +236,7 @@ impl<'a> RecordBatchRowIterator<'a> {
     }
 }
 
-impl<'a> Iterator for RecordBatchRowIterator<'a> {
+impl Iterator for RecordBatchRowIterator<'_> {
     type Item = Vec<Value>;
 
     fn next(&mut self) -> Option<Self::Item> {
diff --git a/src/datatypes/src/scalars.rs b/src/datatypes/src/scalars.rs
index 272d91eebf..2db5bcd3cd 100644
--- a/src/datatypes/src/scalars.rs
+++ b/src/datatypes/src/scalars.rs
@@ -203,7 +203,7 @@ impl Scalar for bool {
     }
 }
 
-impl<'a> ScalarRef<'a> for bool {
+impl ScalarRef<'_> for bool {
     type ScalarType = bool;
 
     #[inline]
@@ -273,7 +273,7 @@ impl Scalar for Date {
     }
 }
 
-impl<'a> ScalarRef<'a> for Date {
+impl ScalarRef<'_> for Date {
     type ScalarType = Date;
 
     fn to_owned_scalar(&self) -> Self::ScalarType {
@@ -294,7 +294,7 @@ impl Scalar for Decimal128 {
     }
 }
 
-impl<'a> ScalarRef<'a> for Decimal128 {
+impl ScalarRef<'_> for Decimal128 {
     type ScalarType = Decimal128;
 
     fn to_owned_scalar(&self) -> Self::ScalarType {
@@ -315,7 +315,7 @@ impl Scalar for DateTime {
     }
 }
 
-impl<'a> ScalarRef<'a> for DateTime {
+impl ScalarRef<'_> for DateTime {
     type ScalarType = DateTime;
 
     fn to_owned_scalar(&self) -> Self::ScalarType {
diff --git a/src/datatypes/src/types/cast.rs b/src/datatypes/src/types/cast.rs
index 702244a142..c369baa4ae 100644
--- a/src/datatypes/src/types/cast.rs
+++ b/src/datatypes/src/types/cast.rs
@@ -82,8 +82,8 @@ pub fn cast_with_opt(
     }
 }
 
-/// Return true if the src_value can be casted to dest_type,
-/// Otherwise, return false.
+/// Return true if the src_value can be casted to dest_type, Otherwise, return false.
+///
 /// Notice: this function does not promise that the `cast_with_opt` will succeed,
 /// it only checks whether the src_value can be casted to dest_type.
 pub fn can_cast_type(src_value: &Value, dest_type: &ConcreteDataType) -> bool {
diff --git a/src/datatypes/src/types/primitive_type.rs b/src/datatypes/src/types/primitive_type.rs
index 86972bd5ee..cc41f3099f 100644
--- a/src/datatypes/src/types/primitive_type.rs
+++ b/src/datatypes/src/types/primitive_type.rs
@@ -83,9 +83,10 @@ pub trait LogicalPrimitiveType: 'static + Sized {
     fn cast_value_ref(value: ValueRef) -> Result<Option<Self::Wrapper>>;
 }
 
-/// A new type for [WrapperType], complement the `Ord` feature for it. Wrapping non ordered
-/// primitive types like `f32` and `f64` in `OrdPrimitive` can make them be used in places that
-/// require `Ord`. For example, in `Median` UDAFs.
+/// A new type for [WrapperType], complement the `Ord` feature for it.
+///
+/// Wrapping non ordered primitive types like `f32` and `f64` in `OrdPrimitive`
+/// can make them be used in places that require `Ord`. For example, in `Median` UDAFs.
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub struct OrdPrimitive<T: WrapperType>(pub T);
 
diff --git a/src/datatypes/src/value.rs b/src/datatypes/src/value.rs
index 76b59a8db7..6bc0c35ee7 100644
--- a/src/datatypes/src/value.rs
+++ b/src/datatypes/src/value.rs
@@ -1087,7 +1087,7 @@ macro_rules! impl_as_for_value_ref {
     };
 }
 
-impl<'a> ValueRef<'a> {
+impl ValueRef<'_> {
     define_data_type_func!(ValueRef);
 
     /// Returns true if this is null.
@@ -1214,13 +1214,13 @@ impl<'a> ValueRef<'a> {
     }
 }
 
-impl<'a> PartialOrd for ValueRef<'a> {
+impl PartialOrd for ValueRef<'_> {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(self.cmp(other))
     }
 }
 
-impl<'a> Ord for ValueRef<'a> {
+impl Ord for ValueRef<'_> {
     fn cmp(&self, other: &Self) -> Ordering {
         impl_ord_for_value_like!(ValueRef, self, other)
     }
@@ -1347,7 +1347,7 @@ pub enum ListValueRef<'a> {
     Ref { val: &'a ListValue },
 }
 
-impl<'a> ListValueRef<'a> {
+impl ListValueRef<'_> {
     /// Convert self to [Value]. This method would clone the underlying data.
     fn to_value(self) -> Value {
         match self {
@@ -1365,7 +1365,7 @@ impl<'a> ListValueRef<'a> {
     }
 }
 
-impl<'a> Serialize for ListValueRef<'a> {
+impl Serialize for ListValueRef<'_> {
     fn serialize<S: Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
         match self {
             ListValueRef::Indexed { vector, idx } => match vector.get(*idx) {
@@ -1377,28 +1377,28 @@ impl<'a> Serialize for ListValueRef<'a> {
     }
 }
 
-impl<'a> PartialEq for ListValueRef<'a> {
+impl PartialEq for ListValueRef<'_> {
     fn eq(&self, other: &Self) -> bool {
         self.to_value().eq(&other.to_value())
     }
 }
 
-impl<'a> Eq for ListValueRef<'a> {}
+impl Eq for ListValueRef<'_> {}
 
-impl<'a> Ord for ListValueRef<'a> {
+impl Ord for ListValueRef<'_> {
     fn cmp(&self, other: &Self) -> Ordering {
         // Respect the order of `Value` by converting into value before comparison.
         self.to_value().cmp(&other.to_value())
     }
 }
 
-impl<'a> PartialOrd for ListValueRef<'a> {
+impl PartialOrd for ListValueRef<'_> {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(self.cmp(other))
     }
 }
 
-impl<'a> ValueRef<'a> {
+impl ValueRef<'_> {
     /// Returns the size of the underlying data in bytes,
     /// The size is estimated and only considers the data size.
     pub fn data_size(&self) -> usize {
diff --git a/src/datatypes/src/vectors/decimal.rs b/src/datatypes/src/vectors/decimal.rs
index 2aef686398..acdf0aadd7 100644
--- a/src/datatypes/src/vectors/decimal.rs
+++ b/src/datatypes/src/vectors/decimal.rs
@@ -247,7 +247,7 @@ pub struct Decimal128Iter<'a> {
     iter: ArrayIter<&'a Decimal128Array>,
 }
 
-impl<'a> Iterator for Decimal128Iter<'a> {
+impl Iterator for Decimal128Iter<'_> {
     type Item = Option<Decimal128>;
 
     fn next(&mut self) -> Option<Self::Item> {
diff --git a/src/datatypes/src/vectors/list.rs b/src/datatypes/src/vectors/list.rs
index 99399edcf4..703a4eb785 100644
--- a/src/datatypes/src/vectors/list.rs
+++ b/src/datatypes/src/vectors/list.rs
@@ -157,7 +157,7 @@ pub struct ListIter<'a> {
 }
 
 impl<'a> ListIter<'a> {
-    fn new(vector: &'a ListVector) -> ListIter {
+    fn new(vector: &'a ListVector) -> ListIter<'a> {
         ListIter { vector, idx: 0 }
     }
 }
diff --git a/src/datatypes/src/vectors/primitive.rs b/src/datatypes/src/vectors/primitive.rs
index fc0007a4b6..b61fedb1b9 100644
--- a/src/datatypes/src/vectors/primitive.rs
+++ b/src/datatypes/src/vectors/primitive.rs
@@ -207,7 +207,7 @@ pub struct PrimitiveIter<'a, T: LogicalPrimitiveType> {
     iter: ArrayIter<&'a PrimitiveArray<T::ArrowPrimitive>>,
 }
 
-impl<'a, T: LogicalPrimitiveType> Iterator for PrimitiveIter<'a, T> {
+impl<T: LogicalPrimitiveType> Iterator for PrimitiveIter<'_, T> {
     type Item = Option<T::Wrapper>;
 
     fn next(&mut self) -> Option<Option<T::Wrapper>> {
diff --git a/src/flow/src/compute/render.rs b/src/flow/src/compute/render.rs
index 90f0d86c73..94f00a1829 100644
--- a/src/flow/src/compute/render.rs
+++ b/src/flow/src/compute/render.rs
@@ -61,7 +61,7 @@ pub struct Context<'referred, 'df> {
     pub err_collector: ErrCollector,
 }
 
-impl<'referred, 'df> Drop for Context<'referred, 'df> {
+impl Drop for Context<'_, '_> {
     fn drop(&mut self) {
         for bundle in std::mem::take(&mut self.input_collection)
             .into_values()
@@ -92,7 +92,7 @@ impl<'referred, 'df> Drop for Context<'referred, 'df> {
     }
 }
 
-impl<'referred, 'df> Context<'referred, 'df> {
+impl Context<'_, '_> {
     pub fn insert_global(&mut self, id: GlobalId, collection: CollectionBundle) {
         self.input_collection.insert(id, collection);
     }
@@ -120,7 +120,7 @@ impl<'referred, 'df> Context<'referred, 'df> {
     }
 }
 
-impl<'referred, 'df> Context<'referred, 'df> {
+impl Context<'_, '_> {
     /// Like `render_plan` but in Batch Mode
     pub fn render_plan_batch(&mut self, plan: TypedPlan) -> Result<CollectionBundle<Batch>, Error> {
         match plan.plan {
diff --git a/src/flow/src/compute/render/map.rs b/src/flow/src/compute/render/map.rs
index d84d1778af..4166523284 100644
--- a/src/flow/src/compute/render/map.rs
+++ b/src/flow/src/compute/render/map.rs
@@ -28,7 +28,7 @@ use crate::plan::TypedPlan;
 use crate::repr::{self, DiffRow, KeyValDiffRow, Row};
 use crate::utils::ArrangeHandler;
 
-impl<'referred, 'df> Context<'referred, 'df> {
+impl Context<'_, '_> {
     /// Like `render_mfp` but in batch mode
     pub fn render_mfp_batch(
         &mut self,
diff --git a/src/flow/src/compute/render/reduce.rs b/src/flow/src/compute/render/reduce.rs
index 8cff13781b..ee20f8ca78 100644
--- a/src/flow/src/compute/render/reduce.rs
+++ b/src/flow/src/compute/render/reduce.rs
@@ -34,7 +34,7 @@ use crate::plan::{AccumulablePlan, AggrWithIndex, KeyValPlan, ReducePlan, TypedP
 use crate::repr::{self, DiffRow, KeyValDiffRow, RelationType, Row};
 use crate::utils::{ArrangeHandler, ArrangeReader, ArrangeWriter, KeyExpiryManager};
 
-impl<'referred, 'df> Context<'referred, 'df> {
+impl Context<'_, '_> {
     const REDUCE_BATCH: &'static str = "reduce_batch";
     /// Like `render_reduce`, but for batch mode, and only barebone implementation
     /// no support for distinct aggregation for now
diff --git a/src/flow/src/compute/render/src_sink.rs b/src/flow/src/compute/render/src_sink.rs
index 3f5bfaf345..62e733420b 100644
--- a/src/flow/src/compute/render/src_sink.rs
+++ b/src/flow/src/compute/render/src_sink.rs
@@ -31,7 +31,7 @@ use crate::expr::{Batch, EvalError};
 use crate::repr::{DiffRow, Row, BROADCAST_CAP};
 
 #[allow(clippy::mutable_key_type)]
-impl<'referred, 'df> Context<'referred, 'df> {
+impl Context<'_, '_> {
     /// simply send the batch to downstream, without fancy features like buffering
     pub fn render_source_batch(
         &mut self,
diff --git a/src/flow/src/df_optimizer.rs b/src/flow/src/df_optimizer.rs
index d73bf91bfb..0b788a79db 100644
--- a/src/flow/src/df_optimizer.rs
+++ b/src/flow/src/df_optimizer.rs
@@ -273,7 +273,7 @@ impl<'a> ExpandAvgRewriter<'a> {
     }
 }
 
-impl<'a> TreeNodeRewriter for ExpandAvgRewriter<'a> {
+impl TreeNodeRewriter for ExpandAvgRewriter<'_> {
     type Node = Expr;
 
     fn f_up(&mut self, expr: Expr) -> Result<Transformed<Expr>, DataFusionError> {
diff --git a/src/index/src/inverted_index/search/fst_values_mapper.rs b/src/index/src/inverted_index/search/fst_values_mapper.rs
index f4d159ed0c..7166531521 100644
--- a/src/index/src/inverted_index/search/fst_values_mapper.rs
+++ b/src/index/src/inverted_index/search/fst_values_mapper.rs
@@ -19,7 +19,9 @@ use crate::inverted_index::error::Result;
 use crate::inverted_index::format::reader::InvertedIndexReader;
 
 /// `FstValuesMapper` maps FST-encoded u64 values to their corresponding bitmaps
-/// within an inverted index. The higher 32 bits of each u64 value represent the
+/// within an inverted index.
+///
+/// The higher 32 bits of each u64 value represent the
 /// bitmap offset and the lower 32 bits represent its size. This mapper uses these
 /// combined offset-size pairs to fetch and union multiple bitmaps into a single `BitVec`.
 pub struct FstValuesMapper<'a> {
diff --git a/src/index/src/inverted_index/search/index_apply/predicates_apply.rs b/src/index/src/inverted_index/search/index_apply/predicates_apply.rs
index 3b21e21dc1..face61bced 100644
--- a/src/index/src/inverted_index/search/index_apply/predicates_apply.rs
+++ b/src/index/src/inverted_index/search/index_apply/predicates_apply.rs
@@ -134,7 +134,7 @@ impl PredicatesIndexApplier {
     fn bitmap_full_range(metadata: &InvertedIndexMetas) -> BitVec {
         let total_count = metadata.total_row_count;
         let segment_count = metadata.segment_row_count;
-        let len = (total_count + segment_count - 1) / segment_count;
+        let len = total_count.div_ceil(segment_count);
         BitVec::repeat(true, len as _)
     }
 }
diff --git a/src/meta-srv/src/lib.rs b/src/meta-srv/src/lib.rs
index a438f4255f..c84e1755ca 100644
--- a/src/meta-srv/src/lib.rs
+++ b/src/meta-srv/src/lib.rs
@@ -16,7 +16,6 @@
 #![feature(result_flattening)]
 #![feature(assert_matches)]
 #![feature(extract_if)]
-#![feature(option_take_if)]
 
 pub mod bootstrap;
 mod cache_invalidator;
diff --git a/src/mito2/src/cache/cache_size.rs b/src/mito2/src/cache/cache_size.rs
index 2b63441e64..3d79bcbe58 100644
--- a/src/mito2/src/cache/cache_size.rs
+++ b/src/mito2/src/cache/cache_size.rs
@@ -137,6 +137,6 @@ mod tests {
     fn test_parquet_meta_size() {
         let metadata = parquet_meta();
 
-        assert_eq!(964, parquet_meta_size(&metadata));
+        assert_eq!(956, parquet_meta_size(&metadata));
     }
 }
diff --git a/src/mito2/src/config.rs b/src/mito2/src/config.rs
index bbdff81ff6..433bc45662 100644
--- a/src/mito2/src/config.rs
+++ b/src/mito2/src/config.rs
@@ -499,7 +499,7 @@ fn divide_num_cpus(divisor: usize) -> usize {
     let cores = common_config::utils::get_cpus();
     debug_assert!(cores > 0);
 
-    (cores + divisor - 1) / divisor
+    cores.div_ceil(divisor)
 }
 
 #[cfg(test)]
diff --git a/src/mito2/src/memtable/key_values.rs b/src/mito2/src/memtable/key_values.rs
index 59812b4619..a98826bdb1 100644
--- a/src/mito2/src/memtable/key_values.rs
+++ b/src/mito2/src/memtable/key_values.rs
@@ -126,7 +126,7 @@ pub struct KeyValue<'a> {
     op_type: OpType,
 }
 
-impl<'a> KeyValue<'a> {
+impl KeyValue<'_> {
     /// Get primary key columns.
     pub fn primary_keys(&self) -> impl Iterator<Item = ValueRef> {
         self.helper.indices[..self.helper.num_primary_key_column]
diff --git a/src/mito2/src/sst/index/inverted_index/applier/builder/between.rs b/src/mito2/src/sst/index/inverted_index/applier/builder/between.rs
index 42742b885c..ae4de2170a 100644
--- a/src/mito2/src/sst/index/inverted_index/applier/builder/between.rs
+++ b/src/mito2/src/sst/index/inverted_index/applier/builder/between.rs
@@ -18,7 +18,7 @@ use index::inverted_index::search::predicate::{Bound, Predicate, Range, RangePre
 use crate::error::Result;
 use crate::sst::index::inverted_index::applier::builder::InvertedIndexApplierBuilder;
 
-impl<'a> InvertedIndexApplierBuilder<'a> {
+impl InvertedIndexApplierBuilder<'_> {
     /// Collects a `BETWEEN` expression in the form of `column BETWEEN lit AND lit`.
     pub(crate) fn collect_between(&mut self, between: &Between) -> Result<()> {
         if between.negated {
diff --git a/src/mito2/src/sst/index/inverted_index/applier/builder/comparison.rs b/src/mito2/src/sst/index/inverted_index/applier/builder/comparison.rs
index dce83cf917..2c9fa861ea 100644
--- a/src/mito2/src/sst/index/inverted_index/applier/builder/comparison.rs
+++ b/src/mito2/src/sst/index/inverted_index/applier/builder/comparison.rs
@@ -19,7 +19,7 @@ use index::inverted_index::Bytes;
 use crate::error::Result;
 use crate::sst::index::inverted_index::applier::builder::InvertedIndexApplierBuilder;
 
-impl<'a> InvertedIndexApplierBuilder<'a> {
+impl InvertedIndexApplierBuilder<'_> {
     /// Collects a comparison expression in the form of
     /// `column < lit`, `column > lit`, `column <= lit`, `column >= lit`,
     /// `lit < column`, `lit > column`, `lit <= column`, `lit >= column`.
diff --git a/src/mito2/src/sst/index/inverted_index/applier/builder/eq_list.rs b/src/mito2/src/sst/index/inverted_index/applier/builder/eq_list.rs
index 85c0eafdb4..6d142d6402 100644
--- a/src/mito2/src/sst/index/inverted_index/applier/builder/eq_list.rs
+++ b/src/mito2/src/sst/index/inverted_index/applier/builder/eq_list.rs
@@ -22,7 +22,7 @@ use index::inverted_index::Bytes;
 use crate::error::Result;
 use crate::sst::index::inverted_index::applier::builder::InvertedIndexApplierBuilder;
 
-impl<'a> InvertedIndexApplierBuilder<'a> {
+impl InvertedIndexApplierBuilder<'_> {
     /// Collects an eq expression in the form of `column = lit`.
     pub(crate) fn collect_eq(&mut self, left: &DfExpr, right: &DfExpr) -> Result<()> {
         let Some(column_name) = Self::column_name(left).or_else(|| Self::column_name(right)) else {
diff --git a/src/mito2/src/sst/index/inverted_index/applier/builder/in_list.rs b/src/mito2/src/sst/index/inverted_index/applier/builder/in_list.rs
index 0c5e2a74af..c8cf9c4d16 100644
--- a/src/mito2/src/sst/index/inverted_index/applier/builder/in_list.rs
+++ b/src/mito2/src/sst/index/inverted_index/applier/builder/in_list.rs
@@ -20,7 +20,7 @@ use index::inverted_index::search::predicate::{InListPredicate, Predicate};
 use crate::error::Result;
 use crate::sst::index::inverted_index::applier::builder::InvertedIndexApplierBuilder;
 
-impl<'a> InvertedIndexApplierBuilder<'a> {
+impl InvertedIndexApplierBuilder<'_> {
     /// Collects an in list expression in the form of `column IN (lit, lit, ...)`.
     pub(crate) fn collect_inlist(&mut self, inlist: &InList) -> Result<()> {
         if inlist.negated {
diff --git a/src/mito2/src/sst/index/inverted_index/applier/builder/regex_match.rs b/src/mito2/src/sst/index/inverted_index/applier/builder/regex_match.rs
index baae6a2c49..a60d9d9c0f 100644
--- a/src/mito2/src/sst/index/inverted_index/applier/builder/regex_match.rs
+++ b/src/mito2/src/sst/index/inverted_index/applier/builder/regex_match.rs
@@ -19,7 +19,7 @@ use index::inverted_index::search::predicate::{Predicate, RegexMatchPredicate};
 use crate::error::Result;
 use crate::sst::index::inverted_index::applier::builder::InvertedIndexApplierBuilder;
 
-impl<'a> InvertedIndexApplierBuilder<'a> {
+impl InvertedIndexApplierBuilder<'_> {
     /// Collects a regex match expression in the form of `column ~ pattern`.
     pub(crate) fn collect_regex_match(&mut self, column: &DfExpr, pattern: &DfExpr) -> Result<()> {
         let Some(column_name) = Self::column_name(column) else {
diff --git a/src/mito2/src/sst/index/store.rs b/src/mito2/src/sst/index/store.rs
index 7dfcdc253c..3c709d5000 100644
--- a/src/mito2/src/sst/index/store.rs
+++ b/src/mito2/src/sst/index/store.rs
@@ -153,7 +153,7 @@ impl<'a, R> InstrumentedAsyncRead<'a, R> {
     }
 }
 
-impl<'a, R: AsyncRead + Unpin + Send> AsyncRead for InstrumentedAsyncRead<'a, R> {
+impl<R: AsyncRead + Unpin + Send> AsyncRead for InstrumentedAsyncRead<'_, R> {
     fn poll_read(
         mut self: Pin<&mut Self>,
         cx: &mut Context<'_>,
@@ -168,7 +168,7 @@ impl<'a, R: AsyncRead + Unpin + Send> AsyncRead for InstrumentedAsyncRead<'a, R>
     }
 }
 
-impl<'a, R: AsyncSeek + Unpin + Send> AsyncSeek for InstrumentedAsyncRead<'a, R> {
+impl<R: AsyncSeek + Unpin + Send> AsyncSeek for InstrumentedAsyncRead<'_, R> {
     fn poll_seek(
         mut self: Pin<&mut Self>,
         cx: &mut Context<'_>,
@@ -209,7 +209,7 @@ impl<'a, W> InstrumentedAsyncWrite<'a, W> {
     }
 }
 
-impl<'a, W: AsyncWrite + Unpin + Send> AsyncWrite for InstrumentedAsyncWrite<'a, W> {
+impl<W: AsyncWrite + Unpin + Send> AsyncWrite for InstrumentedAsyncWrite<'_, W> {
     fn poll_write(
         mut self: Pin<&mut Self>,
         cx: &mut Context<'_>,
@@ -254,7 +254,7 @@ impl<'a> CounterGuard<'a> {
     }
 }
 
-impl<'a> Drop for CounterGuard<'a> {
+impl Drop for CounterGuard<'_> {
     fn drop(&mut self) {
         if self.count > 0 {
             self.counter.inc_by(self.count as _);
diff --git a/src/mito2/src/sst/parquet/metadata.rs b/src/mito2/src/sst/parquet/metadata.rs
index 3cf5a85cf8..e39f259723 100644
--- a/src/mito2/src/sst/parquet/metadata.rs
+++ b/src/mito2/src/sst/parquet/metadata.rs
@@ -35,7 +35,11 @@ pub(crate) struct MetadataLoader<'a> {
 
 impl<'a> MetadataLoader<'a> {
     /// Create a new parquet metadata loader.
-    pub fn new(object_store: ObjectStore, file_path: &'a str, file_size: u64) -> MetadataLoader {
+    pub fn new(
+        object_store: ObjectStore,
+        file_path: &'a str,
+        file_size: u64,
+    ) -> MetadataLoader<'a> {
         Self {
             object_store,
             file_path,
diff --git a/src/mito2/src/sst/parquet/row_group.rs b/src/mito2/src/sst/parquet/row_group.rs
index 991d56943d..73382c06d9 100644
--- a/src/mito2/src/sst/parquet/row_group.rs
+++ b/src/mito2/src/sst/parquet/row_group.rs
@@ -360,7 +360,7 @@ fn cache_uncompressed_pages(column: &ColumnChunkMetaData) -> bool {
     column.uncompressed_size() as usize <= DEFAULT_PAGE_SIZE
 }
 
-impl<'a> RowGroups for InMemoryRowGroup<'a> {
+impl RowGroups for InMemoryRowGroup<'_> {
     fn num_rows(&self) -> usize {
         self.row_count
     }
diff --git a/src/mito2/src/sst/parquet/stats.rs b/src/mito2/src/sst/parquet/stats.rs
index 61360b1f70..09b837698c 100644
--- a/src/mito2/src/sst/parquet/stats.rs
+++ b/src/mito2/src/sst/parquet/stats.rs
@@ -64,7 +64,7 @@ impl<'a, T> RowGroupPruningStats<'a, T> {
     }
 }
 
-impl<'a, T: Borrow<RowGroupMetaData>> PruningStatistics for RowGroupPruningStats<'a, T> {
+impl<T: Borrow<RowGroupMetaData>> PruningStatistics for RowGroupPruningStats<'_, T> {
     fn min_values(&self, column: &Column) -> Option<ArrayRef> {
         let column_id = self.column_id_to_prune(&column.name)?;
         self.read_format.min_values(self.row_groups, column_id)
diff --git a/src/object-store/src/util.rs b/src/object-store/src/util.rs
index e54b67c152..fc0a031ab9 100644
--- a/src/object-store/src/util.rs
+++ b/src/object-store/src/util.rs
@@ -153,7 +153,7 @@ static LOGGING_TARGET: &str = "opendal::services";
 
 struct LoggingContext<'a>(&'a [(&'a str, &'a str)]);
 
-impl<'a> Display for LoggingContext<'a> {
+impl Display for LoggingContext<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         for (i, (k, v)) in self.0.iter().enumerate() {
             if i > 0 {
diff --git a/src/pipeline/src/etl/processor.rs b/src/pipeline/src/etl/processor.rs
index afda8f399e..d78ef671b1 100644
--- a/src/pipeline/src/etl/processor.rs
+++ b/src/pipeline/src/etl/processor.rs
@@ -62,7 +62,8 @@ const TARGET_FIELDS_NAME: &str = "target_fields";
 // const ON_FAILURE_NAME: &str = "on_failure";
 // const TAG_NAME: &str = "tag";
 
-/// Processor trait defines the interface for all processors
+/// Processor trait defines the interface for all processors.
+///
 /// A processor is a transformation that can be applied to a field in a document
 /// It can be used to extract, transform, or enrich data
 /// Now Processor only have one input field. In the future, we may support multiple input fields.
diff --git a/src/pipeline/src/etl/processor/timestamp.rs b/src/pipeline/src/etl/processor/timestamp.rs
index 3f9621c19d..18b6711c1d 100644
--- a/src/pipeline/src/etl/processor/timestamp.rs
+++ b/src/pipeline/src/etl/processor/timestamp.rs
@@ -254,7 +254,7 @@ impl TimestampProcessor {
 }
 
 fn parse_formats(yaml: &yaml_rust::yaml::Yaml) -> Result<Vec<(Arc<String>, Tz)>> {
-    return match yaml.as_vec() {
+    match yaml.as_vec() {
         Some(formats_yaml) => {
             let mut formats = Vec::with_capacity(formats_yaml.len());
             for v in formats_yaml {
@@ -286,7 +286,7 @@ fn parse_formats(yaml: &yaml_rust::yaml::Yaml) -> Result<Vec<(Arc<String>, Tz)>>
             s: format!("{yaml:?}"),
         }
         .fail(),
-    };
+    }
 }
 
 impl TryFrom<&yaml_rust::yaml::Hash> for TimestampProcessorBuilder {
diff --git a/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs b/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs
index a0a96b3562..00b5cc6624 100644
--- a/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs
+++ b/src/pipeline/src/etl/transform/transformer/greptime/coerce.rs
@@ -446,12 +446,10 @@ fn coerce_string_value(s: &String, transform: &Transform) -> Result<Option<Value
             None => CoerceUnsupportedEpochTypeSnafu { ty: "String" }.fail(),
         },
 
-        Value::Array(_) | Value::Map(_) => {
-            return CoerceJsonTypeToSnafu {
-                ty: transform.type_.to_str_type(),
-            }
-            .fail()
+        Value::Array(_) | Value::Map(_) => CoerceJsonTypeToSnafu {
+            ty: transform.type_.to_str_type(),
         }
+        .fail(),
 
         Value::Null => Ok(None),
     }
diff --git a/src/pipeline/src/etl/value.rs b/src/pipeline/src/etl/value.rs
index 5316000256..30505c026f 100644
--- a/src/pipeline/src/etl/value.rs
+++ b/src/pipeline/src/etl/value.rs
@@ -335,7 +335,7 @@ impl TryFrom<&yaml_rust::Yaml> for Value {
     }
 }
 
-impl<'a> From<&Value> for JsonbValue<'a> {
+impl From<&Value> for JsonbValue<'_> {
     fn from(value: &Value) -> Self {
         match value {
             Value::Null => JsonbValue::Null,
@@ -373,7 +373,7 @@ impl<'a> From<&Value> for JsonbValue<'a> {
     }
 }
 
-impl<'a> From<Value> for JsonbValue<'a> {
+impl From<Value> for JsonbValue<'_> {
     fn from(value: Value) -> Self {
         match value {
             Value::Null => JsonbValue::Null,
diff --git a/src/pipeline/src/manager.rs b/src/pipeline/src/manager.rs
index 960197e083..6938d52870 100644
--- a/src/pipeline/src/manager.rs
+++ b/src/pipeline/src/manager.rs
@@ -26,6 +26,7 @@ pub mod table;
 pub mod util;
 
 /// Pipeline version. An optional timestamp with nanosecond precision.
+///
 /// If the version is None, it means the latest version of the pipeline.
 /// User can specify the version by providing a timestamp string formatted as iso8601.
 /// When it used in cache key, it will be converted to i64 meaning the number of nanoseconds since the epoch.
diff --git a/src/promql/src/extension_plan/histogram_fold.rs b/src/promql/src/extension_plan/histogram_fold.rs
index 561fbe7720..bfb745337a 100644
--- a/src/promql/src/extension_plan/histogram_fold.rs
+++ b/src/promql/src/extension_plan/histogram_fold.rs
@@ -44,7 +44,9 @@ use datatypes::vectors::MutableVector;
 use futures::{ready, Stream, StreamExt};
 
 /// `HistogramFold` will fold the conventional (non-native) histogram ([1]) for later
-/// computing. Specifically, it will transform the `le` and `field` column into a complex
+/// computing.
+///
+/// Specifically, it will transform the `le` and `field` column into a complex
 /// type, and samples on other tag columns:
 /// - `le` will become a [ListArray] of [f64]. With each bucket bound parsed
 /// - `field` will become a [ListArray] of [f64]
diff --git a/src/promql/src/extension_plan/scalar_calculate.rs b/src/promql/src/extension_plan/scalar_calculate.rs
index 6383631c30..b206c69b78 100644
--- a/src/promql/src/extension_plan/scalar_calculate.rs
+++ b/src/promql/src/extension_plan/scalar_calculate.rs
@@ -46,7 +46,8 @@ use crate::error::{ColumnNotFoundSnafu, DataFusionPlanningSnafu, DeserializeSnaf
 /// `ScalarCalculate` is the custom logical plan to calculate
 /// [`scalar`](https://prometheus.io/docs/prometheus/latest/querying/functions/#scalar)
 /// in PromQL, return NaN when have multiple time series.
-/// return the time series as scalar value when only have one time series.
+///
+/// Return the time series as scalar value when only have one time series.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct ScalarCalculate {
     start: Millisecond,
diff --git a/src/promql/src/lib.rs b/src/promql/src/lib.rs
index a29fc032e9..42dbdb23a9 100644
--- a/src/promql/src/lib.rs
+++ b/src/promql/src/lib.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(option_get_or_insert_default)]
 #![feature(let_chains)]
 #![feature(map_try_insert)]
 
diff --git a/src/query/src/dist_plan/planner.rs b/src/query/src/dist_plan/planner.rs
index 833ff1c9bc..6f4f7831a9 100644
--- a/src/query/src/dist_plan/planner.rs
+++ b/src/query/src/dist_plan/planner.rs
@@ -40,6 +40,7 @@ use crate::error::{CatalogSnafu, TableNotFoundSnafu};
 use crate::region_query::RegionQueryHandlerRef;
 
 /// Planner for convert merge sort logical plan to physical plan
+///
 /// it is currently a fallback to sort, and doesn't change the execution plan:
 /// `MergeSort(MergeScan) -> Sort(MergeScan) - to physical plan -> ...`
 /// It should be applied after `DistExtensionPlanner`
diff --git a/src/query/src/lib.rs b/src/query/src/lib.rs
index 7eb82a1e60..5b119115e3 100644
--- a/src/query/src/lib.rs
+++ b/src/query/src/lib.rs
@@ -14,7 +14,6 @@
 
 #![feature(let_chains)]
 #![feature(int_roundings)]
-#![feature(option_get_or_insert_default)]
 #![feature(trait_upcasting)]
 
 mod analyze;
diff --git a/src/query/src/range_select/plan_rewrite.rs b/src/query/src/range_select/plan_rewrite.rs
index f7e0a3dd0d..3e6516a9f1 100644
--- a/src/query/src/range_select/plan_rewrite.rs
+++ b/src/query/src/range_select/plan_rewrite.rs
@@ -62,7 +62,7 @@ pub struct RangeExprRewriter<'a> {
     query_ctx: &'a QueryContextRef,
 }
 
-impl<'a> RangeExprRewriter<'a> {
+impl RangeExprRewriter<'_> {
     pub fn get_range_expr(&self, args: &[Expr], i: usize) -> DFResult<Expr> {
         match args.get(i) {
             Some(Expr::Column(column)) => {
@@ -263,7 +263,7 @@ macro_rules! inconsistent_check {
     };
 }
 
-impl<'a> TreeNodeRewriter for RangeExprRewriter<'a> {
+impl TreeNodeRewriter for RangeExprRewriter<'_> {
     type Node = Expr;
 
     fn f_down(&mut self, node: Expr) -> DFResult<Transformed<Expr>> {
diff --git a/src/script/src/python/rspython/vector_impl.rs b/src/script/src/python/rspython/vector_impl.rs
index a543c8295f..9529ce11b8 100644
--- a/src/script/src/python/rspython/vector_impl.rs
+++ b/src/script/src/python/rspython/vector_impl.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//! PyVectors' rustpython specify methods
+
 use arrow::compute::kernels::numeric;
 use common_time::date::Date;
 use common_time::datetime::DateTime;
@@ -40,7 +42,6 @@ use crate::python::ffi_types::vector::{
     arrow_rfloordiv, arrow_rsub, arrow_rtruediv, rspy_is_pyobj_scalar, wrap_result, PyVector,
 };
 use crate::python::rspython::utils::{is_instance, obj_cast_to};
-/// PyVectors' rustpython specify methods
 
 fn to_type_error(vm: &'_ VirtualMachine) -> impl FnOnce(String) -> PyBaseExceptionRef + '_ {
     |msg: String| vm.new_type_error(msg)
diff --git a/src/servers/src/export_metrics.rs b/src/servers/src/export_metrics.rs
index 4834940c0e..3a054a8988 100644
--- a/src/servers/src/export_metrics.rs
+++ b/src/servers/src/export_metrics.rs
@@ -31,7 +31,9 @@ use crate::error::{InvalidExportMetricsConfigSnafu, Result, SendPromRemoteReques
 use crate::prom_store::{snappy_compress, to_grpc_row_insert_requests};
 use crate::query_handler::PromStoreProtocolHandlerRef;
 
-/// Use to export the metrics generated by greptimedb, encoded to Prometheus [RemoteWrite format](https://prometheus.io/docs/concepts/remote_write_spec/),
+/// Use to export the metrics generated by greptimedb.
+///
+/// Encoded to Prometheus [RemoteWrite format](https://prometheus.io/docs/concepts/remote_write_spec/),
 /// and send to Prometheus remote-write compatible receiver (e.g. send to `greptimedb` itself)
 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(default)]
diff --git a/src/servers/src/query_handler.rs b/src/servers/src/query_handler.rs
index a1ad9997ba..618f366eb0 100644
--- a/src/servers/src/query_handler.rs
+++ b/src/servers/src/query_handler.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 //! All query handler traits for various request protocols, like SQL or GRPC.
+//!
 //! Instance that wishes to support certain request protocol, just implement the corresponding
 //! trait, the Server will handle codec for you.
 //!
@@ -131,6 +132,7 @@ pub trait OpenTelemetryProtocolHandler: LogHandler {
 }
 
 /// LogHandler is responsible for handling log related requests.
+///
 /// It should be able to insert logs and manage pipelines.
 /// The pipeline is a series of transformations that can be applied to logs.
 /// The pipeline is stored in the database and can be retrieved by name.
diff --git a/src/sql/src/parser.rs b/src/sql/src/parser.rs
index 56163e378b..bf62a1ad9b 100644
--- a/src/sql/src/parser.rs
+++ b/src/sql/src/parser.rs
@@ -37,9 +37,9 @@ pub struct ParserContext<'a> {
     pub(crate) sql: &'a str,
 }
 
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     /// Construct a new ParserContext.
-    pub fn new(dialect: &'a dyn Dialect, sql: &'a str) -> Result<ParserContext<'a>> {
+    pub fn new<'a>(dialect: &'a dyn Dialect, sql: &'a str) -> Result<ParserContext<'a>> {
         let parser = Parser::new(dialect)
             .with_options(ParserOptions::new().with_trailing_commas(true))
             .try_with_sql(sql)
@@ -55,7 +55,7 @@ impl<'a> ParserContext<'a> {
 
     /// Parses SQL with given dialect
     pub fn create_with_dialect(
-        sql: &'a str,
+        sql: &str,
         dialect: &dyn Dialect,
         _opts: ParseOptions,
     ) -> Result<Vec<Statement>> {
@@ -87,7 +87,7 @@ impl<'a> ParserContext<'a> {
         Ok(stmts)
     }
 
-    pub fn parse_table_name(sql: &'a str, dialect: &dyn Dialect) -> Result<ObjectName> {
+    pub fn parse_table_name(sql: &str, dialect: &dyn Dialect) -> Result<ObjectName> {
         let parser = Parser::new(dialect)
             .with_options(ParserOptions::new().with_trailing_commas(true))
             .try_with_sql(sql)
@@ -106,7 +106,7 @@ impl<'a> ParserContext<'a> {
         Ok(Self::canonicalize_object_name(raw_table_name))
     }
 
-    pub fn parse_function(sql: &'a str, dialect: &dyn Dialect) -> Result<Expr> {
+    pub fn parse_function(sql: &str, dialect: &dyn Dialect) -> Result<Expr> {
         let mut parser = Parser::new(dialect)
             .with_options(ParserOptions::new().with_trailing_commas(true))
             .try_with_sql(sql)
@@ -191,23 +191,20 @@ impl<'a> ParserContext<'a> {
     }
 
     /// Parses MySQL style 'PREPARE stmt_name FROM stmt' into a (stmt_name, stmt) tuple.
-    pub fn parse_mysql_prepare_stmt(
-        sql: &'a str,
-        dialect: &dyn Dialect,
-    ) -> Result<(String, String)> {
+    pub fn parse_mysql_prepare_stmt(sql: &str, dialect: &dyn Dialect) -> Result<(String, String)> {
         ParserContext::new(dialect, sql)?.parse_mysql_prepare()
     }
 
     /// Parses MySQL style 'EXECUTE stmt_name USING param_list' into a stmt_name string and a list of parameters.
     pub fn parse_mysql_execute_stmt(
-        sql: &'a str,
+        sql: &str,
         dialect: &dyn Dialect,
     ) -> Result<(String, Vec<Expr>)> {
         ParserContext::new(dialect, sql)?.parse_mysql_execute()
     }
 
     /// Parses MySQL style 'DEALLOCATE stmt_name' into a stmt_name string.
-    pub fn parse_mysql_deallocate_stmt(sql: &'a str, dialect: &dyn Dialect) -> Result<String> {
+    pub fn parse_mysql_deallocate_stmt(sql: &str, dialect: &dyn Dialect) -> Result<String> {
         ParserContext::new(dialect, sql)?.parse_deallocate()
     }
 
diff --git a/src/sql/src/parsers/admin_parser.rs b/src/sql/src/parsers/admin_parser.rs
index b9a118b2d4..a9644bc39d 100644
--- a/src/sql/src/parsers/admin_parser.rs
+++ b/src/sql/src/parsers/admin_parser.rs
@@ -22,7 +22,7 @@ use crate::statements::statement::Statement;
 
 /// `admin` extension parser: `admin function(arg1, arg2, ...)`
 /// or `admin function`
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     /// Parse `admin function(arg1, arg2, ...)` or `admin function` statement
     pub(crate) fn parse_admin_command(&mut self) -> Result<Statement> {
         let _token = self.parser.next_token();
diff --git a/src/sql/src/parsers/alter_parser.rs b/src/sql/src/parsers/alter_parser.rs
index 0473ef52ce..b6db219626 100644
--- a/src/sql/src/parsers/alter_parser.rs
+++ b/src/sql/src/parsers/alter_parser.rs
@@ -23,7 +23,7 @@ use crate::parser::ParserContext;
 use crate::statements::alter::{AlterTable, AlterTableOperation};
 use crate::statements::statement::Statement;
 
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     pub(crate) fn parse_alter(&mut self) -> Result<Statement> {
         let alter_table = self.parse_alter_table().context(error::SyntaxSnafu)?;
         Ok(Statement::Alter(alter_table))
@@ -46,7 +46,7 @@ impl<'a> ParserContext<'a> {
                 let location = if self.parser.parse_keyword(Keyword::FIRST) {
                     Some(AddColumnLocation::First)
                 } else if let Token::Word(word) = self.parser.peek_token().token {
-                    if word.value.to_ascii_uppercase() == "AFTER" {
+                    if word.value.eq_ignore_ascii_case("AFTER") {
                         let _ = self.parser.next_token();
                         let name = Self::canonicalize_identifier(self.parse_identifier()?);
                         Some(AddColumnLocation::After {
diff --git a/src/sql/src/parsers/copy_parser.rs b/src/sql/src/parsers/copy_parser.rs
index 693e2268bc..5bf270129f 100644
--- a/src/sql/src/parsers/copy_parser.rs
+++ b/src/sql/src/parsers/copy_parser.rs
@@ -28,7 +28,7 @@ pub type With = HashMap<String, String>;
 pub type Connection = HashMap<String, String>;
 
 // COPY tbl TO 'output.parquet';
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     pub(crate) fn parse_copy(&mut self) -> Result<Statement> {
         let _ = self.parser.next_token();
         let next = self.parser.peek_token();
diff --git a/src/sql/src/parsers/create_parser.rs b/src/sql/src/parsers/create_parser.rs
index 8b7eb420d2..764c93573d 100644
--- a/src/sql/src/parsers/create_parser.rs
+++ b/src/sql/src/parsers/create_parser.rs
@@ -630,7 +630,7 @@ impl<'a> ParserContext<'a> {
         })
     }
 
-    fn parse_optional_column_option(parser: &mut Parser<'a>) -> Result<Option<ColumnOption>> {
+    fn parse_optional_column_option(parser: &mut Parser<'_>) -> Result<Option<ColumnOption>> {
         if parser.parse_keywords(&[Keyword::CHARACTER, Keyword::SET]) {
             Ok(Some(ColumnOption::CharacterSet(
                 parser.parse_object_name(false).context(SyntaxSnafu)?,
@@ -681,7 +681,7 @@ impl<'a> ParserContext<'a> {
     }
 
     fn parse_column_extensions(
-        parser: &mut Parser<'a>,
+        parser: &mut Parser<'_>,
         column_name: &Ident,
         column_type: &DataType,
         column_extensions: &mut ColumnExtensions,
diff --git a/src/sql/src/parsers/deallocate_parser.rs b/src/sql/src/parsers/deallocate_parser.rs
index e53337ac9b..3fac296d86 100644
--- a/src/sql/src/parsers/deallocate_parser.rs
+++ b/src/sql/src/parsers/deallocate_parser.rs
@@ -18,7 +18,7 @@ use sqlparser::keywords::Keyword;
 use crate::error::{Result, SyntaxSnafu};
 use crate::parser::ParserContext;
 
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     /// Parses MySQL style 'PREPARE stmt_name' into a stmt_name string.
     pub(crate) fn parse_deallocate(&mut self) -> Result<String> {
         self.parser
diff --git a/src/sql/src/parsers/delete_parser.rs b/src/sql/src/parsers/delete_parser.rs
index 626a8e77c0..b26b709374 100644
--- a/src/sql/src/parsers/delete_parser.rs
+++ b/src/sql/src/parsers/delete_parser.rs
@@ -21,7 +21,7 @@ use crate::statements::delete::Delete;
 use crate::statements::statement::Statement;
 
 /// DELETE statement parser implementation
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     pub(crate) fn parse_delete(&mut self) -> Result<Statement> {
         let _ = self.parser.next_token();
         let spstatement = self.parser.parse_delete().context(error::SyntaxSnafu)?;
diff --git a/src/sql/src/parsers/describe_parser.rs b/src/sql/src/parsers/describe_parser.rs
index 33a65da6fa..f4449cf575 100644
--- a/src/sql/src/parsers/describe_parser.rs
+++ b/src/sql/src/parsers/describe_parser.rs
@@ -21,7 +21,7 @@ use crate::statements::describe::DescribeTable;
 use crate::statements::statement::Statement;
 
 /// DESCRIBE statement parser implementation
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     pub(crate) fn parse_describe(&mut self) -> Result<Statement> {
         if self.matches_keyword(Keyword::TABLE) {
             let _ = self.parser.next_token();
diff --git a/src/sql/src/parsers/drop_parser.rs b/src/sql/src/parsers/drop_parser.rs
index 0934598c6c..d40338e85f 100644
--- a/src/sql/src/parsers/drop_parser.rs
+++ b/src/sql/src/parsers/drop_parser.rs
@@ -22,7 +22,7 @@ use crate::statements::drop::{DropDatabase, DropFlow, DropTable, DropView};
 use crate::statements::statement::Statement;
 
 /// DROP statement parser implementation
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     pub(crate) fn parse_drop(&mut self) -> Result<Statement> {
         let _ = self.parser.next_token();
         match self.parser.peek_token().token {
diff --git a/src/sql/src/parsers/execute_parser.rs b/src/sql/src/parsers/execute_parser.rs
index 67b3e8b669..454b024663 100644
--- a/src/sql/src/parsers/execute_parser.rs
+++ b/src/sql/src/parsers/execute_parser.rs
@@ -20,7 +20,7 @@ use sqlparser::parser::Parser;
 use crate::error::{Result, SyntaxSnafu};
 use crate::parser::ParserContext;
 
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     /// Parses MySQL style 'EXECUTE stmt_name USING param_list' into a stmt_name string and a list of parameters.
     /// Only use for MySQL. for PostgreSQL, use `sqlparser::parser::Parser::parse_execute` instead.
     pub(crate) fn parse_mysql_execute(&mut self) -> Result<(String, Vec<Expr>)> {
diff --git a/src/sql/src/parsers/explain_parser.rs b/src/sql/src/parsers/explain_parser.rs
index 5e57a6c8ad..719a0663cc 100644
--- a/src/sql/src/parsers/explain_parser.rs
+++ b/src/sql/src/parsers/explain_parser.rs
@@ -21,7 +21,7 @@ use crate::statements::explain::Explain;
 use crate::statements::statement::Statement;
 
 /// EXPLAIN statement parser implementation
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     pub(crate) fn parse_explain(&mut self) -> Result<Statement> {
         let explain_statement = self
             .parser
diff --git a/src/sql/src/parsers/insert_parser.rs b/src/sql/src/parsers/insert_parser.rs
index d83c2a102f..c65fbba72b 100644
--- a/src/sql/src/parsers/insert_parser.rs
+++ b/src/sql/src/parsers/insert_parser.rs
@@ -21,7 +21,7 @@ use crate::statements::insert::Insert;
 use crate::statements::statement::Statement;
 
 /// INSERT statement parser implementation
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     pub(crate) fn parse_insert(&mut self) -> Result<Statement> {
         let _ = self.parser.next_token();
         let spstatement = self.parser.parse_insert().context(error::SyntaxSnafu)?;
diff --git a/src/sql/src/parsers/prepare_parser.rs b/src/sql/src/parsers/prepare_parser.rs
index a0fc07456b..c5c8c808b7 100644
--- a/src/sql/src/parsers/prepare_parser.rs
+++ b/src/sql/src/parsers/prepare_parser.rs
@@ -19,7 +19,7 @@ use sqlparser::tokenizer::Token;
 use crate::error::{Result, SyntaxSnafu};
 use crate::parser::ParserContext;
 
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     /// Parses MySQL style 'PREPARE stmt_name FROM stmt' into a (stmt_name, stmt) tuple.
     /// Only use for MySQL. for PostgreSQL, use `sqlparser::parser::Parser::parse_prepare` instead.
     pub(crate) fn parse_mysql_prepare(&mut self) -> Result<(String, String)> {
diff --git a/src/sql/src/parsers/query_parser.rs b/src/sql/src/parsers/query_parser.rs
index c88a0b1e35..f2fbea1b3c 100644
--- a/src/sql/src/parsers/query_parser.rs
+++ b/src/sql/src/parsers/query_parser.rs
@@ -19,7 +19,7 @@ use crate::parser::ParserContext;
 use crate::statements::query::Query;
 use crate::statements::statement::Statement;
 
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     /// Parses select and it's variants.
     pub(crate) fn parse_query(&mut self) -> Result<Statement> {
         let spquery = self.parser.parse_query().context(error::SyntaxSnafu)?;
diff --git a/src/sql/src/parsers/set_var_parser.rs b/src/sql/src/parsers/set_var_parser.rs
index f00ccf2d36..e2a7db9d08 100644
--- a/src/sql/src/parsers/set_var_parser.rs
+++ b/src/sql/src/parsers/set_var_parser.rs
@@ -22,7 +22,7 @@ use crate::statements::set_variables::SetVariables;
 use crate::statements::statement::Statement;
 
 /// SET variables statement parser implementation
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     pub(crate) fn parse_set_variables(&mut self) -> Result<Statement> {
         let _ = self.parser.next_token();
         let spstatement = self.parser.parse_set().context(error::SyntaxSnafu)?;
diff --git a/src/sql/src/parsers/show_parser.rs b/src/sql/src/parsers/show_parser.rs
index 8c7955b4ec..d1530c1fcb 100644
--- a/src/sql/src/parsers/show_parser.rs
+++ b/src/sql/src/parsers/show_parser.rs
@@ -28,7 +28,7 @@ use crate::statements::show::{
 use crate::statements::statement::Statement;
 
 /// SHOW statement parser implementation
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     /// Parses SHOW statements
     /// todo(hl) support `show settings`/`show create`/`show users` etc.
     pub(crate) fn parse_show(&mut self) -> Result<Statement> {
diff --git a/src/sql/src/parsers/tql_parser.rs b/src/sql/src/parsers/tql_parser.rs
index 078cf0f726..2533fe1ef6 100644
--- a/src/sql/src/parsers/tql_parser.rs
+++ b/src/sql/src/parsers/tql_parser.rs
@@ -39,7 +39,7 @@ use crate::parsers::error::{EvaluationSnafu, ParserSnafu, TQLError};
 /// - `TQL EVAL <query>`
 /// - `TQL EXPLAIN [VERBOSE] <query>`
 /// - `TQL ANALYZE [VERBOSE] <query>`
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     pub(crate) fn parse_tql(&mut self) -> Result<Statement> {
         let _ = self.parser.next_token();
 
diff --git a/src/sql/src/parsers/truncate_parser.rs b/src/sql/src/parsers/truncate_parser.rs
index 9cb27e2203..3fa136baff 100644
--- a/src/sql/src/parsers/truncate_parser.rs
+++ b/src/sql/src/parsers/truncate_parser.rs
@@ -21,7 +21,7 @@ use crate::statements::statement::Statement;
 use crate::statements::truncate::TruncateTable;
 
 /// `TRUNCATE [TABLE] table_name;`
-impl<'a> ParserContext<'a> {
+impl ParserContext<'_> {
     pub(crate) fn parse_truncate(&mut self) -> Result<Statement> {
         let _ = self.parser.next_token();
         let _ = self.parser.parse_keyword(Keyword::TABLE);
diff --git a/src/sql/src/statements/tql.rs b/src/sql/src/statements/tql.rs
index a963849a70..0f7a85f95a 100644
--- a/src/sql/src/statements/tql.rs
+++ b/src/sql/src/statements/tql.rs
@@ -131,6 +131,7 @@ impl Display for TqlAnalyze {
 }
 
 /// Intermediate structure used to unify parameter mappings for various TQL operations.
+///
 /// This struct serves as a common parameter container for parsing TQL queries
 /// and constructing corresponding TQL operations: `TqlEval`, `TqlAnalyze` or `TqlExplain`.
 #[derive(Debug)]
diff --git a/src/sql/src/util.rs b/src/sql/src/util.rs
index b81b5c7337..7d8cf3df50 100644
--- a/src/sql/src/util.rs
+++ b/src/sql/src/util.rs
@@ -25,7 +25,7 @@ pub fn format_raw_object_name(name: &ObjectName) -> String {
         name: &'a ObjectName,
     }
 
-    impl<'a> Display for Inner<'a> {
+    impl Display for Inner<'_> {
         fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
             let mut delim = "";
             for ident in self.name.0.iter() {
diff --git a/src/store-api/src/manifest/action.rs b/src/store-api/src/manifest/action.rs
index 2fcc9aca2b..c9b0d3e7d0 100644
--- a/src/store-api/src/manifest/action.rs
+++ b/src/store-api/src/manifest/action.rs
@@ -31,8 +31,9 @@ pub fn supported_protocol_version() -> (ProtocolVersion, ProtocolVersion) {
 }
 
 /// Protocol action that used to block older clients from reading or writing the log when backwards
-/// incompatible changes are made to the protocol. clients should be tolerant of messages and
-/// fields that they do not understand.
+/// incompatible changes are made to the protocol.
+///
+/// clients should be tolerant of messages and fields that they do not understand.
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 pub struct ProtocolAction {
     pub min_reader_version: ProtocolVersion,
diff --git a/src/store-api/src/path_utils.rs b/src/store-api/src/path_utils.rs
index 122a19660d..4585e4b505 100644
--- a/src/store-api/src/path_utils.rs
+++ b/src/store-api/src/path_utils.rs
@@ -13,9 +13,9 @@
 // limitations under the License.
 
 //! Path constants for table engines, cluster states and WAL
+//! All paths relative to data_home(file storage) or root path(S3, OSS etc).
 
 use crate::storage::{RegionId, RegionNumber, TableId};
-/// All paths relative to data_home(file storage) or root path(S3, OSS etc).
 
 /// WAL dir for local file storage
 pub const WAL_DIR: &str = "wal/";
@@ -45,7 +45,9 @@ pub fn region_dir(path: &str, region_id: RegionId) -> String {
     )
 }
 
-/// get_storage_path returns the storage path from the region_dir. It will always return the storage path if the region_dir is valid, otherwise None.
+/// get_storage_path returns the storage path from the region_dir.
+///
+/// It will always return the storage path if the region_dir is valid, otherwise None.
 /// The storage path is constructed from the catalog and schema, which are generated by `common_meta::ddl::utils::region_storage_path`.
 /// We can extract the catalog and schema from the region_dir by following example:
 /// ```
diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs
index 86e96a161b..2a8a0a487f 100644
--- a/src/store-api/src/region_engine.rs
+++ b/src/store-api/src/region_engine.rs
@@ -261,6 +261,7 @@ impl ScannerProperties {
 }
 
 /// A scanner that provides a way to scan the region concurrently.
+///
 /// The scanner splits the region into partitions so that each partition can be scanned concurrently.
 /// You can use this trait to implement an [`ExecutionPlan`](datafusion_physical_plan::ExecutionPlan).
 pub trait RegionScanner: Debug + DisplayAs + Send {
diff --git a/src/table/src/metadata.rs b/src/table/src/metadata.rs
index d01eb82a68..ec9404a8b1 100644
--- a/src/table/src/metadata.rs
+++ b/src/table/src/metadata.rs
@@ -100,7 +100,8 @@ pub struct TableIdent {
     pub version: TableVersion,
 }
 
-/// The table metadata
+/// The table metadata.
+///
 /// Note: if you add new fields to this struct, please ensure 'new_meta_builder' function works.
 /// TODO(dennis): find a better way to ensure 'new_meta_builder' works when adding new fields.
 #[derive(Clone, Debug, Builder, PartialEq, Eq)]
diff --git a/src/table/src/predicate/stats.rs b/src/table/src/predicate/stats.rs
index e1cc48c320..03f84a110b 100644
--- a/src/table/src/predicate/stats.rs
+++ b/src/table/src/predicate/stats.rs
@@ -92,7 +92,7 @@ macro_rules! impl_min_max_values {
     }};
 }
 
-impl<'a> PruningStatistics for RowGroupPruningStatistics<'a> {
+impl PruningStatistics for RowGroupPruningStatistics<'_> {
     fn min_values(&self, column: &Column) -> Option<ArrayRef> {
         impl_min_max_values!(self, column, min)
     }
diff --git a/src/table/src/stats.rs b/src/table/src/stats.rs
index 23b19ddcea..0896556d8c 100644
--- a/src/table/src/stats.rs
+++ b/src/table/src/stats.rs
@@ -14,7 +14,8 @@
 
 use datatypes::value::Value;
 
-/// Statistics for a relation
+/// Statistics for a relation.
+///
 /// Fields are optional and can be inexact because the sources
 /// sometimes provide approximate estimates for performance reasons
 /// and the transformations output are not always predictable.
diff --git a/src/table/src/table_reference.rs b/src/table/src/table_reference.rs
index 3181efddd7..db086240af 100644
--- a/src/table/src/table_reference.rs
+++ b/src/table/src/table_reference.rs
@@ -44,7 +44,7 @@ impl<'a> TableReference<'a> {
     }
 }
 
-impl<'a> Display for TableReference<'a> {
+impl Display for TableReference<'_> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "{}.{}.{}", self.catalog, self.schema, self.table)
     }

From 4e9c25104141c0b2508ed91c34309f2edfc99eaf Mon Sep 17 00:00:00 2001
From: Kaifeng Zheng <100595273+Kev1n8@users.noreply.github.com>
Date: Fri, 25 Oct 2024 11:13:34 +0800
Subject: [PATCH 106/128] feat: add json_path_match udf (#4864)

* add json_path_match udf

* sql tests for json_path_match

* fix clippy & comment

* fix null value behavior

* added null tests

* adjust function's behavior on nulls

* update test cases

* fix null check of json
---
 src/common/function/src/scalars/json.rs       |   2 +
 .../src/scalars/json/json_path_match.rs       | 202 ++++++++++++++++++
 .../common/function/json/json.result          |  57 +++++
 .../standalone/common/function/json/json.sql  |  16 ++
 4 files changed, 277 insertions(+)
 create mode 100644 src/common/function/src/scalars/json/json_path_match.rs

diff --git a/src/common/function/src/scalars/json.rs b/src/common/function/src/scalars/json.rs
index 133eb145a7..2c420c1661 100644
--- a/src/common/function/src/scalars/json.rs
+++ b/src/common/function/src/scalars/json.rs
@@ -16,6 +16,7 @@ use std::sync::Arc;
 mod json_get;
 mod json_is;
 mod json_path_exists;
+mod json_path_match;
 mod json_to_string;
 mod parse_json;
 
@@ -49,5 +50,6 @@ impl JsonFunction {
         registry.register(Arc::new(JsonIsObject));
 
         registry.register(Arc::new(json_path_exists::JsonPathExistsFunction));
+        registry.register(Arc::new(json_path_match::JsonPathMatchFunction));
     }
 }
diff --git a/src/common/function/src/scalars/json/json_path_match.rs b/src/common/function/src/scalars/json/json_path_match.rs
new file mode 100644
index 0000000000..8ea1bf082b
--- /dev/null
+++ b/src/common/function/src/scalars/json/json_path_match.rs
@@ -0,0 +1,202 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fmt::{self, Display};
+
+use common_query::error::{InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu};
+use common_query::prelude::Signature;
+use datafusion::logical_expr::Volatility;
+use datatypes::data_type::ConcreteDataType;
+use datatypes::prelude::VectorRef;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::vectors::{BooleanVectorBuilder, MutableVector};
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+
+/// Check if the given JSON data match the given JSON path's predicate.
+#[derive(Clone, Debug, Default)]
+pub struct JsonPathMatchFunction;
+
+const NAME: &str = "json_path_match";
+
+impl Function for JsonPathMatchFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::boolean_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::exact(
+            vec![
+                ConcreteDataType::json_datatype(),
+                ConcreteDataType::string_datatype(),
+            ],
+            Volatility::Immutable,
+        )
+    }
+
+    fn eval(&self, _func_ctx: FunctionContext, columns: &[VectorRef]) -> Result<VectorRef> {
+        ensure!(
+            columns.len() == 2,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly two, have: {}",
+                    columns.len()
+                ),
+            }
+        );
+        let jsons = &columns[0];
+        let paths = &columns[1];
+
+        let size = jsons.len();
+        let mut results = BooleanVectorBuilder::with_capacity(size);
+
+        for i in 0..size {
+            let json = jsons.get_ref(i);
+            let path = paths.get_ref(i);
+
+            match json.data_type() {
+                // JSON data type uses binary vector
+                ConcreteDataType::Binary(_) => {
+                    let json = json.as_binary();
+                    let path = path.as_string();
+                    let result = match (json, path) {
+                        (Ok(Some(json)), Ok(Some(path))) => {
+                            if !jsonb::is_null(json) {
+                                let json_path = jsonb::jsonpath::parse_json_path(path.as_bytes());
+                                match json_path {
+                                    Ok(json_path) => jsonb::path_match(json, json_path).ok(),
+                                    Err(_) => None,
+                                }
+                            } else {
+                                None
+                            }
+                        }
+                        _ => None,
+                    };
+
+                    results.push(result);
+                }
+
+                _ => {
+                    return UnsupportedInputDataTypeSnafu {
+                        function: NAME,
+                        datatypes: columns.iter().map(|c| c.data_type()).collect::<Vec<_>>(),
+                    }
+                    .fail();
+                }
+            }
+        }
+
+        Ok(results.to_vector())
+    }
+}
+
+impl Display for JsonPathMatchFunction {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "JSON_PATH_MATCH")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use common_query::prelude::TypeSignature;
+    use datatypes::vectors::{BinaryVector, StringVector};
+
+    use super::*;
+
+    #[test]
+    fn test_json_path_match_function() {
+        let json_path_match = JsonPathMatchFunction;
+
+        assert_eq!("json_path_match", json_path_match.name());
+        assert_eq!(
+            ConcreteDataType::boolean_datatype(),
+            json_path_match
+                .return_type(&[ConcreteDataType::json_datatype()])
+                .unwrap()
+        );
+
+        assert!(matches!(json_path_match.signature(),
+                         Signature {
+                             type_signature: TypeSignature::Exact(valid_types),
+                             volatility: Volatility::Immutable
+                         } if valid_types == vec![ConcreteDataType::json_datatype(), ConcreteDataType::string_datatype()],
+        ));
+
+        let json_strings = [
+            Some(r#"{"a": {"b": 2}, "b": 2, "c": 3}"#.to_string()),
+            Some(r#"{"a": 1, "b": [1,2,3]}"#.to_string()),
+            Some(r#"{"a": 1 ,"b": [1,2,3]}"#.to_string()),
+            Some(r#"[1,2,3]"#.to_string()),
+            Some(r#"{"a":1,"b":[1,2,3]}"#.to_string()),
+            Some(r#"null"#.to_string()),
+            Some(r#"null"#.to_string()),
+        ];
+
+        let paths = vec![
+            Some("$.a.b == 2".to_string()),
+            Some("$.b[1 to last] >= 2".to_string()),
+            Some("$.c > 0".to_string()),
+            Some("$[0 to last] > 0".to_string()),
+            Some(r#"null"#.to_string()),
+            Some("$.c > 0".to_string()),
+            Some(r#"null"#.to_string()),
+        ];
+
+        let results = [
+            Some(true),
+            Some(true),
+            Some(false),
+            Some(true),
+            None,
+            None,
+            None,
+        ];
+
+        let jsonbs = json_strings
+            .into_iter()
+            .map(|s| s.map(|json| jsonb::parse_value(json.as_bytes()).unwrap().to_vec()))
+            .collect::<Vec<_>>();
+
+        let json_vector = BinaryVector::from(jsonbs);
+        let path_vector = StringVector::from(paths);
+        let args: Vec<VectorRef> = vec![Arc::new(json_vector), Arc::new(path_vector)];
+        let vector = json_path_match
+            .eval(FunctionContext::default(), &args)
+            .unwrap();
+
+        assert_eq!(7, vector.len());
+        for (i, expected) in results.iter().enumerate() {
+            let result = vector.get_ref(i);
+
+            match expected {
+                Some(expected_value) => {
+                    assert!(!result.is_null());
+                    let result_value = result.as_boolean().unwrap().unwrap();
+                    assert_eq!(*expected_value, result_value);
+                }
+                None => {
+                    assert!(result.is_null());
+                }
+            }
+        }
+    }
+}
diff --git a/tests/cases/standalone/common/function/json/json.result b/tests/cases/standalone/common/function/json/json.result
index bae3313548..42db0b263e 100644
--- a/tests/cases/standalone/common/function/json/json.result
+++ b/tests/cases/standalone/common/function/json/json.result
@@ -31,3 +31,60 @@ SELECT json_path_exists(parse_json('[1, 2]'), '[2]');
 | false                                                    |
 +----------------------------------------------------------+
 
+SELECT json_path_exists(parse_json('[1, 2]'), 'null');
+
++-----------------------------------------------------------+
+| json_path_exists(parse_json(Utf8("[1, 2]")),Utf8("null")) |
++-----------------------------------------------------------+
+| false                                                     |
++-----------------------------------------------------------+
+
+SELECT json_path_exists(parse_json('null'), '$.a');
+
++--------------------------------------------------------+
+| json_path_exists(parse_json(Utf8("null")),Utf8("$.a")) |
++--------------------------------------------------------+
+| false                                                  |
++--------------------------------------------------------+
+
+--- json_path_match ---
+SELECT json_path_match(parse_json('{"a": 1, "b": 2}'), '$.a == 1');
+
++------------------------------------------------------------------------+
+| json_path_match(parse_json(Utf8("{"a": 1, "b": 2}")),Utf8("$.a == 1")) |
++------------------------------------------------------------------------+
+| true                                                                   |
++------------------------------------------------------------------------+
+
+SELECT json_path_match(parse_json('{"a":1,"b":[1,2,3]}'), '$.b[0] > 1');
+
++-----------------------------------------------------------------------------+
+| json_path_match(parse_json(Utf8("{"a":1,"b":[1,2,3]}")),Utf8("$.b[0] > 1")) |
++-----------------------------------------------------------------------------+
+| false                                                                       |
++-----------------------------------------------------------------------------+
+
+SELECT json_path_match(parse_json('{"a":1,"b":[1,2,3]}'), '$.b[1 to last] >= 2');
+
++--------------------------------------------------------------------------------------+
+| json_path_match(parse_json(Utf8("{"a":1,"b":[1,2,3]}")),Utf8("$.b[1 to last] >= 2")) |
++--------------------------------------------------------------------------------------+
+| true                                                                                 |
++--------------------------------------------------------------------------------------+
+
+SELECT json_path_match(parse_json('{"a":1,"b":[1,2,3]}'), 'null');
+
++-----------------------------------------------------------------------+
+| json_path_match(parse_json(Utf8("{"a":1,"b":[1,2,3]}")),Utf8("null")) |
++-----------------------------------------------------------------------+
+|                                                                       |
++-----------------------------------------------------------------------+
+
+SELECT json_path_match(parse_json('null'), '$.a == 1');
+
++------------------------------------------------------------+
+| json_path_match(parse_json(Utf8("null")),Utf8("$.a == 1")) |
++------------------------------------------------------------+
+|                                                            |
++------------------------------------------------------------+
+
diff --git a/tests/cases/standalone/common/function/json/json.sql b/tests/cases/standalone/common/function/json/json.sql
index 76914e2575..8980be33e3 100644
--- a/tests/cases/standalone/common/function/json/json.sql
+++ b/tests/cases/standalone/common/function/json/json.sql
@@ -6,3 +6,19 @@ SELECT json_path_exists(parse_json('{"a": 1, "b": 2}'), '$.c');
 SELECT json_path_exists(parse_json('[1, 2]'), '[0]');
 
 SELECT json_path_exists(parse_json('[1, 2]'), '[2]');
+
+SELECT json_path_exists(parse_json('[1, 2]'), 'null');
+
+SELECT json_path_exists(parse_json('null'), '$.a');
+
+--- json_path_match ---
+
+SELECT json_path_match(parse_json('{"a": 1, "b": 2}'), '$.a == 1');
+
+SELECT json_path_match(parse_json('{"a":1,"b":[1,2,3]}'), '$.b[0] > 1');
+
+SELECT json_path_match(parse_json('{"a":1,"b":[1,2,3]}'), '$.b[1 to last] >= 2');
+
+SELECT json_path_match(parse_json('{"a":1,"b":[1,2,3]}'), 'null');
+
+SELECT json_path_match(parse_json('null'), '$.a == 1');

From 32a0023010fa33d386143a8f314c3a85dd42db91 Mon Sep 17 00:00:00 2001
From: shuiyisong <113876041+shuiyisong@users.noreply.github.com>
Date: Fri, 25 Oct 2024 11:45:24 +0800
Subject: [PATCH 107/128] chore: add schema urls to otlp logs (#4876)

* chore: add schema urls to otlp logs table

* chore: update meter-macros version to remove anymap warning

* chore: change span id and trace id to field
---
 Cargo.lock                        |  26 +--
 Cargo.toml                        |   2 +-
 src/frontend/src/instance/otlp.rs |   1 +
 src/servers/src/http/otlp.rs      |  18 +-
 src/servers/src/otlp/logs.rs      | 288 ++++++++++++++++--------------
 tests-integration/tests/http.rs   |  77 +++++---
 6 files changed, 219 insertions(+), 193 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index f447ea1148..d4db1185f3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -200,12 +200,6 @@ version = "1.0.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6"
 
-[[package]]
-name = "anymap"
-version = "1.0.0-beta.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f1f8f5a6f3d50d89e3797d7593a50f96bb2aaa20ca0cc7be1fb673232c91d72"
-
 [[package]]
 name = "anymap2"
 version = "0.13.0"
@@ -6424,16 +6418,6 @@ dependencies = [
  "url",
 ]
 
-[[package]]
-name = "meter-core"
-version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-meter.git?rev=80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd#80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd"
-dependencies = [
- "anymap",
- "once_cell",
- "parking_lot 0.12.3",
-]
-
 [[package]]
 name = "meter-core"
 version = "0.1.0"
@@ -6447,9 +6431,9 @@ dependencies = [
 [[package]]
 name = "meter-macros"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-meter.git?rev=80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd#80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd"
+source = "git+https://github.com/GreptimeTeam/greptime-meter.git?rev=a10facb353b41460eeb98578868ebf19c2084fac#a10facb353b41460eeb98578868ebf19c2084fac"
 dependencies = [
- "meter-core 0.1.0 (git+https://github.com/GreptimeTeam/greptime-meter.git?rev=80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd)",
+ "meter-core",
 ]
 
 [[package]]
@@ -7614,7 +7598,7 @@ dependencies = [
  "futures-util",
  "lazy_static",
  "meta-client",
- "meter-core 0.1.0 (git+https://github.com/GreptimeTeam/greptime-meter.git?rev=a10facb353b41460eeb98578868ebf19c2084fac)",
+ "meter-core",
  "meter-macros",
  "moka",
  "object-store",
@@ -9018,7 +9002,7 @@ dependencies = [
  "humantime",
  "itertools 0.10.5",
  "lazy_static",
- "meter-core 0.1.0 (git+https://github.com/GreptimeTeam/greptime-meter.git?rev=a10facb353b41460eeb98578868ebf19c2084fac)",
+ "meter-core",
  "meter-macros",
  "num",
  "num-traits",
@@ -10903,7 +10887,7 @@ dependencies = [
  "common-telemetry",
  "common-time",
  "derive_builder 0.12.0",
- "meter-core 0.1.0 (git+https://github.com/GreptimeTeam/greptime-meter.git?rev=a10facb353b41460eeb98578868ebf19c2084fac)",
+ "meter-core",
  "snafu 0.8.5",
  "sql",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index 60d8215634..daf79b17f9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -261,7 +261,7 @@ tokio-rustls = { git = "https://github.com/GreptimeTeam/tokio-rustls" }
 
 [workspace.dependencies.meter-macros]
 git = "https://github.com/GreptimeTeam/greptime-meter.git"
-rev = "80eb97c24c88af4dd9a86f8bbaf50e741d4eb8cd"
+rev = "a10facb353b41460eeb98578868ebf19c2084fac"
 
 [profile.release]
 debug = 1
diff --git a/src/frontend/src/instance/otlp.rs b/src/frontend/src/instance/otlp.rs
index a2245d8fb8..0c12658b37 100644
--- a/src/frontend/src/instance/otlp.rs
+++ b/src/frontend/src/instance/otlp.rs
@@ -113,6 +113,7 @@ impl OpenTelemetryProtocolHandler for Instance {
             .plugins
             .get::<OpenTelemetryProtocolInterceptorRef<servers::error::Error>>();
         interceptor_ref.pre_execute(ctx.clone())?;
+
         let (requests, rows) = otlp::logs::to_grpc_insert_requests(request, pipeline, table_name)?;
         self.handle_log_inserts(requests, ctx)
             .await
diff --git a/src/servers/src/http/otlp.rs b/src/servers/src/http/otlp.rs
index 51f2683e1b..6e5a583c0d 100644
--- a/src/servers/src/http/otlp.rs
+++ b/src/servers/src/http/otlp.rs
@@ -41,7 +41,7 @@ use snafu::prelude::*;
 
 use super::header::constants::GREPTIME_LOG_EXTRACT_KEYS_HEADER_NAME;
 use super::header::{write_cost_header_map, CONTENT_TYPE_PROTOBUF};
-use crate::error::{self, Result};
+use crate::error::{self, PipelineSnafu, Result};
 use crate::http::header::constants::{
     GREPTIME_LOG_PIPELINE_NAME_HEADER_NAME, GREPTIME_LOG_PIPELINE_VERSION_HEADER_NAME,
     GREPTIME_LOG_TABLE_NAME_HEADER_NAME,
@@ -227,15 +227,9 @@ pub async fn logs(
         .start_timer();
     let request = ExportLogsServiceRequest::decode(bytes).context(error::DecodeOtlpRequestSnafu)?;
 
-    let pipeline_way;
-    if let Some(pipeline_name) = &pipeline_info.pipeline_name {
+    let pipeline_way = if let Some(pipeline_name) = &pipeline_info.pipeline_name {
         let pipeline_version =
-            to_pipeline_version(pipeline_info.pipeline_version).map_err(|_| {
-                error::InvalidParameterSnafu {
-                    reason: GREPTIME_LOG_PIPELINE_VERSION_HEADER_NAME,
-                }
-                .build()
-            })?;
+            to_pipeline_version(pipeline_info.pipeline_version).context(PipelineSnafu)?;
         let pipeline = match handler
             .get_pipeline(pipeline_name, pipeline_version, query_ctx.clone())
             .await
@@ -245,10 +239,10 @@ pub async fn logs(
                 return Err(e);
             }
         };
-        pipeline_way = PipelineWay::Custom(pipeline);
+        PipelineWay::Custom(pipeline)
     } else {
-        pipeline_way = PipelineWay::OtlpLog(Box::new(select_info));
-    }
+        PipelineWay::OtlpLog(Box::new(select_info))
+    };
 
     handler
         .logs(request, pipeline_way, table_info.table_name, query_ctx)
diff --git a/src/servers/src/otlp/logs.rs b/src/servers/src/otlp/logs.rs
index 5faaced461..8f31a1db06 100644
--- a/src/servers/src/otlp/logs.rs
+++ b/src/servers/src/otlp/logs.rs
@@ -185,47 +185,6 @@ fn log_to_pipeline_value(
 
 fn build_otlp_logs_identity_schema() -> Vec<ColumnSchema> {
     [
-        (
-            "scope_name",
-            ColumnDataType::String,
-            SemanticType::Tag,
-            None,
-            None,
-        ),
-        (
-            "scope_version",
-            ColumnDataType::String,
-            SemanticType::Field,
-            None,
-            None,
-        ),
-        (
-            "scope_attributes",
-            ColumnDataType::Binary,
-            SemanticType::Field,
-            Some(ColumnDataTypeExtension {
-                type_ext: Some(TypeExt::JsonType(JsonTypeExtension::JsonBinary.into())),
-            }),
-            None,
-        ),
-        (
-            "resource_attributes",
-            ColumnDataType::Binary,
-            SemanticType::Field,
-            Some(ColumnDataTypeExtension {
-                type_ext: Some(TypeExt::JsonType(JsonTypeExtension::JsonBinary.into())),
-            }),
-            None,
-        ),
-        (
-            "log_attributes",
-            ColumnDataType::Binary,
-            SemanticType::Field,
-            Some(ColumnDataTypeExtension {
-                type_ext: Some(TypeExt::JsonType(JsonTypeExtension::JsonBinary.into())),
-            }),
-            None,
-        ),
         (
             "timestamp",
             ColumnDataType::TimestampNanosecond,
@@ -233,30 +192,16 @@ fn build_otlp_logs_identity_schema() -> Vec<ColumnSchema> {
             None,
             None,
         ),
-        (
-            "observed_timestamp",
-            ColumnDataType::TimestampNanosecond,
-            SemanticType::Field,
-            None,
-            None,
-        ),
         (
             "trace_id",
             ColumnDataType::String,
-            SemanticType::Tag,
+            SemanticType::Field,
             None,
             None,
         ),
         (
             "span_id",
             ColumnDataType::String,
-            SemanticType::Tag,
-            None,
-            None,
-        ),
-        (
-            "trace_flags",
-            ColumnDataType::Uint32,
             SemanticType::Field,
             None,
             None,
@@ -287,6 +232,68 @@ fn build_otlp_logs_identity_schema() -> Vec<ColumnSchema> {
                 )]),
             }),
         ),
+        (
+            "log_attributes",
+            ColumnDataType::Binary,
+            SemanticType::Field,
+            Some(ColumnDataTypeExtension {
+                type_ext: Some(TypeExt::JsonType(JsonTypeExtension::JsonBinary.into())),
+            }),
+            None,
+        ),
+        (
+            "trace_flags",
+            ColumnDataType::Uint32,
+            SemanticType::Field,
+            None,
+            None,
+        ),
+        (
+            "scope_name",
+            ColumnDataType::String,
+            SemanticType::Tag,
+            None,
+            None,
+        ),
+        (
+            "scope_version",
+            ColumnDataType::String,
+            SemanticType::Field,
+            None,
+            None,
+        ),
+        (
+            "scope_attributes",
+            ColumnDataType::Binary,
+            SemanticType::Field,
+            Some(ColumnDataTypeExtension {
+                type_ext: Some(TypeExt::JsonType(JsonTypeExtension::JsonBinary.into())),
+            }),
+            None,
+        ),
+        (
+            "scope_schema_url",
+            ColumnDataType::String,
+            SemanticType::Field,
+            None,
+            None,
+        ),
+        (
+            "resource_attributes",
+            ColumnDataType::Binary,
+            SemanticType::Field,
+            Some(ColumnDataTypeExtension {
+                type_ext: Some(TypeExt::JsonType(JsonTypeExtension::JsonBinary.into())),
+            }),
+            None,
+        ),
+        (
+            "resource_schema_url",
+            ColumnDataType::String,
+            SemanticType::Field,
+            None,
+            None,
+        ),
     ]
     .into_iter()
     .map(
@@ -301,39 +308,17 @@ fn build_otlp_logs_identity_schema() -> Vec<ColumnSchema> {
     .collect::<Vec<ColumnSchema>>()
 }
 
-fn build_otlp_build_in_row(
-    log: LogRecord,
-    resource_attr: JsonbValue<'static>,
-    scope_name: Option<String>,
-    scope_version: Option<String>,
-    scope_attrs: JsonbValue<'static>,
-) -> Row {
+fn build_otlp_build_in_row(log: LogRecord, parse_ctx: &mut ParseContext) -> Row {
     let log_attr = key_value_to_jsonb(log.attributes);
+    let ts = if log.time_unix_nano != 0 {
+        log.time_unix_nano
+    } else {
+        log.observed_time_unix_nano
+    };
+
     let row = vec![
         GreptimeValue {
-            value_data: scope_name.map(ValueData::StringValue),
-        },
-        GreptimeValue {
-            value_data: scope_version.map(ValueData::StringValue),
-        },
-        GreptimeValue {
-            value_data: Some(ValueData::BinaryValue(scope_attrs.to_vec())),
-        },
-        GreptimeValue {
-            value_data: Some(ValueData::BinaryValue(resource_attr.to_vec())),
-        },
-        GreptimeValue {
-            value_data: Some(ValueData::BinaryValue(log_attr.to_vec())),
-        },
-        GreptimeValue {
-            value_data: Some(ValueData::TimestampNanosecondValue(
-                log.time_unix_nano as i64,
-            )),
-        },
-        GreptimeValue {
-            value_data: Some(ValueData::TimestampNanosecondValue(
-                log.observed_time_unix_nano as i64,
-            )),
+            value_data: Some(ValueData::TimestampNanosecondValue(ts as i64)),
         },
         GreptimeValue {
             value_data: Some(ValueData::StringValue(bytes_to_hex_string(&log.trace_id))),
@@ -341,9 +326,6 @@ fn build_otlp_build_in_row(
         GreptimeValue {
             value_data: Some(ValueData::StringValue(bytes_to_hex_string(&log.span_id))),
         },
-        GreptimeValue {
-            value_data: Some(ValueData::U32Value(log.flags)),
-        },
         GreptimeValue {
             value_data: Some(ValueData::StringValue(log.severity_text)),
         },
@@ -356,6 +338,30 @@ fn build_otlp_build_in_row(
                 .as_ref()
                 .map(|x| ValueData::StringValue(log_body_to_string(x))),
         },
+        GreptimeValue {
+            value_data: Some(ValueData::BinaryValue(log_attr.to_vec())),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::U32Value(log.flags)),
+        },
+        GreptimeValue {
+            value_data: parse_ctx.scope_name.clone().map(ValueData::StringValue),
+        },
+        GreptimeValue {
+            value_data: parse_ctx.scope_version.clone().map(ValueData::StringValue),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::BinaryValue(parse_ctx.scope_attrs.to_vec())),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::StringValue(parse_ctx.scope_url.clone())),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::BinaryValue(parse_ctx.resource_attr.to_vec())),
+        },
+        GreptimeValue {
+            value_data: Some(ValueData::StringValue(parse_ctx.resource_url.clone())),
+        },
     ];
     Row { values: row }
 }
@@ -363,7 +369,7 @@ fn build_otlp_build_in_row(
 fn extract_field_from_attr_and_combine_schema(
     schema_info: &mut SchemaInfo,
     log_select: &SelectInfo,
-    jsonb: &jsonb::Value<'static>,
+    jsonb: &jsonb::Value,
 ) -> Result<Vec<GreptimeValue>> {
     if log_select.keys.is_empty() {
         return Ok(Vec::new());
@@ -498,13 +504,12 @@ fn parse_export_logs_service_request_to_rows(
     let mut extra_resource_schema = SchemaInfo::default();
     let mut extra_scope_schema = SchemaInfo::default();
     let mut extra_log_schema = SchemaInfo::default();
-    let parse_infos = parse_resource(
-        &select_info,
+    let mut parse_ctx = ParseContext::new(
         &mut extra_resource_schema,
         &mut extra_scope_schema,
         &mut extra_log_schema,
-        request.resource_logs,
-    )?;
+    );
+    let parse_infos = parse_resource(&select_info, &mut parse_ctx, request.resource_logs)?;
 
     // order of schema is important
     // resource < scope < log
@@ -557,28 +562,27 @@ fn parse_export_logs_service_request_to_rows(
 
 fn parse_resource(
     select_info: &SelectInfo,
-    extra_resource_schema: &mut SchemaInfo,
-    extra_scope_schema: &mut SchemaInfo,
-    extra_log_schema: &mut SchemaInfo,
+    parse_ctx: &mut ParseContext,
     resource_logs_vec: Vec<ResourceLogs>,
 ) -> Result<Vec<ParseInfo>> {
     let mut results = Vec::new();
+
     for r in resource_logs_vec {
-        let resource_attr = r
+        parse_ctx.resource_attr = r
             .resource
             .map(|resource| key_value_to_jsonb(resource.attributes))
             .unwrap_or(JsonbValue::Null);
+        parse_ctx.resource_url = r.schema_url;
+
         let resource_extracted_values = extract_field_from_attr_and_combine_schema(
-            extra_resource_schema,
+            parse_ctx.extra_resource_schema,
             select_info,
-            &resource_attr,
+            &parse_ctx.resource_attr,
         )?;
         let rows = parse_scope(
-            extra_scope_schema,
-            extra_log_schema,
             select_info,
             r.scope_logs,
-            resource_attr,
+            parse_ctx,
             resource_extracted_values,
         )?;
         results.extend(rows);
@@ -586,38 +590,65 @@ fn parse_resource(
     Ok(results)
 }
 
-struct ScopeInfo {
+struct ParseContext<'a> {
+    // selector schema
+    extra_resource_schema: &'a mut SchemaInfo,
+    extra_scope_schema: &'a mut SchemaInfo,
+    extra_log_schema: &'a mut SchemaInfo,
+
+    // passdown values
+    resource_url: String,
+    resource_attr: JsonbValue<'a>,
     scope_name: Option<String>,
     scope_version: Option<String>,
-    scope_attrs: JsonbValue<'static>,
+    scope_url: String,
+    scope_attrs: JsonbValue<'a>,
+}
+
+impl<'a> ParseContext<'a> {
+    pub fn new(
+        extra_resource_schema: &'a mut SchemaInfo,
+        extra_scope_schema: &'a mut SchemaInfo,
+        extra_log_schema: &'a mut SchemaInfo,
+    ) -> ParseContext<'a> {
+        ParseContext {
+            extra_resource_schema,
+            extra_scope_schema,
+            extra_log_schema,
+            resource_url: String::new(),
+            resource_attr: JsonbValue::Null,
+            scope_name: None,
+            scope_version: None,
+            scope_url: String::new(),
+            scope_attrs: JsonbValue::Null,
+        }
+    }
 }
 
 fn parse_scope(
-    extra_scope_schema: &mut SchemaInfo,
-    extra_log_schema: &mut SchemaInfo,
     select_info: &SelectInfo,
     scopes_log_vec: Vec<ScopeLogs>,
-    resource_attr: JsonbValue<'static>,
+    parse_ctx: &mut ParseContext,
     resource_extracted_values: Vec<GreptimeValue>,
 ) -> Result<Vec<ParseInfo>> {
     let mut results = Vec::new();
     for scope_logs in scopes_log_vec {
         let (scope_attrs, scope_version, scope_name) = scope_to_jsonb(scope_logs.scope);
+        parse_ctx.scope_name = scope_name;
+        parse_ctx.scope_version = scope_version;
+        parse_ctx.scope_attrs = scope_attrs;
+        parse_ctx.scope_url = scope_logs.schema_url;
+
         let scope_extracted_values = extract_field_from_attr_and_combine_schema(
-            extra_scope_schema,
+            parse_ctx.extra_scope_schema,
             select_info,
-            &scope_attrs,
+            &parse_ctx.scope_attrs,
         )?;
+
         let rows = parse_log(
-            extra_log_schema,
             select_info,
             scope_logs.log_records,
-            &resource_attr,
-            ScopeInfo {
-                scope_name,
-                scope_version,
-                scope_attrs,
-            },
+            parse_ctx,
             &resource_extracted_values,
             &scope_extracted_values,
         )?;
@@ -627,15 +658,9 @@ fn parse_scope(
 }
 
 fn parse_log(
-    extra_log_schema: &mut SchemaInfo,
     select_info: &SelectInfo,
     log_records: Vec<LogRecord>,
-    resource_attr: &JsonbValue<'static>,
-    ScopeInfo {
-        scope_name,
-        scope_version,
-        scope_attrs,
-    }: ScopeInfo,
+    parse_ctx: &mut ParseContext,
     resource_extracted_values: &[GreptimeValue],
     scope_extracted_values: &[GreptimeValue],
 ) -> Result<Vec<ParseInfo>> {
@@ -644,16 +669,13 @@ fn parse_log(
     for log in log_records {
         let log_attr = key_value_to_jsonb(log.attributes.clone());
 
-        let row = build_otlp_build_in_row(
-            log,
-            resource_attr.clone(),
-            scope_name.clone(),
-            scope_version.clone(),
-            scope_attrs.clone(),
-        );
+        let row = build_otlp_build_in_row(log, parse_ctx);
 
-        let log_extracted_values =
-            extract_field_from_attr_and_combine_schema(extra_log_schema, select_info, &log_attr)?;
+        let log_extracted_values = extract_field_from_attr_and_combine_schema(
+            parse_ctx.extra_log_schema,
+            select_info,
+            &log_attr,
+        )?;
 
         let parse_info = ParseInfo {
             values: row,
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index 131ba8363f..c426af4f5f 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -1245,7 +1245,7 @@ pub async fn test_identify_pipeline(store_type: StorageType) {
     );
 
     let expected = r#"[["__source__","String","","YES","","FIELD"],["__time__","Int64","","YES","","FIELD"],["__topic__","String","","YES","","FIELD"],["ip","String","","YES","","FIELD"],["status","String","","YES","","FIELD"],["time","String","","YES","","FIELD"],["url","String","","YES","","FIELD"],["user-agent","String","","YES","","FIELD"],["dongdongdong","String","","YES","","FIELD"],["hasagei","String","","YES","","FIELD"],["greptime_timestamp","TimestampNanosecond","PRI","NO","","TIMESTAMP"]]"#;
-    validate_data(&client, "desc logs", expected).await;
+    validate_data("identity_schema", &client, "desc logs", expected).await;
 
     guard.remove_all().await;
 }
@@ -1527,7 +1527,7 @@ pub async fn test_otlp_metrics(store_type: StorageType) {
 
     // select metrics data
     let expected = r#"[[1726053452870391000,9471.0]]"#;
-    validate_data(&client, "select * from gen;", expected).await;
+    validate_data("otlp_metrics", &client, "select * from gen;", expected).await;
 
     // drop table
     let res = client.get("/v1/sql?sql=drop table gen;").send().await;
@@ -1538,7 +1538,13 @@ pub async fn test_otlp_metrics(store_type: StorageType) {
     assert_eq!(StatusCode::OK, res.status());
 
     // select metrics data again
-    validate_data(&client, "select * from gen;", expected).await;
+    validate_data(
+        "otlp_metrics_with_gzip",
+        &client,
+        "select * from gen;",
+        expected,
+    )
+    .await;
 
     guard.remove_all().await;
 }
@@ -1564,7 +1570,13 @@ pub async fn test_otlp_traces(store_type: StorageType) {
 
     // select traces data
     let expected = r#"[["b5e5fb572cf0a3335dd194a14145fef5","3364d2da58c9fd2b","","{\"service.name\":\"telemetrygen\"}","telemetrygen","","{}","","lets-go","SPAN_KIND_CLIENT","STATUS_CODE_UNSET","","{\"net.peer.ip\":\"1.2.3.4\",\"peer.service\":\"telemetrygen-server\"}","[]","[]",1726631197820927000,1726631197821050000,0.123,1726631197820927000],["b5e5fb572cf0a3335dd194a14145fef5","74c82efa6f628e80","3364d2da58c9fd2b","{\"service.name\":\"telemetrygen\"}","telemetrygen","","{}","","okey-dokey-0","SPAN_KIND_SERVER","STATUS_CODE_UNSET","","{\"net.peer.ip\":\"1.2.3.4\",\"peer.service\":\"telemetrygen-client\"}","[]","[]",1726631197820927000,1726631197821050000,0.123,1726631197820927000]]"#;
-    validate_data(&client, "select * from traces_preview_v01;", expected).await;
+    validate_data(
+        "otlp_traces",
+        &client,
+        "select * from traces_preview_v01;",
+        expected,
+    )
+    .await;
 
     // drop table
     let res = client
@@ -1573,12 +1585,18 @@ pub async fn test_otlp_traces(store_type: StorageType) {
         .await;
     assert_eq!(res.status(), StatusCode::OK);
 
-    // write metrics data with gzip
+    // write traces data with gzip
     let res = send_req(&client, vec![], "/v1/otlp/v1/traces", body.clone(), true).await;
     assert_eq!(StatusCode::OK, res.status());
 
-    // select metrics data again
-    validate_data(&client, "select * from traces_preview_v01;", expected).await;
+    // select traces data again
+    validate_data(
+        "otlp_traces_with_gzip",
+        &client,
+        "select * from traces_preview_v01;",
+        expected,
+    )
+    .await;
 
     guard.remove_all().await;
 }
@@ -1594,8 +1612,27 @@ pub async fn test_otlp_logs(store_type: StorageType) {
 
     let req: ExportLogsServiceRequest = serde_json::from_str(content).unwrap();
     let body = req.encode_to_vec();
+
     {
         // write log data
+        let res = send_req(
+            &client,
+            vec![(
+                HeaderName::from_static("x-greptime-log-table-name"),
+                HeaderValue::from_static("logs1"),
+            )],
+            "/v1/otlp/v1/logs?db=public",
+            body.clone(),
+            false,
+        )
+        .await;
+        assert_eq!(StatusCode::OK, res.status());
+        let expected = r#"[[1581452773000000789,"30","30","Info",9,"something happened",{"customer":"acme","env":"dev"},1,"","",{},"https://opentelemetry.io/schemas/1.0.0/scopeLogs",{"resource-attr":"resource-attr-val-1"},"https://opentelemetry.io/schemas/1.0.0/resourceLogs"],[1581452773000009875,"3038303430323031303030303030303030303030303030303030303030303030","30313032303430383030303030303030","Info",9,"This is a log message",{"app":"server","instance_num":1},1,"","",{},"https://opentelemetry.io/schemas/1.0.0/scopeLogs",{"resource-attr":"resource-attr-val-1"},"https://opentelemetry.io/schemas/1.0.0/resourceLogs"]]"#;
+        validate_data("otlp_logs", &client, "select * from logs1;", expected).await;
+    }
+
+    {
+        // write log data with selector
         let res = send_req(
             &client,
             vec![
@@ -1615,32 +1652,20 @@ pub async fn test_otlp_logs(store_type: StorageType) {
         .await;
         assert_eq!(StatusCode::OK, res.status());
 
-        let expected = r#"[["","",{},{"resource-attr":"resource-attr-val-1"},{"customer":"acme","env":"dev"},1581452773000000789,1581452773000000789,"30","30",1,"Info",9,"something happened",null,null,"resource-attr-val-1"],["","",{},{"resource-attr":"resource-attr-val-1"},{"app":"server","instance_num":1},1581452773000009875,1581452773000009875,"3038303430323031303030303030303030303030303030303030303030303030","30313032303430383030303030303030",1,"Info",9,"This is a log message","server",1,"resource-attr-val-1"]]"#;
-        validate_data(&client, "select * from logs;", expected).await;
-    }
-
-    {
-        // write log data
-        let res = send_req(
+        let expected = r#"[[1581452773000000789,"30","30","Info",9,"something happened",{"customer":"acme","env":"dev"},1,"","",{},"https://opentelemetry.io/schemas/1.0.0/scopeLogs",{"resource-attr":"resource-attr-val-1"},"https://opentelemetry.io/schemas/1.0.0/resourceLogs",null,null,"resource-attr-val-1"],[1581452773000009875,"3038303430323031303030303030303030303030303030303030303030303030","30313032303430383030303030303030","Info",9,"This is a log message",{"app":"server","instance_num":1},1,"","",{},"https://opentelemetry.io/schemas/1.0.0/scopeLogs",{"resource-attr":"resource-attr-val-1"},"https://opentelemetry.io/schemas/1.0.0/resourceLogs","server",1,"resource-attr-val-1"]]"#;
+        validate_data(
+            "otlp_logs_with_selector",
             &client,
-            vec![(
-                HeaderName::from_static("x-greptime-log-table-name"),
-                HeaderValue::from_static("logs1"),
-            )],
-            "/v1/otlp/v1/logs?db=public",
-            body.clone(),
-            false,
+            "select * from logs;",
+            expected,
         )
         .await;
-        assert_eq!(StatusCode::OK, res.status());
-        let expected = r#"[["","",{},{"resource-attr":"resource-attr-val-1"},{"customer":"acme","env":"dev"},1581452773000000789,1581452773000000789,"30","30",1,"Info",9,"something happened"],["","",{},{"resource-attr":"resource-attr-val-1"},{"app":"server","instance_num":1},1581452773000009875,1581452773000009875,"3038303430323031303030303030303030303030303030303030303030303030","30313032303430383030303030303030",1,"Info",9,"This is a log message"]]"#;
-        validate_data(&client, "select * from logs1;", expected).await;
     }
 
     guard.remove_all().await;
 }
 
-async fn validate_data(client: &TestClient, sql: &str, expected: &str) {
+async fn validate_data(test_name: &str, client: &TestClient, sql: &str, expected: &str) {
     let res = client
         .get(format!("/v1/sql?sql={sql}").as_str())
         .send()
@@ -1649,7 +1674,7 @@ async fn validate_data(client: &TestClient, sql: &str, expected: &str) {
     let resp = res.text().await;
     let v = get_rows_from_output(&resp);
 
-    assert_eq!(v, expected);
+    assert_eq!(v, expected, "validate {test_name} fail");
 }
 
 async fn send_req(

From 4f3afb13b69743b153090d4fea538b9da0329e86 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Fri, 25 Oct 2024 15:09:51 +0800
Subject: [PATCH 108/128] fix: fix broken import (#4880)

---
 src/common/runtime/src/runtime.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/common/runtime/src/runtime.rs b/src/common/runtime/src/runtime.rs
index aeba46f24f..cec201bc25 100644
--- a/src/common/runtime/src/runtime.rs
+++ b/src/common/runtime/src/runtime.rs
@@ -210,7 +210,7 @@ impl BuilderBuild<ThrottleableRuntime> for Builder {
 }
 
 #[cfg(tokio_unstable)]
-pub fn register_collector(name: String, handle: &Handle) {
+pub fn register_collector(name: String, handle: &tokio::runtime::Handle) {
     let name = name.replace("-", "_");
     let monitor = tokio_metrics::RuntimeMonitor::new(handle);
     let collector = tokio_metrics_collector::RuntimeCollector::new(monitor, name);

From 2485f660776a51574cf66cf82a7102fb3812d310 Mon Sep 17 00:00:00 2001
From: discord9 <55937128+discord9@users.noreply.github.com>
Date: Fri, 25 Oct 2024 17:29:39 +0800
Subject: [PATCH 109/128] chore: graceful exit on bind fail (#4882)

---
 src/common/greptimedb-telemetry/src/lib.rs |  2 +-
 src/servers/src/error.rs                   | 15 ++++++++++++---
 src/servers/src/http.rs                    |  5 +++--
 3 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/src/common/greptimedb-telemetry/src/lib.rs b/src/common/greptimedb-telemetry/src/lib.rs
index faab69f8b1..19983e71ee 100644
--- a/src/common/greptimedb-telemetry/src/lib.rs
+++ b/src/common/greptimedb-telemetry/src/lib.rs
@@ -325,7 +325,7 @@ mod tests {
             });
             let addr = ([127, 0, 0, 1], port).into();
 
-            let server = Server::bind(&addr).serve(make_svc);
+            let server = Server::try_bind(&addr).unwrap().serve(make_svc);
             let graceful = server.with_graceful_shutdown(async {
                 rx.await.ok();
             });
diff --git a/src/servers/src/error.rs b/src/servers/src/error.rs
index 9e4ef0976b..e564c584be 100644
--- a/src/servers/src/error.rs
+++ b/src/servers/src/error.rs
@@ -36,6 +36,15 @@ use crate::http::error_result::status_code_to_http_status;
 #[snafu(visibility(pub))]
 #[stack_trace_debug]
 pub enum Error {
+    #[snafu(display("Failed to bind address: {}", addr))]
+    AddressBind {
+        addr: SocketAddr,
+        #[snafu(source)]
+        error: hyper::Error,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Arrow error"))]
     Arrow {
         #[snafu(source)]
@@ -565,9 +574,9 @@ impl ErrorExt for Error {
             | Arrow { .. }
             | FileWatch { .. } => StatusCode::Internal,
 
-            AlreadyStarted { .. } | InvalidPromRemoteReadQueryResult { .. } => {
-                StatusCode::IllegalState
-            }
+            AddressBind { .. }
+            | AlreadyStarted { .. }
+            | InvalidPromRemoteReadQueryResult { .. } => StatusCode::IllegalState,
 
             UnsupportedDataType { .. } => StatusCode::Unsupported,
 
diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs
index b0c4843a1b..de4f2cf5ff 100644
--- a/src/servers/src/http.rs
+++ b/src/servers/src/http.rs
@@ -53,7 +53,7 @@ use tower_http::trace::TraceLayer;
 use self::authorize::AuthState;
 use self::table_result::TableResponse;
 use crate::configurator::ConfiguratorRef;
-use crate::error::{AlreadyStartedSnafu, Error, HyperSnafu, Result, ToJsonSnafu};
+use crate::error::{AddressBindSnafu, AlreadyStartedSnafu, Error, HyperSnafu, Result, ToJsonSnafu};
 use crate::http::arrow_result::ArrowResponse;
 use crate::http::csv_result::CsvResponse;
 use crate::http::error_result::ErrorResponse;
@@ -933,7 +933,8 @@ impl Server for HttpServer {
                 app = configurator.config_http(app);
             }
             let app = self.build(app);
-            let server = axum::Server::bind(&listening)
+            let server = axum::Server::try_bind(&listening)
+                .with_context(|_| AddressBindSnafu { addr: listening })?
                 .tcp_nodelay(true)
                 // Enable TCP keepalive to close the dangling established connections.
                 // It's configured to let the keepalive probes first send after the connection sits

From 1008af5324919fef3af1c2eaaa60c30758e46321 Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Sat, 26 Oct 2024 07:36:16 +0800
Subject: [PATCH 110/128] feat!: Divide flush and compaction job pool (#4871)

* feat: divide flush/compact job pool

* feat!: divide bg jobs config

* docs: update config examples

* test: fix tests
---
 config/config.md                        | 12 +++++---
 config/datanode.example.toml            | 25 ++++++++++-----
 config/standalone.example.toml          | 25 ++++++++++-----
 src/mito2/src/compaction/compactor.rs   |  2 +-
 src/mito2/src/config.rs                 | 38 +++++++++++++++++------
 src/mito2/src/engine/compaction_test.rs |  4 +--
 src/mito2/src/worker.rs                 | 41 ++++++++++++++++---------
 tests-integration/tests/http.rs         |  4 ++-
 8 files changed, 103 insertions(+), 48 deletions(-)

diff --git a/config/config.md b/config/config.md
index 8e1fb5837a..65d8fa0c16 100644
--- a/config/config.md
+++ b/config/config.md
@@ -83,7 +83,7 @@
 | `wal.backoff_max` | String | `10s` | The maximum backoff delay.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.backoff_base` | Integer | `2` | The exponential backoff rate, i.e. next backoff = base * current backoff.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.backoff_deadline` | String | `5mins` | The deadline of retries.<br/>**It's only used when the provider is `kafka`**. |
-| `wal.overwrite_entry_start_id` | Bool | `false` | Ignore missing entries during read WAL.<br/>**It's only used when the provider is `kafka`**.<br/><br/>This option ensures that when Kafka messages are deleted, the system <br/>can still successfully replay memtable data without throwing an <br/>out-of-range error. <br/>However, enabling this option might lead to unexpected data loss, <br/>as the system will skip over missing entries instead of treating <br/>them as critical errors. |
+| `wal.overwrite_entry_start_id` | Bool | `false` | Ignore missing entries during read WAL.<br/>**It's only used when the provider is `kafka`**.<br/><br/>This option ensures that when Kafka messages are deleted, the system<br/>can still successfully replay memtable data without throwing an<br/>out-of-range error.<br/>However, enabling this option might lead to unexpected data loss,<br/>as the system will skip over missing entries instead of treating<br/>them as critical errors. |
 | `metadata_store` | -- | -- | Metadata storage options. |
 | `metadata_store.file_size` | String | `256MB` | Kv file size in bytes. |
 | `metadata_store.purge_threshold` | String | `4GB` | Kv purge threshold. |
@@ -116,7 +116,9 @@
 | `region_engine.mito.worker_request_batch_size` | Integer | `64` | Max batch size for a worker to handle requests. |
 | `region_engine.mito.manifest_checkpoint_distance` | Integer | `10` | Number of meta action updated to trigger a new checkpoint for the manifest. |
 | `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). |
-| `region_engine.mito.max_background_jobs` | Integer | `4` | Max number of running background jobs |
+| `region_engine.mito.max_background_flushes` | Integer | Auto | Max number of running background flush jobs (default: 1/2 of cpu cores). |
+| `region_engine.mito.max_background_compactions` | Integer | Auto | Max number of running background compaction jobs (default: 1/4 of cpu cores). |
+| `region_engine.mito.max_background_purges` | Integer | Auto | Max number of running background purge jobs (default: number of cpu cores). |
 | `region_engine.mito.auto_flush_interval` | String | `1h` | Interval to auto flush a region if it has not flushed yet. |
 | `region_engine.mito.global_write_buffer_size` | String | Auto | Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB. |
 | `region_engine.mito.global_write_buffer_reject_size` | String | Auto | Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size`. |
@@ -410,7 +412,7 @@
 | `wal.backoff_deadline` | String | `5mins` | The deadline of retries.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.create_index` | Bool | `true` | Whether to enable WAL index creation.<br/>**It's only used when the provider is `kafka`**. |
 | `wal.dump_index_interval` | String | `60s` | The interval for dumping WAL indexes.<br/>**It's only used when the provider is `kafka`**. |
-| `wal.overwrite_entry_start_id` | Bool | `false` | Ignore missing entries during read WAL.<br/>**It's only used when the provider is `kafka`**.<br/><br/>This option ensures that when Kafka messages are deleted, the system <br/>can still successfully replay memtable data without throwing an <br/>out-of-range error. <br/>However, enabling this option might lead to unexpected data loss, <br/>as the system will skip over missing entries instead of treating <br/>them as critical errors. |
+| `wal.overwrite_entry_start_id` | Bool | `false` | Ignore missing entries during read WAL.<br/>**It's only used when the provider is `kafka`**.<br/><br/>This option ensures that when Kafka messages are deleted, the system<br/>can still successfully replay memtable data without throwing an<br/>out-of-range error.<br/>However, enabling this option might lead to unexpected data loss,<br/>as the system will skip over missing entries instead of treating<br/>them as critical errors. |
 | `storage` | -- | -- | The data storage options. |
 | `storage.data_home` | String | `/tmp/greptimedb/` | The working home directory. |
 | `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
@@ -437,7 +439,9 @@
 | `region_engine.mito.worker_request_batch_size` | Integer | `64` | Max batch size for a worker to handle requests. |
 | `region_engine.mito.manifest_checkpoint_distance` | Integer | `10` | Number of meta action updated to trigger a new checkpoint for the manifest. |
 | `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). |
-| `region_engine.mito.max_background_jobs` | Integer | `4` | Max number of running background jobs |
+| `region_engine.mito.max_background_flushes` | Integer | Auto | Max number of running background flush jobs (default: 1/2 of cpu cores). |
+| `region_engine.mito.max_background_compactions` | Integer | Auto | Max number of running background compaction jobs (default: 1/4 of cpu cores). |
+| `region_engine.mito.max_background_purges` | Integer | Auto | Max number of running background purge jobs (default: number of cpu cores). |
 | `region_engine.mito.auto_flush_interval` | String | `1h` | Interval to auto flush a region if it has not flushed yet. |
 | `region_engine.mito.global_write_buffer_size` | String | Auto | Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB. |
 | `region_engine.mito.global_write_buffer_reject_size` | String | Auto | Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size` |
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index 85b7793ef6..557cd4cef0 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -215,12 +215,12 @@ dump_index_interval = "60s"
 
 ## Ignore missing entries during read WAL.
 ## **It's only used when the provider is `kafka`**.
-## 
-## This option ensures that when Kafka messages are deleted, the system 
-## can still successfully replay memtable data without throwing an 
-## out-of-range error. 
-## However, enabling this option might lead to unexpected data loss, 
-## as the system will skip over missing entries instead of treating 
+##
+## This option ensures that when Kafka messages are deleted, the system
+## can still successfully replay memtable data without throwing an
+## out-of-range error.
+## However, enabling this option might lead to unexpected data loss,
+## as the system will skip over missing entries instead of treating
 ## them as critical errors.
 overwrite_entry_start_id = false
 
@@ -416,8 +416,17 @@ manifest_checkpoint_distance = 10
 ## Whether to compress manifest and checkpoint file by gzip (default false).
 compress_manifest = false
 
-## Max number of running background jobs
-max_background_jobs = 4
+## Max number of running background flush jobs (default: 1/2 of cpu cores).
+## @toml2docs:none-default="Auto"
+#+ max_background_flushes = 4
+
+## Max number of running background compaction jobs (default: 1/4 of cpu cores).
+## @toml2docs:none-default="Auto"
+#+ max_background_compactions = 2
+
+## Max number of running background purge jobs (default: number of cpu cores).
+## @toml2docs:none-default="Auto"
+#+ max_background_purges = 8
 
 ## Interval to auto flush a region if it has not flushed yet.
 auto_flush_interval = "1h"
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index 81f4ee47f0..defd34d8f5 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -239,12 +239,12 @@ backoff_deadline = "5mins"
 
 ## Ignore missing entries during read WAL.
 ## **It's only used when the provider is `kafka`**.
-## 
-## This option ensures that when Kafka messages are deleted, the system 
-## can still successfully replay memtable data without throwing an 
-## out-of-range error. 
-## However, enabling this option might lead to unexpected data loss, 
-## as the system will skip over missing entries instead of treating 
+##
+## This option ensures that when Kafka messages are deleted, the system
+## can still successfully replay memtable data without throwing an
+## out-of-range error.
+## However, enabling this option might lead to unexpected data loss,
+## as the system will skip over missing entries instead of treating
 ## them as critical errors.
 overwrite_entry_start_id = false
 
@@ -454,8 +454,17 @@ manifest_checkpoint_distance = 10
 ## Whether to compress manifest and checkpoint file by gzip (default false).
 compress_manifest = false
 
-## Max number of running background jobs
-max_background_jobs = 4
+## Max number of running background flush jobs (default: 1/2 of cpu cores).
+## @toml2docs:none-default="Auto"
+#+ max_background_flushes = 4
+
+## Max number of running background compaction jobs (default: 1/4 of cpu cores).
+## @toml2docs:none-default="Auto"
+#+ max_background_compactions = 2
+
+## Max number of running background purge jobs (default: number of cpu cores).
+## @toml2docs:none-default="Auto"
+#+ max_background_purges = 8
 
 ## Interval to auto flush a region if it has not flushed yet.
 auto_flush_interval = "1h"
diff --git a/src/mito2/src/compaction/compactor.rs b/src/mito2/src/compaction/compactor.rs
index d919633ba9..004b223053 100644
--- a/src/mito2/src/compaction/compactor.rs
+++ b/src/mito2/src/compaction/compactor.rs
@@ -134,7 +134,7 @@ pub async fn open_compaction_region(
     ));
 
     let file_purger = {
-        let purge_scheduler = Arc::new(LocalScheduler::new(mito_config.max_background_jobs));
+        let purge_scheduler = Arc::new(LocalScheduler::new(mito_config.max_background_purges));
         Arc::new(LocalFilePurger::new(
             purge_scheduler.clone(),
             access_layer.clone(),
diff --git a/src/mito2/src/config.rs b/src/mito2/src/config.rs
index 433bc45662..001d5ffaa8 100644
--- a/src/mito2/src/config.rs
+++ b/src/mito2/src/config.rs
@@ -28,9 +28,6 @@ use crate::error::Result;
 use crate::memtable::MemtableConfig;
 use crate::sst::DEFAULT_WRITE_BUFFER_SIZE;
 
-/// Default max running background job.
-const DEFAULT_MAX_BG_JOB: usize = 4;
-
 const MULTIPART_UPLOAD_MINIMUM_SIZE: ReadableSize = ReadableSize::mb(5);
 /// Default channel size for parallel scan task.
 const DEFAULT_SCAN_CHANNEL_SIZE: usize = 32;
@@ -69,8 +66,12 @@ pub struct MitoConfig {
     pub compress_manifest: bool,
 
     // Background job configs:
-    /// Max number of running background jobs (default 4).
-    pub max_background_jobs: usize,
+    /// Max number of running background flush jobs (default: 1/2 of cpu cores).
+    pub max_background_flushes: usize,
+    /// Max number of running background compaction jobs (default: 1/4 of cpu cores).
+    pub max_background_compactions: usize,
+    /// Max number of running background purge jobs (default: number of cpu cores).
+    pub max_background_purges: usize,
 
     // Flush configs:
     /// Interval to auto flush a region if it has not flushed yet (default 30 min).
@@ -137,7 +138,9 @@ impl Default for MitoConfig {
             worker_request_batch_size: 64,
             manifest_checkpoint_distance: 10,
             compress_manifest: false,
-            max_background_jobs: DEFAULT_MAX_BG_JOB,
+            max_background_flushes: divide_num_cpus(2),
+            max_background_compactions: divide_num_cpus(4),
+            max_background_purges: common_config::utils::get_cpus(),
             auto_flush_interval: Duration::from_secs(30 * 60),
             global_write_buffer_size: ReadableSize::gb(1),
             global_write_buffer_reject_size: ReadableSize::gb(2),
@@ -185,9 +188,26 @@ impl MitoConfig {
             self.worker_channel_size = 1;
         }
 
-        if self.max_background_jobs == 0 {
-            warn!("Sanitize max background jobs 0 to {}", DEFAULT_MAX_BG_JOB);
-            self.max_background_jobs = DEFAULT_MAX_BG_JOB;
+        if self.max_background_flushes == 0 {
+            warn!(
+                "Sanitize max background flushes 0 to {}",
+                divide_num_cpus(2)
+            );
+            self.max_background_flushes = divide_num_cpus(2);
+        }
+        if self.max_background_compactions == 0 {
+            warn!(
+                "Sanitize max background compactions 0 to {}",
+                divide_num_cpus(4)
+            );
+            self.max_background_compactions = divide_num_cpus(4);
+        }
+        if self.max_background_purges == 0 {
+            warn!(
+                "Sanitize max background purges 0 to {}",
+                common_config::utils::get_cpus()
+            );
+            self.max_background_purges = common_config::utils::get_cpus();
         }
 
         if self.global_write_buffer_reject_size <= self.global_write_buffer_size {
diff --git a/src/mito2/src/engine/compaction_test.rs b/src/mito2/src/engine/compaction_test.rs
index 7951331b20..9a80d4f84f 100644
--- a/src/mito2/src/engine/compaction_test.rs
+++ b/src/mito2/src/engine/compaction_test.rs
@@ -272,7 +272,7 @@ async fn test_readonly_during_compaction() {
         .create_engine_with(
             MitoConfig {
                 // Ensure there is only one background worker for purge task.
-                max_background_jobs: 1,
+                max_background_purges: 1,
                 ..Default::default()
             },
             None,
@@ -310,7 +310,7 @@ async fn test_readonly_during_compaction() {
     listener.wake();
 
     let notify = Arc::new(Notify::new());
-    // We already sets max background jobs to 1, so we can submit a task to the
+    // We already sets max background purges to 1, so we can submit a task to the
     // purge scheduler to ensure all purge tasks are finished.
     let job_notify = notify.clone();
     engine
diff --git a/src/mito2/src/worker.rs b/src/mito2/src/worker.rs
index e790ed08c1..eb2cea19d2 100644
--- a/src/mito2/src/worker.rs
+++ b/src/mito2/src/worker.rs
@@ -114,8 +114,10 @@ pub(crate) const MAX_INITIAL_CHECK_DELAY_SECS: u64 = 60 * 3;
 pub(crate) struct WorkerGroup {
     /// Workers of the group.
     workers: Vec<RegionWorker>,
-    /// Global background job scheduelr.
-    scheduler: SchedulerRef,
+    /// Flush background job pool.
+    flush_job_pool: SchedulerRef,
+    /// Compaction background job pool.
+    compact_job_pool: SchedulerRef,
     /// Scheduler for file purgers.
     purge_scheduler: SchedulerRef,
     /// Cache.
@@ -146,10 +148,10 @@ impl WorkerGroup {
         let intermediate_manager = IntermediateManager::init_fs(&config.index.aux_path)
             .await?
             .with_buffer_size(Some(config.index.write_buffer_size.as_bytes() as _));
-        let scheduler = Arc::new(LocalScheduler::new(config.max_background_jobs));
+        let flush_job_pool = Arc::new(LocalScheduler::new(config.max_background_flushes));
+        let compact_job_pool = Arc::new(LocalScheduler::new(config.max_background_compactions));
         // We use another scheduler to avoid purge jobs blocking other jobs.
-        // A purge job is cheaper than other background jobs so they share the same job limit.
-        let purge_scheduler = Arc::new(LocalScheduler::new(config.max_background_jobs));
+        let purge_scheduler = Arc::new(LocalScheduler::new(config.max_background_purges));
         let write_cache = write_cache_from_config(
             &config,
             object_store_manager.clone(),
@@ -178,7 +180,8 @@ impl WorkerGroup {
                     log_store: log_store.clone(),
                     object_store_manager: object_store_manager.clone(),
                     write_buffer_manager: write_buffer_manager.clone(),
-                    scheduler: scheduler.clone(),
+                    flush_job_pool: flush_job_pool.clone(),
+                    compact_job_pool: compact_job_pool.clone(),
                     purge_scheduler: purge_scheduler.clone(),
                     listener: WorkerListener::default(),
                     cache_manager: cache_manager.clone(),
@@ -195,7 +198,8 @@ impl WorkerGroup {
 
         Ok(WorkerGroup {
             workers,
-            scheduler,
+            flush_job_pool,
+            compact_job_pool,
             purge_scheduler,
             cache_manager,
         })
@@ -205,8 +209,11 @@ impl WorkerGroup {
     pub(crate) async fn stop(&self) -> Result<()> {
         info!("Stop region worker group");
 
+        // TODO(yingwen): Do we need to stop gracefully?
         // Stops the scheduler gracefully.
-        self.scheduler.stop(true).await?;
+        self.compact_job_pool.stop(true).await?;
+        // Stops the scheduler gracefully.
+        self.flush_job_pool.stop(true).await?;
         // Stops the purge scheduler gracefully.
         self.purge_scheduler.stop(true).await?;
 
@@ -275,8 +282,9 @@ impl WorkerGroup {
                     .with_notifier(flush_sender.clone()),
             )
         });
-        let scheduler = Arc::new(LocalScheduler::new(config.max_background_jobs));
-        let purge_scheduler = Arc::new(LocalScheduler::new(config.max_background_jobs));
+        let flush_job_pool = Arc::new(LocalScheduler::new(config.max_background_flushes));
+        let compact_job_pool = Arc::new(LocalScheduler::new(config.max_background_compactions));
+        let purge_scheduler = Arc::new(LocalScheduler::new(config.max_background_flushes));
         let puffin_manager_factory = PuffinManagerFactory::new(
             &config.index.aux_path,
             config.index.staging_size.as_bytes(),
@@ -310,7 +318,8 @@ impl WorkerGroup {
                     log_store: log_store.clone(),
                     object_store_manager: object_store_manager.clone(),
                     write_buffer_manager: write_buffer_manager.clone(),
-                    scheduler: scheduler.clone(),
+                    flush_job_pool: flush_job_pool.clone(),
+                    compact_job_pool: compact_job_pool.clone(),
                     purge_scheduler: purge_scheduler.clone(),
                     listener: WorkerListener::new(listener.clone()),
                     cache_manager: cache_manager.clone(),
@@ -327,7 +336,8 @@ impl WorkerGroup {
 
         Ok(WorkerGroup {
             workers,
-            scheduler,
+            flush_job_pool,
+            compact_job_pool,
             purge_scheduler,
             cache_manager,
         })
@@ -382,7 +392,8 @@ struct WorkerStarter<S> {
     log_store: Arc<S>,
     object_store_manager: ObjectStoreManagerRef,
     write_buffer_manager: WriteBufferManagerRef,
-    scheduler: SchedulerRef,
+    compact_job_pool: SchedulerRef,
+    flush_job_pool: SchedulerRef,
     purge_scheduler: SchedulerRef,
     listener: WorkerListener,
     cache_manager: CacheManagerRef,
@@ -423,9 +434,9 @@ impl<S: LogStore> WorkerStarter<S> {
             ),
             purge_scheduler: self.purge_scheduler.clone(),
             write_buffer_manager: self.write_buffer_manager,
-            flush_scheduler: FlushScheduler::new(self.scheduler.clone()),
+            flush_scheduler: FlushScheduler::new(self.flush_job_pool),
             compaction_scheduler: CompactionScheduler::new(
-                self.scheduler,
+                self.compact_job_pool,
                 sender.clone(),
                 self.cache_manager.clone(),
                 self.config,
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index c426af4f5f..823de40d11 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -866,7 +866,6 @@ worker_channel_size = 128
 worker_request_batch_size = 64
 manifest_checkpoint_distance = 10
 compress_manifest = false
-max_background_jobs = 4
 auto_flush_interval = "30m"
 enable_experimental_write_cache = false
 experimental_write_cache_path = ""
@@ -939,6 +938,9 @@ fn drop_lines_with_inconsistent_results(input: String) -> String {
         "content_cache_size =",
         "name =",
         "recovery_parallelism =",
+        "max_background_flushes =",
+        "max_background_compactions =",
+        "max_background_purges =",
     ];
 
     input

From eab9e3a48dd4d5a3bbb6ec4223eac516ac5a9a4c Mon Sep 17 00:00:00 2001
From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com>
Date: Mon, 28 Oct 2024 01:50:10 -0700
Subject: [PATCH 111/128] chore: remove struct size assertion (#4885)

chore/remove-struct-size-assertion: Remove unit tests for parquet_meta_size function in cache_size.rs
---
 src/mito2/src/cache/cache_size.rs | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/src/mito2/src/cache/cache_size.rs b/src/mito2/src/cache/cache_size.rs
index 3d79bcbe58..a60b203bd7 100644
--- a/src/mito2/src/cache/cache_size.rs
+++ b/src/mito2/src/cache/cache_size.rs
@@ -127,16 +127,3 @@ fn parquet_offset_index_heap_size(offset_index: &ParquetOffsetIndex) -> usize {
         })
         .sum()
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::cache::test_util::parquet_meta;
-
-    #[test]
-    fn test_parquet_meta_size() {
-        let metadata = parquet_meta();
-
-        assert_eq!(956, parquet_meta_size(&metadata));
-    }
-}

From 0ee455a9807751823bdf4664320030816fc51d5e Mon Sep 17 00:00:00 2001
From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com>
Date: Mon, 28 Oct 2024 21:47:57 -0700
Subject: [PATCH 112/128] fix: pyo3 ut (#4894)

---
 src/script/src/python/ffi_types/pair_tests/sample_testcases.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/script/src/python/ffi_types/pair_tests/sample_testcases.rs b/src/script/src/python/ffi_types/pair_tests/sample_testcases.rs
index 3088faa4f4..1009a8ee23 100644
--- a/src/script/src/python/ffi_types/pair_tests/sample_testcases.rs
+++ b/src/script/src/python/ffi_types/pair_tests/sample_testcases.rs
@@ -722,6 +722,7 @@ import math
 
 @coprocessor(args=[], returns=["value"], backend="pyo3")
 def test_numpy() -> vector[i64]:
+    from greptime import vector
     try:
         import numpy as np
         import pyarrow as pa
@@ -729,7 +730,6 @@ def test_numpy() -> vector[i64]:
         # Python didn't have numpy or pyarrow
         print("Warning: no pyarrow or numpy found in current python", e)
         return vector([0, 1, 2, 3, 4, 5, 6, 7, 8, 9,])
-    from greptime import vector
     v = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9,])
     v = pa.array(v)
     v = vector.from_pyarrow(v)

From 03f2fa219da7aee3aef8ff13316f39d0b054b3b7 Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Tue, 29 Oct 2024 15:46:05 +0800
Subject: [PATCH 113/128] feat: optimizer rule for windowed sort (#4874)

* basic impl

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* implement physical rule

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* feat: install windowed sort physical rule and optimize partition ranges

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* add logs and sqlness test

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* feat: introduce PartSortExec for partitioned sorting

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* tune exec nodes' properties and metrics

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* clean up

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix typo

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* debug: add more info on very wrong

* debug: also print overlap ranges

* feat: add check when emit PartSort Stream

* dbg: info on overlap working range

* feat: check batch range is inside part range

* set distinguish partition range param

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* chore: more logs

* update sqlness

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* tune optimizer

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* clean up

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix lints

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix windowed sort rule

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix: early terminate sort stream

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* chore: remove min/max check

* chore: remove unused windowed_sort module, uuid feature and refactor region_scanner to synchronous

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* chore: print more fuzz log

* chore: more log

* fix: part sort should skip empty part

* chore: remove insert logs

* tests: empty PartitionRange

* refactor: testcase

* docs: update comment&tests: all empty

* ci: enlarge etcd cpu limit

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
Co-authored-by: discord9 <discord9@163.com>
Co-authored-by: evenyag <realevenyag@gmail.com>
---
 .github/actions/setup-etcd-cluster/action.yml |   2 +-
 Cargo.lock                                    |   1 +
 src/mito2/src/engine.rs                       |   5 +-
 src/mito2/src/read.rs                         |  25 +
 src/mito2/src/read/scan_region.rs             |   2 +-
 src/mito2/src/read/seq_scan.rs                |  12 +-
 src/mito2/src/read/unordered_scan.rs          |  11 +-
 src/query/Cargo.toml                          |   1 +
 src/query/src/lib.rs                          |   1 +
 src/query/src/optimizer.rs                    |   1 +
 src/query/src/optimizer/parallelize_scan.rs   |  13 +-
 src/query/src/optimizer/windowed_sort.rs      | 158 ++++++
 src/query/src/part_sort.rs                    | 267 ++++++++--
 src/query/src/query_engine/state.rs           |   5 +
 src/query/src/window_sort.rs                  | 481 ++++++++++++------
 src/store-api/src/region_engine.rs            |  21 +-
 src/table/src/table/scan.rs                   |  43 +-
 .../targets/fuzz_insert_logical_table.rs      |   3 +
 .../common/order/windowed_sort.result         |  81 +++
 .../standalone/common/order/windowed_sort.sql |  26 +
 .../common/tql-explain-analyze/explain.result |   1 +
 21 files changed, 930 insertions(+), 230 deletions(-)
 create mode 100644 src/query/src/optimizer/windowed_sort.rs
 create mode 100644 tests/cases/standalone/common/order/windowed_sort.result
 create mode 100644 tests/cases/standalone/common/order/windowed_sort.sql

diff --git a/.github/actions/setup-etcd-cluster/action.yml b/.github/actions/setup-etcd-cluster/action.yml
index 7d0d8d01ab..e6abb96bc2 100644
--- a/.github/actions/setup-etcd-cluster/action.yml
+++ b/.github/actions/setup-etcd-cluster/action.yml
@@ -18,7 +18,7 @@ runs:
         --set replicaCount=${{ inputs.etcd-replicas }} \
         --set resources.requests.cpu=50m \
         --set resources.requests.memory=128Mi \
-        --set resources.limits.cpu=1000m \
+        --set resources.limits.cpu=1500m \
         --set resources.limits.memory=2Gi \
         --set auth.rbac.create=false \
         --set auth.rbac.token.enabled=false \
diff --git a/Cargo.lock b/Cargo.lock
index d4db1185f3..c5af069e7c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9030,6 +9030,7 @@ dependencies = [
  "table",
  "tokio",
  "tokio-stream",
+ "uuid",
 ]
 
 [[package]]
diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs
index ed8cc92909..8e62877270 100644
--- a/src/mito2/src/engine.rs
+++ b/src/mito2/src/engine.rs
@@ -163,13 +163,13 @@ impl MitoEngine {
     }
 
     /// Returns a region scanner to scan the region for `request`.
-    async fn region_scanner(
+    fn region_scanner(
         &self,
         region_id: RegionId,
         request: ScanRequest,
     ) -> Result<RegionScannerRef> {
         let scanner = self.scanner(region_id, request)?;
-        scanner.region_scanner().await
+        scanner.region_scanner()
     }
 
     /// Scans a region.
@@ -527,7 +527,6 @@ impl RegionEngine for MitoEngine {
         request: ScanRequest,
     ) -> Result<RegionScannerRef, BoxedError> {
         self.region_scanner(region_id, request)
-            .await
             .map_err(BoxedError::new)
     }
 
diff --git a/src/mito2/src/read.rs b/src/mito2/src/read.rs
index f7ed4ca958..e1ce9572c4 100644
--- a/src/mito2/src/read.rs
+++ b/src/mito2/src/read.rs
@@ -532,10 +532,24 @@ impl Batch {
 #[derive(Default)]
 pub(crate) struct BatchChecker {
     last_batch: Option<Batch>,
+    start: Option<Timestamp>,
+    end: Option<Timestamp>,
 }
 
 #[cfg(debug_assertions)]
 impl BatchChecker {
+    /// Attaches the given start timestamp to the checker.
+    pub(crate) fn with_start(mut self, start: Option<Timestamp>) -> Self {
+        self.start = start;
+        self
+    }
+
+    /// Attaches the given end timestamp to the checker.
+    pub(crate) fn with_end(mut self, end: Option<Timestamp>) -> Self {
+        self.end = end;
+        self
+    }
+
     /// Returns true if the given batch is monotonic and behind
     /// the last batch.
     pub(crate) fn check_monotonic(&mut self, batch: &Batch) -> bool {
@@ -543,6 +557,17 @@ impl BatchChecker {
             return false;
         }
 
+        if let (Some(start), Some(first)) = (self.start, batch.first_timestamp()) {
+            if start > first {
+                return false;
+            }
+        }
+        if let (Some(end), Some(last)) = (self.end, batch.last_timestamp()) {
+            if end <= last {
+                return false;
+            }
+        }
+
         // Checks the batch is behind the last batch.
         // Then Updates the last batch.
         let is_behind = self
diff --git a/src/mito2/src/read/scan_region.rs b/src/mito2/src/read/scan_region.rs
index 1c0dd50d0b..2c61124180 100644
--- a/src/mito2/src/read/scan_region.rs
+++ b/src/mito2/src/read/scan_region.rs
@@ -74,7 +74,7 @@ impl Scanner {
 
     /// Returns a [RegionScanner] to scan the region.
     #[tracing::instrument(level = tracing::Level::DEBUG, skip_all)]
-    pub(crate) async fn region_scanner(self) -> Result<RegionScannerRef> {
+    pub(crate) fn region_scanner(self) -> Result<RegionScannerRef> {
         match self {
             Scanner::Seq(seq_scan) => Ok(Box::new(seq_scan)),
             Scanner::Unordered(unordered_scan) => Ok(Box::new(unordered_scan)),
diff --git a/src/mito2/src/read/seq_scan.rs b/src/mito2/src/read/seq_scan.rs
index 8d867d7d92..5049ea8a47 100644
--- a/src/mito2/src/read/seq_scan.rs
+++ b/src/mito2/src/read/seq_scan.rs
@@ -232,7 +232,10 @@ impl SeqScan {
                 let mut metrics = ScannerMetrics::default();
                 let mut fetch_start = Instant::now();
                 #[cfg(debug_assertions)]
-                let mut checker = crate::read::BatchChecker::default();
+                let mut checker = crate::read::BatchChecker::default()
+                    .with_start(Some(part_range.start))
+                    .with_end(Some(part_range.end));
+
                 while let Some(batch) = reader
                     .next_batch()
                     .await
@@ -304,8 +307,13 @@ impl RegionScanner for SeqScan {
         self.scan_partition_impl(partition)
     }
 
-    fn prepare(&mut self, ranges: Vec<Vec<PartitionRange>>) -> Result<(), BoxedError> {
+    fn prepare(
+        &mut self,
+        ranges: Vec<Vec<PartitionRange>>,
+        distinguish_partition_range: bool,
+    ) -> Result<(), BoxedError> {
         self.properties.partitions = ranges;
+        self.properties.distinguish_partition_range = distinguish_partition_range;
         Ok(())
     }
 
diff --git a/src/mito2/src/read/unordered_scan.rs b/src/mito2/src/read/unordered_scan.rs
index c51f349cbd..72bf424020 100644
--- a/src/mito2/src/read/unordered_scan.rs
+++ b/src/mito2/src/read/unordered_scan.rs
@@ -140,7 +140,9 @@ impl UnorderedScan {
                 let mut metrics = ScannerMetrics::default();
                 let mut fetch_start = Instant::now();
                 #[cfg(debug_assertions)]
-                let mut checker = crate::read::BatchChecker::default();
+                let mut checker = crate::read::BatchChecker::default()
+                    .with_start(Some(part_range.start))
+                    .with_end(Some(part_range.end));
 
                 let stream = Self::scan_partition_range(
                     stream_ctx.clone(),
@@ -209,8 +211,13 @@ impl RegionScanner for UnorderedScan {
         self.stream_ctx.input.mapper.output_schema()
     }
 
-    fn prepare(&mut self, ranges: Vec<Vec<PartitionRange>>) -> Result<(), BoxedError> {
+    fn prepare(
+        &mut self,
+        ranges: Vec<Vec<PartitionRange>>,
+        distinguish_partition_range: bool,
+    ) -> Result<(), BoxedError> {
         self.properties.partitions = ranges;
+        self.properties.distinguish_partition_range = distinguish_partition_range;
         Ok(())
     }
 
diff --git a/src/query/Cargo.toml b/src/query/Cargo.toml
index 3f983bb9e3..863a5a1c33 100644
--- a/src/query/Cargo.toml
+++ b/src/query/Cargo.toml
@@ -64,6 +64,7 @@ store-api.workspace = true
 substrait.workspace = true
 table.workspace = true
 tokio.workspace = true
+uuid.workspace = true
 
 [dev-dependencies]
 approx_eq = "0.1"
diff --git a/src/query/src/lib.rs b/src/query/src/lib.rs
index 5b119115e3..18b0b22582 100644
--- a/src/query/src/lib.rs
+++ b/src/query/src/lib.rs
@@ -15,6 +15,7 @@
 #![feature(let_chains)]
 #![feature(int_roundings)]
 #![feature(trait_upcasting)]
+#![feature(try_blocks)]
 
 mod analyze;
 pub mod dataframe;
diff --git a/src/query/src/optimizer.rs b/src/query/src/optimizer.rs
index 2f97a9bd32..7dbeae4b98 100644
--- a/src/query/src/optimizer.rs
+++ b/src/query/src/optimizer.rs
@@ -20,6 +20,7 @@ pub mod string_normalization;
 #[cfg(test)]
 pub(crate) mod test_util;
 pub mod type_conversion;
+pub mod windowed_sort;
 
 use datafusion_common::config::ConfigOptions;
 use datafusion_common::Result;
diff --git a/src/query/src/optimizer/parallelize_scan.rs b/src/query/src/optimizer/parallelize_scan.rs
index d679702796..19f5db39d3 100644
--- a/src/query/src/optimizer/parallelize_scan.rs
+++ b/src/query/src/optimizer/parallelize_scan.rs
@@ -51,16 +51,27 @@ impl ParallelizeScan {
         let result = plan
             .transform_down(|plan| {
                 if let Some(region_scan_exec) = plan.as_any().downcast_ref::<RegionScanExec>() {
+                    if region_scan_exec.is_partition_set() {
+                        return Ok(Transformed::no(plan));
+                    }
+
                     let ranges = region_scan_exec.get_partition_ranges();
                     let total_range_num = ranges.len();
                     let expected_partition_num = config.execution.target_partitions;
 
                     // assign ranges to each partition
-                    let partition_ranges =
+                    let mut partition_ranges =
                         Self::assign_partition_range(ranges, expected_partition_num);
                     debug!(
                         "Assign {total_range_num} ranges to {expected_partition_num} partitions"
                     );
+
+                    // sort the ranges in each partition
+                    // TODO(ruihang): smart sort!
+                    for ranges in partition_ranges.iter_mut() {
+                        ranges.sort_by(|a, b| a.start.cmp(&b.start));
+                    }
+
                     // update the partition ranges
                     let new_exec = region_scan_exec
                         .with_new_partitions(partition_ranges)
diff --git a/src/query/src/optimizer/windowed_sort.rs b/src/query/src/optimizer/windowed_sort.rs
new file mode 100644
index 0000000000..62d4495cf3
--- /dev/null
+++ b/src/query/src/optimizer/windowed_sort.rs
@@ -0,0 +1,158 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use datafusion::physical_optimizer::PhysicalOptimizerRule;
+use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
+use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
+use datafusion::physical_plan::repartition::RepartitionExec;
+use datafusion::physical_plan::sorts::sort::SortExec;
+use datafusion::physical_plan::ExecutionPlan;
+use datafusion_common::tree_node::{Transformed, TreeNode};
+use datafusion_common::Result as DataFusionResult;
+use datafusion_physical_expr::expressions::Column as PhysicalColumn;
+use store_api::region_engine::PartitionRange;
+use table::table::scan::RegionScanExec;
+
+use crate::part_sort::PartSortExec;
+use crate::window_sort::WindowedSortExec;
+
+/// Optimize rule for windowed sort.
+///
+/// This is expected to run after [`ScanHint`] and [`ParallelizeScan`].
+/// It would change the original sort to a custom plan. To make sure
+/// other rules are applied correctly, this rule can be run as later as
+/// possible.
+///
+/// [`ScanHint`]: crate::optimizer::scan_hint::ScanHintRule
+/// [`ParallelizeScan`]: crate::optimizer::parallelize_scan::ParallelizeScan
+pub struct WindowedSortPhysicalRule;
+
+impl PhysicalOptimizerRule for WindowedSortPhysicalRule {
+    fn optimize(
+        &self,
+        plan: Arc<dyn ExecutionPlan>,
+        config: &datafusion::config::ConfigOptions,
+    ) -> DataFusionResult<Arc<dyn ExecutionPlan>> {
+        Self::do_optimize(plan, config)
+    }
+
+    fn name(&self) -> &str {
+        "WindowedSortRule"
+    }
+
+    fn schema_check(&self) -> bool {
+        false
+    }
+}
+
+impl WindowedSortPhysicalRule {
+    fn do_optimize(
+        plan: Arc<dyn ExecutionPlan>,
+        _config: &datafusion::config::ConfigOptions,
+    ) -> DataFusionResult<Arc<dyn ExecutionPlan>> {
+        let result = plan
+            .transform_down(|plan| {
+                if let Some(sort_exec) = plan.as_any().downcast_ref::<SortExec>() {
+                    // TODO: support multiple expr in windowed sort
+                    if !sort_exec.preserve_partitioning() || sort_exec.expr().len() != 1 {
+                        return Ok(Transformed::no(plan));
+                    }
+
+                    let Some(scanner_info) = fetch_partition_range(sort_exec.input().clone())?
+                    else {
+                        return Ok(Transformed::no(plan));
+                    };
+
+                    if let Some(first_sort_expr) = sort_exec.expr().first()
+                        && !first_sort_expr.options.descending
+                        && let Some(column_expr) = first_sort_expr
+                            .expr
+                            .as_any()
+                            .downcast_ref::<PhysicalColumn>()
+                        && column_expr.name() == scanner_info.time_index
+                    {
+                    } else {
+                        return Ok(Transformed::no(plan));
+                    }
+
+                    let first_sort_expr = sort_exec.expr().first().unwrap().clone();
+                    let part_sort_exec = Arc::new(PartSortExec::new(
+                        first_sort_expr.clone(),
+                        scanner_info.partition_ranges.clone(),
+                        sort_exec.input().clone(),
+                    ));
+                    let windowed_sort_exec = WindowedSortExec::try_new(
+                        first_sort_expr,
+                        sort_exec.fetch(),
+                        scanner_info.partition_ranges,
+                        part_sort_exec,
+                    )?;
+
+                    return Ok(Transformed {
+                        data: Arc::new(windowed_sort_exec),
+                        transformed: true,
+                        tnr: datafusion_common::tree_node::TreeNodeRecursion::Stop,
+                    });
+                }
+
+                Ok(Transformed::no(plan))
+            })?
+            .data;
+
+        Ok(result)
+    }
+}
+
+struct ScannerInfo {
+    partition_ranges: Vec<Vec<PartitionRange>>,
+    time_index: String,
+}
+
+fn fetch_partition_range(input: Arc<dyn ExecutionPlan>) -> DataFusionResult<Option<ScannerInfo>> {
+    let mut partition_ranges = None;
+    let mut time_index = None;
+
+    input.transform_up(|plan| {
+        // Unappliable case, reset the state.
+        if plan.as_any().is::<RepartitionExec>()
+            || plan.as_any().is::<CoalesceBatchesExec>()
+            || plan.as_any().is::<CoalescePartitionsExec>()
+            || plan.as_any().is::<SortExec>()
+            || plan.as_any().is::<WindowedSortExec>()
+        {
+            partition_ranges = None;
+        }
+
+        if let Some(region_scan_exec) = plan.as_any().downcast_ref::<RegionScanExec>() {
+            partition_ranges = Some(region_scan_exec.get_uncollapsed_partition_ranges());
+            time_index = region_scan_exec.time_index();
+
+            // set distinguish_partition_ranges to true, this is an incorrect workaround
+            region_scan_exec.with_distinguish_partition_range(true);
+        }
+
+        Ok(Transformed::no(plan))
+    })?;
+
+    let result = try {
+        ScannerInfo {
+            partition_ranges: partition_ranges?,
+            time_index: time_index?,
+        }
+    };
+
+    Ok(result)
+}
diff --git a/src/query/src/part_sort.rs b/src/query/src/part_sort.rs
index 965dfa9c5a..c4b9d35b20 100644
--- a/src/query/src/part_sort.rs
+++ b/src/query/src/part_sort.rs
@@ -17,6 +17,7 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::task::{Context, Poll};
 
+use arrow::array::ArrayRef;
 use arrow::compute::{concat, take_record_batch};
 use arrow_schema::SchemaRef;
 use common_recordbatch::{DfRecordBatch, DfSendableRecordBatchStream};
@@ -24,7 +25,7 @@ use datafusion::common::arrow::compute::sort_to_indices;
 use datafusion::execution::memory_pool::{MemoryConsumer, MemoryReservation};
 use datafusion::execution::{RecordBatchStream, TaskContext};
 use datafusion::physical_plan::coalesce_batches::concat_batches;
-use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet};
+use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet};
 use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties,
 };
@@ -33,8 +34,9 @@ use datafusion_physical_expr::PhysicalSortExpr;
 use futures::Stream;
 use itertools::Itertools;
 use snafu::location;
+use store_api::region_engine::PartitionRange;
 
-use crate::error::Result;
+use crate::downcast_ts_array;
 
 /// Sort input within given PartitionRange
 ///
@@ -48,11 +50,16 @@ pub struct PartSortExec {
     input: Arc<dyn ExecutionPlan>,
     /// Execution metrics
     metrics: ExecutionPlanMetricsSet,
+    partition_ranges: Vec<Vec<PartitionRange>>,
     properties: PlanProperties,
 }
 
 impl PartSortExec {
-    pub fn try_new(expression: PhysicalSortExpr, input: Arc<dyn ExecutionPlan>) -> Result<Self> {
+    pub fn new(
+        expression: PhysicalSortExpr,
+        partition_ranges: Vec<Vec<PartitionRange>>,
+        input: Arc<dyn ExecutionPlan>,
+    ) -> Self {
         let metrics = ExecutionPlanMetricsSet::new();
         let properties = PlanProperties::new(
             input.equivalence_properties().clone(),
@@ -60,12 +67,13 @@ impl PartSortExec {
             input.execution_mode(),
         );
 
-        Ok(Self {
+        Self {
             expression,
             input,
             metrics,
+            partition_ranges,
             properties,
-        })
+        }
     }
 
     pub fn to_stream(
@@ -76,7 +84,21 @@ impl PartSortExec {
         let input_stream: DfSendableRecordBatchStream =
             self.input.execute(partition, context.clone())?;
 
-        let df_stream = Box::pin(PartSortStream::new(context, self, input_stream)) as _;
+        if partition >= self.partition_ranges.len() {
+            internal_err!(
+                "Partition index out of range: {} >= {}",
+                partition,
+                self.partition_ranges.len()
+            )?;
+        }
+
+        let df_stream = Box::pin(PartSortStream::new(
+            context,
+            self,
+            input_stream,
+            self.partition_ranges[partition].clone(),
+            partition,
+        )) as _;
 
         Ok(df_stream)
     }
@@ -114,10 +136,11 @@ impl ExecutionPlan for PartSortExec {
         } else {
             internal_err!("No children found")?
         };
-        Ok(Arc::new(Self::try_new(
+        Ok(Arc::new(Self::new(
             self.expression.clone(),
+            self.partition_ranges.clone(),
             new_input.clone(),
-        )?))
+        )))
     }
 
     fn execute(
@@ -131,6 +154,15 @@ impl ExecutionPlan for PartSortExec {
     fn metrics(&self) -> Option<MetricsSet> {
         Some(self.metrics.clone_inner())
     }
+
+    /// # Explain
+    ///
+    /// This plan needs to be executed on each partition independently,
+    /// and is expected to run directly on storage engine's output
+    /// distribution / partition.
+    fn benefits_from_input_partitioning(&self) -> Vec<bool> {
+        vec![false]
+    }
 }
 
 struct PartSortStream {
@@ -142,6 +174,10 @@ struct PartSortStream {
     input: DfSendableRecordBatchStream,
     input_complete: bool,
     schema: SchemaRef,
+    partition_ranges: Vec<PartitionRange>,
+    partition: usize,
+    cur_part_idx: usize,
+    metrics: BaselineMetrics,
 }
 
 impl PartSortStream {
@@ -149,6 +185,8 @@ impl PartSortStream {
         context: Arc<TaskContext>,
         sort: &PartSortExec,
         input: DfSendableRecordBatchStream,
+        partition_ranges: Vec<PartitionRange>,
+        partition: usize,
     ) -> Self {
         Self {
             reservation: MemoryConsumer::new("PartSortStream".to_string())
@@ -159,17 +197,85 @@ impl PartSortStream {
             input,
             input_complete: false,
             schema: sort.input.schema(),
+            partition_ranges,
+            partition,
+            cur_part_idx: 0,
+            metrics: BaselineMetrics::new(&sort.metrics, partition),
         }
     }
 }
 
+macro_rules! array_check_helper {
+    ($t:ty, $unit:expr, $arr:expr, $cur_range:expr, $min_max_idx:expr) => {{
+            if $cur_range.start.unit().as_arrow_time_unit() != $unit
+            || $cur_range.end.unit().as_arrow_time_unit() != $unit
+        {
+            internal_err!(
+                "PartitionRange unit mismatch, expect {:?}, found {:?}",
+                $cur_range.start.unit(),
+                $unit
+            )?;
+        }
+        let arr = $arr
+            .as_any()
+            .downcast_ref::<arrow::array::PrimitiveArray<$t>>()
+            .unwrap();
+
+        let min = arr.value($min_max_idx.0);
+        let max = arr.value($min_max_idx.1);
+        let (min, max) = if min < max{
+            (min, max)
+        } else {
+            (max, min)
+        };
+        let cur_min = $cur_range.start.value();
+        let cur_max = $cur_range.end.value();
+        // note that PartitionRange is left inclusive and right exclusive
+        if !(min >= cur_min && max < cur_max) {
+            internal_err!(
+                "Sort column min/max value out of partition range: sort_column.min_max=[{:?}, {:?}] not in PartitionRange=[{:?}, {:?}]",
+                min,
+                max,
+                cur_min,
+                cur_max
+            )?;
+        }
+    }};
+}
+
 impl PartSortStream {
+    /// check whether the sort column's min/max value is within the partition range
+    fn check_in_range(
+        &self,
+        sort_column: &ArrayRef,
+        min_max_idx: (usize, usize),
+    ) -> datafusion_common::Result<()> {
+        if self.cur_part_idx >= self.partition_ranges.len() {
+            internal_err!(
+                "Partition index out of range: {} >= {}",
+                self.cur_part_idx,
+                self.partition_ranges.len()
+            )?;
+        }
+        let cur_range = self.partition_ranges[self.cur_part_idx];
+
+        downcast_ts_array!(
+            sort_column.data_type() => (array_check_helper, sort_column, cur_range, min_max_idx),
+            _ => internal_err!(
+                "Unsupported data type for sort column: {:?}",
+                sort_column.data_type()
+            )?,
+        );
+
+        Ok(())
+    }
+
     /// Sort and clear the buffer and return the sorted record batch
     ///
-    /// this function should return None if RecordBatch is empty
-    fn sort_buffer(&mut self) -> datafusion_common::Result<Option<DfRecordBatch>> {
-        if self.buffer.iter().map(|r| r.num_rows()).sum::<usize>() == 0 {
-            return Ok(None);
+    /// this function should return a empty record batch if the buffer is empty
+    fn sort_buffer(&mut self) -> datafusion_common::Result<DfRecordBatch> {
+        if self.buffer.is_empty() {
+            return Ok(DfRecordBatch::new_empty(self.schema.clone()));
         }
         let mut sort_columns = Vec::with_capacity(self.buffer.len());
         let mut opt = None;
@@ -194,6 +300,24 @@ impl PartSortStream {
             )
         })?;
 
+        self.check_in_range(
+            &sort_column,
+            (
+                indices.value(0) as usize,
+                indices.value(indices.len() - 1) as usize,
+            ),
+        )
+        .inspect_err(|_e| {
+            #[cfg(debug_assertions)]
+            common_telemetry::error!(
+                "Fail to check sort column in range at {}, current_idx: {}, num_rows: {}, err: {}",
+                self.partition,
+                self.cur_part_idx,
+                sort_column.len(),
+                _e
+            );
+        })?;
+
         // reserve memory for the concat input and sorted output
         let total_mem: usize = self.buffer.iter().map(|r| r.get_array_memory_size()).sum();
         self.reservation.try_grow(total_mem * 2)?;
@@ -229,7 +353,7 @@ impl PartSortStream {
         drop(full_input);
         // here remove both buffer and full_input memory
         self.reservation.shrink(2 * total_mem);
-        Ok(Some(sorted))
+        Ok(sorted)
     }
 
     pub fn poll_next_inner(
@@ -241,7 +365,7 @@ impl PartSortStream {
                 if self.buffer.is_empty() {
                     return Poll::Ready(None);
                 } else {
-                    return Poll::Ready(self.sort_buffer().transpose());
+                    return Poll::Ready(Some(self.sort_buffer()));
                 }
             }
             let res = self.input.as_mut().poll_next(cx);
@@ -249,7 +373,13 @@ impl PartSortStream {
                 Poll::Ready(Some(Ok(batch))) => {
                     if batch.num_rows() == 0 {
                         // mark end of current PartitionRange
-                        return Poll::Ready(self.sort_buffer().transpose());
+                        let sorted_batch = self.sort_buffer()?;
+                        self.cur_part_idx += 1;
+                        if sorted_batch.num_rows() == 0 {
+                            // Current part is empty, continue polling next part.
+                            continue;
+                        }
+                        return Poll::Ready(Some(Ok(sorted_batch)));
                     }
                     self.buffer.push(batch);
                     // keep polling until boundary(a empty RecordBatch) is reached
@@ -271,10 +401,11 @@ impl Stream for PartSortStream {
     type Item = datafusion_common::Result<DfRecordBatch>;
 
     fn poll_next(
-        self: Pin<&mut Self>,
+        mut self: Pin<&mut Self>,
         cx: &mut Context<'_>,
     ) -> Poll<Option<datafusion_common::Result<DfRecordBatch>>> {
-        self.poll_next_inner(cx)
+        let result = self.as_mut().poll_next_inner(cx);
+        self.metrics.record_poll(result)
     }
 }
 
@@ -290,7 +421,6 @@ mod test {
 
     use arrow::json::ArrayWriter;
     use arrow_schema::{DataType, Field, Schema, SortOptions, TimeUnit};
-    use common_telemetry::error;
     use common_time::Timestamp;
     use datafusion_physical_expr::expressions::Column;
     use futures::StreamExt;
@@ -311,6 +441,8 @@ mod test {
         let mut rng = fastrand::Rng::new();
         rng.seed(1337);
 
+        let mut test_cases = Vec::new();
+
         for case_id in 0..test_cnt {
             let mut bound_val: Option<i64> = None;
             let descending = rng.bool();
@@ -359,21 +491,23 @@ mod test {
                 };
                 assert!(start < end);
 
-                let mut sort_data = vec![];
+                let mut per_part_sort_data = vec![];
                 let mut batches = vec![];
                 for _batch_idx in 0..rng.usize(1..batch_cnt_bound) {
-                    let cnt = rng.usize(0..batch_size_bound) + 2;
-                    let iter = 0..rng.usize(1..cnt);
+                    let cnt = rng.usize(0..batch_size_bound) + 1;
+                    let iter = 0..rng.usize(0..cnt);
                     let data_gen = iter
                         .map(|_| rng.i64(start.value()..end.value()))
                         .collect_vec();
-                    sort_data.extend(data_gen.clone());
+                    if data_gen.is_empty() {
+                        // current batch is empty, skip
+                        continue;
+                    }
+                    per_part_sort_data.extend(data_gen.clone());
                     let arr = new_ts_array(unit.clone(), data_gen.clone());
-
                     let batch = DfRecordBatch::try_new(schema.clone(), vec![arr]).unwrap();
                     batches.push(batch);
                 }
-                assert!(batches.iter().all(|i| i.num_rows() >= 1));
 
                 let range = PartitionRange {
                     start,
@@ -384,12 +518,14 @@ mod test {
                 input_ranged_data.push((range, batches));
 
                 if descending {
-                    sort_data.sort_by(|a, b| b.cmp(a));
+                    per_part_sort_data.sort_by(|a, b| b.cmp(a));
                 } else {
-                    sort_data.sort();
+                    per_part_sort_data.sort();
                 }
-
-                output_data.push(sort_data);
+                if per_part_sort_data.is_empty() {
+                    continue;
+                }
+                output_data.push(per_part_sort_data);
             }
 
             let expected_output = output_data
@@ -399,8 +535,17 @@ mod test {
                         .unwrap()
                 })
                 .collect_vec();
+            test_cases.push((
+                case_id,
+                unit,
+                input_ranged_data,
+                schema,
+                opt,
+                expected_output,
+            ));
+        }
 
-            assert!(!expected_output.is_empty());
+        for (case_id, _unit, input_ranged_data, schema, opt, expected_output) in test_cases {
             run_test(case_id, input_ranged_data, schema, opt, expected_output).await;
         }
     }
@@ -412,25 +557,50 @@ mod test {
                 TimeUnit::Millisecond,
                 vec![
                     ((0, 10), vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8, 9]]),
-                    ((5, 10), vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8]]),
+                    ((5, 10), vec![vec![5, 6], vec![7, 8]]),
                 ],
                 false,
-                vec![
-                    vec![1, 2, 3, 4, 5, 6, 7, 8, 9],
-                    vec![1, 2, 3, 4, 5, 6, 7, 8],
-                ],
+                vec![vec![1, 2, 3, 4, 5, 6, 7, 8, 9], vec![5, 6, 7, 8]],
             ),
             (
                 TimeUnit::Millisecond,
                 vec![
-                    ((5, 10), vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8, 9]]),
+                    ((5, 10), vec![vec![5, 6], vec![7, 8, 9]]),
                     ((0, 10), vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8]]),
                 ],
                 true,
+                vec![vec![9, 8, 7, 6, 5], vec![8, 7, 6, 5, 4, 3, 2, 1]],
+            ),
+            (
+                TimeUnit::Millisecond,
                 vec![
-                    vec![9, 8, 7, 6, 5, 4, 3, 2, 1],
-                    vec![8, 7, 6, 5, 4, 3, 2, 1],
+                    ((5, 10), vec![]),
+                    ((0, 10), vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8]]),
                 ],
+                true,
+                vec![vec![8, 7, 6, 5, 4, 3, 2, 1]],
+            ),
+            (
+                TimeUnit::Millisecond,
+                vec![
+                    ((15, 20), vec![vec![17, 18, 19]]),
+                    ((10, 15), vec![]),
+                    ((5, 10), vec![]),
+                    ((0, 10), vec![vec![1, 2, 3], vec![4, 5, 6], vec![7, 8]]),
+                ],
+                true,
+                vec![vec![19, 18, 17], vec![8, 7, 6, 5, 4, 3, 2, 1]],
+            ),
+            (
+                TimeUnit::Millisecond,
+                vec![
+                    ((15, 20), vec![]),
+                    ((10, 15), vec![]),
+                    ((5, 10), vec![]),
+                    ((0, 10), vec![]),
+                ],
+                true,
+                vec![],
             ),
         ];
 
@@ -487,7 +657,7 @@ mod test {
         opt: SortOptions,
         expected_output: Vec<DfRecordBatch>,
     ) {
-        let (_ranges, batches): (Vec<_>, Vec<_>) = input_ranged_data.clone().into_iter().unzip();
+        let (ranges, batches): (Vec<_>, Vec<_>) = input_ranged_data.clone().into_iter().unzip();
 
         let batches = batches
             .into_iter()
@@ -498,14 +668,14 @@ mod test {
             .collect_vec();
         let mock_input = MockInputExec::new(batches, schema.clone());
 
-        let exec = PartSortExec::try_new(
+        let exec = PartSortExec::new(
             PhysicalSortExpr {
                 expr: Arc::new(Column::new("ts", 0)),
                 options: opt,
             },
+            vec![ranges],
             Arc::new(mock_input),
-        )
-        .unwrap();
+        );
 
         let exec_stream = exec.execute(0, Arc::new(TaskContext::default())).unwrap();
 
@@ -513,6 +683,7 @@ mod test {
 
         // a makeshift solution for compare large data
         if real_output != expected_output {
+            let mut full_msg = String::new();
             {
                 let mut buf = Vec::with_capacity(10 * real_output.len());
                 for batch in &real_output {
@@ -523,8 +694,9 @@ mod test {
                     buf.append(&mut rb_json);
                     buf.push(b',');
                 }
-                let buf = String::from_utf8_lossy(&buf);
-                error!("case_id:{case_id}, real_output: [{buf}]");
+                // TODO(discord9): better ways to print buf
+                let _buf = String::from_utf8_lossy(&buf);
+                full_msg += &format!("case_id:{case_id}, real_output");
             }
             {
                 let mut buf = Vec::with_capacity(10 * real_output.len());
@@ -536,10 +708,13 @@ mod test {
                     buf.append(&mut rb_json);
                     buf.push(b',');
                 }
-                let buf = String::from_utf8_lossy(&buf);
-                error!("case_id:{case_id}, expected_output: [{buf}]");
+                let _buf = String::from_utf8_lossy(&buf);
+                full_msg += &format!("case_id:{case_id}, expected_output");
             }
-            panic!("case_{} failed, opt: {:?}", case_id, opt);
+            panic!(
+                "case_{} failed, opt: {:?}, full msg: {}",
+                case_id, opt, full_msg
+            );
         }
     }
 }
diff --git a/src/query/src/query_engine/state.rs b/src/query/src/query_engine/state.rs
index 97386e9b2d..edc10e076a 100644
--- a/src/query/src/query_engine/state.rs
+++ b/src/query/src/query_engine/state.rs
@@ -49,6 +49,7 @@ use crate::optimizer::remove_duplicate::RemoveDuplicate;
 use crate::optimizer::scan_hint::ScanHintRule;
 use crate::optimizer::string_normalization::StringNormalizationRule;
 use crate::optimizer::type_conversion::TypeConversionRule;
+use crate::optimizer::windowed_sort::WindowedSortPhysicalRule;
 use crate::optimizer::ExtensionAnalyzerRule;
 use crate::query_engine::options::QueryOptions;
 use crate::query_engine::DefaultSerializer;
@@ -120,6 +121,10 @@ impl QueryEngineState {
         physical_optimizer
             .rules
             .insert(0, Arc::new(ParallelizeScan));
+        // Add rule for windowed sort
+        physical_optimizer
+            .rules
+            .push(Arc::new(WindowedSortPhysicalRule));
         // Add rule to remove duplicate nodes generated by other rules. Run this in the last.
         physical_optimizer.rules.push(Arc::new(RemoveDuplicate));
 
diff --git a/src/query/src/window_sort.rs b/src/query/src/window_sort.rs
index 25dd4b5fb8..305585b267 100644
--- a/src/query/src/window_sort.rs
+++ b/src/query/src/window_sort.rs
@@ -21,10 +21,6 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::task::{Context, Poll};
 
-use arrow::array::types::{
-    TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
-    TimestampSecondType,
-};
 use arrow::array::{Array, ArrayRef, PrimitiveArray};
 use arrow::compute::SortColumn;
 use arrow_schema::{DataType, SchemaRef, SortOptions};
@@ -58,30 +54,57 @@ use crate::error::{QueryExecutionSnafu, Result};
 ///
 /// internally, it call [`streaming_merge`] multiple times to merge multiple sorted "working ranges"
 ///
-/// the input stream must be concated in the order of `PartitionRange` in `ranges`(and each `PartitionRange` is sorted within itself), or else the result will be incorrect
+/// # Invariant Promise on Input Stream
+/// 1. The input stream must be sorted by timestamp and
+/// 2. in the order of `PartitionRange` in `ranges`
+/// 3. Each `PartitionRange` is sorted within itself(ascending or descending) but no need to be sorted across ranges
+/// 4. There can't be any RecordBatch that is cross multiple `PartitionRange` in the input stream
+///
+///  TODO(discord9): fix item 4, but since only use `PartSort` as input, this might not be a problem
+
 #[derive(Debug, Clone)]
 pub struct WindowedSortExec {
     /// Physical sort expressions(that is, sort by timestamp)
     expression: PhysicalSortExpr,
     /// Optional number of rows to fetch. Stops producing rows after this fetch
     fetch: Option<usize>,
-    /// The input ranges indicate input stream will be composed of those ranges in given order
-    ranges: Vec<PartitionRange>,
+    /// The input ranges indicate input stream will be composed of those ranges in given order.
+    ///
+    /// Each partition has one vector of `PartitionRange`.
+    ranges: Vec<Vec<PartitionRange>>,
     /// All available working ranges and their corresponding working set
     ///
-    /// working ranges promise once input stream get a value out of current range, future values will never be in this range
-    all_avail_working_range: Vec<(TimeRange, BTreeSet<usize>)>,
+    /// working ranges promise once input stream get a value out of current range, future values will never
+    /// be in this range. Each partition has one vector of ranges.
+    all_avail_working_range: Vec<Vec<(TimeRange, BTreeSet<usize>)>>,
     input: Arc<dyn ExecutionPlan>,
     /// Execution metrics
     metrics: ExecutionPlanMetricsSet,
     properties: PlanProperties,
 }
 
-fn check_partition_range_monotonicity(ranges: &[PartitionRange], descending: bool) -> bool {
-    if descending {
-        ranges.windows(2).all(|w| w[0].end >= w[1].end)
+fn check_partition_range_monotonicity(
+    ranges: &[Vec<PartitionRange>],
+    descending: bool,
+) -> Result<()> {
+    let is_valid = ranges.iter().all(|r| {
+        if descending {
+            r.windows(2).all(|w| w[0].end >= w[1].end)
+        } else {
+            r.windows(2).all(|w| w[0].start <= w[1].start)
+        }
+    });
+
+    if !is_valid {
+        let msg = if descending {
+            "Input `PartitionRange`s's upper bound is not monotonic non-increase"
+        } else {
+            "Input `PartitionRange`s's lower bound is not monotonic non-decrease"
+        };
+        let plain_error = PlainError::new(msg.to_string(), StatusCode::Unexpected);
+        Err(BoxedError::new(plain_error)).context(QueryExecutionSnafu {})
     } else {
-        ranges.windows(2).all(|w| w[0].start <= w[1].start)
+        Ok(())
     }
 }
 
@@ -89,28 +112,28 @@ impl WindowedSortExec {
     pub fn try_new(
         expression: PhysicalSortExpr,
         fetch: Option<usize>,
-        ranges: Vec<PartitionRange>,
+        ranges: Vec<Vec<PartitionRange>>,
         input: Arc<dyn ExecutionPlan>,
     ) -> Result<Self> {
-        if !check_partition_range_monotonicity(&ranges, expression.options.descending) {
-            let msg = if expression.options.descending {
-                "Input `PartitionRange`s's upper bound is not monotonic non-increase"
-            } else {
-                "Input `PartitionRange`s's lower bound is not monotonic non-decrease"
-            };
-            let plain_error = PlainError::new(msg.to_string(), StatusCode::Unexpected);
-            return Err(BoxedError::new(plain_error)).context(QueryExecutionSnafu {});
-        }
+        check_partition_range_monotonicity(&ranges, expression.options.descending)?;
 
         let properties = PlanProperties::new(
-            input.equivalence_properties().clone(),
+            input
+                .equivalence_properties()
+                .clone()
+                .with_reorder(vec![expression.clone()]),
             input.output_partitioning().clone(),
             input.execution_mode(),
         );
 
-        let overlap_counts = split_overlapping_ranges(&ranges);
-        let all_avail_working_range =
-            compute_all_working_ranges(&overlap_counts, expression.options.descending);
+        let mut all_avail_working_range = Vec::with_capacity(ranges.len());
+        for r in &ranges {
+            let overlap_counts = split_overlapping_ranges(r);
+            let working_ranges =
+                compute_all_working_ranges(&overlap_counts, expression.options.descending);
+            all_avail_working_range.push(working_ranges);
+        }
+
         Ok(Self {
             expression,
             fetch,
@@ -123,7 +146,8 @@ impl WindowedSortExec {
     }
 
     /// During receiving partial-sorted RecordBatch, we need to update the working set which is the
-    /// `PartitionRange` we think those RecordBatch belongs to. And when we receive something outside of working set, we can merge results before whenever possible.
+    /// `PartitionRange` we think those RecordBatch belongs to. And when we receive something outside
+    /// of working set, we can merge results before whenever possible.
     pub fn to_stream(
         &self,
         context: Arc<TaskContext>,
@@ -132,7 +156,12 @@ impl WindowedSortExec {
         let input_stream: DfSendableRecordBatchStream =
             self.input.execute(partition, context.clone())?;
 
-        let df_stream = Box::pin(WindowedSortStream::new(context, self, input_stream)) as _;
+        let df_stream = Box::pin(WindowedSortStream::new(
+            context,
+            self,
+            input_stream,
+            partition,
+        )) as _;
 
         Ok(df_stream)
     }
@@ -190,6 +219,15 @@ impl ExecutionPlan for WindowedSortExec {
     fn metrics(&self) -> Option<MetricsSet> {
         Some(self.metrics.clone_inner())
     }
+
+    /// # Explain
+    ///
+    /// This plan needs to be executed on each partition independently,
+    /// and is expected to run directly on storage engine's output
+    /// distribution / partition.
+    fn benefits_from_input_partitioning(&self) -> Vec<bool> {
+        vec![false; self.ranges.len()]
+    }
 }
 
 /// The core logic of merging sort multiple sorted ranges
@@ -215,8 +253,8 @@ pub struct WindowedSortStream {
     working_idx: usize,
     /// input stream assumed reading in order of `PartitionRange`
     input: DfSendableRecordBatchStream,
-    /// Whether the input stream is terminated, if it did, we should poll input again
-    is_input_terminated: bool,
+    /// Whether this stream is terminated. For reasons like limit reached or input stream is done.
+    is_terminated: bool,
     /// Output Schema, which is the same as input schema, since this is a sort plan
     schema: SchemaRef,
     /// Physical sort expressions(that is, sort by timestamp)
@@ -231,8 +269,10 @@ pub struct WindowedSortStream {
     ///
     /// working ranges promise once input stream get a value out of current range, future values will never be in this range
     all_avail_working_range: Vec<(TimeRange, BTreeSet<usize>)>,
+    /// The input partition ranges
+    ranges: Vec<PartitionRange>,
     /// Execution metrics
-    metrics: ExecutionPlanMetricsSet,
+    metrics: BaselineMetrics,
 }
 
 impl WindowedSortStream {
@@ -240,6 +280,7 @@ impl WindowedSortStream {
         context: Arc<TaskContext>,
         exec: &WindowedSortExec,
         input: DfSendableRecordBatchStream,
+        partition: usize,
     ) -> Self {
         Self {
             memory_pool: context.runtime_env().memory_pool.clone(),
@@ -251,18 +292,79 @@ impl WindowedSortStream {
             working_idx: 0,
             schema: input.schema(),
             input,
-            is_input_terminated: false,
+            is_terminated: false,
             expression: exec.expression.clone(),
             fetch: exec.fetch,
             produced: 0,
             batch_size: context.session_config().batch_size(),
-            all_avail_working_range: exec.all_avail_working_range.clone(),
-            metrics: exec.metrics.clone(),
+            all_avail_working_range: exec.all_avail_working_range[partition].clone(),
+            ranges: exec.ranges[partition].clone(),
+            metrics: BaselineMetrics::new(&exec.metrics, partition),
         }
     }
 }
 
 impl WindowedSortStream {
+    #[cfg(debug_assertions)]
+    fn check_subset_ranges(&self, cur_range: &TimeRange) {
+        let cur_is_subset_to = self
+            .ranges
+            .iter()
+            .filter(|r| cur_range.is_subset(&TimeRange::from(*r)))
+            .collect_vec();
+        if cur_is_subset_to.is_empty() {
+            error!("Current range is not a subset of any PartitionRange");
+            // found in all ranges that are subset of current range
+            let subset_ranges = self
+                .ranges
+                .iter()
+                .filter(|r| TimeRange::from(*r).is_subset(cur_range))
+                .collect_vec();
+            let only_overlap = self
+                .ranges
+                .iter()
+                .filter(|r| {
+                    let r = TimeRange::from(*r);
+                    r.is_overlapping(cur_range) && !r.is_subset(cur_range)
+                })
+                .collect_vec();
+            error!(
+                "Bad input, found {} ranges that are subset of current range, also found {} ranges that only overlap, subset ranges are: {:?}; overlap ranges are: {:?}",
+                subset_ranges.len(),
+                only_overlap.len(),
+                subset_ranges,
+                only_overlap
+            );
+        } else {
+            let only_overlap = self
+                .ranges
+                .iter()
+                .filter(|r| {
+                    let r = TimeRange::from(*r);
+                    r.is_overlapping(cur_range) && !cur_range.is_subset(&r)
+                })
+                .collect_vec();
+            error!(
+                "Found current range to be subset of {} ranges, also found {} ranges that only overlap, of subset ranges are:{:?}; overlap ranges are: {:?}",
+                cur_is_subset_to.len(),
+                only_overlap.len(),
+                cur_is_subset_to,
+                only_overlap
+            );
+        }
+        let all_overlap_working_range = self
+            .all_avail_working_range
+            .iter()
+            .filter(|(range, _)| range.is_overlapping(cur_range))
+            .map(|(range, _)| range)
+            .collect_vec();
+        error!(
+            "Found {} working ranges that overlap with current range: {:?}",
+            all_overlap_working_range.len(),
+            all_overlap_working_range
+        );
+    }
+
     /// Poll the next RecordBatch from the merge-sort's output stream
     fn poll_result_stream(
         &mut self,
@@ -273,6 +375,7 @@ impl WindowedSortStream {
                 Poll::Ready(Some(Ok(batch))) => {
                     let ret = if let Some(remaining) = self.remaining_fetch() {
                         if remaining == 0 {
+                            self.is_terminated = true;
                             None
                         } else if remaining < batch.num_rows() {
                             self.produced += remaining;
@@ -304,6 +407,7 @@ impl WindowedSortStream {
         // if no output stream is available
         Poll::Ready(None)
     }
+
     /// The core logic of merging sort multiple sorted ranges
     ///
     /// We try to maximize the number of sorted runs we can merge in one go, while emit the result as soon as possible.
@@ -314,7 +418,7 @@ impl WindowedSortStream {
         // first check and send out the merge result
         match self.poll_result_stream(cx) {
             Poll::Ready(None) => {
-                if self.is_input_terminated {
+                if self.is_terminated {
                     return Poll::Ready(None);
                 }
             }
@@ -322,7 +426,7 @@ impl WindowedSortStream {
         };
 
         // consume input stream
-        while !self.is_input_terminated {
+        while !self.is_terminated {
             // then we get a new RecordBatch from input stream
             let new_input_rbs = match self.input.as_mut().poll_next(cx) {
                 Poll::Ready(Some(Ok(batch))) => {
@@ -333,128 +437,114 @@ impl WindowedSortStream {
                 }
                 Poll::Ready(None) => {
                     // input stream is done, we need to merge sort the remaining working set
-                    self.is_input_terminated = true;
+                    self.is_terminated = true;
                     None
                 }
                 Poll::Pending => return Poll::Pending,
             };
 
+            let Some(SortedRunSet {
+                runs_with_batch,
+                sort_column,
+            }) = new_input_rbs
+            else {
+                // input stream is done, we need to merge sort the remaining working set
+                self.build_sorted_stream()?;
+                self.start_new_merge_sort()?;
+                continue;
+            };
             // The core logic to eargerly merge sort the working set
-            match new_input_rbs {
-                Some(SortedRunSet {
-                    runs_with_batch,
-                    sort_column,
-                }) => {
-                    // compare with last_value to find boundary, then merge runs if needed
 
-                    // iterate over runs_with_batch to merge sort, might create zero or more stream to put to `sort_partition_rbs`
+            // compare with last_value to find boundary, then merge runs if needed
 
-                    let mut last_remaining = None;
-                    let mut run_iter = runs_with_batch.into_iter();
-                    loop {
-                        let (sorted_rb, run_info) = if let Some(r) = last_remaining.take() {
-                            r
-                        } else if let Some(r) = run_iter.next() {
-                            r
-                        } else {
-                            break;
-                        };
-                        if sorted_rb.num_rows() == 0 {
-                            continue;
-                        }
-                        // determine if this batch is in current working range
-                        let cur_range = {
-                            match run_info.get_time_range() {
-                                Some(r) => r,
-                                _ => internal_err!("Found NULL in time index column")?,
-                            }
-                        };
-                        let working_range = if let Some(r) = self.get_working_range() {
-                            r
-                        } else {
-                            internal_err!("No working range found")?
-                        };
-
-                        if sort_column.options.unwrap_or_default().descending {
-                            if cur_range.end > working_range.end {
-                                error!("Invalid range: {:?} > {:?}", cur_range, working_range);
-                                internal_err!("Current batch have data on the right side of working range, something is very wrong")?;
-                            }
-                        } else if cur_range.start < working_range.start {
-                            error!("Invalid range: {:?} < {:?}", cur_range, working_range);
-                            internal_err!("Current batch have data on the left side of working range, something is very wrong")?;
-                        }
-
-                        if cur_range.is_subset(&working_range) {
-                            // data still in range, can't merge sort yet
-                            // see if can concat entire sorted rb, merge sort need to wait
-                            self.try_concat_batch(
-                                sorted_rb.clone(),
-                                &run_info,
-                                sort_column.options,
-                            )?;
-                        } else if let Some(intersection) = cur_range.intersection(&working_range) {
-                            // slice rb by intersection and concat it then merge sort
-                            let cur_sort_column =
-                                sort_column.values.slice(run_info.offset, run_info.len);
-                            let (offset, len) = find_slice_from_range(
-                                &SortColumn {
-                                    values: cur_sort_column.clone(),
-                                    options: sort_column.options,
-                                },
-                                &intersection,
-                            )?;
-
-                            if offset != 0 {
-                                internal_err!("Current batch have data on the left side of working range, something is very wrong")?;
-                            }
-
-                            let sliced_rb = sorted_rb.slice(offset, len);
-
-                            // try to concat the sliced input batch to the current `in_progress` run
-                            self.try_concat_batch(sliced_rb, &run_info, sort_column.options)?;
-                            // since no more sorted data in this range will come in now, build stream now
-                            self.build_sorted_stream()?;
-
-                            // since we are crossing working range, we need to merge sort the working set
-                            self.start_new_merge_sort()?;
-
-                            let (r_offset, r_len) =
-                                (offset + len, sorted_rb.num_rows() - offset - len);
-                            if r_len != 0 {
-                                // we have remaining data in this batch, put it back to input queue
-                                let remaining_rb = sorted_rb.slice(r_offset, r_len);
-                                let new_first_val =
-                                    get_timestamp_from_idx(&cur_sort_column, r_offset)?;
-                                let new_run_info = SucRun {
-                                    offset: run_info.offset + r_offset,
-                                    len: r_len,
-                                    first_val: new_first_val,
-                                    last_val: run_info.last_val,
-                                };
-                                last_remaining = Some((remaining_rb, new_run_info));
-                            }
-                            // deal with remaining batch cross working range problem
-                            // i.e: this example require more slice, and we are currently at point A
-                            // |---1---|       |---3---|
-                            // |-------A--2------------|
-                            //  put the remaining batch back to iter and deal it in next loop
-                        } else {
-                            // no overlap, we can merge sort the working set
-
-                            self.build_sorted_stream()?;
-                            self.start_new_merge_sort()?;
-
-                            // always put it back to input queue until some batch is in working range
-                            last_remaining = Some((sorted_rb, run_info));
-                        }
-                    }
+            // iterate over runs_with_batch to merge sort, might create zero or more stream to put to `sort_partition_rbs`
+            let mut last_remaining = None;
+            let mut run_iter = runs_with_batch.into_iter();
+            loop {
+                let Some((sorted_rb, run_info)) = last_remaining.take().or(run_iter.next()) else {
+                    break;
+                };
+                if sorted_rb.num_rows() == 0 {
+                    continue;
                 }
-                None => {
-                    // input stream is done, we need to merge sort the remaining working set
-                    // and emit the result
+                // determine if this batch is in current working range
+                let Some(cur_range) = run_info.get_time_range() else {
+                    internal_err!("Found NULL in time index column")?
+                };
+                let Some(working_range) = self.get_working_range() else {
+                    internal_err!("No working range found")?
+                };
+
+                // ensure the current batch is in the working range
+                if sort_column.options.unwrap_or_default().descending {
+                    if cur_range.end > working_range.end {
+                        error!("Invalid range: {:?} > {:?}", cur_range, working_range);
+                        #[cfg(debug_assertions)]
+                        self.check_subset_ranges(&cur_range);
+                        internal_err!("Current batch have data on the right side of working range, something is very wrong")?;
+                    }
+                } else if cur_range.start < working_range.start {
+                    error!("Invalid range: {:?} < {:?}", cur_range, working_range);
+                    #[cfg(debug_assertions)]
+                    self.check_subset_ranges(&cur_range);
+                    internal_err!("Current batch have data on the left side of working range, something is very wrong")?;
+                }
+
+                if cur_range.is_subset(&working_range) {
+                    // data still in range, can't merge sort yet
+                    // see if can concat entire sorted rb, merge sort need to wait
+                    self.try_concat_batch(sorted_rb.clone(), &run_info, sort_column.options)?;
+                } else if let Some(intersection) = cur_range.intersection(&working_range) {
+                    // slice rb by intersection and concat it then merge sort
+                    let cur_sort_column = sort_column.values.slice(run_info.offset, run_info.len);
+                    let (offset, len) = find_slice_from_range(
+                        &SortColumn {
+                            values: cur_sort_column.clone(),
+                            options: sort_column.options,
+                        },
+                        &intersection,
+                    )?;
+
+                    if offset != 0 {
+                        internal_err!("Current batch have data on the left side of working range, something is very wrong")?;
+                    }
+
+                    let sliced_rb = sorted_rb.slice(offset, len);
+
+                    // try to concat the sliced input batch to the current `in_progress` run
+                    self.try_concat_batch(sliced_rb, &run_info, sort_column.options)?;
+                    // since no more sorted data in this range will come in now, build stream now
+                    self.build_sorted_stream()?;
+
+                    // since we are crossing working range, we need to merge sort the working set
+                    self.start_new_merge_sort()?;
+
+                    let (r_offset, r_len) = (offset + len, sorted_rb.num_rows() - offset - len);
+                    if r_len != 0 {
+                        // we have remaining data in this batch, put it back to input queue
+                        let remaining_rb = sorted_rb.slice(r_offset, r_len);
+                        let new_first_val = get_timestamp_from_idx(&cur_sort_column, r_offset)?;
+                        let new_run_info = SucRun {
+                            offset: run_info.offset + r_offset,
+                            len: r_len,
+                            first_val: new_first_val,
+                            last_val: run_info.last_val,
+                        };
+                        last_remaining = Some((remaining_rb, new_run_info));
+                    }
+                    // deal with remaining batch cross working range problem
+                    // i.e: this example require more slice, and we are currently at point A
+                    // |---1---|       |---3---|
+                    // |-------A--2------------|
+                    //  put the remaining batch back to iter and deal it in next loop
+                } else {
+                    // no overlap, we can merge sort the working set
+
                     self.build_sorted_stream()?;
                     self.start_new_merge_sort()?;
+
+                    // always put it back to input queue until some batch is in working range
+                    last_remaining = Some((sorted_rb, run_info));
                 }
             }
         }
@@ -540,7 +630,7 @@ impl WindowedSortStream {
             streams,
             self.schema(),
             &[self.expression.clone()],
-            BaselineMetrics::new(&self.metrics, 0),
+            self.metrics.clone(),
             self.batch_size,
             fetch,
             reservation,
@@ -562,10 +652,11 @@ impl Stream for WindowedSortStream {
     type Item = datafusion_common::Result<DfRecordBatch>;
 
     fn poll_next(
-        self: Pin<&mut Self>,
+        mut self: Pin<&mut Self>,
         cx: &mut Context<'_>,
     ) -> Poll<Option<datafusion_common::Result<DfRecordBatch>>> {
-        self.poll_next_inner(cx)
+        let result = self.as_mut().poll_next_inner(cx);
+        self.metrics.record_poll(result)
     }
 }
 
@@ -624,21 +715,22 @@ fn split_batch_to_sorted_run(
 /// Downcast a temporal array to a specific type
 ///
 /// usage similar to `downcast_primitive!` in `arrow-array` crate
+#[macro_export]
 macro_rules! downcast_ts_array {
     ($data_type:expr => ($m:path $(, $args:tt)*), $($p:pat => $fallback:expr $(,)*)*) =>
     {
         match $data_type {
             arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Second, _) => {
-                $m!(TimestampSecondType $(, $args)*)
+                $m!(arrow::datatypes::TimestampSecondType, arrow_schema::TimeUnit::Second $(, $args)*)
             }
             arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Millisecond, _) => {
-                $m!(TimestampMillisecondType $(, $args)*)
+                $m!(arrow::datatypes::TimestampMillisecondType, arrow_schema::TimeUnit::Millisecond $(, $args)*)
             }
             arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Microsecond, _) => {
-                $m!(TimestampMicrosecondType $(, $args)*)
+                $m!(arrow::datatypes::TimestampMicrosecondType, arrow_schema::TimeUnit::Microsecond $(, $args)*)
             }
             arrow_schema::DataType::Timestamp(arrow_schema::TimeUnit::Nanosecond, _) => {
-                $m!(TimestampNanosecondType $(, $args)*)
+                $m!(arrow::datatypes::TimestampNanosecondType, arrow_schema::TimeUnit::Nanosecond $(, $args)*)
             }
             $($p => $fallback,)*
         }
@@ -711,7 +803,7 @@ fn find_slice_from_range(
 }
 
 macro_rules! array_iter_helper {
-    ($t:ty, $arr:expr) => {{
+    ($t:ty, $unit:expr, $arr:expr) => {{
         let typed = $arr.as_any().downcast_ref::<PrimitiveArray<$t>>().unwrap();
         let iter = typed.iter().enumerate();
         Box::new(iter) as Box<dyn Iterator<Item = (usize, Option<i64>)>>
@@ -993,7 +1085,8 @@ fn compute_all_working_ranges(
         match &mut cur_range_set {
             None => cur_range_set = Some((*range, BTreeSet::from_iter(set.iter().cloned()))),
             Some((working_range, working_set)) => {
-                // if next overlap range have Partition that's is not last one in `working_set`(hence need to be read before merge sorting), and `working_set` have >1 count
+                // if next overlap range have Partition that's is not last one in `working_set`(hence need
+                // to be read before merge sorting), and `working_set` have >1 count
                 // we have to expand current working range to cover it(and add it's `set` to `working_set`)
                 // so that merge sort is possible
                 let need_expand = {
@@ -2410,7 +2503,7 @@ mod test {
             let exec = WindowedSortExec::try_new(
                 self.expression.clone(),
                 self.fetch,
-                ranges,
+                vec![ranges],
                 Arc::new(mock_input),
             )
             .unwrap();
@@ -2441,6 +2534,78 @@ mod test {
                 vec![],
                 vec![],
             ),
+            TestStream::new(
+                Column::new("ts", 0),
+                SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                },
+                None,
+                vec![Field::new(
+                    "ts",
+                    DataType::Timestamp(TimeUnit::Millisecond, None),
+                    false,
+                )],
+                vec![
+                    // test one empty
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(2),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(3),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([2]))],
+                    ),
+                ],
+                vec![vec![Arc::new(TimestampMillisecondArray::from_iter_values(
+                    [2],
+                ))]],
+            ),
+            TestStream::new(
+                Column::new("ts", 0),
+                SortOptions {
+                    descending: false,
+                    nulls_first: true,
+                },
+                None,
+                vec![Field::new(
+                    "ts",
+                    DataType::Timestamp(TimeUnit::Millisecond, None),
+                    false,
+                )],
+                vec![
+                    // test one empty
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(2),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([]))],
+                    ),
+                    (
+                        PartitionRange {
+                            start: Timestamp::new_millisecond(1),
+                            end: Timestamp::new_millisecond(3),
+                            num_rows: 1,
+                            identifier: 0,
+                        },
+                        vec![Arc::new(TimestampMillisecondArray::from_iter_values([]))],
+                    ),
+                ],
+                vec![],
+            ),
             TestStream::new(
                 Column::new("ts", 0),
                 SortOptions {
diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs
index 2a8a0a487f..785e66d37f 100644
--- a/src/store-api/src/region_engine.rs
+++ b/src/store-api/src/region_engine.rs
@@ -205,7 +205,7 @@ pub struct ScannerProperties {
     total_rows: usize,
 
     /// Whether to yield an empty batch to distinguish partition ranges.
-    distinguish_partition_range: bool,
+    pub distinguish_partition_range: bool,
 }
 
 impl ScannerProperties {
@@ -227,12 +227,6 @@ impl ScannerProperties {
         self
     }
 
-    /// Sets distinguish partition range for scanner.
-    pub fn with_distinguish_partition_range(mut self, distinguish_partition_range: bool) -> Self {
-        self.distinguish_partition_range = distinguish_partition_range;
-        self
-    }
-
     /// Creates a new [`ScannerProperties`] with the given partitioning.
     pub fn new(partitions: Vec<Vec<PartitionRange>>, append_mode: bool, total_rows: usize) -> Self {
         Self {
@@ -274,7 +268,11 @@ pub trait RegionScanner: Debug + DisplayAs + Send {
     /// Prepares the scanner with the given partition ranges.
     ///
     /// This method is for the planner to adjust the scanner's behavior based on the partition ranges.
-    fn prepare(&mut self, ranges: Vec<Vec<PartitionRange>>) -> Result<(), BoxedError>;
+    fn prepare(
+        &mut self,
+        ranges: Vec<Vec<PartitionRange>>,
+        distinguish_partition_range: bool,
+    ) -> Result<(), BoxedError>;
 
     /// Scans the partition and returns a stream of record batches.
     ///
@@ -441,8 +439,13 @@ impl RegionScanner for SinglePartitionScanner {
         self.schema.clone()
     }
 
-    fn prepare(&mut self, ranges: Vec<Vec<PartitionRange>>) -> Result<(), BoxedError> {
+    fn prepare(
+        &mut self,
+        ranges: Vec<Vec<PartitionRange>>,
+        distinguish_partition_range: bool,
+    ) -> Result<(), BoxedError> {
         self.properties.partitions = ranges;
+        self.properties.distinguish_partition_range = distinguish_partition_range;
         Ok(())
     }
 
diff --git a/src/table/src/table/scan.rs b/src/table/src/table/scan.rs
index e67c6dc032..9207c1e86d 100644
--- a/src/table/src/table/scan.rs
+++ b/src/table/src/table/scan.rs
@@ -22,6 +22,7 @@ use common_error::ext::BoxedError;
 use common_recordbatch::{DfRecordBatch, DfSendableRecordBatchStream, SendableRecordBatchStream};
 use common_telemetry::tracing::Span;
 use common_telemetry::tracing_context::TracingContext;
+use common_telemetry::warn;
 use datafusion::error::Result as DfResult;
 use datafusion::execution::context::TaskContext;
 use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet};
@@ -49,6 +50,7 @@ pub struct RegionScanExec {
     properties: PlanProperties,
     append_mode: bool,
     total_rows: usize,
+    is_partition_set: bool,
 }
 
 impl RegionScanExec {
@@ -77,15 +79,10 @@ impl RegionScanExec {
             properties,
             append_mode,
             total_rows,
+            is_partition_set: false,
         }
     }
 
-    /// Set the expected output ordering for the plan.
-    pub fn with_output_ordering(mut self, output_ordering: Vec<PhysicalSortExpr>) -> Self {
-        self.output_ordering = Some(output_ordering);
-        self
-    }
-
     /// Get the partition ranges of the scanner. This method will collapse the ranges into
     /// a single vector.
     pub fn get_partition_ranges(&self) -> Vec<PartitionRange> {
@@ -101,18 +98,33 @@ impl RegionScanExec {
         ranges
     }
 
+    /// Similar to [`Self::get_partition_ranges`] but don't collapse the ranges.
+    pub fn get_uncollapsed_partition_ranges(&self) -> Vec<Vec<PartitionRange>> {
+        let scanner = self.scanner.lock().unwrap();
+        scanner.properties().partitions.clone()
+    }
+
+    pub fn is_partition_set(&self) -> bool {
+        self.is_partition_set
+    }
+
     /// Update the partition ranges of underlying scanner.
     pub fn with_new_partitions(
         &self,
         partitions: Vec<Vec<PartitionRange>>,
     ) -> Result<Self, BoxedError> {
+        if self.is_partition_set {
+            warn!("Setting partition ranges more than once for RegionScanExec");
+        }
+
         let num_partitions = partitions.len();
         let mut properties = self.properties.clone();
         properties.partitioning = Partitioning::UnknownPartitioning(num_partitions);
 
         {
             let mut scanner = self.scanner.lock().unwrap();
-            scanner.prepare(partitions)?;
+            let distinguish_partition_range = scanner.properties().distinguish_partition_range();
+            scanner.prepare(partitions, distinguish_partition_range)?;
         }
 
         Ok(Self {
@@ -123,8 +135,25 @@ impl RegionScanExec {
             properties,
             append_mode: self.append_mode,
             total_rows: self.total_rows,
+            is_partition_set: true,
         })
     }
+
+    pub fn with_distinguish_partition_range(&self, distinguish_partition_range: bool) {
+        let mut scanner = self.scanner.lock().unwrap();
+        let partition_ranges = scanner.properties().partitions.clone();
+        // set distinguish_partition_range won't fail
+        let _ = scanner.prepare(partition_ranges, distinguish_partition_range);
+    }
+
+    pub fn time_index(&self) -> Option<String> {
+        self.scanner
+            .lock()
+            .unwrap()
+            .schema()
+            .timestamp_column()
+            .map(|x| x.name.clone())
+    }
 }
 
 impl ExecutionPlan for RegionScanExec {
diff --git a/tests-fuzz/targets/fuzz_insert_logical_table.rs b/tests-fuzz/targets/fuzz_insert_logical_table.rs
index abafdef9a8..ee2035acd6 100644
--- a/tests-fuzz/targets/fuzz_insert_logical_table.rs
+++ b/tests-fuzz/targets/fuzz_insert_logical_table.rs
@@ -189,6 +189,7 @@ async fn validate_values(
         "SELECT {} FROM {} ORDER BY {}",
         column_list, logical_table_ctx.name, primary_keys_column_list
     );
+    info!("Select SQL: {select_sql}");
     let fetched_rows = validator::row::fetch_values(&ctx.greptime, select_sql.as_str()).await?;
     let mut expected_rows =
         replace_default(&insert_expr.values_list, &logical_table_ctx, insert_expr);
@@ -214,6 +215,8 @@ async fn insert_values<R: Rng + 'static>(
         .await
         .context(error::ExecuteQuerySnafu { sql: &sql })?;
 
+    info!("Insert values, result: {result:?}");
+
     ensure!(
         result.rows_affected() == rows as u64,
         error::AssertSnafu {
diff --git a/tests/cases/standalone/common/order/windowed_sort.result b/tests/cases/standalone/common/order/windowed_sort.result
new file mode 100644
index 0000000000..1cc0ab7720
--- /dev/null
+++ b/tests/cases/standalone/common/order/windowed_sort.result
@@ -0,0 +1,81 @@
+CREATE TABLE test(i INTEGER, t TIMESTAMP TIME INDEX);
+
+Affected Rows: 0
+
+INSERT INTO test VALUES (1, 1), (NULL, 2), (1, 3);
+
+Affected Rows: 3
+
+ADMIN FLUSH_TABLE('test');
+
++---------------------------+
+| ADMIN FLUSH_TABLE('test') |
++---------------------------+
+| 0                         |
++---------------------------+
+
+INSERT INTO test VALUES (2, 4), (2, 5), (NULL, 6);
+
+Affected Rows: 3
+
+ADMIN FLUSH_TABLE('test');
+
++---------------------------+
+| ADMIN FLUSH_TABLE('test') |
++---------------------------+
+| 0                         |
++---------------------------+
+
+INSERT INTO test VALUES (3, 7), (3, 8), (3, 9);
+
+Affected Rows: 3
+
+ADMIN FLUSH_TABLE('test');
+
++---------------------------+
+| ADMIN FLUSH_TABLE('test') |
++---------------------------+
+| 0                         |
++---------------------------+
+
+INSERT INTO test VALUES (4, 10), (4, 11), (4, 12);
+
+Affected Rows: 3
+
+SELECT * FROM test ORDER BY t LIMIT 5;
+
++---+-------------------------+
+| i | t                       |
++---+-------------------------+
+| 1 | 1970-01-01T00:00:00.001 |
+|   | 1970-01-01T00:00:00.002 |
+| 1 | 1970-01-01T00:00:00.003 |
+| 2 | 1970-01-01T00:00:00.004 |
+| 2 | 1970-01-01T00:00:00.005 |
++---+-------------------------+
+
+-- SQLNESS REPLACE (-+) -
+-- SQLNESS REPLACE (\s\s+) _
+-- SQLNESS REPLACE (peers.*) REDACTED
+-- SQLNESS REPLACE (metrics.*) REDACTED
+-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
+EXPLAIN ANALYZE SELECT * FROM test ORDER BY t LIMIT 5;
+
++-+-+-+
+| stage | node | plan_|
++-+-+-+
+| 0_| 0_|_MergeScanExec: REDACTED
+|_|_|_|
+| 1_| 0_|_GlobalLimitExec: skip=0, fetch=5 REDACTED
+|_|_|_SortPreservingMergeExec: [t@1 ASC NULLS LAST] REDACTED
+|_|_|_WindowedSortExec REDACTED
+|_|_|_PartSortExec t@1 ASC NULLS LAST REDACTED
+|_|_|_SeqScan: region=REDACTED, partition_count=2 (1 memtable ranges, 1 file 1 ranges) REDACTED
+|_|_|_|
+|_|_| Total rows: 5_|
++-+-+-+
+
+DROP TABLE test;
+
+Affected Rows: 0
+
diff --git a/tests/cases/standalone/common/order/windowed_sort.sql b/tests/cases/standalone/common/order/windowed_sort.sql
new file mode 100644
index 0000000000..7767825e3d
--- /dev/null
+++ b/tests/cases/standalone/common/order/windowed_sort.sql
@@ -0,0 +1,26 @@
+CREATE TABLE test(i INTEGER, t TIMESTAMP TIME INDEX);
+
+INSERT INTO test VALUES (1, 1), (NULL, 2), (1, 3);
+
+ADMIN FLUSH_TABLE('test');
+
+INSERT INTO test VALUES (2, 4), (2, 5), (NULL, 6);
+
+ADMIN FLUSH_TABLE('test');
+
+INSERT INTO test VALUES (3, 7), (3, 8), (3, 9);
+
+ADMIN FLUSH_TABLE('test');
+
+INSERT INTO test VALUES (4, 10), (4, 11), (4, 12);
+
+SELECT * FROM test ORDER BY t LIMIT 5;
+
+-- SQLNESS REPLACE (-+) -
+-- SQLNESS REPLACE (\s\s+) _
+-- SQLNESS REPLACE (peers.*) REDACTED
+-- SQLNESS REPLACE (metrics.*) REDACTED
+-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
+EXPLAIN ANALYZE SELECT * FROM test ORDER BY t LIMIT 5;
+
+DROP TABLE test;
diff --git a/tests/cases/standalone/common/tql-explain-analyze/explain.result b/tests/cases/standalone/common/tql-explain-analyze/explain.result
index bcaf39803f..4cdb5a8790 100644
--- a/tests/cases/standalone/common/tql-explain-analyze/explain.result
+++ b/tests/cases/standalone/common/tql-explain-analyze/explain.result
@@ -183,6 +183,7 @@ TQL EXPLAIN VERBOSE (0, 10, '5s') test;
 | physical_plan after LimitAggregation_| SAME TEXT AS ABOVE_|
 | physical_plan after ProjectionPushdown_| SAME TEXT AS ABOVE_|
 | physical_plan after PipelineChecker_| SAME TEXT AS ABOVE_|
+| physical_plan after WindowedSortRule_| SAME TEXT AS ABOVE_|
 | physical_plan after RemoveDuplicateRule_| SAME TEXT AS ABOVE_|
 | physical_plan_| PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[j]_|
 |_|_PromSeriesNormalizeExec: offset=[0], time index=[j], filter NaN: [false]_|

From 9c79baca4bd90b6f1c849a9652ccde0cef271bec Mon Sep 17 00:00:00 2001
From: Zhenchi <zhongzc_arch@outlook.com>
Date: Tue, 29 Oct 2024 15:57:17 +0800
Subject: [PATCH 114/128] feat(index): support building inverted index for the
 field column on Mito (#4887)

feat(index): support building inverted index for the field column

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>
---
 .../search/index_apply/predicates_apply.rs    |   8 +-
 src/mito2/src/read/scan_region.rs             |  25 ++-
 src/mito2/src/sst/index.rs                    |  18 +-
 .../index/inverted_index/applier/builder.rs   |  21 +-
 .../inverted_index/applier/builder/between.rs |  31 ++-
 .../applier/builder/comparison.rs             |  26 ++-
 .../inverted_index/applier/builder/eq_list.rs |  28 ++-
 .../inverted_index/applier/builder/in_list.rs |  22 +-
 .../applier/builder/regex_match.rs            |  19 +-
 .../src/sst/index/inverted_index/creator.rs   | 200 ++++++++++++------
 10 files changed, 271 insertions(+), 127 deletions(-)

diff --git a/src/index/src/inverted_index/search/index_apply/predicates_apply.rs b/src/index/src/inverted_index/search/index_apply/predicates_apply.rs
index face61bced..bead0761d7 100644
--- a/src/index/src/inverted_index/search/index_apply/predicates_apply.rs
+++ b/src/index/src/inverted_index/search/index_apply/predicates_apply.rs
@@ -114,17 +114,17 @@ impl PredicatesIndexApplier {
             .partition_in_place(|(_, ps)| ps.iter().any(|p| matches!(p, Predicate::InList(_))));
         let mut iter = predicates.into_iter();
         for _ in 0..in_list_index {
-            let (tag_name, predicates) = iter.next().unwrap();
+            let (column_name, predicates) = iter.next().unwrap();
             let fst_applier = Box::new(KeysFstApplier::try_from(predicates)?) as _;
-            fst_appliers.push((tag_name, fst_applier));
+            fst_appliers.push((column_name, fst_applier));
         }
 
-        for (tag_name, predicates) in iter {
+        for (column_name, predicates) in iter {
             if predicates.is_empty() {
                 continue;
             }
             let fst_applier = Box::new(IntersectionFstApplier::try_from(predicates)?) as _;
-            fst_appliers.push((tag_name, fst_applier));
+            fst_appliers.push((column_name, fst_applier));
         }
 
         Ok(PredicatesIndexApplier { fst_appliers })
diff --git a/src/mito2/src/read/scan_region.rs b/src/mito2/src/read/scan_region.rs
index 2c61124180..6e2a034433 100644
--- a/src/mito2/src/read/scan_region.rs
+++ b/src/mito2/src/read/scan_region.rs
@@ -393,20 +393,29 @@ impl ScanRegion {
             .and_then(|c| c.index_cache())
             .cloned();
 
+        // TODO(zhongzc): currently we only index tag columns, need to support field columns.
+        let ignore_column_ids = &self
+            .version
+            .options
+            .index_options
+            .inverted_index
+            .ignore_column_ids;
+        let indexed_column_ids = self
+            .version
+            .metadata
+            .primary_key
+            .iter()
+            .filter(|id| !ignore_column_ids.contains(id))
+            .copied()
+            .collect::<HashSet<_>>();
+
         InvertedIndexApplierBuilder::new(
             self.access_layer.region_dir().to_string(),
             self.access_layer.object_store().clone(),
             file_cache,
             index_cache,
             self.version.metadata.as_ref(),
-            self.version
-                .options
-                .index_options
-                .inverted_index
-                .ignore_column_ids
-                .iter()
-                .copied()
-                .collect(),
+            indexed_column_ids,
             self.access_layer.puffin_manager_factory().clone(),
         )
         .build(&self.request.filters)
diff --git a/src/mito2/src/sst/index.rs b/src/mito2/src/sst/index.rs
index 2fcfd8ee8c..f0ee66ab01 100644
--- a/src/mito2/src/sst/index.rs
+++ b/src/mito2/src/sst/index.rs
@@ -20,6 +20,7 @@ pub(crate) mod puffin_manager;
 mod statistics;
 mod store;
 
+use std::collections::HashSet;
 use std::num::NonZeroUsize;
 
 use common_telemetry::{debug, warn};
@@ -212,13 +213,28 @@ impl<'a> IndexerBuilder<'a> {
             segment_row_count = row_group_size;
         }
 
+        // TODO(zhongzc): currently we only index tag columns, need to support field columns.
+        let indexed_column_ids = self
+            .metadata
+            .primary_key
+            .iter()
+            .filter(|id| {
+                !self
+                    .index_options
+                    .inverted_index
+                    .ignore_column_ids
+                    .contains(id)
+            })
+            .copied()
+            .collect::<HashSet<_>>();
+
         let indexer = InvertedIndexer::new(
             self.file_id,
             self.metadata,
             self.intermediate_manager.clone(),
             self.inverted_index_config.mem_threshold_on_create(),
             segment_row_count,
-            &self.index_options.inverted_index.ignore_column_ids,
+            indexed_column_ids,
         );
 
         Some(indexer)
diff --git a/src/mito2/src/sst/index/inverted_index/applier/builder.rs b/src/mito2/src/sst/index/inverted_index/applier/builder.rs
index 6d37ffc023..603cf5aa23 100644
--- a/src/mito2/src/sst/index/inverted_index/applier/builder.rs
+++ b/src/mito2/src/sst/index/inverted_index/applier/builder.rs
@@ -20,7 +20,6 @@ mod regex_match;
 
 use std::collections::{HashMap, HashSet};
 
-use api::v1::SemanticType;
 use common_telemetry::warn;
 use datafusion_common::ScalarValue;
 use datafusion_expr::{BinaryExpr, Expr, Operator};
@@ -55,8 +54,8 @@ pub(crate) struct InvertedIndexApplierBuilder<'a> {
     /// Metadata of the region, used to get metadata like column type.
     metadata: &'a RegionMetadata,
 
-    /// Column ids to ignore.
-    ignore_column_ids: HashSet<ColumnId>,
+    /// Column ids of the columns that are indexed.
+    indexed_column_ids: HashSet<ColumnId>,
 
     /// Stores predicates during traversal on the Expr tree.
     output: HashMap<ColumnId, Vec<Predicate>>,
@@ -76,7 +75,7 @@ impl<'a> InvertedIndexApplierBuilder<'a> {
         file_cache: Option<FileCacheRef>,
         index_cache: Option<InvertedIndexCacheRef>,
         metadata: &'a RegionMetadata,
-        ignore_column_ids: HashSet<ColumnId>,
+        indexed_column_ids: HashSet<ColumnId>,
         puffin_manager_factory: PuffinManagerFactory,
     ) -> Self {
         Self {
@@ -84,7 +83,7 @@ impl<'a> InvertedIndexApplierBuilder<'a> {
             object_store,
             file_cache,
             metadata,
-            ignore_column_ids,
+            indexed_column_ids,
             output: HashMap::default(),
             index_cache,
             puffin_manager_factory,
@@ -156,9 +155,9 @@ impl<'a> InvertedIndexApplierBuilder<'a> {
         self.output.entry(column_id).or_default().push(predicate);
     }
 
-    /// Helper function to get the column id and the column type of a tag column.
+    /// Helper function to get the column id and the column type of a column.
     /// Returns `None` if the column is not a tag column or if the column is ignored.
-    fn tag_column_id_and_type(
+    fn column_id_and_type(
         &self,
         column_name: &str,
     ) -> Result<Option<(ColumnId, ConcreteDataType)>> {
@@ -169,11 +168,7 @@ impl<'a> InvertedIndexApplierBuilder<'a> {
                 column: column_name,
             })?;
 
-        if self.ignore_column_ids.contains(&column.column_id) {
-            return Ok(None);
-        }
-
-        if column.semantic_type != SemanticType::Tag {
+        if !self.indexed_column_ids.contains(&column.column_id) {
             return Ok(None);
         }
 
@@ -330,7 +325,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
diff --git a/src/mito2/src/sst/index/inverted_index/applier/builder/between.rs b/src/mito2/src/sst/index/inverted_index/applier/builder/between.rs
index ae4de2170a..0a196e6f1a 100644
--- a/src/mito2/src/sst/index/inverted_index/applier/builder/between.rs
+++ b/src/mito2/src/sst/index/inverted_index/applier/builder/between.rs
@@ -28,7 +28,7 @@ impl InvertedIndexApplierBuilder<'_> {
         let Some(column_name) = Self::column_name(&between.expr) else {
             return Ok(());
         };
-        let Some((column_id, data_type)) = self.tag_column_id_and_type(column_name)? else {
+        let Some((column_id, data_type)) = self.column_id_and_type(column_name)? else {
             return Ok(());
         };
         let Some(low) = Self::nonnull_lit(&between.low) else {
@@ -78,7 +78,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -121,7 +121,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -147,7 +147,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -159,7 +159,24 @@ mod tests {
         };
 
         builder.collect_between(&between).unwrap();
-        assert!(builder.output.is_empty());
+
+        let predicates = builder.output.get(&3).unwrap();
+        assert_eq!(predicates.len(), 1);
+        assert_eq!(
+            predicates[0],
+            Predicate::Range(RangePredicate {
+                range: Range {
+                    lower: Some(Bound {
+                        inclusive: true,
+                        value: encoded_string("abc"),
+                    }),
+                    upper: Some(Bound {
+                        inclusive: true,
+                        value: encoded_string("def"),
+                    }),
+                }
+            })
+        );
     }
 
     #[test]
@@ -173,7 +190,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -200,7 +217,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
diff --git a/src/mito2/src/sst/index/inverted_index/applier/builder/comparison.rs b/src/mito2/src/sst/index/inverted_index/applier/builder/comparison.rs
index 2c9fa861ea..cdaec9f94e 100644
--- a/src/mito2/src/sst/index/inverted_index/applier/builder/comparison.rs
+++ b/src/mito2/src/sst/index/inverted_index/applier/builder/comparison.rs
@@ -114,7 +114,7 @@ impl InvertedIndexApplierBuilder<'_> {
         let Some(lit) = Self::nonnull_lit(literal) else {
             return Ok(());
         };
-        let Some((column_id, data_type)) = self.tag_column_id_and_type(column_name)? else {
+        let Some((column_id, data_type)) = self.column_id_and_type(column_name)? else {
             return Ok(());
         };
 
@@ -234,7 +234,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -263,7 +263,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -283,14 +283,28 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
         builder
             .collect_comparison_expr(&field_column(), &Operator::Lt, &string_lit("abc"))
             .unwrap();
-        assert!(builder.output.is_empty());
+
+        let predicates = builder.output.get(&3).unwrap();
+        assert_eq!(predicates.len(), 1);
+        assert_eq!(
+            predicates[0],
+            Predicate::Range(RangePredicate {
+                range: Range {
+                    lower: None,
+                    upper: Some(Bound {
+                        inclusive: false,
+                        value: encoded_string("abc"),
+                    }),
+                }
+            })
+        );
     }
 
     #[test]
@@ -304,7 +318,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
diff --git a/src/mito2/src/sst/index/inverted_index/applier/builder/eq_list.rs b/src/mito2/src/sst/index/inverted_index/applier/builder/eq_list.rs
index 6d142d6402..1d07cca487 100644
--- a/src/mito2/src/sst/index/inverted_index/applier/builder/eq_list.rs
+++ b/src/mito2/src/sst/index/inverted_index/applier/builder/eq_list.rs
@@ -31,7 +31,7 @@ impl InvertedIndexApplierBuilder<'_> {
         let Some(lit) = Self::nonnull_lit(right).or_else(|| Self::nonnull_lit(left)) else {
             return Ok(());
         };
-        let Some((column_id, data_type)) = self.tag_column_id_and_type(column_name)? else {
+        let Some((column_id, data_type)) = self.column_id_and_type(column_name)? else {
             return Ok(());
         };
 
@@ -59,7 +59,7 @@ impl InvertedIndexApplierBuilder<'_> {
         let Some(lit) = Self::nonnull_lit(right).or_else(|| Self::nonnull_lit(left)) else {
             return Ok(());
         };
-        let Some((column_id, data_type)) = self.tag_column_id_and_type(column_name)? else {
+        let Some((column_id, data_type)) = self.column_id_and_type(column_name)? else {
             return Ok(());
         };
 
@@ -140,7 +140,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -178,14 +178,22 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
         builder
             .collect_eq(&field_column(), &string_lit("abc"))
             .unwrap();
-        assert!(builder.output.is_empty());
+
+        let predicates = builder.output.get(&3).unwrap();
+        assert_eq!(predicates.len(), 1);
+        assert_eq!(
+            predicates[0],
+            Predicate::InList(InListPredicate {
+                list: HashSet::from_iter([encoded_string("abc")])
+            })
+        );
     }
 
     #[test]
@@ -199,7 +207,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -219,7 +227,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -239,7 +247,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -298,7 +306,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -336,7 +344,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
diff --git a/src/mito2/src/sst/index/inverted_index/applier/builder/in_list.rs b/src/mito2/src/sst/index/inverted_index/applier/builder/in_list.rs
index c8cf9c4d16..6a520ba401 100644
--- a/src/mito2/src/sst/index/inverted_index/applier/builder/in_list.rs
+++ b/src/mito2/src/sst/index/inverted_index/applier/builder/in_list.rs
@@ -29,7 +29,7 @@ impl InvertedIndexApplierBuilder<'_> {
         let Some(column_name) = Self::column_name(&inlist.expr) else {
             return Ok(());
         };
-        let Some((column_id, data_type)) = self.tag_column_id_and_type(column_name)? else {
+        let Some((column_id, data_type)) = self.column_id_and_type(column_name)? else {
             return Ok(());
         };
 
@@ -71,7 +71,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -104,7 +104,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -129,7 +129,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -140,7 +140,15 @@ mod tests {
         };
 
         builder.collect_inlist(&in_list).unwrap();
-        assert!(builder.output.is_empty());
+
+        let predicates = builder.output.get(&3).unwrap();
+        assert_eq!(predicates.len(), 1);
+        assert_eq!(
+            predicates[0],
+            Predicate::InList(InListPredicate {
+                list: HashSet::from_iter([encoded_string("foo"), encoded_string("bar")])
+            })
+        );
     }
 
     #[test]
@@ -154,7 +162,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -181,7 +189,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
diff --git a/src/mito2/src/sst/index/inverted_index/applier/builder/regex_match.rs b/src/mito2/src/sst/index/inverted_index/applier/builder/regex_match.rs
index a60d9d9c0f..7fdf7f3de5 100644
--- a/src/mito2/src/sst/index/inverted_index/applier/builder/regex_match.rs
+++ b/src/mito2/src/sst/index/inverted_index/applier/builder/regex_match.rs
@@ -25,7 +25,7 @@ impl InvertedIndexApplierBuilder<'_> {
         let Some(column_name) = Self::column_name(column) else {
             return Ok(());
         };
-        let Some((column_id, data_type)) = self.tag_column_id_and_type(column_name)? else {
+        let Some((column_id, data_type)) = self.column_id_and_type(column_name)? else {
             return Ok(());
         };
         if !data_type.is_string() {
@@ -65,7 +65,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -94,7 +94,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -102,7 +102,14 @@ mod tests {
             .collect_regex_match(&field_column(), &string_lit("abc"))
             .unwrap();
 
-        assert!(builder.output.is_empty());
+        let predicates = builder.output.get(&3).unwrap();
+        assert_eq!(predicates.len(), 1);
+        assert_eq!(
+            predicates[0],
+            Predicate::RegexMatch(RegexMatchPredicate {
+                pattern: "abc".to_string()
+            })
+        );
     }
 
     #[test]
@@ -116,7 +123,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
@@ -138,7 +145,7 @@ mod tests {
             None,
             None,
             &metadata,
-            HashSet::default(),
+            HashSet::from_iter([1, 2, 3]),
             facotry,
         );
 
diff --git a/src/mito2/src/sst/index/inverted_index/creator.rs b/src/mito2/src/sst/index/inverted_index/creator.rs
index b2b1104819..6db1ef6e0b 100644
--- a/src/mito2/src/sst/index/inverted_index/creator.rs
+++ b/src/mito2/src/sst/index/inverted_index/creator.rs
@@ -36,6 +36,7 @@ use crate::error::{
     PushIndexValueSnafu, Result,
 };
 use crate::read::Batch;
+use crate::row_converter::SortField;
 use crate::sst::file::FileId;
 use crate::sst::index::intermediate::{IntermediateLocation, IntermediateManager};
 use crate::sst::index::inverted_index::codec::{IndexValueCodec, IndexValuesCodec};
@@ -72,7 +73,7 @@ pub struct InvertedIndexer {
     memory_usage: Arc<AtomicUsize>,
 
     /// Ids of indexed columns.
-    column_ids: HashSet<ColumnId>,
+    indexed_column_ids: HashSet<ColumnId>,
 }
 
 impl InvertedIndexer {
@@ -84,7 +85,7 @@ impl InvertedIndexer {
         intermediate_manager: IntermediateManager,
         memory_usage_threshold: Option<usize>,
         segment_row_count: NonZeroUsize,
-        ignore_column_ids: &[ColumnId],
+        indexed_column_ids: HashSet<ColumnId>,
     ) -> Self {
         let temp_file_provider = Arc::new(TempFileProvider::new(
             IntermediateLocation::new(&metadata.region_id, &sst_file_id),
@@ -102,14 +103,6 @@ impl InvertedIndexer {
         let index_creator = Box::new(SortIndexCreator::new(sorter, segment_row_count));
 
         let codec = IndexValuesCodec::from_tag_columns(metadata.primary_key_columns());
-        let mut column_ids = metadata
-            .primary_key_columns()
-            .map(|c| c.column_id)
-            .collect::<HashSet<_>>();
-        for id in ignore_column_ids {
-            column_ids.remove(id);
-        }
-
         Self {
             codec,
             index_creator,
@@ -118,7 +111,7 @@ impl InvertedIndexer {
             stats: Statistics::new(TYPE_INVERTED_INDEX),
             aborted: false,
             memory_usage,
-            column_ids,
+            indexed_column_ids,
         }
     }
 
@@ -189,7 +182,7 @@ impl InvertedIndexer {
         guard.inc_row_count(n);
 
         for ((col_id, col_id_str), field, value) in self.codec.decode(batch.primary_key())? {
-            if !self.column_ids.contains(col_id) {
+            if !self.indexed_column_ids.contains(col_id) {
                 continue;
             }
 
@@ -210,6 +203,32 @@ impl InvertedIndexer {
                 .context(PushIndexValueSnafu)?;
         }
 
+        for field in batch.fields() {
+            if !self.indexed_column_ids.contains(&field.column_id) {
+                continue;
+            }
+
+            let sort_field = SortField::new(field.data.data_type());
+            let col_id_str = field.column_id.to_string();
+            for i in 0..n {
+                self.value_buf.clear();
+                let value = field.data.get_ref(i);
+
+                if value.is_null() {
+                    self.index_creator
+                        .push_with_name(&col_id_str, None)
+                        .await
+                        .context(PushIndexValueSnafu)?;
+                } else {
+                    IndexValueCodec::encode_nonnull_value(value, &sort_field, &mut self.value_buf)?;
+                    self.index_creator
+                        .push_with_name(&col_id_str, Some(&self.value_buf))
+                        .await
+                        .context(PushIndexValueSnafu)?;
+                }
+            }
+        }
+
         Ok(())
     }
 
@@ -269,7 +288,7 @@ impl InvertedIndexer {
     }
 
     pub fn column_ids(&self) -> impl Iterator<Item = ColumnId> + '_ {
-        self.column_ids.iter().copied()
+        self.indexed_column_ids.iter().copied()
     }
 
     pub fn memory_usage(&self) -> usize {
@@ -297,6 +316,7 @@ mod tests {
 
     use super::*;
     use crate::cache::index::InvertedIndexCache;
+    use crate::read::BatchColumn;
     use crate::row_converter::{McmpRowCodec, RowCodec, SortField};
     use crate::sst::index::inverted_index::applier::builder::InvertedIndexApplierBuilder;
     use crate::sst::index::puffin_manager::PuffinManagerFactory;
@@ -340,12 +360,25 @@ mod tests {
                 semantic_type: SemanticType::Timestamp,
                 column_id: 3,
             })
+            .push_column_metadata(ColumnMetadata {
+                column_schema: ColumnSchema::new(
+                    "field_u64",
+                    ConcreteDataType::uint64_datatype(),
+                    false,
+                ),
+                semantic_type: SemanticType::Field,
+                column_id: 4,
+            })
             .primary_key(vec![1, 2]);
 
         Arc::new(builder.build().unwrap())
     }
 
-    fn new_batch(num_rows: usize, str_tag: impl AsRef<str>, i32_tag: impl Into<i32>) -> Batch {
+    fn new_batch(
+        str_tag: impl AsRef<str>,
+        i32_tag: impl Into<i32>,
+        u64_field: impl IntoIterator<Item = u64>,
+    ) -> Batch {
         let fields = vec![
             SortField::new(ConcreteDataType::string_datatype()),
             SortField::new(ConcreteDataType::int32_datatype()),
@@ -354,6 +387,12 @@ mod tests {
         let row: [ValueRef; 2] = [str_tag.as_ref().into(), i32_tag.into().into()];
         let primary_key = codec.encode(row.into_iter()).unwrap();
 
+        let u64_field = BatchColumn {
+            column_id: 4,
+            data: Arc::new(UInt64Vector::from_iter_values(u64_field)),
+        };
+        let num_rows = u64_field.data.len();
+
         Batch::new(
             primary_key,
             Arc::new(UInt64Vector::from_iter_values(
@@ -365,14 +404,14 @@ mod tests {
             Arc::new(UInt8Vector::from_iter_values(
                 iter::repeat(1).take(num_rows),
             )),
-            vec![],
+            vec![u64_field],
         )
         .unwrap()
     }
 
     async fn build_applier_factory(
         prefix: &str,
-        tags: BTreeSet<(&'static str, i32)>,
+        rows: BTreeSet<(&'static str, i32, [u64; 2])>,
     ) -> impl Fn(DfExpr) -> BoxFuture<'static, Vec<usize>> {
         let (d, factory) = PuffinManagerFactory::new_for_test_async(prefix).await;
         let region_dir = "region0".to_string();
@@ -383,6 +422,7 @@ mod tests {
         let intm_mgr = new_intm_mgr(d.path().to_string_lossy()).await;
         let memory_threshold = None;
         let segment_row_count = 2;
+        let indexed_column_ids = HashSet::from_iter([1, 2, 4]);
 
         let mut creator = InvertedIndexer::new(
             sst_file_id,
@@ -390,18 +430,18 @@ mod tests {
             intm_mgr,
             memory_threshold,
             NonZeroUsize::new(segment_row_count).unwrap(),
-            &[],
+            indexed_column_ids.clone(),
         );
 
-        for (str_tag, i32_tag) in &tags {
-            let batch = new_batch(segment_row_count, str_tag, *i32_tag);
+        for (str_tag, i32_tag, u64_field) in &rows {
+            let batch = new_batch(str_tag, *i32_tag, u64_field.iter().copied());
             creator.update(&batch).await.unwrap();
         }
 
         let puffin_manager = factory.build(object_store.clone());
         let mut writer = puffin_manager.writer(&file_path).await.unwrap();
         let (row_count, _) = creator.finish(&mut writer).await.unwrap();
-        assert_eq!(row_count, tags.len() * segment_row_count);
+        assert_eq!(row_count, rows.len() * segment_row_count);
         writer.finish().await.unwrap();
 
         move |expr| {
@@ -413,7 +453,7 @@ mod tests {
                 None,
                 Some(cache),
                 &region_metadata,
-                Default::default(),
+                indexed_column_ids.clone(),
                 factory.clone(),
             )
             .build(&[expr])
@@ -433,19 +473,19 @@ mod tests {
 
     #[tokio::test]
     async fn test_create_and_query_get_key() {
-        let tags = BTreeSet::from_iter([
-            ("aaa", 1),
-            ("aaa", 2),
-            ("aaa", 3),
-            ("aab", 1),
-            ("aab", 2),
-            ("aab", 3),
-            ("abc", 1),
-            ("abc", 2),
-            ("abc", 3),
+        let rows = BTreeSet::from_iter([
+            ("aaa", 1, [1, 2]),
+            ("aaa", 2, [2, 3]),
+            ("aaa", 3, [3, 4]),
+            ("aab", 1, [4, 5]),
+            ("aab", 2, [5, 6]),
+            ("aab", 3, [6, 7]),
+            ("abc", 1, [7, 8]),
+            ("abc", 2, [8, 9]),
+            ("abc", 3, [9, 10]),
         ]);
 
-        let applier_factory = build_applier_factory("test_create_and_query_get_key_", tags).await;
+        let applier_factory = build_applier_factory("test_create_and_query_get_key_", rows).await;
 
         let expr = col("tag_str").eq(lit("aaa"));
         let res = applier_factory(expr).await;
@@ -468,23 +508,27 @@ mod tests {
         let expr = col("tag_str").in_list(vec![lit("aaa"), lit("abc")], false);
         let res = applier_factory(expr).await;
         assert_eq!(res, vec![0, 1, 2, 6, 7, 8]);
+
+        let expr = col("field_u64").eq(lit(2u64));
+        let res = applier_factory(expr).await;
+        assert_eq!(res, vec![0, 1]);
     }
 
     #[tokio::test]
     async fn test_create_and_query_range() {
-        let tags = BTreeSet::from_iter([
-            ("aaa", 1),
-            ("aaa", 2),
-            ("aaa", 3),
-            ("aab", 1),
-            ("aab", 2),
-            ("aab", 3),
-            ("abc", 1),
-            ("abc", 2),
-            ("abc", 3),
+        let rows = BTreeSet::from_iter([
+            ("aaa", 1, [1, 2]),
+            ("aaa", 2, [2, 3]),
+            ("aaa", 3, [3, 4]),
+            ("aab", 1, [4, 5]),
+            ("aab", 2, [5, 6]),
+            ("aab", 3, [6, 7]),
+            ("abc", 1, [7, 8]),
+            ("abc", 2, [8, 9]),
+            ("abc", 3, [9, 10]),
         ]);
 
-        let applier_factory = build_applier_factory("test_create_and_query_range_", tags).await;
+        let applier_factory = build_applier_factory("test_create_and_query_range_", rows).await;
 
         let expr = col("tag_str").between(lit("aaa"), lit("aab"));
         let res = applier_factory(expr).await;
@@ -501,24 +545,28 @@ mod tests {
         let expr = col("tag_i32").between(lit(2), lit(2));
         let res = applier_factory(expr).await;
         assert_eq!(res, vec![1, 4, 7]);
+
+        let expr = col("field_u64").between(lit(2u64), lit(5u64));
+        let res = applier_factory(expr).await;
+        assert_eq!(res, vec![0, 1, 2, 3, 4]);
     }
 
     #[tokio::test]
     async fn test_create_and_query_comparison() {
-        let tags = BTreeSet::from_iter([
-            ("aaa", 1),
-            ("aaa", 2),
-            ("aaa", 3),
-            ("aab", 1),
-            ("aab", 2),
-            ("aab", 3),
-            ("abc", 1),
-            ("abc", 2),
-            ("abc", 3),
+        let rows = BTreeSet::from_iter([
+            ("aaa", 1, [1, 2]),
+            ("aaa", 2, [2, 3]),
+            ("aaa", 3, [3, 4]),
+            ("aab", 1, [4, 5]),
+            ("aab", 2, [5, 6]),
+            ("aab", 3, [6, 7]),
+            ("abc", 1, [7, 8]),
+            ("abc", 2, [8, 9]),
+            ("abc", 3, [9, 10]),
         ]);
 
         let applier_factory =
-            build_applier_factory("test_create_and_query_comparison_", tags).await;
+            build_applier_factory("test_create_and_query_comparison_", rows).await;
 
         let expr = col("tag_str").lt(lit("aab"));
         let res = applier_factory(expr).await;
@@ -528,6 +576,10 @@ mod tests {
         let res = applier_factory(expr).await;
         assert_eq!(res, vec![0, 3, 6]);
 
+        let expr = col("field_u64").lt(lit(2u64));
+        let res = applier_factory(expr).await;
+        assert_eq!(res, vec![0]);
+
         let expr = col("tag_str").gt(lit("aab"));
         let res = applier_factory(expr).await;
         assert_eq!(res, vec![6, 7, 8]);
@@ -536,6 +588,10 @@ mod tests {
         let res = applier_factory(expr).await;
         assert_eq!(res, vec![2, 5, 8]);
 
+        let expr = col("field_u64").gt(lit(8u64));
+        let res = applier_factory(expr).await;
+        assert_eq!(res, vec![7, 8]);
+
         let expr = col("tag_str").lt_eq(lit("aab"));
         let res = applier_factory(expr).await;
         assert_eq!(res, vec![0, 1, 2, 3, 4, 5]);
@@ -544,6 +600,10 @@ mod tests {
         let res = applier_factory(expr).await;
         assert_eq!(res, vec![0, 1, 3, 4, 6, 7]);
 
+        let expr = col("field_u64").lt_eq(lit(2u64));
+        let res = applier_factory(expr).await;
+        assert_eq!(res, vec![0, 1]);
+
         let expr = col("tag_str").gt_eq(lit("aab"));
         let res = applier_factory(expr).await;
         assert_eq!(res, vec![3, 4, 5, 6, 7, 8]);
@@ -552,6 +612,10 @@ mod tests {
         let res = applier_factory(expr).await;
         assert_eq!(res, vec![1, 2, 4, 5, 7, 8]);
 
+        let expr = col("field_u64").gt_eq(lit(8u64));
+        let res = applier_factory(expr).await;
+        assert_eq!(res, vec![6, 7, 8]);
+
         let expr = col("tag_str")
             .gt(lit("aaa"))
             .and(col("tag_str").lt(lit("abc")));
@@ -561,23 +625,29 @@ mod tests {
         let expr = col("tag_i32").gt(lit(1)).and(col("tag_i32").lt(lit(3)));
         let res = applier_factory(expr).await;
         assert_eq!(res, vec![1, 4, 7]);
+
+        let expr = col("field_u64")
+            .gt(lit(2u64))
+            .and(col("field_u64").lt(lit(9u64)));
+        let res = applier_factory(expr).await;
+        assert_eq!(res, vec![1, 2, 3, 4, 5, 6, 7]);
     }
 
     #[tokio::test]
     async fn test_create_and_query_regex() {
-        let tags = BTreeSet::from_iter([
-            ("aaa", 1),
-            ("aaa", 2),
-            ("aaa", 3),
-            ("aab", 1),
-            ("aab", 2),
-            ("aab", 3),
-            ("abc", 1),
-            ("abc", 2),
-            ("abc", 3),
+        let rows = BTreeSet::from_iter([
+            ("aaa", 1, [1, 2]),
+            ("aaa", 2, [2, 3]),
+            ("aaa", 3, [3, 4]),
+            ("aab", 1, [4, 5]),
+            ("aab", 2, [5, 6]),
+            ("aab", 3, [6, 7]),
+            ("abc", 1, [7, 8]),
+            ("abc", 2, [8, 9]),
+            ("abc", 3, [9, 10]),
         ]);
 
-        let applier_factory = build_applier_factory("test_create_and_query_regex_", tags).await;
+        let applier_factory = build_applier_factory("test_create_and_query_regex_", rows).await;
 
         let expr = binary_expr(col("tag_str"), Operator::RegexMatch, lit(".*"));
         let res = applier_factory(expr).await;

From f3e5a5a7aa1e25bec17e6a9d66dbc9dabe51919b Mon Sep 17 00:00:00 2001
From: discord9 <55937128+discord9@users.noreply.github.com>
Date: Tue, 29 Oct 2024 15:57:40 +0800
Subject: [PATCH 115/128] ci: install numpy in CI (#4895)

chore: install numpy in CI
---
 .github/actions/build-windows-artifacts/action.yml | 2 +-
 .github/workflows/develop.yml                      | 2 +-
 .github/workflows/nightly-ci.yml                   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/actions/build-windows-artifacts/action.yml b/.github/actions/build-windows-artifacts/action.yml
index 0cb8884cbd..0d92e1512e 100644
--- a/.github/actions/build-windows-artifacts/action.yml
+++ b/.github/actions/build-windows-artifacts/action.yml
@@ -40,7 +40,7 @@ runs:
 
     - name: Install PyArrow Package
       shell: pwsh
-      run: pip install pyarrow
+      run: pip install pyarrow numpy
 
     - name: Install WSL distribution
       uses: Vampire/setup-wsl@v2
diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
index 13c155d617..5d9972dd4f 100644
--- a/.github/workflows/develop.yml
+++ b/.github/workflows/develop.yml
@@ -688,7 +688,7 @@ jobs:
         with:
           python-version: '3.10'
       - name: Install PyArrow Package
-        run: pip install pyarrow
+        run: pip install pyarrow numpy
       - name: Setup etcd server
         working-directory: tests-integration/fixtures/etcd
         run: docker compose -f docker-compose-standalone.yml up -d --wait
diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml
index a2379e0e39..5b47b49ff0 100644
--- a/.github/workflows/nightly-ci.yml
+++ b/.github/workflows/nightly-ci.yml
@@ -92,7 +92,7 @@ jobs:
         with:
           python-version: "3.10"
       - name: Install PyArrow Package
-        run: pip install pyarrow
+        run: pip install pyarrow numpy
       - name: Install WSL distribution
         uses: Vampire/setup-wsl@v2
         with:

From 702a55a2356da679429af3879ca97c3530af6f85 Mon Sep 17 00:00:00 2001
From: discord9 <55937128+discord9@users.noreply.github.com>
Date: Tue, 29 Oct 2024 17:32:28 +0800
Subject: [PATCH 116/128] chore: update proto depend (#4899)

---
 Cargo.lock | 2 +-
 Cargo.toml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index c5af069e7c..0c8114f7de 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4531,7 +4531,7 @@ dependencies = [
 [[package]]
 name = "greptime-proto"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=b4d301184eb0d01fd4d1042fcc7c5dfb54f3c1e3#b4d301184eb0d01fd4d1042fcc7c5dfb54f3c1e3"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=fc45ac4cce7e91c9438c2e5a497a4d99531397f1#fc45ac4cce7e91c9438c2e5a497a4d99531397f1"
 dependencies = [
  "prost 0.12.6",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index daf79b17f9..48bf291f62 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -121,7 +121,7 @@ etcd-client = { version = "0.13" }
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "b4d301184eb0d01fd4d1042fcc7c5dfb54f3c1e3" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "fc45ac4cce7e91c9438c2e5a497a4d99531397f1" }
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"

From 9ded3149056a3ea6d675f887316492f67ea66a04 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Tue, 29 Oct 2024 20:37:53 +0800
Subject: [PATCH 117/128] feat: add json datatype for grpc protocol (#4897)

* chore: update greptime-proto

* feat: add json datatype for grpc protocol
---
 Cargo.lock                             | 2 +-
 Cargo.toml                             | 2 +-
 src/api/src/helper.rs                  | 5 +++++
 src/operator/src/req_convert/common.rs | 1 +
 4 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 0c8114f7de..40b5948bcc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4531,7 +4531,7 @@ dependencies = [
 [[package]]
 name = "greptime-proto"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=fc45ac4cce7e91c9438c2e5a497a4d99531397f1#fc45ac4cce7e91c9438c2e5a497a4d99531397f1"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=9faf45e8bd83cba106ddfb09bba85784bf9ade2a#9faf45e8bd83cba106ddfb09bba85784bf9ade2a"
 dependencies = [
  "prost 0.12.6",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index 48bf291f62..6795e8927f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -121,7 +121,7 @@ etcd-client = { version = "0.13" }
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "fc45ac4cce7e91c9438c2e5a497a4d99531397f1" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "9faf45e8bd83cba106ddfb09bba85784bf9ade2a" }
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"
diff --git a/src/api/src/helper.rs b/src/api/src/helper.rs
index 4281201c42..9d8bbacff9 100644
--- a/src/api/src/helper.rs
+++ b/src/api/src/helper.rs
@@ -116,6 +116,7 @@ impl From<ColumnDataTypeWrapper> for ConcreteDataType {
                     ConcreteDataType::binary_datatype()
                 }
             }
+            ColumnDataType::Json => ConcreteDataType::json_datatype(),
             ColumnDataType::String => ConcreteDataType::string_datatype(),
             ColumnDataType::Date => ConcreteDataType::date_datatype(),
             ColumnDataType::Datetime => ConcreteDataType::datetime_datatype(),
@@ -417,6 +418,10 @@ pub fn values_with_capacity(datatype: ColumnDataType, capacity: usize) -> Values
             decimal128_values: Vec::with_capacity(capacity),
             ..Default::default()
         },
+        ColumnDataType::Json => Values {
+            string_values: Vec::with_capacity(capacity),
+            ..Default::default()
+        },
     }
 }
 
diff --git a/src/operator/src/req_convert/common.rs b/src/operator/src/req_convert/common.rs
index 3994b32fc7..355a908d2a 100644
--- a/src/operator/src/req_convert/common.rs
+++ b/src/operator/src/req_convert/common.rs
@@ -114,6 +114,7 @@ fn push_column_to_rows(column: Column, rows: &mut [Row]) -> Result<()> {
         (Float64, F64Value, f64_values),
         (Binary, BinaryValue, binary_values),
         (String, StringValue, string_values),
+        (Json, StringValue, string_values),
         (Date, DateValue, date_values),
         (Datetime, DatetimeValue, datetime_values),
         (

From 8ed5bc5305cd65d840db19458aafaaec22416aed Mon Sep 17 00:00:00 2001
From: Yohan Wal <59358312+CookiePieWw@users.noreply.github.com>
Date: Tue, 29 Oct 2024 23:46:24 +0800
Subject: [PATCH 118/128] refactor: json conversion (#4893)

* refactor: json type update

* test: update test

* fix: convert when needed

* revert: leave sqlness tests unchanged

* fix: fmt

* refactor: just refactor

* Apply suggestions from code review

Co-authored-by: Weny Xu <wenymedia@gmail.com>

* refactor: parse jsonb first

* test: add bad cases

* Update src/datatypes/src/vectors/binary.rs

Co-authored-by: Weny Xu <wenymedia@gmail.com>

* fix: fmt

* fix: fix clippy/check

---------

Co-authored-by: Weny Xu <wenymedia@gmail.com>
---
 Cargo.lock                              |  2 +-
 Cargo.toml                              |  2 +-
 src/datatypes/src/error.rs              | 10 +++-
 src/datatypes/src/lib.rs                |  1 +
 src/datatypes/src/vectors/binary.rs     | 80 +++++++++++++++++++++++++
 src/datatypes/src/vectors/operations.rs |  8 +++
 src/servers/src/postgres/types.rs       |  4 +-
 tests-integration/tests/sql.rs          | 35 +++++++++--
 8 files changed, 132 insertions(+), 10 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 40b5948bcc..0d8058f6a9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5524,7 +5524,7 @@ dependencies = [
 [[package]]
 name = "jsonb"
 version = "0.4.1"
-source = "git+https://github.com/datafuselabs/jsonb.git?rev=46ad50fc71cf75afbf98eec455f7892a6387c1fc#46ad50fc71cf75afbf98eec455f7892a6387c1fc"
+source = "git+https://github.com/databendlabs/jsonb.git?rev=46ad50fc71cf75afbf98eec455f7892a6387c1fc#46ad50fc71cf75afbf98eec455f7892a6387c1fc"
 dependencies = [
  "byteorder",
  "fast-float",
diff --git a/Cargo.toml b/Cargo.toml
index 6795e8927f..f06c83a458 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -125,7 +125,7 @@ greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", r
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"
-jsonb = { git = "https://github.com/datafuselabs/jsonb.git", rev = "46ad50fc71cf75afbf98eec455f7892a6387c1fc", default-features = false }
+jsonb = { git = "https://github.com/databendlabs/jsonb.git", rev = "46ad50fc71cf75afbf98eec455f7892a6387c1fc", default-features = false }
 lazy_static = "1.4"
 meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "a10facb353b41460eeb98578868ebf19c2084fac" }
 mockall = "0.11.4"
diff --git a/src/datatypes/src/error.rs b/src/datatypes/src/error.rs
index 5a255dc0a6..aca9b883a9 100644
--- a/src/datatypes/src/error.rs
+++ b/src/datatypes/src/error.rs
@@ -189,6 +189,13 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Invalid JSON text: {}", value))]
+    InvalidJson {
+        value: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Value exceeds the precision {} bound", precision))]
     ValueExceedsPrecision {
         precision: u8,
@@ -222,7 +229,8 @@ impl ErrorExt for Error {
             | DefaultValueType { .. }
             | DuplicateMeta { .. }
             | InvalidTimestampPrecision { .. }
-            | InvalidPrecisionOrScale { .. } => StatusCode::InvalidArguments,
+            | InvalidPrecisionOrScale { .. }
+            | InvalidJson { .. } => StatusCode::InvalidArguments,
 
             ValueExceedsPrecision { .. }
             | CastType { .. }
diff --git a/src/datatypes/src/lib.rs b/src/datatypes/src/lib.rs
index 3766cebf87..3ce78322fe 100644
--- a/src/datatypes/src/lib.rs
+++ b/src/datatypes/src/lib.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #![feature(let_chains)]
+#![feature(assert_matches)]
 
 pub mod arrow_array;
 pub mod data_type;
diff --git a/src/datatypes/src/vectors/binary.rs b/src/datatypes/src/vectors/binary.rs
index e2074f949c..9a690ea411 100644
--- a/src/datatypes/src/vectors/binary.rs
+++ b/src/datatypes/src/vectors/binary.rs
@@ -36,6 +36,36 @@ impl BinaryVector {
     pub(crate) fn as_arrow(&self) -> &dyn Array {
         &self.array
     }
+
+    /// Creates a new binary vector of JSONB from a binary vector.
+    /// The binary vector must contain valid JSON strings.
+    pub fn convert_binary_to_json(&self) -> Result<BinaryVector> {
+        let arrow_array = self.to_arrow_array();
+        let mut vector = vec![];
+        for binary in arrow_array
+            .as_any()
+            .downcast_ref::<BinaryArray>()
+            .unwrap()
+            .iter()
+        {
+            let jsonb = if let Some(binary) = binary {
+                match jsonb::from_slice(binary) {
+                    Ok(jsonb) => Some(jsonb.to_vec()),
+                    Err(_) => {
+                        let s = String::from_utf8_lossy(binary);
+                        return error::InvalidJsonSnafu {
+                            value: s.to_string(),
+                        }
+                        .fail();
+                    }
+                }
+            } else {
+                None
+            };
+            vector.push(jsonb);
+        }
+        Ok(BinaryVector::from(vector))
+    }
 }
 
 impl From<BinaryArray> for BinaryVector {
@@ -233,6 +263,8 @@ vectors::impl_try_from_arrow_array_for_vector!(BinaryArray, BinaryVector);
 
 #[cfg(test)]
 mod tests {
+    use std::assert_matches::assert_matches;
+
     use arrow::datatypes::DataType as ArrowDataType;
     use common_base::bytes::Bytes;
     use serde_json;
@@ -383,4 +415,52 @@ mod tests {
         assert_eq!(b"four", vector.get_data(3).unwrap());
         assert_eq!(builder.len(), 4);
     }
+
+    #[test]
+    fn test_binary_json_conversion() {
+        // json strings
+        let json_strings = vec![
+            b"{\"hello\": \"world\"}".to_vec(),
+            b"{\"foo\": 1}".to_vec(),
+            b"123".to_vec(),
+        ];
+        let json_vector = BinaryVector::from(json_strings.clone())
+            .convert_binary_to_json()
+            .unwrap();
+        let jsonbs = json_strings
+            .iter()
+            .map(|v| jsonb::parse_value(v).unwrap().to_vec())
+            .collect::<Vec<_>>();
+        for i in 0..3 {
+            assert_eq!(
+                json_vector.get_ref(i).as_binary().unwrap().unwrap(),
+                jsonbs.get(i).unwrap().as_slice()
+            );
+        }
+
+        // jsonb
+        let json_vector = BinaryVector::from(jsonbs.clone())
+            .convert_binary_to_json()
+            .unwrap();
+        for i in 0..3 {
+            assert_eq!(
+                json_vector.get_ref(i).as_binary().unwrap().unwrap(),
+                jsonbs.get(i).unwrap().as_slice()
+            );
+        }
+
+        // binary with jsonb header (0x80, 0x40, 0x20)
+        let binary_with_jsonb_header: Vec<u8> = [0x80, 0x23, 0x40, 0x22].to_vec();
+        let error = BinaryVector::from(vec![binary_with_jsonb_header])
+            .convert_binary_to_json()
+            .unwrap_err();
+        assert_matches!(error, error::Error::InvalidJson { .. });
+
+        // invalid json string
+        let json_strings = vec![b"{\"hello\": \"world\"".to_vec()];
+        let error = BinaryVector::from(json_strings)
+            .convert_binary_to_json()
+            .unwrap_err();
+        assert_matches!(error, error::Error::InvalidJson { .. });
+    }
 }
diff --git a/src/datatypes/src/vectors/operations.rs b/src/datatypes/src/vectors/operations.rs
index b2de83c6e6..caa0730f72 100644
--- a/src/datatypes/src/vectors/operations.rs
+++ b/src/datatypes/src/vectors/operations.rs
@@ -18,6 +18,8 @@ mod find_unique;
 mod replicate;
 mod take;
 
+use std::sync::Arc;
+
 use common_base::BitVec;
 
 use crate::error::{self, Result};
@@ -89,6 +91,12 @@ macro_rules! impl_scalar_vector_op {
             }
 
             fn cast(&self, to_type: &ConcreteDataType) -> Result<VectorRef> {
+                if to_type == &ConcreteDataType::json_datatype() {
+                    if let Some(vector) = self.as_any().downcast_ref::<BinaryVector>() {
+                        let json_vector = vector.convert_binary_to_json()?;
+                        return Ok(Arc::new(json_vector) as VectorRef);
+                    }
+                }
                 cast::cast_non_constant!(self, to_type)
             }
 
diff --git a/src/servers/src/postgres/types.rs b/src/servers/src/postgres/types.rs
index c268b893d8..a7ce2252b9 100644
--- a/src/servers/src/postgres/types.rs
+++ b/src/servers/src/postgres/types.rs
@@ -961,7 +961,7 @@ pub(super) fn parameters_to_scalar_values(
                 if let Some(server_type) = &server_type {
                     match server_type {
                         ConcreteDataType::Binary(_) => {
-                            ScalarValue::Binary(data.map(|d| jsonb::Value::from(d).to_vec()))
+                            ScalarValue::Binary(data.map(|d| d.to_string().into_bytes()))
                         }
                         _ => {
                             return Err(invalid_parameter_error(
@@ -971,7 +971,7 @@ pub(super) fn parameters_to_scalar_values(
                         }
                     }
                 } else {
-                    ScalarValue::Binary(data.map(|d| jsonb::Value::from(d).to_vec()))
+                    ScalarValue::Binary(data.map(|d| d.to_string().into_bytes()))
                 }
             }
             _ => Err(invalid_parameter_error(
diff --git a/tests-integration/tests/sql.rs b/tests-integration/tests/sql.rs
index 19acc37ea6..af9374a2a8 100644
--- a/tests-integration/tests/sql.rs
+++ b/tests-integration/tests/sql.rs
@@ -145,7 +145,7 @@ pub async fn test_mysql_crud(store_type: StorageType) {
         .unwrap();
 
     sqlx::query(
-        "create table demo(i bigint, ts timestamp time index default current_timestamp, d date default null, dt datetime default null, b blob default null)",
+        "create table demo(i bigint, ts timestamp time index default current_timestamp, d date default null, dt datetime default null, b blob default null, j json default null)",
     )
     .execute(&pool)
     .await
@@ -158,18 +158,30 @@ pub async fn test_mysql_crud(store_type: StorageType) {
         let d = NaiveDate::from_yo_opt(2015, 100).unwrap();
         let hello = format!("hello{i}");
         let bytes = hello.as_bytes();
-        sqlx::query("insert into demo values(?, ?, ?, ?, ?)")
+        let jsons = serde_json::json!({
+            "code": i,
+            "success": true,
+            "payload": {
+                "features": [
+                    "serde",
+                    "json"
+                ],
+                "homepage": null
+            }
+        });
+        sqlx::query("insert into demo values(?, ?, ?, ?, ?, ?)")
             .bind(i)
             .bind(i)
             .bind(d)
             .bind(dt)
             .bind(bytes)
+            .bind(jsons)
             .execute(&pool)
             .await
             .unwrap();
     }
 
-    let rows = sqlx::query("select i, d, dt, b from demo")
+    let rows = sqlx::query("select i, d, dt, b, j from demo")
         .fetch_all(&pool)
         .await
         .unwrap();
@@ -180,6 +192,7 @@ pub async fn test_mysql_crud(store_type: StorageType) {
         let d: NaiveDate = row.get("d");
         let dt: DateTime<Utc> = row.get("dt");
         let bytes: Vec<u8> = row.get("b");
+        let json: serde_json::Value = row.get("j");
         assert_eq!(ret, i as i64);
         let expected_d = NaiveDate::from_yo_opt(2015, 100).unwrap();
         assert_eq!(expected_d, d);
@@ -194,6 +207,18 @@ pub async fn test_mysql_crud(store_type: StorageType) {
             format!("{}", dt.format("%Y-%m-%d %H:%M:%S"))
         );
         assert_eq!(format!("hello{i}"), String::from_utf8_lossy(&bytes));
+        let expected_j = serde_json::json!({
+            "code": i,
+            "success": true,
+            "payload": {
+                "features": [
+                    "serde",
+                    "json"
+                ],
+                "homepage": null
+            }
+        });
+        assert_eq!(json, expected_j);
     }
 
     let rows = sqlx::query("select i from demo where i=?")
@@ -396,7 +421,7 @@ pub async fn test_postgres_crud(store_type: StorageType) {
         let dt = d.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis();
         let bytes = "hello".as_bytes();
         let json = serde_json::json!({
-            "code": 200,
+            "code": i,
             "success": true,
             "payload": {
                 "features": [
@@ -444,7 +469,7 @@ pub async fn test_postgres_crud(store_type: StorageType) {
         assert_eq!("hello".as_bytes(), bytes);
 
         let expected_j = serde_json::json!({
-            "code": 200,
+            "code": i,
             "success": true,
             "payload": {
                 "features": [

From 83eb777d217b7991cea4f5f083b1dd4e03c56476 Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Tue, 29 Oct 2024 23:47:48 +0800
Subject: [PATCH 119/128] test: add fuzz test for metric region migration
 (#4862)

* test: add fuzz tests for migrate metric regions

* test: insert values before migrating metric region

* feat: correct table num

* chore: apply suggestions from CR
---
 .github/workflows/develop.yml                 |   8 +-
 tests-fuzz/Cargo.toml                         |   7 +
 .../migration/fuzz_migrate_metric_regions.rs  | 475 ++++++++++++++++++
 3 files changed, 489 insertions(+), 1 deletion(-)
 create mode 100644 tests-fuzz/targets/migration/fuzz_migrate_metric_regions.rs

diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
index 5d9972dd4f..80a4f042c0 100644
--- a/.github/workflows/develop.yml
+++ b/.github/workflows/develop.yml
@@ -436,7 +436,7 @@ jobs:
     timeout-minutes: 60
     strategy:
       matrix:
-        target: ["fuzz_migrate_mito_regions", "fuzz_failover_mito_regions", "fuzz_failover_metric_regions"]
+        target: ["fuzz_migrate_mito_regions", "fuzz_migrate_metric_regions", "fuzz_failover_mito_regions", "fuzz_failover_metric_regions"]
         mode:
           - name: "Remote WAL"
             minio: true
@@ -449,6 +449,12 @@ jobs:
               minio: true
               kafka: false
               values: "with-minio.yaml"
+          - target: "fuzz_migrate_metric_regions"
+            mode:
+              name: "Local WAL"
+              minio: true
+              kafka: false
+              values: "with-minio.yaml"
     steps:
       - name: Remove unused software
         run: |
diff --git a/tests-fuzz/Cargo.toml b/tests-fuzz/Cargo.toml
index b0c33a1e28..cbac9df713 100644
--- a/tests-fuzz/Cargo.toml
+++ b/tests-fuzz/Cargo.toml
@@ -142,3 +142,10 @@ path = "targets/migration/fuzz_migrate_mito_regions.rs"
 test = false
 bench = false
 doc = false
+
+[[bin]]
+name = "fuzz_migrate_metric_regions"
+path = "targets/migration/fuzz_migrate_metric_regions.rs"
+test = false
+bench = false
+doc = false
diff --git a/tests-fuzz/targets/migration/fuzz_migrate_metric_regions.rs b/tests-fuzz/targets/migration/fuzz_migrate_metric_regions.rs
new file mode 100644
index 0000000000..d4fa4d08fd
--- /dev/null
+++ b/tests-fuzz/targets/migration/fuzz_migrate_metric_regions.rs
@@ -0,0 +1,475 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![no_main]
+
+use std::collections::{BTreeMap, HashMap, HashSet};
+use std::sync::Arc;
+use std::time::Duration;
+
+use arbitrary::{Arbitrary, Unstructured};
+use common_telemetry::info;
+use libfuzzer_sys::fuzz_target;
+use rand::{Rng, SeedableRng};
+use rand_chacha::{ChaCha20Rng, ChaChaRng};
+use snafu::{ensure, ResultExt};
+use sqlx::{Executor, MySql, Pool};
+use store_api::storage::RegionId;
+use tests_fuzz::context::{TableContext, TableContextRef};
+use tests_fuzz::error::{self, Result};
+use tests_fuzz::fake::{
+    merge_two_word_map_fn, random_capitalize_map, uppercase_and_keyword_backtick_map,
+    MappedGenerator, WordGenerator,
+};
+use tests_fuzz::generator::create_expr::{
+    CreateLogicalTableExprGeneratorBuilder, CreatePhysicalTableExprGeneratorBuilder,
+};
+use tests_fuzz::generator::insert_expr::InsertExprGeneratorBuilder;
+use tests_fuzz::generator::Generator;
+use tests_fuzz::ir::{
+    generate_random_timestamp_for_mysql, generate_random_value, CreateTableExpr, Ident,
+    InsertIntoExpr,
+};
+use tests_fuzz::translator::mysql::create_expr::CreateTableExprTranslator;
+use tests_fuzz::translator::mysql::insert_expr::InsertIntoExprTranslator;
+use tests_fuzz::translator::DslTranslator;
+use tests_fuzz::utils::cluster_info::{fetch_nodes, PEER_TYPE_DATANODE};
+use tests_fuzz::utils::migration::migrate_region;
+use tests_fuzz::utils::partition::{fetch_partition, fetch_partitions, region_distribution};
+use tests_fuzz::utils::procedure::procedure_state;
+use tests_fuzz::utils::wait::wait_condition_fn;
+use tests_fuzz::utils::{
+    compact_table, flush_memtable, get_gt_fuzz_input_max_rows, get_gt_fuzz_input_max_tables,
+    init_greptime_connections_via_env, Connections,
+};
+use tests_fuzz::validator::row::count_values;
+
+struct FuzzContext {
+    greptime: Pool<MySql>,
+}
+
+impl FuzzContext {
+    async fn close(self) {
+        self.greptime.close().await;
+    }
+}
+
+#[derive(Copy, Clone, Debug)]
+struct FuzzInput {
+    seed: u64,
+    rows: usize,
+    tables: usize,
+}
+
+impl Arbitrary<'_> for FuzzInput {
+    fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result<Self> {
+        let seed = u.int_in_range(u64::MIN..=u64::MAX)?;
+        let mut rng = ChaChaRng::seed_from_u64(seed);
+        let max_rows = get_gt_fuzz_input_max_rows();
+        let rows = rng.gen_range(2..max_rows);
+        let max_tables = get_gt_fuzz_input_max_tables();
+        let tables = rng.gen_range(1..max_tables);
+
+        Ok(FuzzInput { rows, seed, tables })
+    }
+}
+
+fn generate_create_physical_table_expr<R: Rng + 'static>(rng: &mut R) -> Result<CreateTableExpr> {
+    let physical_table_if_not_exists = rng.gen_bool(0.5);
+    let create_physical_table_expr = CreatePhysicalTableExprGeneratorBuilder::default()
+        .name_generator(Box::new(MappedGenerator::new(
+            WordGenerator,
+            merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map),
+        )))
+        .if_not_exists(physical_table_if_not_exists)
+        .build()
+        .unwrap();
+    create_physical_table_expr.generate(rng)
+}
+
+async fn create_physical_table<R: Rng + 'static>(
+    ctx: &FuzzContext,
+    rng: &mut R,
+) -> Result<TableContextRef> {
+    // Create a physical table and a logical table on top of it
+    let create_physical_table_expr = generate_create_physical_table_expr(rng).unwrap();
+    let translator = CreateTableExprTranslator;
+    let sql = translator.translate(&create_physical_table_expr)?;
+    let result = sqlx::query(&sql)
+        .execute(&ctx.greptime)
+        .await
+        .context(error::ExecuteQuerySnafu { sql: &sql })?;
+    info!("Create physical table: {sql}, result: {result:?}");
+
+    Ok(Arc::new(TableContext::from(&create_physical_table_expr)))
+}
+
+fn generate_create_logical_table_expr<R: Rng + 'static>(
+    physical_table_ctx: TableContextRef,
+    rng: &mut R,
+) -> Result<CreateTableExpr> {
+    let labels = rng.gen_range(1..=5);
+    let logical_table_if_not_exists = rng.gen_bool(0.5);
+
+    let create_logical_table_expr = CreateLogicalTableExprGeneratorBuilder::default()
+        .name_generator(Box::new(MappedGenerator::new(
+            WordGenerator,
+            merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map),
+        )))
+        .physical_table_ctx(physical_table_ctx)
+        .labels(labels)
+        .if_not_exists(logical_table_if_not_exists)
+        .build()
+        .unwrap();
+    create_logical_table_expr.generate(rng)
+}
+
+fn generate_insert_expr<R: Rng + 'static>(
+    rows: usize,
+    rng: &mut R,
+    table_ctx: TableContextRef,
+) -> Result<InsertIntoExpr> {
+    let insert_generator = InsertExprGeneratorBuilder::default()
+        .omit_column_list(false)
+        .table_ctx(table_ctx)
+        .rows(rows)
+        .value_generator(Box::new(generate_random_value))
+        .ts_value_generator(Box::new(generate_random_timestamp_for_mysql))
+        .build()
+        .unwrap();
+    insert_generator.generate(rng)
+}
+
+async fn insert_values<R: Rng + 'static>(
+    rows: usize,
+    ctx: &FuzzContext,
+    rng: &mut R,
+    logical_table_ctx: TableContextRef,
+) -> Result<InsertIntoExpr> {
+    let insert_expr = generate_insert_expr(rows, rng, logical_table_ctx.clone())?;
+    let translator = InsertIntoExprTranslator;
+    let sql = translator.translate(&insert_expr)?;
+    let result = ctx
+        .greptime
+        // unprepared query, see <https://github.com/GreptimeTeam/greptimedb/issues/3500>
+        .execute(sql.as_str())
+        .await
+        .context(error::ExecuteQuerySnafu { sql: &sql })?;
+
+    ensure!(
+        result.rows_affected() == rows as u64,
+        error::AssertSnafu {
+            reason: format!(
+                "expected rows affected: {}, actual: {}",
+                rows,
+                result.rows_affected(),
+            )
+        }
+    );
+
+    Ok(insert_expr)
+}
+
+#[derive(Debug, Clone, Copy)]
+struct Migration {
+    from_peer: u64,
+    to_peer: u64,
+    region_id: RegionId,
+}
+
+async fn create_logical_table_and_insert_values(
+    ctx: &FuzzContext,
+    input: &FuzzInput,
+    rng: &mut ChaCha20Rng,
+    physical_table_ctx: &TableContextRef,
+    table_num: usize,
+    tables: &mut HashMap<Ident, (TableContextRef, InsertIntoExpr)>,
+) -> Result<()> {
+    for _ in 0..table_num {
+        let translator = CreateTableExprTranslator;
+        let create_logical_table_expr =
+            generate_create_logical_table_expr(physical_table_ctx.clone(), rng).unwrap();
+        if tables.contains_key(&create_logical_table_expr.table_name) {
+            // Ignores same name logical table.
+            continue;
+        }
+        let sql = translator.translate(&create_logical_table_expr)?;
+        let result = sqlx::query(&sql)
+            .execute(&ctx.greptime)
+            .await
+            .context(error::ExecuteQuerySnafu { sql: &sql })?;
+        info!("Create logical table: {sql}, result: {result:?}");
+        let logical_table_ctx = Arc::new(TableContext::from(&create_logical_table_expr));
+
+        let insert_expr = insert_values(input.rows, ctx, rng, logical_table_ctx.clone()).await?;
+        if rng.gen_bool(0.1) {
+            flush_memtable(&ctx.greptime, &physical_table_ctx.name).await?;
+        }
+        if rng.gen_bool(0.1) {
+            compact_table(&ctx.greptime, &physical_table_ctx.name).await?;
+        }
+
+        tables.insert(
+            logical_table_ctx.name.clone(),
+            (logical_table_ctx.clone(), insert_expr),
+        );
+    }
+
+    Ok(())
+}
+
+async fn execute_migration(ctx: FuzzContext, input: FuzzInput) -> Result<()> {
+    let mut rng = ChaCha20Rng::seed_from_u64(input.seed);
+    // Creates a physical table.
+    let physical_table_ctx = create_physical_table(&ctx, &mut rng).await?;
+    let mut tables = HashMap::with_capacity(input.tables);
+
+    let table_num = (input.tables / 3).max(1);
+
+    // Creates more logical tables and inserts values
+    create_logical_table_and_insert_values(
+        &ctx,
+        &input,
+        &mut rng,
+        &physical_table_ctx,
+        table_num,
+        &mut tables,
+    )
+    .await?;
+
+    // Fetches region distribution
+    let partitions = fetch_partitions(&ctx.greptime, physical_table_ctx.name.clone()).await?;
+    let num_partitions = partitions.len();
+    let region_distribution = region_distribution(partitions);
+    info!("Region distribution: {region_distribution:?}");
+    let datanodes = fetch_nodes(&ctx.greptime)
+        .await?
+        .into_iter()
+        .flat_map(|node| {
+            if node.peer_type == PEER_TYPE_DATANODE {
+                Some(node)
+            } else {
+                None
+            }
+        })
+        .collect::<Vec<_>>();
+    info!("List datanodes: {:?}", datanodes);
+
+    // Generates region migration task.
+    let mut migrations = Vec::with_capacity(num_partitions);
+    let mut new_distribution: BTreeMap<u64, HashSet<_>> = BTreeMap::new();
+    for (datanode_id, regions) in region_distribution {
+        let step = rng.gen_range(1..datanodes.len());
+        for region in regions {
+            let to_peer = (datanode_id + step as u64) % datanodes.len() as u64;
+            new_distribution.entry(to_peer).or_default().insert(region);
+            migrations.push(Migration {
+                from_peer: datanode_id,
+                to_peer,
+                region_id: region,
+            })
+        }
+    }
+
+    let mut procedure_ids = Vec::with_capacity(migrations.len());
+    // Triggers region migrations
+    for Migration {
+        from_peer,
+        to_peer,
+        region_id,
+    } in &migrations
+    {
+        let procedure_id =
+            migrate_region(&ctx.greptime, region_id.as_u64(), *from_peer, *to_peer, 120).await;
+        info!("Migrating region: {region_id} from {from_peer} to {to_peer}, procedure: {procedure_id}");
+        procedure_ids.push(procedure_id);
+    }
+    info!("Excepted new region distribution: {new_distribution:?}");
+    for (migration, procedure_id) in migrations.clone().into_iter().zip(procedure_ids) {
+        info!("Waits for migration: {migration:?}");
+        let region_id = migration.region_id.as_u64();
+        wait_condition_fn(
+            Duration::from_secs(120),
+            || {
+                let greptime = ctx.greptime.clone();
+                let procedure_id = procedure_id.to_string();
+                Box::pin(async move {
+                    {
+                        let output = procedure_state(&greptime, &procedure_id).await;
+                        info!("Checking procedure: {procedure_id}, output: {output}");
+                        fetch_partition(&greptime, region_id).await.unwrap()
+                    }
+                })
+            },
+            |partition| {
+                info!("Region: {region_id},  datanode: {}", partition.datanode_id);
+                partition.datanode_id == migration.to_peer
+            },
+            Duration::from_secs(1),
+        )
+        .await;
+    }
+
+    // Validates value rows
+    info!("Validates num of rows");
+    for (table_ctx, insert_expr) in tables.values() {
+        let sql = format!("select count(1) as count from {}", table_ctx.name);
+        let values = count_values(&ctx.greptime, &sql).await?;
+        let expected_rows = insert_expr.values_list.len() as u64;
+        assert_eq!(
+            values.count as u64, expected_rows,
+            "Expected rows: {}, got: {}, table: {}",
+            expected_rows, values.count, table_ctx.name
+        );
+    }
+
+    // Creates more logical tables and inserts values
+    create_logical_table_and_insert_values(
+        &ctx,
+        &input,
+        &mut rng,
+        &physical_table_ctx,
+        table_num,
+        &mut tables,
+    )
+    .await?;
+
+    // Validates value rows
+    info!("Validates num of rows");
+    for (table_ctx, insert_expr) in tables.values() {
+        let sql = format!("select count(1) as count from {}", table_ctx.name);
+        let values = count_values(&ctx.greptime, &sql).await?;
+        let expected_rows = insert_expr.values_list.len() as u64;
+        assert_eq!(
+            values.count as u64, expected_rows,
+            "Expected rows: {}, got: {}, table: {}",
+            expected_rows, values.count, table_ctx.name
+        );
+    }
+
+    // Recovers region distribution
+    let migrations = migrations
+        .into_iter()
+        .map(
+            |Migration {
+                 from_peer,
+                 to_peer,
+                 region_id,
+             }| Migration {
+                from_peer: to_peer,
+                to_peer: from_peer,
+                region_id,
+            },
+        )
+        .collect::<Vec<_>>();
+
+    let mut procedure_ids = Vec::with_capacity(migrations.len());
+    // Triggers region migrations
+    for Migration {
+        from_peer,
+        to_peer,
+        region_id,
+    } in &migrations
+    {
+        let procedure_id =
+            migrate_region(&ctx.greptime, region_id.as_u64(), *from_peer, *to_peer, 120).await;
+        info!("Migrating region: {region_id} from {from_peer} to {to_peer}, procedure: {procedure_id}");
+        procedure_ids.push(procedure_id);
+    }
+    info!("Excepted new region distribution: {new_distribution:?}");
+    for (migration, procedure_id) in migrations.into_iter().zip(procedure_ids) {
+        info!("Waits for migration: {migration:?}");
+        let region_id = migration.region_id.as_u64();
+        wait_condition_fn(
+            Duration::from_secs(120),
+            || {
+                let greptime = ctx.greptime.clone();
+                let procedure_id = procedure_id.to_string();
+                Box::pin(async move {
+                    {
+                        let output = procedure_state(&greptime, &procedure_id).await;
+                        info!("Checking procedure: {procedure_id}, output: {output}");
+                        fetch_partition(&greptime, region_id).await.unwrap()
+                    }
+                })
+            },
+            |partition| {
+                info!("Region: {region_id},  datanode: {}", partition.datanode_id);
+                partition.datanode_id == migration.to_peer
+            },
+            Duration::from_secs(1),
+        )
+        .await;
+    }
+
+    // Creates more logical tables and inserts values
+    create_logical_table_and_insert_values(
+        &ctx,
+        &input,
+        &mut rng,
+        &physical_table_ctx,
+        table_num,
+        &mut tables,
+    )
+    .await?;
+
+    // Validates value rows
+    info!("Validates num of rows");
+    for (table_ctx, insert_expr) in tables.values() {
+        let sql = format!("select count(1) as count from {}", table_ctx.name);
+        let values = count_values(&ctx.greptime, &sql).await?;
+        let expected_rows = insert_expr.values_list.len() as u64;
+        assert_eq!(
+            values.count as u64, expected_rows,
+            "Expected rows: {}, got: {}, table: {}",
+            expected_rows, values.count, table_ctx.name
+        );
+    }
+
+    // Clean up
+    for (table_ctx, _) in tables.values() {
+        let sql = format!("DROP TABLE {}", table_ctx.name);
+        let result = sqlx::query(&sql)
+            .execute(&ctx.greptime)
+            .await
+            .context(error::ExecuteQuerySnafu { sql })?;
+        info!("Drop table: {}\n\nResult: {result:?}\n\n", table_ctx.name);
+    }
+
+    let sql = format!("DROP TABLE {}", physical_table_ctx.name);
+    let result = sqlx::query(&sql)
+        .execute(&ctx.greptime)
+        .await
+        .context(error::ExecuteQuerySnafu { sql })?;
+    info!(
+        "Drop table: {}\n\nResult: {result:?}\n\n",
+        physical_table_ctx.name
+    );
+
+    ctx.close().await;
+    Ok(())
+}
+
+fuzz_target!(|input: FuzzInput| {
+    common_telemetry::init_default_ut_logging();
+    common_runtime::block_on_global(async {
+        let Connections { mysql } = init_greptime_connections_via_env().await;
+        let ctx = FuzzContext {
+            greptime: mysql.expect("mysql connection init must be succeed"),
+        };
+        execute_migration(ctx, input)
+            .await
+            .unwrap_or_else(|err| panic!("fuzz test must be succeed: {err:?}"));
+    })
+});

From d275cdd570a3579e695c16b60f2c13cff9f94747 Mon Sep 17 00:00:00 2001
From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com>
Date: Tue, 29 Oct 2024 21:39:48 -0700
Subject: [PATCH 120/128] feat: Support altering table TTL (#4848)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* feat/alter-ttl:
 Update greptime-proto source and add ChangeTableOptions handling

 - Change greptime-proto source repository and revision in Cargo.lock and Cargo.toml
 - Implement handling for ChangeTableOptions in grpc-expr and meta modules
 - Add support for parsing and applying region option changes in mito2
 - Introduce new error type for invalid change table option requests
 - Add humantime dependency to store-api
 - Fix SQL syntax in tests for changing column types

* chore: remove write buffer size option handling since we don't support specifying write_buffer_size for single table or region

* persist ttl to manifest

* chore: add sqlness

* fix: sqlness

* fix: typo and toml format

* fix: tests

* update: change alter syntax

* feat/alter-ttl: Add Clone trait to RegionFlushRequest and remove redundant Default derive in region_request.rs.

* feat/alter-ttl: Refactor code to replace 'ChangeTableOption' with 'ChangeRegionOption' and handle TTL as a region option

 • Rename ChangeTableOption to ChangeRegionOption across various files.
 • Update AlterKind::ChangeTableOptions to AlterKind::ChangeRegionOptions.
 • Modify TTL handling to treat '0d' as None for TTL in table options.
 • Adjust related function names and comments to reflect the change from table to region options.
 • Include test case updates to verify the new TTL handling behavior.

* chore: update format

* refactor: update region options in DatanodeTableValue

* feat/alter-ttl:
 Remove TTL handling from RegionManifest and related structures

 - Eliminate TTL fields from `RegionManifest`, `RegionChange`, and associated handling logic.
 - Update tests and checksums to reflect removal of TTL.
 - Refactor `RegionOpener` and `handle_alter` to adjust to TTL removal.
 - Simplify `RegionChangeResult` by replacing `change` with `new_meta`.

* chore: fmt

* remove useless delete op

* feat/alter-ttl: Updated Cargo.lock and gRPC expression Cargo.toml to include store-api dependency. Refactored alter.rs to use ChangeOption from store-api instead of ChangeTableOptionRequest.
Adjusted error handling in error.rs to use MetadataError. Modified handle_alter.rs to handle TTL changes with ChangeOption. Simplified region_request.rs by replacing
ChangeRegionOption with ChangeOption and removing redundant code. Removed UnsupportedTableOptionChange error in table/src/error.rs. Updated metadata.rs to use ChangeOption for table
options. Removed ChangeTableOptionRequest enum and related conversion code from requests.rs.

* feat/alter-ttl: Update greptime-proto dependency to revision 53ab9a9553

* chore: format code

* chore: update greptime-proto
---
 Cargo.lock                                    |  4 +-
 Cargo.toml                                    |  2 +-
 src/common/grpc-expr/Cargo.toml               |  1 +
 src/common/grpc-expr/src/alter.rs             | 14 ++-
 src/common/grpc-expr/src/error.rs             |  8 ++
 .../alter_logical_tables/update_metadata.rs   |  4 +-
 src/common/meta/src/ddl/alter_table.rs        | 20 +++-
 .../src/ddl/alter_table/region_request.rs     |  1 +
 .../src/ddl/alter_table/update_metadata.rs    | 13 ++-
 .../create_logical_tables/update_metadata.rs  |  4 +-
 .../src/ddl/test_util/datanode_handler.rs     |  5 +-
 src/common/meta/src/ddl/tests/alter_table.rs  | 91 ++++++++++++++++++-
 src/common/meta/src/error.rs                  | 13 +++
 src/common/meta/src/key.rs                    | 29 ++++--
 src/common/meta/src/key/datanode_table.rs     | 45 ++++++++-
 src/common/meta/src/key/table_info.rs         |  6 ++
 src/mito2/src/region/version.rs               | 12 +++
 src/mito2/src/worker/handle_alter.rs          | 55 ++++++++++-
 src/mito2/src/worker/handle_open.rs           |  3 +-
 src/operator/src/expr_factory.rs              | 21 +++--
 src/sql/src/parsers/alter_parser.rs           | 70 +++++++++++++-
 src/sql/src/statements/alter.rs               | 28 +++++-
 src/store-api/Cargo.toml                      |  1 +
 src/store-api/src/metadata.rs                 | 11 +++
 src/store-api/src/region_request.rs           | 49 +++++++++-
 src/table/src/error.rs                        |  4 +
 src/table/src/metadata.rs                     | 35 ++++++-
 src/table/src/requests.rs                     | 30 +++++-
 .../common/alter/alter_table_options.result   | 88 ++++++++++++++++++
 .../common/alter/alter_table_options.sql      | 19 ++++
 .../common/alter/change_col_type.result       | 12 +--
 .../common/alter/change_col_type.sql          | 12 +--
 .../alter/change_col_type_not_null.result     |  2 +-
 .../common/alter/change_col_type_not_null.sql |  2 +-
 34 files changed, 648 insertions(+), 66 deletions(-)
 create mode 100644 tests/cases/standalone/common/alter/alter_table_options.result
 create mode 100644 tests/cases/standalone/common/alter/alter_table_options.sql

diff --git a/Cargo.lock b/Cargo.lock
index 0d8058f6a9..264ceb2de2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2149,6 +2149,7 @@ dependencies = [
  "paste",
  "prost 0.12.6",
  "snafu 0.8.5",
+ "store-api",
  "table",
 ]
 
@@ -4531,7 +4532,7 @@ dependencies = [
 [[package]]
 name = "greptime-proto"
 version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=9faf45e8bd83cba106ddfb09bba85784bf9ade2a#9faf45e8bd83cba106ddfb09bba85784bf9ade2a"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=255f87a3318ace3f88a67f76995a0e14910983f4#255f87a3318ace3f88a67f76995a0e14910983f4"
 dependencies = [
  "prost 0.12.6",
  "serde",
@@ -11497,6 +11498,7 @@ dependencies = [
  "datatypes",
  "derive_builder 0.12.0",
  "futures",
+ "humantime",
  "serde",
  "serde_json",
  "snafu 0.8.5",
diff --git a/Cargo.toml b/Cargo.toml
index f06c83a458..d89289b304 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -121,7 +121,7 @@ etcd-client = { version = "0.13" }
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "9faf45e8bd83cba106ddfb09bba85784bf9ade2a" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "255f87a3318ace3f88a67f76995a0e14910983f4" }
 humantime = "2.1"
 humantime-serde = "1.1"
 itertools = "0.10"
diff --git a/src/common/grpc-expr/Cargo.toml b/src/common/grpc-expr/Cargo.toml
index 246a999a84..650b726325 100644
--- a/src/common/grpc-expr/Cargo.toml
+++ b/src/common/grpc-expr/Cargo.toml
@@ -18,6 +18,7 @@ common-time.workspace = true
 datatypes.workspace = true
 prost.workspace = true
 snafu.workspace = true
+store-api.workspace = true
 table.workspace = true
 
 [dev-dependencies]
diff --git a/src/common/grpc-expr/src/alter.rs b/src/common/grpc-expr/src/alter.rs
index ac49069412..5dcc861e9d 100644
--- a/src/common/grpc-expr/src/alter.rs
+++ b/src/common/grpc-expr/src/alter.rs
@@ -22,12 +22,13 @@ use api::v1::{
 use common_query::AddColumnLocation;
 use datatypes::schema::{ColumnSchema, RawSchema};
 use snafu::{ensure, OptionExt, ResultExt};
+use store_api::region_request::ChangeOption;
 use table::metadata::TableId;
 use table::requests::{AddColumnRequest, AlterKind, AlterTableRequest, ChangeColumnTypeRequest};
 
 use crate::error::{
-    InvalidColumnDefSnafu, MissingFieldSnafu, MissingTimestampColumnSnafu, Result,
-    UnknownLocationTypeSnafu,
+    InvalidChangeTableOptionRequestSnafu, InvalidColumnDefSnafu, MissingFieldSnafu,
+    MissingTimestampColumnSnafu, Result, UnknownLocationTypeSnafu,
 };
 
 const LOCATION_TYPE_FIRST: i32 = LocationType::First as i32;
@@ -92,6 +93,15 @@ pub fn alter_expr_to_request(table_id: TableId, expr: AlterExpr) -> Result<Alter
         Kind::RenameTable(RenameTable { new_table_name }) => {
             AlterKind::RenameTable { new_table_name }
         }
+        Kind::ChangeTableOptions(api::v1::ChangeTableOptions {
+            change_table_options,
+        }) => AlterKind::ChangeTableOptions {
+            options: change_table_options
+                .iter()
+                .map(ChangeOption::try_from)
+                .collect::<std::result::Result<Vec<_>, _>>()
+                .context(InvalidChangeTableOptionRequestSnafu)?,
+        },
     };
 
     let request = AlterTableRequest {
diff --git a/src/common/grpc-expr/src/error.rs b/src/common/grpc-expr/src/error.rs
index f025c4d5a5..5d59c46983 100644
--- a/src/common/grpc-expr/src/error.rs
+++ b/src/common/grpc-expr/src/error.rs
@@ -19,6 +19,7 @@ use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use common_macro::stack_trace_debug;
 use snafu::{Location, Snafu};
+use store_api::metadata::MetadataError;
 
 #[derive(Snafu)]
 #[snafu(visibility(pub))]
@@ -118,6 +119,12 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Invalid change table option request"))]
+    InvalidChangeTableOptionRequest {
+        #[snafu(source)]
+        error: MetadataError,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -141,6 +148,7 @@ impl ErrorExt for Error {
             Error::UnknownColumnDataType { .. } | Error::InvalidFulltextColumnType { .. } => {
                 StatusCode::InvalidArguments
             }
+            Error::InvalidChangeTableOptionRequest { .. } => StatusCode::InvalidArguments,
         }
     }
 
diff --git a/src/common/meta/src/ddl/alter_logical_tables/update_metadata.rs b/src/common/meta/src/ddl/alter_logical_tables/update_metadata.rs
index 33be226dda..7338968153 100644
--- a/src/common/meta/src/ddl/alter_logical_tables/update_metadata.rs
+++ b/src/common/meta/src/ddl/alter_logical_tables/update_metadata.rs
@@ -43,10 +43,10 @@ impl AlterLogicalTablesProcedure {
             &self.data.physical_columns,
         );
 
-        // Updates physical table's metadata
+        // Updates physical table's metadata, and we don't need to touch per-region settings.
         self.context
             .table_metadata_manager
-            .update_table_info(physical_table_info, new_raw_table_info)
+            .update_table_info(physical_table_info, None, new_raw_table_info)
             .await?;
 
         Ok(())
diff --git a/src/common/meta/src/ddl/alter_table.rs b/src/common/meta/src/ddl/alter_table.rs
index 8bece74e78..ae0d6dc6bd 100644
--- a/src/common/meta/src/ddl/alter_table.rs
+++ b/src/common/meta/src/ddl/alter_table.rs
@@ -43,10 +43,10 @@ use crate::ddl::DdlContext;
 use crate::error::{Error, Result};
 use crate::instruction::CacheIdent;
 use crate::key::table_info::TableInfoValue;
-use crate::key::DeserializedValueWithBytes;
+use crate::key::{DeserializedValueWithBytes, RegionDistribution};
 use crate::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
 use crate::rpc::ddl::AlterTableTask;
-use crate::rpc::router::{find_leader_regions, find_leaders};
+use crate::rpc::router::{find_leader_regions, find_leaders, region_distribution};
 use crate::{metrics, ClusterId};
 
 /// The alter table procedure
@@ -101,6 +101,9 @@ impl AlterTableProcedure {
             .get_physical_table_route(table_id)
             .await?;
 
+        self.data.region_distribution =
+            Some(region_distribution(&physical_table_route.region_routes));
+
         let leaders = find_leaders(&physical_table_route.region_routes);
         let mut alter_region_tasks = Vec::with_capacity(leaders.len());
 
@@ -161,8 +164,14 @@ impl AlterTableProcedure {
             self.on_update_metadata_for_rename(new_table_name.to_string(), table_info_value)
                 .await?;
         } else {
-            self.on_update_metadata_for_alter(new_info.into(), table_info_value)
-                .await?;
+            // region distribution is set in submit_alter_region_requests
+            let region_distribution = self.data.region_distribution.as_ref().unwrap().clone();
+            self.on_update_metadata_for_alter(
+                new_info.into(),
+                region_distribution,
+                table_info_value,
+            )
+            .await?;
         }
 
         info!("Updated table metadata for table {table_ref}, table_id: {table_id}");
@@ -271,6 +280,8 @@ pub struct AlterTableData {
     table_id: TableId,
     /// Table info value before alteration.
     table_info_value: Option<DeserializedValueWithBytes<TableInfoValue>>,
+    /// Region distribution for table in case we need to update region options.
+    region_distribution: Option<RegionDistribution>,
 }
 
 impl AlterTableData {
@@ -281,6 +292,7 @@ impl AlterTableData {
             table_id,
             cluster_id,
             table_info_value: None,
+            region_distribution: None,
         }
     }
 
diff --git a/src/common/meta/src/ddl/alter_table/region_request.rs b/src/common/meta/src/ddl/alter_table/region_request.rs
index 0756360395..41bd1481c9 100644
--- a/src/common/meta/src/ddl/alter_table/region_request.rs
+++ b/src/common/meta/src/ddl/alter_table/region_request.rs
@@ -106,6 +106,7 @@ fn create_proto_alter_kind(
             })))
         }
         Kind::RenameTable(_) => Ok(None),
+        Kind::ChangeTableOptions(v) => Ok(Some(alter_request::Kind::ChangeTableOptions(v.clone()))),
     }
 }
 
diff --git a/src/common/meta/src/ddl/alter_table/update_metadata.rs b/src/common/meta/src/ddl/alter_table/update_metadata.rs
index 6ef6e2e7fc..6d27dabe27 100644
--- a/src/common/meta/src/ddl/alter_table/update_metadata.rs
+++ b/src/common/meta/src/ddl/alter_table/update_metadata.rs
@@ -20,7 +20,7 @@ use table::requests::AlterKind;
 use crate::ddl::alter_table::AlterTableProcedure;
 use crate::error::{self, Result};
 use crate::key::table_info::TableInfoValue;
-use crate::key::DeserializedValueWithBytes;
+use crate::key::{DeserializedValueWithBytes, RegionDistribution};
 
 impl AlterTableProcedure {
     /// Builds new_meta
@@ -51,7 +51,9 @@ impl AlterTableProcedure {
             AlterKind::RenameTable { new_table_name } => {
                 new_info.name = new_table_name.to_string();
             }
-            AlterKind::DropColumns { .. } | AlterKind::ChangeColumnTypes { .. } => {}
+            AlterKind::DropColumns { .. }
+            | AlterKind::ChangeColumnTypes { .. }
+            | AlterKind::ChangeTableOptions { .. } => {}
         }
 
         Ok(new_info)
@@ -75,11 +77,16 @@ impl AlterTableProcedure {
     pub(crate) async fn on_update_metadata_for_alter(
         &self,
         new_table_info: RawTableInfo,
+        region_distribution: RegionDistribution,
         current_table_info_value: &DeserializedValueWithBytes<TableInfoValue>,
     ) -> Result<()> {
         let table_metadata_manager = &self.context.table_metadata_manager;
         table_metadata_manager
-            .update_table_info(current_table_info_value, new_table_info)
+            .update_table_info(
+                current_table_info_value,
+                Some(region_distribution),
+                new_table_info,
+            )
             .await?;
 
         Ok(())
diff --git a/src/common/meta/src/ddl/create_logical_tables/update_metadata.rs b/src/common/meta/src/ddl/create_logical_tables/update_metadata.rs
index 0309a04613..75d235812f 100644
--- a/src/common/meta/src/ddl/create_logical_tables/update_metadata.rs
+++ b/src/common/meta/src/ddl/create_logical_tables/update_metadata.rs
@@ -58,10 +58,10 @@ impl CreateLogicalTablesProcedure {
             &new_table_info.name,
         );
 
-        // Update physical table's metadata
+        // Update physical table's metadata and we don't need to touch per-region settings.
         self.context
             .table_metadata_manager
-            .update_table_info(&physical_table_info, new_table_info)
+            .update_table_info(&physical_table_info, None, new_table_info)
             .await?;
 
         // Invalid physical table cache
diff --git a/src/common/meta/src/ddl/test_util/datanode_handler.rs b/src/common/meta/src/ddl/test_util/datanode_handler.rs
index 5cb7d4a0f2..b82609b985 100644
--- a/src/common/meta/src/ddl/test_util/datanode_handler.rs
+++ b/src/common/meta/src/ddl/test_util/datanode_handler.rs
@@ -29,7 +29,10 @@ use crate::test_util::MockDatanodeHandler;
 #[async_trait::async_trait]
 impl MockDatanodeHandler for () {
     async fn handle(&self, _peer: &Peer, _request: RegionRequest) -> Result<RegionResponse> {
-        unreachable!()
+        Ok(RegionResponse {
+            affected_rows: 0,
+            extensions: Default::default(),
+        })
     }
 
     async fn handle_query(
diff --git a/src/common/meta/src/ddl/tests/alter_table.rs b/src/common/meta/src/ddl/tests/alter_table.rs
index 36a1ff0ece..7ede16b125 100644
--- a/src/common/meta/src/ddl/tests/alter_table.rs
+++ b/src/common/meta/src/ddl/tests/alter_table.rs
@@ -19,13 +19,14 @@ use std::sync::Arc;
 use api::v1::alter_expr::Kind;
 use api::v1::region::{region_request, RegionRequest};
 use api::v1::{
-    AddColumn, AddColumns, AlterExpr, ColumnDataType, ColumnDef as PbColumnDef, DropColumn,
-    DropColumns, SemanticType,
+    AddColumn, AddColumns, AlterExpr, ChangeTableOption, ChangeTableOptions, ColumnDataType,
+    ColumnDef as PbColumnDef, DropColumn, DropColumns, SemanticType,
 };
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use store_api::storage::RegionId;
+use table::requests::TTL_KEY;
 use tokio::sync::mpsc::{self};
 
 use crate::ddl::alter_table::AlterTableProcedure;
@@ -34,6 +35,7 @@ use crate::ddl::test_util::create_table::test_create_table_task;
 use crate::ddl::test_util::datanode_handler::{
     DatanodeWatcher, RequestOutdatedErrorDatanodeHandler,
 };
+use crate::key::datanode_table::DatanodeTableKey;
 use crate::key::table_name::TableNameKey;
 use crate::key::table_route::TableRouteValue;
 use crate::peer::Peer;
@@ -293,12 +295,21 @@ async fn test_on_update_metadata_add_columns() {
     let table_name = "foo";
     let table_id = 1024;
     let task = test_create_table_task(table_name, table_id);
+
+    let region_id = RegionId::new(table_id, 0);
+    let mock_table_routes = vec![RegionRoute {
+        region: Region::new_test(region_id),
+        leader_peer: Some(Peer::default()),
+        follower_peers: vec![],
+        leader_state: None,
+        leader_down_since: None,
+    }];
     // Puts a value to table name key.
     ddl_context
         .table_metadata_manager
         .create_table_metadata(
             task.table_info.clone(),
-            TableRouteValue::physical(vec![]),
+            TableRouteValue::physical(mock_table_routes),
             HashMap::new(),
         )
         .await
@@ -326,6 +337,7 @@ async fn test_on_update_metadata_add_columns() {
     let mut procedure =
         AlterTableProcedure::new(cluster_id, table_id, task, ddl_context.clone()).unwrap();
     procedure.on_prepare().await.unwrap();
+    procedure.submit_alter_region_requests().await.unwrap();
     procedure.on_update_metadata().await.unwrap();
 
     let table_info = ddl_context
@@ -343,3 +355,76 @@ async fn test_on_update_metadata_add_columns() {
         table_info.meta.next_column_id
     );
 }
+
+#[tokio::test]
+async fn test_on_update_table_options() {
+    let node_manager = Arc::new(MockDatanodeManager::new(()));
+    let ddl_context = new_ddl_context(node_manager);
+    let cluster_id = 1;
+    let table_name = "foo";
+    let table_id = 1024;
+    let task = test_create_table_task(table_name, table_id);
+
+    let region_id = RegionId::new(table_id, 0);
+    let mock_table_routes = vec![RegionRoute {
+        region: Region::new_test(region_id),
+        leader_peer: Some(Peer::default()),
+        follower_peers: vec![],
+        leader_state: None,
+        leader_down_since: None,
+    }];
+    // Puts a value to table name key.
+    ddl_context
+        .table_metadata_manager
+        .create_table_metadata(
+            task.table_info.clone(),
+            TableRouteValue::physical(mock_table_routes),
+            HashMap::new(),
+        )
+        .await
+        .unwrap();
+
+    let task = AlterTableTask {
+        alter_table: AlterExpr {
+            catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+            table_name: table_name.to_string(),
+            kind: Some(Kind::ChangeTableOptions(ChangeTableOptions {
+                change_table_options: vec![ChangeTableOption {
+                    key: TTL_KEY.to_string(),
+                    value: "1d".to_string(),
+                }],
+            })),
+        },
+    };
+    let mut procedure =
+        AlterTableProcedure::new(cluster_id, table_id, task, ddl_context.clone()).unwrap();
+    procedure.on_prepare().await.unwrap();
+    procedure.submit_alter_region_requests().await.unwrap();
+    procedure.on_update_metadata().await.unwrap();
+
+    let table_info = ddl_context
+        .table_metadata_manager
+        .table_info_manager()
+        .get(table_id)
+        .await
+        .unwrap()
+        .unwrap()
+        .into_inner()
+        .table_info;
+
+    let datanode_key = DatanodeTableKey::new(0, table_id);
+    let region_info = ddl_context
+        .table_metadata_manager
+        .datanode_table_manager()
+        .get(&datanode_key)
+        .await
+        .unwrap()
+        .unwrap()
+        .region_info;
+
+    assert_eq!(
+        region_info.region_options,
+        HashMap::from(&table_info.meta.options)
+    );
+}
diff --git a/src/common/meta/src/error.rs b/src/common/meta/src/error.rs
index 0e7709df0b..51dc74e646 100644
--- a/src/common/meta/src/error.rs
+++ b/src/common/meta/src/error.rs
@@ -652,6 +652,18 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display(
+        "Datanode table info not found, table id: {}, datanode id: {}",
+        table_id,
+        datanode_id
+    ))]
+    DatanodeTableInfoNotFound {
+        datanode_id: DatanodeId,
+        table_id: TableId,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -752,6 +764,7 @@ impl ErrorExt for Error {
             PostgresExecution { .. } => StatusCode::Internal,
             #[cfg(feature = "pg_kvbackend")]
             ConnectPostgres { .. } => StatusCode::Internal,
+            Error::DatanodeTableInfoNotFound { .. } => StatusCode::Internal,
         }
     }
 
diff --git a/src/common/meta/src/key.rs b/src/common/meta/src/key.rs
index 0f703b9430..39ca065393 100644
--- a/src/common/meta/src/key.rs
+++ b/src/common/meta/src/key.rs
@@ -133,7 +133,6 @@ use self::flow::flow_name::FlowNameValue;
 use self::schema_name::{SchemaManager, SchemaNameKey, SchemaNameValue};
 use self::table_route::{TableRouteManager, TableRouteValue};
 use self::tombstone::TombstoneManager;
-use crate::ddl::utils::region_storage_path;
 use crate::error::{self, Result, SerdeJsonSnafu};
 use crate::key::node_address::NodeAddressValue;
 use crate::key::table_route::TableRouteKey;
@@ -593,8 +592,6 @@ impl TableMetadataManager {
         table_info.meta.region_numbers = region_numbers;
         let table_id = table_info.ident.table_id;
         let engine = table_info.meta.engine.clone();
-        let region_storage_path =
-            region_storage_path(&table_info.catalog_name, &table_info.schema_name);
 
         // Creates table name.
         let table_name = TableNameKey::new(
@@ -606,7 +603,7 @@ impl TableMetadataManager {
             .table_name_manager()
             .build_create_txn(&table_name, table_id)?;
 
-        let region_options = (&table_info.meta.options).into();
+        let region_options = table_info.to_region_options();
         // Creates table info.
         let table_info_value = TableInfoValue::new(table_info);
         let (create_table_info_txn, on_create_table_info_failure) = self
@@ -625,6 +622,7 @@ impl TableMetadataManager {
         ]);
 
         if let TableRouteValue::Physical(x) = &table_route_value {
+            let region_storage_path = table_info_value.region_storage_path();
             let create_datanode_table_txn = self.datanode_table_manager().build_create_txn(
                 table_id,
                 &engine,
@@ -926,13 +924,15 @@ impl TableMetadataManager {
     }
 
     /// Updates table info and returns an error if different metadata exists.
+    /// And cascade-ly update all redundant table options for each region
+    /// if region_distribution is present.
     pub async fn update_table_info(
         &self,
         current_table_info_value: &DeserializedValueWithBytes<TableInfoValue>,
+        region_distribution: Option<RegionDistribution>,
         new_table_info: RawTableInfo,
     ) -> Result<()> {
         let table_id = current_table_info_value.table_info.ident.table_id;
-
         let new_table_info_value = current_table_info_value.update(new_table_info);
 
         // Updates table info.
@@ -940,8 +940,19 @@ impl TableMetadataManager {
             .table_info_manager()
             .build_update_txn(table_id, current_table_info_value, &new_table_info_value)?;
 
-        let mut r = self.kv_backend.txn(update_table_info_txn).await?;
+        let txn = if let Some(region_distribution) = region_distribution {
+            // region options induced from table info.
+            let new_region_options = new_table_info_value.table_info.to_region_options();
+            let update_datanode_table_options_txn = self
+                .datanode_table_manager
+                .build_update_table_options_txn(table_id, region_distribution, new_region_options)
+                .await?;
+            Txn::merge_all([update_table_info_txn, update_datanode_table_options_txn])
+        } else {
+            update_table_info_txn
+        };
 
+        let mut r = self.kv_backend.txn(txn).await?;
         // Checks whether metadata was already updated.
         if !r.succeeded {
             let mut set = TxnOpGetResponseSet::from(&mut r.responses);
@@ -1669,12 +1680,12 @@ mod tests {
             DeserializedValueWithBytes::from_inner(TableInfoValue::new(table_info.clone()));
         // should be ok.
         table_metadata_manager
-            .update_table_info(&current_table_info_value, new_table_info.clone())
+            .update_table_info(&current_table_info_value, None, new_table_info.clone())
             .await
             .unwrap();
         // if table info was updated, it should be ok.
         table_metadata_manager
-            .update_table_info(&current_table_info_value, new_table_info.clone())
+            .update_table_info(&current_table_info_value, None, new_table_info.clone())
             .await
             .unwrap();
 
@@ -1696,7 +1707,7 @@ mod tests {
         // if the current_table_info_value is wrong, it should return an error.
         // The ABA problem.
         assert!(table_metadata_manager
-            .update_table_info(&wrong_table_info_value, new_table_info)
+            .update_table_info(&wrong_table_info_value, None, new_table_info)
             .await
             .is_err())
     }
diff --git a/src/common/meta/src/key/datanode_table.rs b/src/common/meta/src/key/datanode_table.rs
index e904983cc8..a0f0e9e511 100644
--- a/src/common/meta/src/key/datanode_table.rs
+++ b/src/common/meta/src/key/datanode_table.rs
@@ -23,7 +23,7 @@ use store_api::storage::RegionNumber;
 use table::metadata::TableId;
 
 use super::MetadataKey;
-use crate::error::{InvalidMetadataSnafu, Result};
+use crate::error::{DatanodeTableInfoNotFoundSnafu, InvalidMetadataSnafu, Result};
 use crate::key::{
     MetadataValue, RegionDistribution, DATANODE_TABLE_KEY_PATTERN, DATANODE_TABLE_KEY_PREFIX,
 };
@@ -209,6 +209,49 @@ impl DatanodeTableManager {
         Ok(txn)
     }
 
+    /// Builds a transaction to updates the redundant table options (including WAL options)
+    /// for given table id, if provided.
+    ///
+    /// Note that the provided `new_region_options` must be a
+    /// complete set of all options rather than incremental changes.
+    pub(crate) async fn build_update_table_options_txn(
+        &self,
+        table_id: TableId,
+        region_distribution: RegionDistribution,
+        new_region_options: HashMap<String, String>,
+    ) -> Result<Txn> {
+        assert!(!region_distribution.is_empty());
+        // safety: region_distribution must not be empty
+        let (any_datanode, _) = region_distribution.first_key_value().unwrap();
+
+        let mut region_info = self
+            .kv_backend
+            .get(&DatanodeTableKey::new(*any_datanode, table_id).to_bytes())
+            .await
+            .transpose()
+            .context(DatanodeTableInfoNotFoundSnafu {
+                datanode_id: *any_datanode,
+                table_id,
+            })?
+            .and_then(|r| DatanodeTableValue::try_from_raw_value(&r.value))?
+            .region_info;
+        // substitute region options only.
+        region_info.region_options = new_region_options;
+
+        let mut txns = Vec::with_capacity(region_distribution.len());
+
+        for (datanode, regions) in region_distribution.into_iter() {
+            let key = DatanodeTableKey::new(datanode, table_id);
+            let key_bytes = key.to_bytes();
+            let value_bytes = DatanodeTableValue::new(table_id, regions, region_info.clone())
+                .try_as_raw_value()?;
+            txns.push(TxnOp::Put(key_bytes, value_bytes));
+        }
+
+        let txn = Txn::new().and_then(txns);
+        Ok(txn)
+    }
+
     /// Builds the update datanode table transactions. It only executes while the primary keys comparing successes.
     pub(crate) fn build_update_txn(
         &self,
diff --git a/src/common/meta/src/key/table_info.rs b/src/common/meta/src/key/table_info.rs
index b3ca42b6ff..615043f853 100644
--- a/src/common/meta/src/key/table_info.rs
+++ b/src/common/meta/src/key/table_info.rs
@@ -23,6 +23,7 @@ use table::table_name::TableName;
 use table::table_reference::TableReference;
 
 use super::TABLE_INFO_KEY_PATTERN;
+use crate::ddl::utils::region_storage_path;
 use crate::error::{InvalidMetadataSnafu, Result};
 use crate::key::txn_helper::TxnOpGetResponseSet;
 use crate::key::{DeserializedValueWithBytes, MetadataKey, MetadataValue, TABLE_INFO_KEY_PREFIX};
@@ -125,6 +126,11 @@ impl TableInfoValue {
             table_name: self.table_info.name.to_string(),
         }
     }
+
+    /// Builds storage path for all regions in table.
+    pub fn region_storage_path(&self) -> String {
+        region_storage_path(&self.table_info.catalog_name, &self.table_info.schema_name)
+    }
 }
 
 pub type TableInfoManagerRef = Arc<TableInfoManager>;
diff --git a/src/mito2/src/region/version.rs b/src/mito2/src/region/version.rs
index 15928e1e22..8e78fa4ab3 100644
--- a/src/mito2/src/region/version.rs
+++ b/src/mito2/src/region/version.rs
@@ -98,6 +98,18 @@ impl VersionControl {
         Ok(())
     }
 
+    /// Applies region option changes and generates a new version.
+    pub(crate) fn alter_options(&self, options: RegionOptions) {
+        let version = self.current().version;
+        let new_version = Arc::new(
+            VersionBuilder::from_version(version)
+                .options(options)
+                .build(),
+        );
+        let mut version_data = self.data.write().unwrap();
+        version_data.version = new_version;
+    }
+
     /// Apply edit to current version.
     pub(crate) fn apply_edit(
         &self,
diff --git a/src/mito2/src/worker/handle_alter.rs b/src/mito2/src/worker/handle_alter.rs
index 27aadbfc0d..d73b24c46a 100644
--- a/src/mito2/src/worker/handle_alter.rs
+++ b/src/mito2/src/worker/handle_alter.rs
@@ -19,7 +19,7 @@ use std::sync::Arc;
 use common_telemetry::{debug, info};
 use snafu::ResultExt;
 use store_api::metadata::{RegionMetadata, RegionMetadataBuilder, RegionMetadataRef};
-use store_api::region_request::RegionAlterRequest;
+use store_api::region_request::{AlterKind, ChangeOption, RegionAlterRequest};
 use store_api::storage::RegionId;
 
 use crate::error::{
@@ -27,6 +27,8 @@ use crate::error::{
 };
 use crate::flush::FlushReason;
 use crate::manifest::action::RegionChange;
+use crate::region::version::VersionRef;
+use crate::region::MitoRegionRef;
 use crate::request::{DdlRequest, OptionOutputTx, SenderDdlRequest};
 use crate::worker::RegionWorkerLoop;
 
@@ -45,6 +47,13 @@ impl<S> RegionWorkerLoop<S> {
 
         // Get the version before alter.
         let version = region.version();
+
+        // fast path for memory state changes like options.
+        if let AlterKind::ChangeRegionOptions { options } = request.kind {
+            self.handle_alter_region_options(region, version, options, sender);
+            return;
+        }
+
         if version.metadata.schema_version != request.schema_version {
             // This is possible if we retry the request.
             debug!(
@@ -67,6 +76,7 @@ impl<S> RegionWorkerLoop<S> {
             sender.send(Err(e).context(InvalidRegionRequestSnafu));
             return;
         }
+
         // Checks whether we need to alter the region.
         if !request.need_alter(&version.metadata) {
             debug!(
@@ -111,7 +121,17 @@ impl<S> RegionWorkerLoop<S> {
             version.metadata.schema_version,
             region.metadata().schema_version
         );
+        self.handle_alter_region_metadata(region, version, request, sender);
+    }
 
+    /// Handles region metadata changes.
+    fn handle_alter_region_metadata(
+        &mut self,
+        region: MitoRegionRef,
+        version: VersionRef,
+        request: RegionAlterRequest,
+        sender: OptionOutputTx,
+    ) {
         let new_meta = match metadata_after_alteration(&version.metadata, request) {
             Ok(new_meta) => new_meta,
             Err(e) => {
@@ -120,11 +140,38 @@ impl<S> RegionWorkerLoop<S> {
             }
         };
         // Persist the metadata to region's manifest.
-        let change = RegionChange {
-            metadata: new_meta.clone(),
-        };
+        let change = RegionChange { metadata: new_meta };
         self.handle_manifest_region_change(region, change, sender)
     }
+
+    /// Handles requests that changes region options, like TTL. It only affects memory state
+    /// since changes are persisted in the `DatanodeTableValue` in metasrv.
+    fn handle_alter_region_options(
+        &mut self,
+        region: MitoRegionRef,
+        version: VersionRef,
+        options: Vec<ChangeOption>,
+        sender: OptionOutputTx,
+    ) {
+        let mut current_options = version.options.clone();
+        for option in options {
+            match option {
+                ChangeOption::TTL(new_ttl) => {
+                    info!(
+                        "Update region ttl: {}, previous: {:?} new: {:?}",
+                        region.region_id, current_options.ttl, new_ttl
+                    );
+                    if new_ttl.is_zero() {
+                        current_options.ttl = None;
+                    } else {
+                        current_options.ttl = Some(new_ttl);
+                    }
+                }
+            }
+        }
+        region.version_control.alter_options(current_options);
+        sender.send(Ok(0));
+    }
 }
 
 /// Creates a metadata after applying the alter `request` to the old `metadata`.
diff --git a/src/mito2/src/worker/handle_open.rs b/src/mito2/src/worker/handle_open.rs
index 925e2dc8f5..d4a13a1345 100644
--- a/src/mito2/src/worker/handle_open.rs
+++ b/src/mito2/src/worker/handle_open.rs
@@ -22,6 +22,7 @@ use snafu::{OptionExt, ResultExt};
 use store_api::logstore::LogStore;
 use store_api::region_request::RegionOpenRequest;
 use store_api::storage::RegionId;
+use table::requests::STORAGE_KEY;
 
 use crate::error::{
     ObjectStoreNotFoundSnafu, OpenDalSnafu, OpenRegionSnafu, RegionNotFoundSnafu, Result,
@@ -38,7 +39,7 @@ impl<S: LogStore> RegionWorkerLoop<S> {
         region_id: RegionId,
         request: &RegionOpenRequest,
     ) -> Result<()> {
-        let object_store = if let Some(storage_name) = request.options.get("storage") {
+        let object_store = if let Some(storage_name) = request.options.get(STORAGE_KEY) {
             self.object_store_manager
                 .find(storage_name)
                 .context(ObjectStoreNotFoundSnafu {
diff --git a/src/operator/src/expr_factory.rs b/src/operator/src/expr_factory.rs
index e9be2e712f..499441603f 100644
--- a/src/operator/src/expr_factory.rs
+++ b/src/operator/src/expr_factory.rs
@@ -18,9 +18,9 @@ use api::helper::ColumnDataTypeWrapper;
 use api::v1::alter_expr::Kind;
 use api::v1::column_def::options_from_column_schema;
 use api::v1::{
-    AddColumn, AddColumns, AlterExpr, ChangeColumnType, ChangeColumnTypes, ColumnDataType,
-    ColumnDataTypeExtension, CreateFlowExpr, CreateTableExpr, CreateViewExpr, DropColumn,
-    DropColumns, ExpireAfter, RenameTable, SemanticType, TableName,
+    AddColumn, AddColumns, AlterExpr, ChangeColumnType, ChangeColumnTypes, ChangeTableOptions,
+    ColumnDataType, ColumnDataTypeExtension, CreateFlowExpr, CreateTableExpr, CreateViewExpr,
+    DropColumn, DropColumns, ExpireAfter, RenameTable, SemanticType, TableName,
 };
 use common_error::ext::BoxedError;
 use common_grpc_expr::util::ColumnExpr;
@@ -438,7 +438,7 @@ pub(crate) fn to_alter_expr(
             .map_err(BoxedError::new)
             .context(ExternalSnafu)?;
 
-    let kind = match alter_table.alter_operation() {
+    let kind = match alter_table.alter_operation {
         AlterTableOperation::AddConstraint(_) => {
             return NotSupportedSnafu {
                 feat: "ADD CONSTRAINT",
@@ -451,7 +451,7 @@ pub(crate) fn to_alter_expr(
         } => Kind::AddColumns(AddColumns {
             add_columns: vec![AddColumn {
                 column_def: Some(
-                    sql_column_def_to_grpc_column_def(column_def, Some(&query_ctx.timezone()))
+                    sql_column_def_to_grpc_column_def(&column_def, Some(&query_ctx.timezone()))
                         .map_err(BoxedError::new)
                         .context(ExternalSnafu)?,
                 ),
@@ -463,13 +463,13 @@ pub(crate) fn to_alter_expr(
             target_type,
         } => {
             let target_type =
-                sql_data_type_to_concrete_data_type(target_type).context(ParseSqlSnafu)?;
+                sql_data_type_to_concrete_data_type(&target_type).context(ParseSqlSnafu)?;
             let (target_type, target_type_extension) = ColumnDataTypeWrapper::try_from(target_type)
                 .map(|w| w.to_parts())
                 .context(ColumnDataTypeSnafu)?;
             Kind::ChangeColumnTypes(ChangeColumnTypes {
                 change_column_types: vec![ChangeColumnType {
-                    column_name: column_name.value.to_string(),
+                    column_name: column_name.value,
                     target_type: target_type as i32,
                     target_type_extension,
                 }],
@@ -483,6 +483,11 @@ pub(crate) fn to_alter_expr(
         AlterTableOperation::RenameTable { new_table_name } => Kind::RenameTable(RenameTable {
             new_table_name: new_table_name.to_string(),
         }),
+        AlterTableOperation::ChangeTableOptions { options } => {
+            Kind::ChangeTableOptions(ChangeTableOptions {
+                change_table_options: options.into_iter().map(Into::into).collect(),
+            })
+        }
     };
 
     Ok(AlterExpr {
@@ -744,7 +749,7 @@ mod tests {
 
     #[test]
     fn test_to_alter_change_column_type_expr() {
-        let sql = "ALTER TABLE monitor MODIFY mem_usage STRING;";
+        let sql = "ALTER TABLE monitor MODIFY COLUMN mem_usage STRING;";
         let stmt =
             ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
                 .unwrap()
diff --git a/src/sql/src/parsers/alter_parser.rs b/src/sql/src/parsers/alter_parser.rs
index b6db219626..eb231e6df4 100644
--- a/src/sql/src/parsers/alter_parser.rs
+++ b/src/sql/src/parsers/alter_parser.rs
@@ -15,12 +15,12 @@
 use common_query::AddColumnLocation;
 use snafu::ResultExt;
 use sqlparser::keywords::Keyword;
-use sqlparser::parser::ParserError;
+use sqlparser::parser::{Parser, ParserError};
 use sqlparser::tokenizer::Token;
 
 use crate::error::{self, Result};
 use crate::parser::ParserContext;
-use crate::statements::alter::{AlterTable, AlterTableOperation};
+use crate::statements::alter::{AlterTable, AlterTableOperation, ChangeTableOption};
 use crate::statements::statement::Statement;
 
 impl ParserContext<'_> {
@@ -94,6 +94,14 @@ impl ParserContext<'_> {
                 }
             };
             AlterTableOperation::RenameTable { new_table_name }
+        } else if self.parser.parse_keyword(Keyword::SET) {
+            let options = self
+                .parser
+                .parse_comma_separated(parse_string_options)?
+                .into_iter()
+                .map(|(key, value)| ChangeTableOption { key, value })
+                .collect();
+            AlterTableOperation::ChangeTableOptions { options }
         } else {
             return Err(ParserError::ParserError(format!(
                 "expect keyword ADD or DROP or MODIFY or RENAME after ALTER TABLE, found {}",
@@ -104,6 +112,22 @@ impl ParserContext<'_> {
     }
 }
 
+fn parse_string_options(parser: &mut Parser) -> std::result::Result<(String, String), ParserError> {
+    let name = parser.parse_literal_string()?;
+    parser.expect_token(&Token::Eq)?;
+    let value = if parser.parse_keyword(Keyword::NULL) {
+        "".to_string()
+    } else if let Ok(v) = parser.parse_literal_string() {
+        v
+    } else {
+        return Err(ParserError::ParserError(format!(
+            "Unexpected option value for alter table statements, expect string literal or NULL, got: `{}`",
+            parser.next_token()
+        )));
+    };
+    Ok((name, value))
+}
+
 #[cfg(test)]
 mod tests {
     use std::assert_matches::assert_matches;
@@ -272,7 +296,7 @@ mod tests {
         )
         .unwrap();
 
-        let sql_2 = "ALTER TABLE my_metric_1 MODIFY a STRING";
+        let sql_2 = "ALTER TABLE my_metric_1 MODIFY COLUMN a STRING";
         let mut result_2 = ParserContext::create_with_dialect(
             sql_2,
             &GreptimeDbDialect {},
@@ -406,4 +430,44 @@ mod tests {
             _ => unreachable!(),
         }
     }
+
+    fn check_parse_alter_table(sql: &str, expected: &[(&str, &str)]) {
+        let result =
+            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
+                .unwrap();
+        assert_eq!(1, result.len());
+        let Statement::Alter(alter) = &result[0] else {
+            unreachable!()
+        };
+        assert_eq!("test_table", alter.table_name.0[0].value);
+        let AlterTableOperation::ChangeTableOptions { options } = &alter.alter_operation else {
+            unreachable!()
+        };
+        let res = options
+            .iter()
+            .map(|o| (o.key.as_str(), o.value.as_str()))
+            .collect::<Vec<_>>();
+        assert_eq!(expected, &res);
+    }
+
+    #[test]
+    fn test_parse_alter_column() {
+        check_parse_alter_table("ALTER TABLE test_table SET 'a'='A';", &[("a", "A")]);
+        check_parse_alter_table(
+            "ALTER TABLE test_table SET 'a'='A','b'='B'",
+            &[("a", "A"), ("b", "B")],
+        );
+        check_parse_alter_table(
+            "ALTER TABLE test_table SET 'a'='A','b'='B','c'='C';",
+            &[("a", "A"), ("b", "B"), ("c", "C")],
+        );
+        check_parse_alter_table("ALTER TABLE test_table SET 'a'=NULL;", &[("a", "")]);
+
+        ParserContext::create_with_dialect(
+            "ALTER TABLE test_table SET a INTEGER",
+            &GreptimeDbDialect {},
+            ParseOptions::default(),
+        )
+        .unwrap_err();
+    }
 }
diff --git a/src/sql/src/statements/alter.rs b/src/sql/src/statements/alter.rs
index 5d679549a1..3270d002c8 100644
--- a/src/sql/src/statements/alter.rs
+++ b/src/sql/src/statements/alter.rs
@@ -14,14 +14,15 @@
 
 use std::fmt::{Debug, Display};
 
+use api::v1;
 use common_query::AddColumnLocation;
 use sqlparser::ast::{ColumnDef, DataType, Ident, ObjectName, TableConstraint};
 use sqlparser_derive::{Visit, VisitMut};
 
 #[derive(Debug, Clone, PartialEq, Eq, Visit, VisitMut)]
 pub struct AlterTable {
-    table_name: ObjectName,
-    alter_operation: AlterTableOperation,
+    pub table_name: ObjectName,
+    pub alter_operation: AlterTableOperation,
 }
 
 impl AlterTable {
@@ -67,6 +68,8 @@ pub enum AlterTableOperation {
         column_name: Ident,
         target_type: DataType,
     },
+    /// `MODIFY <table attrs key> = <table attr value>`
+    ChangeTableOptions { options: Vec<ChangeTableOption> },
     /// `DROP COLUMN <name>`
     DropColumn { name: Ident },
     /// `RENAME <new_table_name>`
@@ -97,6 +100,27 @@ impl Display for AlterTableOperation {
             } => {
                 write!(f, r#"MODIFY COLUMN {column_name} {target_type}"#)
             }
+            AlterTableOperation::ChangeTableOptions { options } => {
+                for ChangeTableOption { key, value } in options {
+                    write!(f, r#"MODIFY '{key}'='{value}', "#)?;
+                }
+                Ok(())
+            }
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Visit, VisitMut)]
+pub struct ChangeTableOption {
+    pub key: String,
+    pub value: String,
+}
+
+impl From<ChangeTableOption> for v1::ChangeTableOption {
+    fn from(c: ChangeTableOption) -> Self {
+        v1::ChangeTableOption {
+            key: c.key,
+            value: c.value,
         }
     }
 }
diff --git a/src/store-api/Cargo.toml b/src/store-api/Cargo.toml
index c3ee5c3930..2d11b31ebe 100644
--- a/src/store-api/Cargo.toml
+++ b/src/store-api/Cargo.toml
@@ -22,6 +22,7 @@ datafusion-physical-plan.workspace = true
 datatypes.workspace = true
 derive_builder.workspace = true
 futures.workspace = true
+humantime.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 snafu.workspace = true
diff --git a/src/store-api/src/metadata.rs b/src/store-api/src/metadata.rs
index a94879675e..3a08523b90 100644
--- a/src/store-api/src/metadata.rs
+++ b/src/store-api/src/metadata.rs
@@ -523,6 +523,9 @@ impl RegionMetadataBuilder {
             AlterKind::AddColumns { columns } => self.add_columns(columns)?,
             AlterKind::DropColumns { names } => self.drop_columns(&names),
             AlterKind::ChangeColumnTypes { columns } => self.change_column_types(columns),
+            AlterKind::ChangeRegionOptions { options: _ } => {
+                // nothing to be done with RegionMetadata
+            }
         }
         Ok(self)
     }
@@ -738,6 +741,14 @@ pub enum MetadataError {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Invalid region option change request, key: {}, value: {}", key, value))]
+    InvalidRegionOptionChangeRequest {
+        key: String,
+        value: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
 }
 
 impl ErrorExt for MetadataError {
diff --git a/src/store-api/src/region_request.rs b/src/store-api/src/region_request.rs
index 2453b83340..867f231754 100644
--- a/src/store-api/src/region_request.rs
+++ b/src/store-api/src/region_request.rs
@@ -14,25 +14,29 @@
 
 use std::collections::HashMap;
 use std::fmt;
+use std::time::Duration;
 
 use api::helper::ColumnDataTypeWrapper;
 use api::v1::add_column_location::LocationType;
+use api::v1::region::alter_request::Kind;
 use api::v1::region::{
     alter_request, compact_request, region_request, AlterRequest, AlterRequests, CloseRequest,
     CompactRequest, CreateRequest, CreateRequests, DeleteRequests, DropRequest, DropRequests,
     FlushRequest, InsertRequests, OpenRequest, TruncateRequest,
 };
-use api::v1::{self, Rows, SemanticType};
+use api::v1::{self, ChangeTableOption, Rows, SemanticType};
 pub use common_base::AffectedRows;
 use datatypes::data_type::ConcreteDataType;
+use serde::{Deserialize, Serialize};
 use snafu::{ensure, OptionExt};
 use strum::IntoStaticStr;
 
 use crate::logstore::entry;
 use crate::metadata::{
-    ColumnMetadata, InvalidRawRegionRequestSnafu, InvalidRegionRequestSnafu, MetadataError,
-    RegionMetadata, Result,
+    ColumnMetadata, InvalidRawRegionRequestSnafu, InvalidRegionOptionChangeRequestSnafu,
+    InvalidRegionRequestSnafu, MetadataError, RegionMetadata, Result,
 };
+use crate::mito_engine_options::TTL_KEY;
 use crate::path_utils::region_dir;
 use crate::storage::{ColumnId, RegionId, ScanRequest};
 
@@ -389,6 +393,8 @@ pub enum AlterKind {
         /// Columns to change.
         columns: Vec<ChangeColumnType>,
     },
+    /// Change region options.
+    ChangeRegionOptions { options: Vec<ChangeOption> },
 }
 
 impl AlterKind {
@@ -412,6 +418,7 @@ impl AlterKind {
                     col_to_change.validate(metadata)?;
                 }
             }
+            AlterKind::ChangeRegionOptions { .. } => {}
         }
         Ok(())
     }
@@ -429,6 +436,11 @@ impl AlterKind {
             AlterKind::ChangeColumnTypes { columns } => columns
                 .iter()
                 .any(|col_to_change| col_to_change.need_alter(metadata)),
+            AlterKind::ChangeRegionOptions { .. } => {
+                // we need to update region options for `ChangeTableOptions`.
+                // todo: we need to check if ttl has ever changed.
+                true
+            }
         }
     }
 
@@ -473,6 +485,13 @@ impl TryFrom<alter_request::Kind> for AlterKind {
                 let names = x.drop_columns.into_iter().map(|x| x.name).collect();
                 AlterKind::DropColumns { names }
             }
+            Kind::ChangeTableOptions(change_options) => AlterKind::ChangeRegionOptions {
+                options: change_options
+                    .change_table_options
+                    .iter()
+                    .map(TryFrom::try_from)
+                    .collect::<Result<Vec<_>>>()?,
+            },
         };
 
         Ok(alter_kind)
@@ -639,6 +658,30 @@ impl From<v1::ChangeColumnType> for ChangeColumnType {
     }
 }
 
+#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)]
+pub enum ChangeOption {
+    TTL(Duration),
+}
+
+impl TryFrom<&ChangeTableOption> for ChangeOption {
+    type Error = MetadataError;
+
+    fn try_from(value: &ChangeTableOption) -> std::result::Result<Self, Self::Error> {
+        let ChangeTableOption { key, value } = value;
+        if key == TTL_KEY {
+            let ttl = if value.is_empty() {
+                Duration::from_secs(0)
+            } else {
+                humantime::parse_duration(value)
+                    .map_err(|_| InvalidRegionOptionChangeRequestSnafu { key, value }.build())?
+            };
+            Ok(Self::TTL(ttl))
+        } else {
+            InvalidRegionOptionChangeRequestSnafu { key, value }.fail()
+        }
+    }
+}
+
 #[derive(Debug, Clone, Default)]
 pub struct RegionFlushRequest {
     pub row_group_size: Option<usize>,
diff --git a/src/table/src/error.rs b/src/table/src/error.rs
index 9b633bb5bb..e6ea1e1932 100644
--- a/src/table/src/error.rs
+++ b/src/table/src/error.rs
@@ -137,6 +137,9 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Table options value is not valid, key: `{}`, value: `{}`", key, value))]
+    InvalidTableOptionValue { key: String, value: String },
 }
 
 impl ErrorExt for Error {
@@ -157,6 +160,7 @@ impl ErrorExt for Error {
             Error::Unsupported { .. } => StatusCode::Unsupported,
             Error::ParseTableOption { .. } => StatusCode::InvalidArguments,
             Error::MissingTimeIndexColumn { .. } => StatusCode::IllegalState,
+            Error::InvalidTableOptionValue { .. } => StatusCode::InvalidArguments,
         }
     }
 
diff --git a/src/table/src/metadata.rs b/src/table/src/metadata.rs
index ec9404a8b1..3e75217508 100644
--- a/src/table/src/metadata.rs
+++ b/src/table/src/metadata.rs
@@ -24,6 +24,7 @@ use datatypes::schema::{ColumnSchema, RawSchema, Schema, SchemaBuilder, SchemaRe
 use derive_builder::Builder;
 use serde::{Deserialize, Serialize};
 use snafu::{ensure, OptionExt, ResultExt};
+use store_api::region_request::ChangeOption;
 use store_api::storage::{ColumnDescriptor, ColumnDescriptorBuilder, ColumnId, RegionId};
 
 use crate::error::{self, Result};
@@ -209,9 +210,35 @@ impl TableMeta {
                     .next_column_id(self.next_column_id);
                 Ok(meta_builder)
             }
+            AlterKind::ChangeTableOptions { options } => self.change_table_options(options),
         }
     }
 
+    /// Creates a [TableMetaBuilder] with modified table options.
+    fn change_table_options(&self, requests: &[ChangeOption]) -> Result<TableMetaBuilder> {
+        let mut new_options = self.options.clone();
+
+        for request in requests {
+            match request {
+                ChangeOption::TTL(new_ttl) => {
+                    if new_ttl.is_zero() {
+                        new_options.ttl = None;
+                    } else {
+                        new_options.ttl = Some(*new_ttl);
+                    }
+                }
+            }
+        }
+        let mut builder = TableMetaBuilder::default();
+        builder
+            .options(new_options)
+            .schema(self.schema.clone())
+            .primary_key_indices(self.primary_key_indices.clone())
+            .engine(self.engine.clone())
+            .next_column_id(self.next_column_id);
+        Ok(builder)
+    }
+
     /// Allocate a new column for the table.
     ///
     /// This method would bump the `next_column_id` of the meta.
@@ -823,6 +850,13 @@ impl RawTableInfo {
         self.meta.primary_key_indices = primary_key_indices;
         self.meta.value_indices = value_indices;
     }
+
+    /// Extracts region options from table info.
+    ///
+    /// All "region options" are actually a copy of table options for redundancy.
+    pub fn to_region_options(&self) -> HashMap<String, String> {
+        HashMap::from(&self.meta.options)
+    }
 }
 
 impl From<TableInfo> for RawTableInfo {
@@ -857,7 +891,6 @@ impl TryFrom<RawTableInfo> for TableInfo {
 
 #[cfg(test)]
 mod tests {
-
     use common_error::ext::ErrorExt;
     use common_error::status_code::StatusCode;
     use datatypes::data_type::ConcreteDataType;
diff --git a/src/table/src/requests.rs b/src/table/src/requests.rs
index 2aebc47055..a4fcce7bcd 100644
--- a/src/table/src/requests.rs
+++ b/src/table/src/requests.rs
@@ -30,6 +30,7 @@ use greptime_proto::v1::region::compact_request;
 use serde::{Deserialize, Serialize};
 use store_api::metric_engine_consts::{LOGICAL_TABLE_METADATA_KEY, PHYSICAL_TABLE_METADATA_KEY};
 use store_api::mito_engine_options::is_mito_engine_option_key;
+use store_api::region_request::ChangeOption;
 
 use crate::error::{ParseTableOptionSnafu, Result};
 use crate::metadata::{TableId, TableVersion};
@@ -80,7 +81,7 @@ pub struct TableOptions {
 }
 
 pub const WRITE_BUFFER_SIZE_KEY: &str = "write_buffer_size";
-pub const TTL_KEY: &str = "ttl";
+pub const TTL_KEY: &str = store_api::mito_engine_options::TTL_KEY;
 pub const STORAGE_KEY: &str = "storage";
 pub const COMMENT_KEY: &str = "comment";
 pub const AUTO_CREATE_TABLE_KEY: &str = "auto_create_table";
@@ -212,8 +213,35 @@ pub enum AlterKind {
     RenameTable {
         new_table_name: String,
     },
+    ChangeTableOptions {
+        options: Vec<ChangeOption>,
+    },
 }
 
+// #[derive(Debug, Clone, Serialize, Deserialize)]
+// pub enum ChangeTableOptionRequest {
+//     TTL(Duration),
+// }
+
+// impl TryFrom<&ChangeTableOption> for ChangeTableOptionRequest {
+//     type Error = Error;
+//
+//     fn try_from(value: &ChangeTableOption) -> std::result::Result<Self, Self::Error> {
+//         let ChangeTableOption { key, value } = value;
+//         if key == TTL_KEY {
+//             let ttl = if value.is_empty() {
+//                 Duration::from_secs(0)
+//             } else {
+//                 humantime::parse_duration(value)
+//                     .map_err(|_| error::InvalidTableOptionValueSnafu { key, value }.build())?
+//             };
+//             Ok(Self::TTL(ttl))
+//         } else {
+//             UnsupportedTableOptionChangeSnafu { key }.fail()
+//         }
+//     }
+// }
+
 #[derive(Debug)]
 pub struct InsertRequest {
     pub catalog_name: String,
diff --git a/tests/cases/standalone/common/alter/alter_table_options.result b/tests/cases/standalone/common/alter/alter_table_options.result
new file mode 100644
index 0000000000..13daf0f89b
--- /dev/null
+++ b/tests/cases/standalone/common/alter/alter_table_options.result
@@ -0,0 +1,88 @@
+CREATE TABLE ato(i INTEGER, j TIMESTAMP TIME INDEX);
+
+Affected Rows: 0
+
+ALTER TABLE ato SET 'ttl'='1d';
+
+Affected Rows: 0
+
+SHOW CREATE TABLE ato;
+
++-------+------------------------------------+
+| Table | Create Table                       |
++-------+------------------------------------+
+| ato   | CREATE TABLE IF NOT EXISTS "ato" ( |
+|       |   "i" INT NULL,                    |
+|       |   "j" TIMESTAMP(3) NOT NULL,       |
+|       |   TIME INDEX ("j")                 |
+|       | )                                  |
+|       |                                    |
+|       | ENGINE=mito                        |
+|       | WITH(                              |
+|       |   ttl = '1day'                     |
+|       | )                                  |
++-------+------------------------------------+
+
+ALTER TABLE ato SET 'ttl'='2d';
+
+Affected Rows: 0
+
+SHOW CREATE TABLE ato;
+
++-------+------------------------------------+
+| Table | Create Table                       |
++-------+------------------------------------+
+| ato   | CREATE TABLE IF NOT EXISTS "ato" ( |
+|       |   "i" INT NULL,                    |
+|       |   "j" TIMESTAMP(3) NOT NULL,       |
+|       |   TIME INDEX ("j")                 |
+|       | )                                  |
+|       |                                    |
+|       | ENGINE=mito                        |
+|       | WITH(                              |
+|       |   ttl = '2days'                    |
+|       | )                                  |
++-------+------------------------------------+
+
+ALTER TABLE ato SET 'ttl'=NULL;
+
+Affected Rows: 0
+
+SHOW CREATE TABLE ato;
+
++-------+------------------------------------+
+| Table | Create Table                       |
++-------+------------------------------------+
+| ato   | CREATE TABLE IF NOT EXISTS "ato" ( |
+|       |   "i" INT NULL,                    |
+|       |   "j" TIMESTAMP(3) NOT NULL,       |
+|       |   TIME INDEX ("j")                 |
+|       | )                                  |
+|       |                                    |
+|       | ENGINE=mito                        |
+|       |                                    |
++-------+------------------------------------+
+
+ALTER TABLE ato SET 'ttl'='0d';
+
+Affected Rows: 0
+
+SHOW CREATE TABLE ato;
+
++-------+------------------------------------+
+| Table | Create Table                       |
++-------+------------------------------------+
+| ato   | CREATE TABLE IF NOT EXISTS "ato" ( |
+|       |   "i" INT NULL,                    |
+|       |   "j" TIMESTAMP(3) NOT NULL,       |
+|       |   TIME INDEX ("j")                 |
+|       | )                                  |
+|       |                                    |
+|       | ENGINE=mito                        |
+|       |                                    |
++-------+------------------------------------+
+
+DROP TABLE ato;
+
+Affected Rows: 0
+
diff --git a/tests/cases/standalone/common/alter/alter_table_options.sql b/tests/cases/standalone/common/alter/alter_table_options.sql
new file mode 100644
index 0000000000..4b30a8c28f
--- /dev/null
+++ b/tests/cases/standalone/common/alter/alter_table_options.sql
@@ -0,0 +1,19 @@
+CREATE TABLE ato(i INTEGER, j TIMESTAMP TIME INDEX);
+
+ALTER TABLE ato SET 'ttl'='1d';
+
+SHOW CREATE TABLE ato;
+
+ALTER TABLE ato SET 'ttl'='2d';
+
+SHOW CREATE TABLE ato;
+
+ALTER TABLE ato SET 'ttl'=NULL;
+
+SHOW CREATE TABLE ato;
+
+ALTER TABLE ato SET 'ttl'='0d';
+
+SHOW CREATE TABLE ato;
+
+DROP TABLE ato;
\ No newline at end of file
diff --git a/tests/cases/standalone/common/alter/change_col_type.result b/tests/cases/standalone/common/alter/change_col_type.result
index f0a641d28b..aa992a292c 100644
--- a/tests/cases/standalone/common/alter/change_col_type.result
+++ b/tests/cases/standalone/common/alter/change_col_type.result
@@ -6,23 +6,23 @@ INSERT INTO test VALUES (1, 1, 1, false), (2, 2, 2, true);
 
 Affected Rows: 2
 
-ALTER TABLE test MODIFY "I" STRING;
+ALTER TABLE test MODIFY COLUMN "I" STRING;
 
 Error: 4002(TableColumnNotFound), Column I not exists in table test
 
-ALTER TABLE test MODIFY k DATE;
+ALTER TABLE test MODIFY COLUMN k DATE;
 
 Error: 1004(InvalidArguments), Invalid alter table(test) request: column 'k' cannot be cast automatically to type 'Date'
 
-ALTER TABLE test MODIFY id STRING;
+ALTER TABLE test MODIFY COLUMN id STRING;
 
 Error: 1004(InvalidArguments), Invalid alter table(test) request: Not allowed to change primary key index column 'id'
 
-ALTER TABLE test MODIFY j STRING;
+ALTER TABLE test MODIFY COLUMN j STRING;
 
 Error: 1004(InvalidArguments), Invalid alter table(test) request: Not allowed to change timestamp index column 'j' datatype
 
-ALTER TABLE test MODIFY I STRING;
+ALTER TABLE test MODIFY COLUMN I STRING;
 
 Affected Rows: 0
 
@@ -61,7 +61,7 @@ DESCRIBE test;
 | k      | Boolean              |     | YES  |         | FIELD         |
 +--------+----------------------+-----+------+---------+---------------+
 
-ALTER TABLE test MODIFY I INTEGER;
+ALTER TABLE test MODIFY COLUMN I INTEGER;
 
 Affected Rows: 0
 
diff --git a/tests/cases/standalone/common/alter/change_col_type.sql b/tests/cases/standalone/common/alter/change_col_type.sql
index 0fe8c28e9b..3282385c87 100644
--- a/tests/cases/standalone/common/alter/change_col_type.sql
+++ b/tests/cases/standalone/common/alter/change_col_type.sql
@@ -2,15 +2,15 @@ CREATE TABLE test(id INTEGER PRIMARY KEY, i INTEGER NULL, j TIMESTAMP TIME INDEX
 
 INSERT INTO test VALUES (1, 1, 1, false), (2, 2, 2, true);
 
-ALTER TABLE test MODIFY "I" STRING;
+ALTER TABLE test MODIFY COLUMN "I" STRING;
 
-ALTER TABLE test MODIFY k DATE;
+ALTER TABLE test MODIFY COLUMN k DATE;
 
-ALTER TABLE test MODIFY id STRING;
+ALTER TABLE test MODIFY COLUMN id STRING;
 
-ALTER TABLE test MODIFY j STRING;
+ALTER TABLE test MODIFY COLUMN j STRING;
 
-ALTER TABLE test MODIFY I STRING;
+ALTER TABLE test MODIFY COLUMN I STRING;
 
 SELECT * FROM test;
 
@@ -21,7 +21,7 @@ SELECT * FROM test;
 
 DESCRIBE test;
 
-ALTER TABLE test MODIFY I INTEGER;
+ALTER TABLE test MODIFY COLUMN I INTEGER;
 
 -- SQLNESS SORT_RESULT 3 1
 SELECT * FROM test;
diff --git a/tests/cases/standalone/common/alter/change_col_type_not_null.result b/tests/cases/standalone/common/alter/change_col_type_not_null.result
index 79f03c9cb0..497fb5dca6 100644
--- a/tests/cases/standalone/common/alter/change_col_type_not_null.result
+++ b/tests/cases/standalone/common/alter/change_col_type_not_null.result
@@ -15,7 +15,7 @@ SELECT * FROM test;
 | 1970-01-01T00:00:00.002 | 2 |
 +-------------------------+---+
 
-ALTER TABLE test MODIFY j STRING;
+ALTER TABLE test MODIFY COLUMN j STRING;
 
 Error: 1004(InvalidArguments), Invalid alter table(test) request: column 'j' must be nullable to ensure safe conversion.
 
diff --git a/tests/cases/standalone/common/alter/change_col_type_not_null.sql b/tests/cases/standalone/common/alter/change_col_type_not_null.sql
index c91ae44a2c..5d2e6f954b 100644
--- a/tests/cases/standalone/common/alter/change_col_type_not_null.sql
+++ b/tests/cases/standalone/common/alter/change_col_type_not_null.sql
@@ -4,7 +4,7 @@ INSERT INTO test VALUES (1, 1), (2, 2);
 
 SELECT * FROM test;
 
-ALTER TABLE test MODIFY j STRING;
+ALTER TABLE test MODIFY COLUMN j STRING;
 
 SELECT * FROM test;
 

From 971229517705f26b2a11212f5a85f279cab2283e Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Wed, 30 Oct 2024 16:31:31 +0800
Subject: [PATCH 121/128] fix(config): update tracing section headers in
 example TOML files (#4898)

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 config/datanode.example.toml   | 4 ++--
 config/flownode.example.toml   | 4 ++--
 config/frontend.example.toml   | 4 ++--
 config/metasrv.example.toml    | 4 ++--
 config/standalone.example.toml | 4 ++--
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index 557cd4cef0..6e426e89cd 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -646,7 +646,7 @@ url = ""
 headers = { }
 
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
-[tracing]
+#+ [tracing]
 ## The tokio console address.
 ## @toml2docs:none-default
-tokio_console_addr = "127.0.0.1"
+#+ tokio_console_addr = "127.0.0.1"
diff --git a/config/flownode.example.toml b/config/flownode.example.toml
index 34825542fa..ffa9924365 100644
--- a/config/flownode.example.toml
+++ b/config/flownode.example.toml
@@ -101,8 +101,8 @@ threshold = "10s"
 sample_ratio = 1.0
 
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
-[tracing]
+#+ [tracing]
 ## The tokio console address.
 ## @toml2docs:none-default
-tokio_console_addr = "127.0.0.1"
+#+ tokio_console_addr = "127.0.0.1"
 
diff --git a/config/frontend.example.toml b/config/frontend.example.toml
index 83e7808d46..1fb372a6d1 100644
--- a/config/frontend.example.toml
+++ b/config/frontend.example.toml
@@ -231,7 +231,7 @@ url = ""
 headers = { }
 
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
-[tracing]
+#+ [tracing]
 ## The tokio console address.
 ## @toml2docs:none-default
-tokio_console_addr = "127.0.0.1"
+#+ tokio_console_addr = "127.0.0.1"
diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml
index 416f5ee6ef..b80d1c164e 100644
--- a/config/metasrv.example.toml
+++ b/config/metasrv.example.toml
@@ -218,7 +218,7 @@ url = ""
 headers = { }
 
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
-[tracing]
+#+ [tracing]
 ## The tokio console address.
 ## @toml2docs:none-default
-tokio_console_addr = "127.0.0.1"
+#+ tokio_console_addr = "127.0.0.1"
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index defd34d8f5..52f6d5b694 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -690,7 +690,7 @@ url = ""
 headers = { }
 
 ## The tracing options. Only effect when compiled with `tokio-console` feature.
-[tracing]
+#+ [tracing]
 ## The tokio console address.
 ## @toml2docs:none-default
-tokio_console_addr = "127.0.0.1"
+#+ tokio_console_addr = "127.0.0.1"

From fd8eba36a8d03d7ce7b9566ebc17220d728f9fd5 Mon Sep 17 00:00:00 2001
From: LFC <990479+MichaelScofield@users.noreply.github.com>
Date: Wed, 30 Oct 2024 17:16:46 +0800
Subject: [PATCH 122/128] refactor: make use of the "pre_execute" in sql
 execution interceptor (#4875)

* feat: dynamic definition of plugin options

* rebase

* revert

* fix ci
---
 Cargo.lock                                 |  1 -
 Cargo.toml                                 |  5 +-
 src/client/Cargo.toml                      |  1 -
 src/cmd/Cargo.toml                         |  2 +-
 src/cmd/src/cli/repl.rs                    |  2 +-
 src/cmd/src/standalone.rs                  | 14 +++-
 src/common/meta/Cargo.toml                 |  2 +-
 src/common/telemetry/Cargo.toml            |  4 +-
 src/flow/src/df_optimizer.rs               |  2 +-
 src/flow/src/transform.rs                  |  4 +-
 src/frontend/src/error.rs                  | 13 +++-
 src/frontend/src/instance.rs               | 54 +++++++++++----
 src/meta-client/Cargo.toml                 |  2 +-
 src/meta-srv/Cargo.toml                    |  4 +-
 src/operator/src/statement.rs              | 17 +++--
 src/operator/src/statement/ddl.rs          |  2 +-
 src/operator/src/statement/tql.rs          |  2 +-
 src/query/src/datafusion.rs                |  6 +-
 src/query/src/planner.rs                   | 10 +--
 src/query/src/promql/planner.rs            | 76 +++++++++-------------
 src/query/src/range_select/plan_rewrite.rs |  2 +-
 src/query/src/tests.rs                     |  2 +-
 src/query/src/tests/query_engine_test.rs   |  4 +-
 src/script/src/python/engine.rs            |  2 +-
 src/script/src/python/ffi_types/copr.rs    |  2 +-
 src/servers/tests/mod.rs                   |  4 +-
 tests-integration/src/grpc.rs              |  2 +-
 tests-integration/src/instance.rs          |  2 +-
 28 files changed, 141 insertions(+), 102 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 264ceb2de2..26554d7f81 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1788,7 +1788,6 @@ dependencies = [
  "tokio-stream",
  "tonic 0.11.0",
  "tracing",
- "tracing-subscriber",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index d89289b304..2e7f70d2ab 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -180,13 +180,16 @@ sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "5
 ] }
 strum = { version = "0.25", features = ["derive"] }
 tempfile = "3"
-tokio = { version = "1.36", features = ["full"] }
+tokio = { version = "1.40", features = ["full"] }
 tokio-postgres = "0.7"
 tokio-stream = { version = "0.1" }
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
 toml = "0.8.8"
 tonic = { version = "0.11", features = ["tls", "gzip", "zstd"] }
 tower = { version = "0.4" }
+tracing-appender = "0.2"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "fmt"] }
+typetag = "0.2"
 uuid = { version = "1.7", features = ["serde", "v4", "fast-rng"] }
 zstd = "0.13"
 
diff --git a/src/client/Cargo.toml b/src/client/Cargo.toml
index 501c2f82d8..9d198ab9fb 100644
--- a/src/client/Cargo.toml
+++ b/src/client/Cargo.toml
@@ -45,7 +45,6 @@ common-grpc-expr.workspace = true
 datanode.workspace = true
 derive-new = "0.5"
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 [dev-dependencies.substrait_proto]
 package = "substrait"
diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml
index 501f40a047..6309d64294 100644
--- a/src/cmd/Cargo.toml
+++ b/src/cmd/Cargo.toml
@@ -78,7 +78,7 @@ table.workspace = true
 tokio.workspace = true
 toml.workspace = true
 tonic.workspace = true
-tracing-appender = "0.2"
+tracing-appender.workspace = true
 
 [target.'cfg(not(windows))'.dependencies]
 tikv-jemallocator = "0.6"
diff --git a/src/cmd/src/cli/repl.rs b/src/cmd/src/cli/repl.rs
index 0138471227..0f8e5b0450 100644
--- a/src/cmd/src/cli/repl.rs
+++ b/src/cmd/src/cli/repl.rs
@@ -174,7 +174,7 @@ impl Repl {
 
             let plan = query_engine
                 .planner()
-                .plan(stmt, query_ctx.clone())
+                .plan(&stmt, query_ctx.clone())
                 .await
                 .context(PlanStatementSnafu)?;
 
diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
index a72aea3c9a..54d6e4d72c 100644
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::net::SocketAddr;
 use std::sync::Arc;
 use std::{fs, path};
 
@@ -250,6 +251,13 @@ pub struct Instance {
     _guard: Vec<WorkerGuard>,
 }
 
+impl Instance {
+    /// Find the socket addr of a server by its `name`.
+    pub async fn server_addr(&self, name: &str) -> Option<SocketAddr> {
+        self.frontend.server_handlers().addr(name).await
+    }
+}
+
 #[async_trait]
 impl App for Instance {
     fn name(&self) -> &str {
@@ -340,7 +348,8 @@ pub struct StartCommand {
 }
 
 impl StartCommand {
-    fn load_options(
+    /// Load the GreptimeDB options from various sources (command line, config file or env).
+    pub fn load_options(
         &self,
         global_options: &GlobalOptions,
     ) -> Result<GreptimeOptions<StandaloneOptions>> {
@@ -430,7 +439,8 @@ impl StartCommand {
     #[allow(unreachable_code)]
     #[allow(unused_variables)]
     #[allow(clippy::diverging_sub_expression)]
-    async fn build(&self, opts: GreptimeOptions<StandaloneOptions>) -> Result<Instance> {
+    /// Build GreptimeDB instance with the loaded options.
+    pub async fn build(&self, opts: GreptimeOptions<StandaloneOptions>) -> Result<Instance> {
         common_runtime::init_global_runtimes(&opts.runtime);
 
         let guard = common_telemetry::init_global_logging(
diff --git a/src/common/meta/Cargo.toml b/src/common/meta/Cargo.toml
index 591055d472..d165a403f5 100644
--- a/src/common/meta/Cargo.toml
+++ b/src/common/meta/Cargo.toml
@@ -60,7 +60,7 @@ table.workspace = true
 tokio.workspace = true
 tokio-postgres = { workspace = true, optional = true }
 tonic.workspace = true
-typetag = "0.2"
+typetag.workspace = true
 
 [dev-dependencies]
 chrono.workspace = true
diff --git a/src/common/telemetry/Cargo.toml b/src/common/telemetry/Cargo.toml
index da044436c8..0d5d188a06 100644
--- a/src/common/telemetry/Cargo.toml
+++ b/src/common/telemetry/Cargo.toml
@@ -32,7 +32,7 @@ serde.workspace = true
 serde_json.workspace = true
 tokio.workspace = true
 tracing = "0.1"
-tracing-appender = "0.2"
+tracing-appender.workspace = true
 tracing-log = "0.1"
 tracing-opentelemetry = "0.22.0"
-tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "fmt"] }
+tracing-subscriber.workspace = true
diff --git a/src/flow/src/df_optimizer.rs b/src/flow/src/df_optimizer.rs
index 0b788a79db..bb296cba70 100644
--- a/src/flow/src/df_optimizer.rs
+++ b/src/flow/src/df_optimizer.rs
@@ -106,7 +106,7 @@ pub async fn sql_to_flow_plan(
         .context(ExternalSnafu)?;
     let plan = engine
         .planner()
-        .plan(stmt, query_ctx)
+        .plan(&stmt, query_ctx)
         .await
         .map_err(BoxedError::new)
         .context(ExternalSnafu)?;
diff --git a/src/flow/src/transform.rs b/src/flow/src/transform.rs
index e33b023560..94878115cf 100644
--- a/src/flow/src/transform.rs
+++ b/src/flow/src/transform.rs
@@ -278,7 +278,7 @@ mod test {
         let stmt = QueryLanguageParser::parse_sql(sql, &QueryContext::arc()).unwrap();
         let plan = engine
             .planner()
-            .plan(stmt, QueryContext::arc())
+            .plan(&stmt, QueryContext::arc())
             .await
             .unwrap();
         let plan = apply_df_optimizer(plan).await.unwrap();
@@ -300,7 +300,7 @@ mod test {
         let stmt = QueryLanguageParser::parse_sql(sql, &QueryContext::arc()).unwrap();
         let plan = engine
             .planner()
-            .plan(stmt, QueryContext::arc())
+            .plan(&stmt, QueryContext::arc())
             .await
             .unwrap();
         let plan = apply_df_optimizer(plan).await;
diff --git a/src/frontend/src/error.rs b/src/frontend/src/error.rs
index 742deaf808..c6e7218a38 100644
--- a/src/frontend/src/error.rs
+++ b/src/frontend/src/error.rs
@@ -313,6 +313,14 @@ pub enum Error {
         #[snafu(implicit)]
         location: Location,
     },
+
+    #[snafu(display("Failed to init plugin"))]
+    // this comment is to bypass the unused snafu check in "check-snafu.py"
+    InitPlugin {
+        #[snafu(implicit)]
+        location: Location,
+        source: BoxedError,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -375,8 +383,9 @@ impl ErrorExt for Error {
             | Error::ExecLogicalPlan { source, .. } => source.status_code(),
 
             Error::InvokeRegionServer { source, .. } => source.status_code(),
-
-            Error::External { source, .. } => source.status_code(),
+            Error::External { source, .. } | Error::InitPlugin { source, .. } => {
+                source.status_code()
+            }
             Error::FindTableRoute { source, .. } => source.status_code(),
 
             #[cfg(feature = "python")]
diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs
index fc46ae9e59..bd9f8959fe 100644
--- a/src/frontend/src/instance.rs
+++ b/src/frontend/src/instance.rs
@@ -225,11 +225,45 @@ impl Instance {
     async fn query_statement(&self, stmt: Statement, query_ctx: QueryContextRef) -> Result<Output> {
         check_permission(self.plugins.clone(), &stmt, &query_ctx)?;
 
-        let stmt = QueryStatement::Sql(stmt);
-        self.statement_executor
-            .execute_stmt(stmt, query_ctx)
-            .await
-            .context(TableOperationSnafu)
+        let query_interceptor = self.plugins.get::<SqlQueryInterceptorRef<Error>>();
+        let query_interceptor = query_interceptor.as_ref();
+
+        let output = match stmt {
+            Statement::Query(_) | Statement::Explain(_) | Statement::Delete(_) => {
+                let stmt = QueryStatement::Sql(stmt);
+                let plan = self
+                    .statement_executor
+                    .plan(&stmt, query_ctx.clone())
+                    .await?;
+
+                let QueryStatement::Sql(stmt) = stmt else {
+                    unreachable!()
+                };
+                query_interceptor.pre_execute(&stmt, Some(&plan), query_ctx.clone())?;
+
+                self.statement_executor.exec_plan(plan, query_ctx).await
+            }
+            Statement::Tql(tql) => {
+                let plan = self
+                    .statement_executor
+                    .plan_tql(tql.clone(), &query_ctx)
+                    .await?;
+
+                query_interceptor.pre_execute(
+                    &Statement::Tql(tql),
+                    Some(&plan),
+                    query_ctx.clone(),
+                )?;
+
+                self.statement_executor.exec_plan(plan, query_ctx).await
+            }
+            _ => {
+                query_interceptor.pre_execute(&stmt, None, query_ctx.clone())?;
+
+                self.statement_executor.execute_sql(stmt, query_ctx).await
+            }
+        };
+        output.context(TableOperationSnafu)
     }
 }
 
@@ -255,14 +289,6 @@ impl SqlQueryHandler for Instance {
             Ok(stmts) => {
                 let mut results = Vec::with_capacity(stmts.len());
                 for stmt in stmts {
-                    // TODO(sunng87): figure out at which stage we can call
-                    // this hook after ArrowFlight adoption. We need to provide
-                    // LogicalPlan as to this hook.
-                    if let Err(e) = query_interceptor.pre_execute(&stmt, None, query_ctx.clone()) {
-                        results.push(Err(e));
-                        break;
-                    }
-
                     if let Err(e) = checker
                         .check_permission(
                             query_ctx.current_user(),
@@ -341,7 +367,7 @@ impl SqlQueryHandler for Instance {
             let plan = self
                 .query_engine
                 .planner()
-                .plan(QueryStatement::Sql(stmt), query_ctx.clone())
+                .plan(&QueryStatement::Sql(stmt), query_ctx.clone())
                 .await
                 .context(PlanStatementSnafu)?;
             self.query_engine
diff --git a/src/meta-client/Cargo.toml b/src/meta-client/Cargo.toml
index c0cb0d3e89..7304274499 100644
--- a/src/meta-client/Cargo.toml
+++ b/src/meta-client/Cargo.toml
@@ -29,4 +29,4 @@ futures = "0.3"
 meta-srv = { workspace = true, features = ["mock"] }
 tower.workspace = true
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tracing-subscriber.workspace = true
diff --git a/src/meta-srv/Cargo.toml b/src/meta-srv/Cargo.toml
index 6d051c2eee..d436867582 100644
--- a/src/meta-srv/Cargo.toml
+++ b/src/meta-srv/Cargo.toml
@@ -59,7 +59,7 @@ tokio-stream = { workspace = true, features = ["net"] }
 toml.workspace = true
 tonic.workspace = true
 tower.workspace = true
-typetag = "0.2"
+typetag.workspace = true
 url = "2.3"
 
 [dev-dependencies]
@@ -69,4 +69,4 @@ common-meta = { workspace = true, features = ["testing"] }
 common-procedure-test.workspace = true
 session.workspace = true
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tracing-subscriber.workspace = true
diff --git a/src/operator/src/statement.rs b/src/operator/src/statement.rs
index 033bd14e9c..b9063781fe 100644
--- a/src/operator/src/statement.rs
+++ b/src/operator/src/statement.rs
@@ -363,7 +363,7 @@ impl StatementExecutor {
 
     pub async fn plan(
         &self,
-        stmt: QueryStatement,
+        stmt: &QueryStatement,
         query_ctx: QueryContextRef,
     ) -> Result<LogicalPlan> {
         self.query_engine
@@ -373,6 +373,14 @@ impl StatementExecutor {
             .context(PlanStatementSnafu)
     }
 
+    /// Execute [`LogicalPlan`] directly.
+    pub async fn exec_plan(&self, plan: LogicalPlan, query_ctx: QueryContextRef) -> Result<Output> {
+        self.query_engine
+            .execute(plan, query_ctx)
+            .await
+            .context(ExecLogicalPlanSnafu)
+    }
+
     pub fn optimize_logical_plan(&self, plan: LogicalPlan) -> Result<LogicalPlan> {
         self.query_engine
             .planner()
@@ -382,11 +390,8 @@ impl StatementExecutor {
 
     #[tracing::instrument(skip_all)]
     async fn plan_exec(&self, stmt: QueryStatement, query_ctx: QueryContextRef) -> Result<Output> {
-        let plan = self.plan(stmt, query_ctx.clone()).await?;
-        self.query_engine
-            .execute(plan, query_ctx)
-            .await
-            .context(ExecLogicalPlanSnafu)
+        let plan = self.plan(&stmt, query_ctx.clone()).await?;
+        self.exec_plan(plan, query_ctx).await
     }
 
     async fn get_table(&self, table_ref: &TableReference<'_>) -> Result<TableRef> {
diff --git a/src/operator/src/statement/ddl.rs b/src/operator/src/statement/ddl.rs
index af0f6883d8..cf10a45652 100644
--- a/src/operator/src/statement/ddl.rs
+++ b/src/operator/src/statement/ddl.rs
@@ -391,7 +391,7 @@ impl StatementExecutor {
         let logical_plan = match &*create_view.query {
             Statement::Query(query) => {
                 self.plan(
-                    QueryStatement::Sql(Statement::Query(query.clone())),
+                    &QueryStatement::Sql(Statement::Query(query.clone())),
                     ctx.clone(),
                 )
                 .await?
diff --git a/src/operator/src/statement/tql.rs b/src/operator/src/statement/tql.rs
index 008aba0d78..5a5d3808e7 100644
--- a/src/operator/src/statement/tql.rs
+++ b/src/operator/src/statement/tql.rs
@@ -90,7 +90,7 @@ impl StatementExecutor {
         };
         self.query_engine
             .planner()
-            .plan(stmt, query_ctx.clone())
+            .plan(&stmt, query_ctx.clone())
             .await
             .context(PlanStatementSnafu)
     }
diff --git a/src/query/src/datafusion.rs b/src/query/src/datafusion.rs
index 888ebbba83..f295a2c9b3 100644
--- a/src/query/src/datafusion.rs
+++ b/src/query/src/datafusion.rs
@@ -570,7 +570,7 @@ mod tests {
         let stmt = QueryLanguageParser::parse_sql(sql, &QueryContext::arc()).unwrap();
         let plan = engine
             .planner()
-            .plan(stmt, QueryContext::arc())
+            .plan(&stmt, QueryContext::arc())
             .await
             .unwrap();
 
@@ -592,7 +592,7 @@ mod tests {
         let stmt = QueryLanguageParser::parse_sql(sql, &QueryContext::arc()).unwrap();
         let plan = engine
             .planner()
-            .plan(stmt, QueryContext::arc())
+            .plan(&stmt, QueryContext::arc())
             .await
             .unwrap();
 
@@ -671,7 +671,7 @@ mod tests {
 
         let plan = engine
             .planner()
-            .plan(stmt, QueryContext::arc())
+            .plan(&stmt, QueryContext::arc())
             .await
             .unwrap();
 
diff --git a/src/query/src/planner.rs b/src/query/src/planner.rs
index 0f4f74133a..29a0a364ea 100644
--- a/src/query/src/planner.rs
+++ b/src/query/src/planner.rs
@@ -39,7 +39,7 @@ use crate::{DfContextProviderAdapter, QueryEngineContext};
 
 #[async_trait]
 pub trait LogicalPlanner: Send + Sync {
-    async fn plan(&self, stmt: QueryStatement, query_ctx: QueryContextRef) -> Result<LogicalPlan>;
+    async fn plan(&self, stmt: &QueryStatement, query_ctx: QueryContextRef) -> Result<LogicalPlan>;
 
     fn optimize(&self, plan: LogicalPlan) -> Result<LogicalPlan>;
 
@@ -61,8 +61,8 @@ impl DfLogicalPlanner {
     }
 
     #[tracing::instrument(skip_all)]
-    async fn plan_sql(&self, stmt: Statement, query_ctx: QueryContextRef) -> Result<LogicalPlan> {
-        let df_stmt = (&stmt).try_into().context(SqlSnafu)?;
+    async fn plan_sql(&self, stmt: &Statement, query_ctx: QueryContextRef) -> Result<LogicalPlan> {
+        let df_stmt = stmt.try_into().context(SqlSnafu)?;
 
         let table_provider = DfTableSourceProvider::new(
             self.engine_state.catalog_manager().clone(),
@@ -142,7 +142,7 @@ impl DfLogicalPlanner {
     }
 
     #[tracing::instrument(skip_all)]
-    async fn plan_pql(&self, stmt: EvalStmt, query_ctx: QueryContextRef) -> Result<LogicalPlan> {
+    async fn plan_pql(&self, stmt: &EvalStmt, query_ctx: QueryContextRef) -> Result<LogicalPlan> {
         let plan_decoder = Arc::new(DefaultPlanDecoder::new(
             self.session_state.clone(),
             &query_ctx,
@@ -175,7 +175,7 @@ impl DfLogicalPlanner {
 #[async_trait]
 impl LogicalPlanner for DfLogicalPlanner {
     #[tracing::instrument(skip_all)]
-    async fn plan(&self, stmt: QueryStatement, query_ctx: QueryContextRef) -> Result<LogicalPlan> {
+    async fn plan(&self, stmt: &QueryStatement, query_ctx: QueryContextRef) -> Result<LogicalPlan> {
         match stmt {
             QueryStatement::Sql(stmt) => self.plan_sql(stmt, query_ctx).await,
             QueryStatement::Promql(stmt) => self.plan_pql(stmt, query_ctx).await,
diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs
index bafd69aa95..001e41ca99 100644
--- a/src/query/src/promql/planner.rs
+++ b/src/query/src/promql/planner.rs
@@ -157,29 +157,29 @@ pub struct PromPlanner {
 impl PromPlanner {
     pub async fn stmt_to_plan(
         table_provider: DfTableSourceProvider,
-        stmt: EvalStmt,
+        stmt: &EvalStmt,
         session_state: &SessionState,
     ) -> Result<LogicalPlan> {
         let mut planner = Self {
             table_provider,
-            ctx: PromPlannerContext::from_eval_stmt(&stmt),
+            ctx: PromPlannerContext::from_eval_stmt(stmt),
         };
 
-        planner.prom_expr_to_plan(stmt.expr, session_state).await
+        planner.prom_expr_to_plan(&stmt.expr, session_state).await
     }
 
     #[async_recursion]
     pub async fn prom_expr_to_plan(
         &mut self,
-        prom_expr: PromExpr,
+        prom_expr: &PromExpr,
         session_state: &SessionState,
     ) -> Result<LogicalPlan> {
-        let res = match &prom_expr {
+        let res = match prom_expr {
             PromExpr::Aggregate(expr) => self.prom_aggr_expr_to_plan(session_state, expr).await?,
             PromExpr::Unary(expr) => self.prom_unary_expr_to_plan(session_state, expr).await?,
             PromExpr::Binary(expr) => self.prom_binary_expr_to_plan(session_state, expr).await?,
             PromExpr::Paren(ParenExpr { expr }) => {
-                self.prom_expr_to_plan(*expr.clone(), session_state).await?
+                self.prom_expr_to_plan(expr, session_state).await?
             }
             PromExpr::Subquery(SubqueryExpr { .. }) => UnsupportedExprSnafu {
                 name: "Prom Subquery",
@@ -212,7 +212,7 @@ impl PromPlanner {
             modifier,
         } = aggr_expr;
 
-        let input = self.prom_expr_to_plan(*expr.clone(), session_state).await?;
+        let input = self.prom_expr_to_plan(expr, session_state).await?;
 
         // calculate columns to group by
         // Need to append time index column into group by columns
@@ -242,7 +242,7 @@ impl PromPlanner {
     ) -> Result<LogicalPlan> {
         let UnaryExpr { expr } = unary_expr;
         // Unary Expr in PromQL implys the `-` operator
-        let input = self.prom_expr_to_plan(*expr.clone(), session_state).await?;
+        let input = self.prom_expr_to_plan(expr, session_state).await?;
         self.projection_for_each_field_column(input, |col| {
             Ok(DfExpr::Negative(Box::new(DfExpr::Column(col.into()))))
         })
@@ -305,7 +305,7 @@ impl PromPlanner {
             }
             // lhs is a literal, rhs is a column
             (Some(mut expr), None) => {
-                let input = self.prom_expr_to_plan(*rhs.clone(), session_state).await?;
+                let input = self.prom_expr_to_plan(rhs, session_state).await?;
                 // check if the literal is a special time expr
                 if let Some(time_expr) = Self::try_build_special_time_expr(
                     lhs,
@@ -334,7 +334,7 @@ impl PromPlanner {
             }
             // lhs is a column, rhs is a literal
             (None, Some(mut expr)) => {
-                let input = self.prom_expr_to_plan(*lhs.clone(), session_state).await?;
+                let input = self.prom_expr_to_plan(lhs, session_state).await?;
                 // check if the literal is a special time expr
                 if let Some(time_expr) = Self::try_build_special_time_expr(
                     rhs,
@@ -363,14 +363,14 @@ impl PromPlanner {
             }
             // both are columns. join them on time index
             (None, None) => {
-                let left_input = self.prom_expr_to_plan(*lhs.clone(), session_state).await?;
+                let left_input = self.prom_expr_to_plan(lhs, session_state).await?;
                 let left_field_columns = self.ctx.field_columns.clone();
                 let mut left_table_ref = self
                     .table_ref()
                     .unwrap_or_else(|_| TableReference::bare(""));
                 let left_context = self.ctx.clone();
 
-                let right_input = self.prom_expr_to_plan(*rhs.clone(), session_state).await?;
+                let right_input = self.prom_expr_to_plan(rhs, session_state).await?;
                 let right_field_columns = self.ctx.field_columns.clone();
                 let mut right_table_ref = self
                     .table_ref()
@@ -589,7 +589,7 @@ impl PromPlanner {
         // transform function arguments
         let args = self.create_function_args(&args.args)?;
         let input = if let Some(prom_expr) = args.input {
-            self.prom_expr_to_plan(prom_expr, session_state).await?
+            self.prom_expr_to_plan(&prom_expr, session_state).await?
         } else {
             self.ctx.time_index_column = Some(SPECIAL_TIME_FUNCTION.to_string());
             self.ctx.reset_table_name_and_schema();
@@ -628,9 +628,7 @@ impl PromPlanner {
         // let promql_parser::parser::ast::Extension { expr } = ext_expr;
         let expr = &ext_expr.expr;
         let children = expr.children();
-        let plan = self
-            .prom_expr_to_plan(children[0].clone(), session_state)
-            .await?;
+        let plan = self.prom_expr_to_plan(&children[0], session_state).await?;
         // Wrapper for the explanation/analyze of the existing plan
         // https://docs.rs/datafusion-expr/latest/datafusion_expr/logical_plan/builder/struct.LogicalPlanBuilder.html#method.explain
         // if `analyze` is true, runs the actual plan and produces
@@ -1548,7 +1546,7 @@ impl PromPlanner {
             }
         })?;
         let input = args.args[1].as_ref().clone();
-        let input_plan = self.prom_expr_to_plan(input, session_state).await?;
+        let input_plan = self.prom_expr_to_plan(&input, session_state).await?;
 
         if !self.ctx.has_le_tag() {
             return ColumnNotFoundSnafu {
@@ -1633,9 +1631,7 @@ impl PromPlanner {
                 fn_name: SCALAR_FUNCTION
             }
         );
-        let input = self
-            .prom_expr_to_plan(args.args[0].as_ref().clone(), session_state)
-            .await?;
+        let input = self.prom_expr_to_plan(&args.args[0], session_state).await?;
         ensure!(
             self.ctx.field_columns.len() == 1,
             MultiFieldsNotSupportedSnafu {
@@ -2420,7 +2416,7 @@ mod test {
             1,
         )
         .await;
-        let plan = PromPlanner::stmt_to_plan(table_provider, eval_stmt, &build_session_state())
+        let plan = PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_session_state())
             .await
             .unwrap();
 
@@ -2630,10 +2626,9 @@ mod test {
             2,
         )
         .await;
-        let plan =
-            PromPlanner::stmt_to_plan(table_provider, eval_stmt.clone(), &build_session_state())
-                .await
-                .unwrap();
+        let plan = PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_session_state())
+            .await
+            .unwrap();
         let expected_no_without = String::from(
             "Sort: some_metric.tag_1 ASC NULLS LAST, some_metric.timestamp ASC NULLS LAST [tag_1:Utf8, timestamp:Timestamp(Millisecond, None), TEMPLATE(some_metric.field_0):Float64;N, TEMPLATE(some_metric.field_1):Float64;N]\
             \n  Aggregate: groupBy=[[some_metric.tag_1, some_metric.timestamp]], aggr=[[TEMPLATE(some_metric.field_0), TEMPLATE(some_metric.field_1)]] [tag_1:Utf8, timestamp:Timestamp(Millisecond, None), TEMPLATE(some_metric.field_0):Float64;N, TEMPLATE(some_metric.field_1):Float64;N]\
@@ -2661,7 +2656,7 @@ mod test {
             2,
         )
         .await;
-        let plan = PromPlanner::stmt_to_plan(table_provider, eval_stmt, &build_session_state())
+        let plan = PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_session_state())
             .await
             .unwrap();
         let expected_without = String::from(
@@ -2786,7 +2781,7 @@ mod test {
             1,
         )
         .await;
-        let plan = PromPlanner::stmt_to_plan(table_provider, eval_stmt, &build_session_state())
+        let plan = PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_session_state())
             .await
             .unwrap();
 
@@ -2836,7 +2831,7 @@ mod test {
             1,
         )
         .await;
-        let plan = PromPlanner::stmt_to_plan(table_provider, eval_stmt, &build_session_state())
+        let plan = PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_session_state())
             .await
             .unwrap();
 
@@ -3080,13 +3075,10 @@ mod test {
                 3,
             )
             .await;
-            let plan = PromPlanner::stmt_to_plan(
-                table_provider,
-                eval_stmt.clone(),
-                &build_session_state(),
-            )
-            .await
-            .unwrap();
+            let plan =
+                PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_session_state())
+                    .await
+                    .unwrap();
             let mut fields = plan.schema().field_names();
             let mut expected = case.1.into_iter().map(String::from).collect::<Vec<_>>();
             fields.sort();
@@ -3108,12 +3100,8 @@ mod test {
                 3,
             )
             .await;
-            let plan = PromPlanner::stmt_to_plan(
-                table_provider,
-                eval_stmt.clone(),
-                &build_session_state(),
-            )
-            .await;
+            let plan =
+                PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_session_state()).await;
             assert!(plan.is_err(), "case: {:?}", case);
         }
     }
@@ -3166,7 +3154,7 @@ mod test {
             .await;
 
             let plan =
-                PromPlanner::stmt_to_plan(table_provider, eval_stmt, &build_session_state()).await;
+                PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_session_state()).await;
             assert!(plan.is_err(), "query: {:?}", query);
         }
     }
@@ -3224,7 +3212,7 @@ mod test {
                 DummyDecoder::arc(),
                 true,
             ),
-            EvalStmt {
+            &EvalStmt {
                 expr: parser::parse("metrics{tag = \"1\"}").unwrap(),
                 start: UNIX_EPOCH,
                 end: UNIX_EPOCH
@@ -3254,7 +3242,7 @@ mod test {
                 DummyDecoder::arc(),
                 true,
             ),
-            EvalStmt {
+            &EvalStmt {
                 expr: parser::parse("avg_over_time(metrics{tag = \"1\"}[5s])").unwrap(),
                 start: UNIX_EPOCH,
                 end: UNIX_EPOCH
diff --git a/src/query/src/range_select/plan_rewrite.rs b/src/query/src/range_select/plan_rewrite.rs
index 3e6516a9f1..56cad37ef2 100644
--- a/src/query/src/range_select/plan_rewrite.rs
+++ b/src/query/src/range_select/plan_rewrite.rs
@@ -634,7 +634,7 @@ mod test {
     async fn do_query(sql: &str) -> Result<LogicalPlan> {
         let stmt = QueryLanguageParser::parse_sql(sql, &QueryContext::arc()).unwrap();
         let engine = create_test_engine().await;
-        engine.planner().plan(stmt, QueryContext::arc()).await
+        engine.planner().plan(&stmt, QueryContext::arc()).await
     }
 
     async fn query_plan_compare(sql: &str, expected: String) {
diff --git a/src/query/src/tests.rs b/src/query/src/tests.rs
index 1fee632073..2bebdbad58 100644
--- a/src/query/src/tests.rs
+++ b/src/query/src/tests.rs
@@ -39,7 +39,7 @@ async fn exec_selection(engine: QueryEngineRef, sql: &str) -> Vec<RecordBatch> {
     let stmt = QueryLanguageParser::parse_sql(sql, &query_ctx).unwrap();
     let plan = engine
         .planner()
-        .plan(stmt, query_ctx.clone())
+        .plan(&stmt, query_ctx.clone())
         .await
         .unwrap();
     let OutputData::Stream(stream) = engine.execute(plan, query_ctx).await.unwrap().data else {
diff --git a/src/query/src/tests/query_engine_test.rs b/src/query/src/tests/query_engine_test.rs
index 687346dbcb..d46d7afd9d 100644
--- a/src/query/src/tests/query_engine_test.rs
+++ b/src/query/src/tests/query_engine_test.rs
@@ -134,7 +134,7 @@ async fn test_query_validate() -> Result<()> {
             .unwrap();
     assert!(engine
         .planner()
-        .plan(stmt, QueryContext::arc())
+        .plan(&stmt, QueryContext::arc())
         .await
         .is_ok());
 
@@ -145,7 +145,7 @@ async fn test_query_validate() -> Result<()> {
     .unwrap();
     assert!(engine
         .planner()
-        .plan(stmt, QueryContext::arc())
+        .plan(&stmt, QueryContext::arc())
         .await
         .is_err());
     Ok(())
diff --git a/src/script/src/python/engine.rs b/src/script/src/python/engine.rs
index d7d3b1326a..0398a02f43 100644
--- a/src/script/src/python/engine.rs
+++ b/src/script/src/python/engine.rs
@@ -315,7 +315,7 @@ impl Script for PyScript {
             let plan = self
                 .query_engine
                 .planner()
-                .plan(stmt, ctx.query_ctx.clone())
+                .plan(&stmt, ctx.query_ctx.clone())
                 .await
                 .context(DatabaseQuerySnafu)?;
             let res = self
diff --git a/src/script/src/python/ffi_types/copr.rs b/src/script/src/python/ffi_types/copr.rs
index b81fc9d576..1a9a88466b 100644
--- a/src/script/src/python/ffi_types/copr.rs
+++ b/src/script/src/python/ffi_types/copr.rs
@@ -414,7 +414,7 @@ impl PyQueryEngine {
                     let ctx = Arc::new(QueryContextBuilder::default().build());
                     let plan = engine
                         .planner()
-                        .plan(stmt, ctx.clone())
+                        .plan(&stmt, ctx.clone())
                         .await
                         .map_err(|e| e.to_string())?;
                     let res = engine
diff --git a/src/servers/tests/mod.rs b/src/servers/tests/mod.rs
index ca098546aa..6a280783e4 100644
--- a/src/servers/tests/mod.rs
+++ b/src/servers/tests/mod.rs
@@ -70,7 +70,7 @@ impl SqlQueryHandler for DummyInstance {
         let plan = self
             .query_engine
             .planner()
-            .plan(stmt, query_ctx.clone())
+            .plan(&stmt, query_ctx.clone())
             .await
             .unwrap();
         let output = self.query_engine.execute(plan, query_ctx).await.unwrap();
@@ -98,7 +98,7 @@ impl SqlQueryHandler for DummyInstance {
             let plan = self
                 .query_engine
                 .planner()
-                .plan(QueryStatement::Sql(stmt), query_ctx.clone())
+                .plan(&QueryStatement::Sql(stmt), query_ctx.clone())
                 .await
                 .unwrap();
             let schema = self.query_engine.describe(plan, query_ctx).await.unwrap();
diff --git a/tests-integration/src/grpc.rs b/tests-integration/src/grpc.rs
index a4ef632b33..b785f4ad37 100644
--- a/tests-integration/src/grpc.rs
+++ b/tests-integration/src/grpc.rs
@@ -542,7 +542,7 @@ CREATE TABLE {table_name} (
         let plan = instance
             .frontend()
             .statement_executor()
-            .plan(stmt, QueryContext::arc())
+            .plan(&stmt, QueryContext::arc())
             .await
             .unwrap();
         let plan = DFLogicalSubstraitConvertor
diff --git a/tests-integration/src/instance.rs b/tests-integration/src/instance.rs
index 096be44e0c..fb4a001467 100644
--- a/tests-integration/src/instance.rs
+++ b/tests-integration/src/instance.rs
@@ -236,7 +236,7 @@ mod tests {
         let plan = instance
             .frontend()
             .statement_executor()
-            .plan(stmt, QueryContext::arc())
+            .plan(&stmt, QueryContext::arc())
             .await
             .unwrap();
         let plan = DFLogicalSubstraitConvertor

From a34035a1f2cc2e2e2438c142e0c17317af54a3ce Mon Sep 17 00:00:00 2001
From: dennis zhuang <killme2008@gmail.com>
Date: Wed, 30 Oct 2024 18:59:13 +0800
Subject: [PATCH 123/128] fix: set transaction variables not working in mysql
 protocol (#4912)

---
 src/servers/src/mysql/federated.rs |  2 ++
 tests-integration/tests/sql.rs     | 20 ++++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/src/servers/src/mysql/federated.rs b/src/servers/src/mysql/federated.rs
index 63faa22d54..71733dcc12 100644
--- a/src/servers/src/mysql/federated.rs
+++ b/src/servers/src/mysql/federated.rs
@@ -60,6 +60,8 @@ static OTHER_NOT_SUPPORTED_STMT: Lazy<RegexSet> = Lazy::new(|| {
         "(?i)^(SET FOREIGN_KEY_CHECKS(.*))",
         "(?i)^(SET AUTOCOMMIT(.*))",
         "(?i)^(SET SQL_LOG_BIN(.*))",
+        "(?i)^(SET SESSION TRANSACTION(.*))",
+        "(?i)^(SET TRANSACTION(.*))",
         "(?i)^(SET sql_mode(.*))",
         "(?i)^(SET SQL_SELECT_LIMIT(.*))",
         "(?i)^(SET @@(.*))",
diff --git a/tests-integration/tests/sql.rs b/tests-integration/tests/sql.rs
index af9374a2a8..1a481e533d 100644
--- a/tests-integration/tests/sql.rs
+++ b/tests-integration/tests/sql.rs
@@ -60,6 +60,7 @@ macro_rules! sql_tests {
                 $service,
 
                 test_mysql_auth,
+                test_mysql_stmts,
                 test_mysql_crud,
                 test_mysql_timezone,
                 test_mysql_async_timestamp,
@@ -133,6 +134,25 @@ pub async fn test_mysql_auth(store_type: StorageType) {
     guard.remove_all().await;
 }
 
+pub async fn test_mysql_stmts(store_type: StorageType) {
+    common_telemetry::init_default_ut_logging();
+
+    let (addr, mut guard, fe_mysql_server) = setup_mysql_server(store_type, "sql_crud").await;
+
+    let mut conn = MySqlConnection::connect(&format!("mysql://{addr}/public"))
+        .await
+        .unwrap();
+
+    conn.execute("SET SESSION TRANSACTION READ ONLY")
+        .await
+        .unwrap();
+
+    conn.execute("SET TRANSACTION READ ONLY").await.unwrap();
+
+    let _ = fe_mysql_server.shutdown().await;
+    guard.remove_all().await;
+}
+
 pub async fn test_mysql_crud(store_type: StorageType) {
     common_telemetry::init_default_ut_logging();
 

From dcc08f6b3ed45813470d3b64f4e02d2c38e870e9 Mon Sep 17 00:00:00 2001
From: dennis zhuang <killme2008@gmail.com>
Date: Wed, 30 Oct 2024 19:12:58 +0800
Subject: [PATCH 124/128] feat: adds the number of rows and index files size to
 region_statistics table (#4909)

* feat: adds index size to region statistics

* feat: adds the number of rows for region statistics

* test: adds sqlness test for region_statistics

* fix: test
---
 .../information_schema/region_statistics.rs   | 26 ++++++++++++-
 src/cmd/src/standalone.rs                     |  4 +-
 src/common/meta/src/datanode.rs               | 12 ++++--
 src/meta-srv/src/handler/failure_handler.rs   |  2 +
 .../src/handler/region_lease_handler.rs       |  2 +
 src/meta-srv/src/selector/weight_compute.rs   |  6 +++
 src/mito2/src/engine/basic_test.rs            |  3 +-
 src/mito2/src/memtable/time_partition.rs      | 10 +++++
 src/mito2/src/memtable/version.rs             |  9 +++++
 src/mito2/src/region.rs                       | 16 +++++---
 src/mito2/src/sst/version.rs                  | 39 ++++++++++++++++++-
 src/store-api/src/region_engine.rs            | 10 ++++-
 .../region_statistics.result                  | 39 +++++++++++++++++++
 .../information_schema/region_statistics.sql  | 25 ++++++++++++
 .../common/system/information_schema.result   | 13 ++++---
 tests/conf/datanode-test.toml.template        |  3 ++
 16 files changed, 198 insertions(+), 21 deletions(-)
 create mode 100644 tests/cases/standalone/common/information_schema/region_statistics.result
 create mode 100644 tests/cases/standalone/common/information_schema/region_statistics.sql

diff --git a/src/catalog/src/system_schema/information_schema/region_statistics.rs b/src/catalog/src/system_schema/information_schema/region_statistics.rs
index e92558acd0..a98a70cb23 100644
--- a/src/catalog/src/system_schema/information_schema/region_statistics.rs
+++ b/src/catalog/src/system_schema/information_schema/region_statistics.rs
@@ -39,9 +39,12 @@ use crate::CatalogManager;
 const REGION_ID: &str = "region_id";
 const TABLE_ID: &str = "table_id";
 const REGION_NUMBER: &str = "region_number";
+const REGION_ROWS: &str = "region_rows";
+const DISK_SIZE: &str = "disk_size";
 const MEMTABLE_SIZE: &str = "memtable_size";
 const MANIFEST_SIZE: &str = "manifest_size";
 const SST_SIZE: &str = "sst_size";
+const INDEX_SIZE: &str = "index_size";
 const ENGINE: &str = "engine";
 const REGION_ROLE: &str = "region_role";
 
@@ -52,9 +55,12 @@ const INIT_CAPACITY: usize = 42;
 /// - `region_id`: The region id.
 /// - `table_id`: The table id.
 /// - `region_number`: The region number.
+/// - `region_rows`: The number of rows in region.
 /// - `memtable_size`: The memtable size in bytes.
+/// - `disk_size`: The approximate disk size in bytes.
 /// - `manifest_size`: The manifest size in bytes.
-/// - `sst_size`: The sst size in bytes.
+/// - `sst_size`: The sst data files size in bytes.
+/// - `index_size`: The sst index files size in bytes.
 /// - `engine`: The engine type.
 /// - `region_role`: The region role.
 ///
@@ -76,9 +82,12 @@ impl InformationSchemaRegionStatistics {
             ColumnSchema::new(REGION_ID, ConcreteDataType::uint64_datatype(), false),
             ColumnSchema::new(TABLE_ID, ConcreteDataType::uint32_datatype(), false),
             ColumnSchema::new(REGION_NUMBER, ConcreteDataType::uint32_datatype(), false),
+            ColumnSchema::new(REGION_ROWS, ConcreteDataType::uint64_datatype(), true),
+            ColumnSchema::new(DISK_SIZE, ConcreteDataType::uint64_datatype(), true),
             ColumnSchema::new(MEMTABLE_SIZE, ConcreteDataType::uint64_datatype(), true),
             ColumnSchema::new(MANIFEST_SIZE, ConcreteDataType::uint64_datatype(), true),
             ColumnSchema::new(SST_SIZE, ConcreteDataType::uint64_datatype(), true),
+            ColumnSchema::new(INDEX_SIZE, ConcreteDataType::uint64_datatype(), true),
             ColumnSchema::new(ENGINE, ConcreteDataType::string_datatype(), true),
             ColumnSchema::new(REGION_ROLE, ConcreteDataType::string_datatype(), true),
         ]))
@@ -135,9 +144,12 @@ struct InformationSchemaRegionStatisticsBuilder {
     region_ids: UInt64VectorBuilder,
     table_ids: UInt32VectorBuilder,
     region_numbers: UInt32VectorBuilder,
+    region_rows: UInt64VectorBuilder,
+    disk_sizes: UInt64VectorBuilder,
     memtable_sizes: UInt64VectorBuilder,
     manifest_sizes: UInt64VectorBuilder,
     sst_sizes: UInt64VectorBuilder,
+    index_sizes: UInt64VectorBuilder,
     engines: StringVectorBuilder,
     region_roles: StringVectorBuilder,
 }
@@ -150,9 +162,12 @@ impl InformationSchemaRegionStatisticsBuilder {
             region_ids: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
             table_ids: UInt32VectorBuilder::with_capacity(INIT_CAPACITY),
             region_numbers: UInt32VectorBuilder::with_capacity(INIT_CAPACITY),
+            region_rows: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
+            disk_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
             memtable_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
             manifest_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
             sst_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
+            index_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
             engines: StringVectorBuilder::with_capacity(INIT_CAPACITY),
             region_roles: StringVectorBuilder::with_capacity(INIT_CAPACITY),
         }
@@ -177,9 +192,12 @@ impl InformationSchemaRegionStatisticsBuilder {
             (REGION_ID, &Value::from(region_stat.id.as_u64())),
             (TABLE_ID, &Value::from(region_stat.id.table_id())),
             (REGION_NUMBER, &Value::from(region_stat.id.region_number())),
+            (REGION_ROWS, &Value::from(region_stat.num_rows)),
+            (DISK_SIZE, &Value::from(region_stat.approximate_bytes)),
             (MEMTABLE_SIZE, &Value::from(region_stat.memtable_size)),
             (MANIFEST_SIZE, &Value::from(region_stat.manifest_size)),
             (SST_SIZE, &Value::from(region_stat.sst_size)),
+            (INDEX_SIZE, &Value::from(region_stat.index_size)),
             (ENGINE, &Value::from(region_stat.engine.as_str())),
             (REGION_ROLE, &Value::from(region_stat.role.to_string())),
         ];
@@ -192,9 +210,12 @@ impl InformationSchemaRegionStatisticsBuilder {
         self.table_ids.push(Some(region_stat.id.table_id()));
         self.region_numbers
             .push(Some(region_stat.id.region_number()));
+        self.region_rows.push(Some(region_stat.num_rows));
+        self.disk_sizes.push(Some(region_stat.approximate_bytes));
         self.memtable_sizes.push(Some(region_stat.memtable_size));
         self.manifest_sizes.push(Some(region_stat.manifest_size));
         self.sst_sizes.push(Some(region_stat.sst_size));
+        self.index_sizes.push(Some(region_stat.index_size));
         self.engines.push(Some(&region_stat.engine));
         self.region_roles.push(Some(&region_stat.role.to_string()));
     }
@@ -204,9 +225,12 @@ impl InformationSchemaRegionStatisticsBuilder {
             Arc::new(self.region_ids.finish()),
             Arc::new(self.table_ids.finish()),
             Arc::new(self.region_numbers.finish()),
+            Arc::new(self.region_rows.finish()),
+            Arc::new(self.disk_sizes.finish()),
             Arc::new(self.memtable_sizes.finish()),
             Arc::new(self.manifest_sizes.finish()),
             Arc::new(self.sst_sizes.finish()),
+            Arc::new(self.index_sizes.finish()),
             Arc::new(self.engines.finish()),
             Arc::new(self.region_roles.finish()),
         ];
diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
index 54d6e4d72c..251957dd28 100644
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -736,12 +736,14 @@ impl InformationExtension for StandaloneInformationExtension {
                     id: stat.region_id,
                     rcus: 0,
                     wcus: 0,
-                    approximate_bytes: region_stat.estimated_disk_size() as i64,
+                    approximate_bytes: region_stat.estimated_disk_size(),
                     engine: stat.engine,
                     role: RegionRole::from(stat.role).into(),
+                    num_rows: region_stat.num_rows,
                     memtable_size: region_stat.memtable_size,
                     manifest_size: region_stat.manifest_size,
                     sst_size: region_stat.sst_size,
+                    index_size: region_stat.index_size,
                 }
             })
             .collect::<Vec<_>>();
diff --git a/src/common/meta/src/datanode.rs b/src/common/meta/src/datanode.rs
index 4551b8de2f..869af96a28 100644
--- a/src/common/meta/src/datanode.rs
+++ b/src/common/meta/src/datanode.rs
@@ -78,17 +78,21 @@ pub struct RegionStat {
     /// The write capacity units during this period
     pub wcus: i64,
     /// Approximate bytes of this region
-    pub approximate_bytes: i64,
+    pub approximate_bytes: u64,
     /// The engine name.
     pub engine: String,
     /// The region role.
     pub role: RegionRole,
+    /// The number of rows
+    pub num_rows: u64,
     /// The size of the memtable in bytes.
     pub memtable_size: u64,
     /// The size of the manifest in bytes.
     pub manifest_size: u64,
-    /// The size of the SST files in bytes.
+    /// The size of the SST data files in bytes.
     pub sst_size: u64,
+    /// The size of the SST index files in bytes.
+    pub index_size: u64,
 }
 
 impl Stat {
@@ -178,12 +182,14 @@ impl From<&api::v1::meta::RegionStat> for RegionStat {
             id: RegionId::from_u64(value.region_id),
             rcus: value.rcus,
             wcus: value.wcus,
-            approximate_bytes: value.approximate_bytes,
+            approximate_bytes: value.approximate_bytes as u64,
             engine: value.engine.to_string(),
             role: RegionRole::from(value.role()),
+            num_rows: region_stat.num_rows,
             memtable_size: region_stat.memtable_size,
             manifest_size: region_stat.manifest_size,
             sst_size: region_stat.sst_size,
+            index_size: region_stat.index_size,
         }
     }
 }
diff --git a/src/meta-srv/src/handler/failure_handler.rs b/src/meta-srv/src/handler/failure_handler.rs
index 02f423c4b4..ae38f887f4 100644
--- a/src/meta-srv/src/handler/failure_handler.rs
+++ b/src/meta-srv/src/handler/failure_handler.rs
@@ -93,9 +93,11 @@ mod tests {
                 approximate_bytes: 0,
                 engine: default_engine().to_string(),
                 role: RegionRole::Follower,
+                num_rows: 0,
                 memtable_size: 0,
                 manifest_size: 0,
                 sst_size: 0,
+                index_size: 0,
             }
         }
         acc.stat = Some(Stat {
diff --git a/src/meta-srv/src/handler/region_lease_handler.rs b/src/meta-srv/src/handler/region_lease_handler.rs
index de491da371..98a74f67bb 100644
--- a/src/meta-srv/src/handler/region_lease_handler.rs
+++ b/src/meta-srv/src/handler/region_lease_handler.rs
@@ -135,9 +135,11 @@ mod test {
             wcus: 0,
             approximate_bytes: 0,
             engine: String::new(),
+            num_rows: 0,
             memtable_size: 0,
             manifest_size: 0,
             sst_size: 0,
+            index_size: 0,
         }
     }
 
diff --git a/src/meta-srv/src/selector/weight_compute.rs b/src/meta-srv/src/selector/weight_compute.rs
index 09d8833e2e..7f3b28a364 100644
--- a/src/meta-srv/src/selector/weight_compute.rs
+++ b/src/meta-srv/src/selector/weight_compute.rs
@@ -198,9 +198,11 @@ mod tests {
                 approximate_bytes: 1,
                 engine: "mito2".to_string(),
                 role: RegionRole::Leader,
+                num_rows: 0,
                 memtable_size: 0,
                 manifest_size: 0,
                 sst_size: 0,
+                index_size: 0,
             }],
             ..Default::default()
         }
@@ -217,9 +219,11 @@ mod tests {
                 approximate_bytes: 1,
                 engine: "mito2".to_string(),
                 role: RegionRole::Leader,
+                num_rows: 0,
                 memtable_size: 0,
                 manifest_size: 0,
                 sst_size: 0,
+                index_size: 0,
             }],
             ..Default::default()
         }
@@ -236,9 +240,11 @@ mod tests {
                 approximate_bytes: 1,
                 engine: "mito2".to_string(),
                 role: RegionRole::Leader,
+                num_rows: 0,
                 memtable_size: 0,
                 manifest_size: 0,
                 sst_size: 0,
+                index_size: 0,
             }],
             ..Default::default()
         }
diff --git a/src/mito2/src/engine/basic_test.rs b/src/mito2/src/engine/basic_test.rs
index 533b6a2ea1..785c914e3e 100644
--- a/src/mito2/src/engine/basic_test.rs
+++ b/src/mito2/src/engine/basic_test.rs
@@ -580,7 +580,8 @@ async fn test_region_usage() {
     flush_region(&engine, region_id, None).await;
 
     let region_stat = region.region_statistic();
-    assert_eq!(region_stat.sst_size, 3010);
+    assert_eq!(region_stat.sst_size, 2790);
+    assert_eq!(region_stat.num_rows, 10);
 
     // region total usage
     // Some memtables may share items.
diff --git a/src/mito2/src/memtable/time_partition.rs b/src/mito2/src/memtable/time_partition.rs
index 6d92488a7b..7fa03ae1be 100644
--- a/src/mito2/src/memtable/time_partition.rs
+++ b/src/mito2/src/memtable/time_partition.rs
@@ -216,6 +216,16 @@ impl TimePartitions {
             .sum()
     }
 
+    /// Returns the number of rows.
+    pub(crate) fn num_rows(&self) -> u64 {
+        let inner = self.inner.lock().unwrap();
+        inner
+            .parts
+            .iter()
+            .map(|part| part.memtable.stats().num_rows as u64)
+            .sum()
+    }
+
     /// Append memtables in partitions to small vec.
     pub(crate) fn list_memtables_to_small_vec(&self, memtables: &mut SmallMemtableVec) {
         let inner = self.inner.lock().unwrap();
diff --git a/src/mito2/src/memtable/version.rs b/src/mito2/src/memtable/version.rs
index 9e18edc673..1c7f2b7d4a 100644
--- a/src/mito2/src/memtable/version.rs
+++ b/src/mito2/src/memtable/version.rs
@@ -115,6 +115,15 @@ impl MemtableVersion {
             .sum()
     }
 
+    /// Returns the number of rows in memtables.
+    pub(crate) fn num_rows(&self) -> u64 {
+        self.immutables
+            .iter()
+            .map(|mem| mem.stats().num_rows as u64)
+            .sum::<u64>()
+            + self.mutable.num_rows()
+    }
+
     /// Returns true if the memtable version is empty.
     ///
     /// The version is empty when mutable memtable is empty and there is no
diff --git a/src/mito2/src/region.rs b/src/mito2/src/region.rs
index b05daf3da0..4ce633e6a6 100644
--- a/src/mito2/src/region.rs
+++ b/src/mito2/src/region.rs
@@ -277,15 +277,19 @@ impl MitoRegion {
         let memtable_usage = (memtables.mutable_usage() + memtables.immutables_usage()) as u64;
 
         let sst_usage = version.ssts.sst_usage();
+        let index_usage = version.ssts.index_usage();
 
         let wal_usage = self.estimated_wal_usage(memtable_usage);
         let manifest_usage = self.stats.total_manifest_size();
+        let num_rows = version.ssts.num_rows() + version.memtables.num_rows();
 
         RegionStatistic {
+            num_rows,
             memtable_size: memtable_usage,
             wal_size: wal_usage,
             manifest_size: manifest_usage,
             sst_size: sst_usage,
+            index_size: index_usage,
         }
     }
 
@@ -422,15 +426,15 @@ impl ManifestContext {
     /// Sets the [`RegionRole`].
     ///
     /// ```
-    ///     +------------------------------------------+    
-    ///     |                      +-----------------+ |    
-    ///     |                      |                 | |    
+    ///     +------------------------------------------+
+    ///     |                      +-----------------+ |
+    ///     |                      |                 | |
     /// +---+------+       +-------+-----+        +--v-v---+
     /// | Follower |       | Downgrading |        | Leader |
     /// +---^-^----+       +-----+-^-----+        +--+-+---+
-    ///     | |                  | |                 | |    
-    ///     | +------------------+ +-----------------+ |    
-    ///     +------------------------------------------+    
+    ///     | |                  | |                 | |
+    ///     | +------------------+ +-----------------+ |
+    ///     +------------------------------------------+
     ///
     /// Transition:
     /// - Follower -> Leader
diff --git a/src/mito2/src/sst/version.rs b/src/mito2/src/sst/version.rs
index 07d6bee9d9..c677a95413 100644
--- a/src/mito2/src/sst/version.rs
+++ b/src/mito2/src/sst/version.rs
@@ -84,7 +84,25 @@ impl SstVersion {
         }
     }
 
-    /// Returns SST files'space occupied in current version.
+    /// Returns the number of rows in SST files.
+    /// For historical reasons, the result is not precise for old SST files.
+    pub(crate) fn num_rows(&self) -> u64 {
+        self.levels
+            .iter()
+            .map(|level_meta| {
+                level_meta
+                    .files
+                    .values()
+                    .map(|file_handle| {
+                        let meta = file_handle.meta_ref();
+                        meta.num_rows
+                    })
+                    .sum::<u64>()
+            })
+            .sum()
+    }
+
+    /// Returns SST data files'space occupied in current version.
     pub(crate) fn sst_usage(&self) -> u64 {
         self.levels
             .iter()
@@ -94,7 +112,24 @@ impl SstVersion {
                     .values()
                     .map(|file_handle| {
                         let meta = file_handle.meta_ref();
-                        meta.file_size + meta.index_file_size
+                        meta.file_size
+                    })
+                    .sum::<u64>()
+            })
+            .sum()
+    }
+
+    /// Returns SST index files'space occupied in current version.
+    pub(crate) fn index_usage(&self) -> u64 {
+        self.levels
+            .iter()
+            .map(|level_meta| {
+                level_meta
+                    .files
+                    .values()
+                    .map(|file_handle| {
+                        let meta = file_handle.meta_ref();
+                        meta.index_file_size
                     })
                     .sum::<u64>()
             })
diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs
index 785e66d37f..0832385c93 100644
--- a/src/store-api/src/region_engine.rs
+++ b/src/store-api/src/region_engine.rs
@@ -291,14 +291,20 @@ pub type BatchResponses = Vec<(RegionId, Result<RegionResponse, BoxedError>)>;
 /// Represents the statistics of a region.
 #[derive(Debug, Deserialize, Serialize, Default)]
 pub struct RegionStatistic {
+    /// The number of rows
+    #[serde(default)]
+    pub num_rows: u64,
     /// The size of memtable in bytes.
     pub memtable_size: u64,
     /// The size of WAL in bytes.
     pub wal_size: u64,
     /// The size of manifest in bytes.
     pub manifest_size: u64,
-    /// The size of SST files in bytes.
+    /// The size of SST data files in bytes.
     pub sst_size: u64,
+    /// The size of SST index files in bytes.
+    #[serde(default)]
+    pub index_size: u64,
 }
 
 impl RegionStatistic {
@@ -320,7 +326,7 @@ impl RegionStatistic {
 impl RegionStatistic {
     /// Returns the estimated disk size of the region.
     pub fn estimated_disk_size(&self) -> u64 {
-        self.wal_size + self.sst_size + self.manifest_size
+        self.wal_size + self.sst_size + self.manifest_size + self.index_size
     }
 }
 
diff --git a/tests/cases/standalone/common/information_schema/region_statistics.result b/tests/cases/standalone/common/information_schema/region_statistics.result
new file mode 100644
index 0000000000..0c62b4ad6c
--- /dev/null
+++ b/tests/cases/standalone/common/information_schema/region_statistics.result
@@ -0,0 +1,39 @@
+USE public;
+
+Affected Rows: 0
+
+CREATE TABLE test (
+    a int primary key,
+    b string,
+    ts timestamp time index,
+) PARTITION ON COLUMNS (a) (
+    a < 10,
+    a >= 10 AND a < 20,
+    a >= 20,
+);
+
+Affected Rows: 0
+
+INSERT INTO test VALUES
+       (1, 'a', 1),
+       (11, 'b', 11),
+       (21, 'c', 21);
+
+Affected Rows: 3
+
+-- SQLNESS SLEEP 11s
+-- FIXME(dennis): we need to wait the datanode reporting stats info to metasrv.
+SELECT SUM(region_rows), SUM(disk_size), SUM(sst_size), SUM(index_size)
+       FROM INFORMATION_SCHEMA.REGION_STATISTICS WHERE table_id
+       IN (SELECT TABLE_ID FROM INFORMATION_SCHEMA.TABLES WHERE table_name = 'test' and table_schema = 'public');
+
++-------------------------------------------------------+-----------------------------------------------------+----------------------------------------------------+------------------------------------------------------+
+| SUM(information_schema.region_statistics.region_rows) | SUM(information_schema.region_statistics.disk_size) | SUM(information_schema.region_statistics.sst_size) | SUM(information_schema.region_statistics.index_size) |
++-------------------------------------------------------+-----------------------------------------------------+----------------------------------------------------+------------------------------------------------------+
+| 3                                                     | 2145                                                | 0                                                  | 0                                                    |
++-------------------------------------------------------+-----------------------------------------------------+----------------------------------------------------+------------------------------------------------------+
+
+DROP TABLE test;
+
+Affected Rows: 0
+
diff --git a/tests/cases/standalone/common/information_schema/region_statistics.sql b/tests/cases/standalone/common/information_schema/region_statistics.sql
new file mode 100644
index 0000000000..cbc4424683
--- /dev/null
+++ b/tests/cases/standalone/common/information_schema/region_statistics.sql
@@ -0,0 +1,25 @@
+USE public;
+
+CREATE TABLE test (
+    a int primary key,
+    b string,
+    ts timestamp time index,
+) PARTITION ON COLUMNS (a) (
+    a < 10,
+    a >= 10 AND a < 20,
+    a >= 20,
+);
+
+
+INSERT INTO test VALUES
+       (1, 'a', 1),
+       (11, 'b', 11),
+       (21, 'c', 21);
+
+-- SQLNESS SLEEP 11s
+-- FIXME(dennis): we need to wait the datanode reporting stats info to metasrv.
+SELECT SUM(region_rows), SUM(disk_size), SUM(sst_size), SUM(index_size)
+       FROM INFORMATION_SCHEMA.REGION_STATISTICS WHERE table_id
+       IN (SELECT TABLE_ID FROM INFORMATION_SCHEMA.TABLES WHERE table_name = 'test' and table_schema = 'public');
+
+DROP TABLE test;
diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result
index 4264bd8df6..b1c8c93295 100644
--- a/tests/cases/standalone/common/system/information_schema.result
+++ b/tests/cases/standalone/common/system/information_schema.result
@@ -299,13 +299,16 @@ select * from information_schema.columns order by table_schema, table_name, colu
 | greptime      | information_schema | region_peers                          | peer_id                           | 2                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | Yes         | bigint unsigned |                |        |
 | greptime      | information_schema | region_peers                          | region_id                         | 1                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | No          | bigint unsigned |                |        |
 | greptime      | information_schema | region_peers                          | status                            | 5                | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | Yes         | string          |                |        |
-| greptime      | information_schema | region_statistics                     | engine                            | 7                | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | Yes         | string          |                |        |
-| greptime      | information_schema | region_statistics                     | manifest_size                     | 5                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | Yes         | bigint unsigned |                |        |
-| greptime      | information_schema | region_statistics                     | memtable_size                     | 4                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | Yes         | bigint unsigned |                |        |
+| greptime      | information_schema | region_statistics                     | disk_size                         | 5                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | Yes         | bigint unsigned |                |        |
+| greptime      | information_schema | region_statistics                     | engine                            | 10               | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | Yes         | string          |                |        |
+| greptime      | information_schema | region_statistics                     | index_size                        | 9                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | Yes         | bigint unsigned |                |        |
+| greptime      | information_schema | region_statistics                     | manifest_size                     | 7                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | Yes         | bigint unsigned |                |        |
+| greptime      | information_schema | region_statistics                     | memtable_size                     | 6                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | Yes         | bigint unsigned |                |        |
 | greptime      | information_schema | region_statistics                     | region_id                         | 1                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | No          | bigint unsigned |                |        |
 | greptime      | information_schema | region_statistics                     | region_number                     | 3                |                          |                        | 10                | 0             |                    |                    |                |            |       | select,insert |                       | UInt32               | int unsigned    | FIELD         |                | No          | int unsigned    |                |        |
-| greptime      | information_schema | region_statistics                     | region_role                       | 8                | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | Yes         | string          |                |        |
-| greptime      | information_schema | region_statistics                     | sst_size                          | 6                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | Yes         | bigint unsigned |                |        |
+| greptime      | information_schema | region_statistics                     | region_role                       | 11               | 2147483647               | 2147483647             |                   |               |                    | utf8               | utf8_bin       |            |       | select,insert |                       | String               | string          | FIELD         |                | Yes         | string          |                |        |
+| greptime      | information_schema | region_statistics                     | region_rows                       | 4                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | Yes         | bigint unsigned |                |        |
+| greptime      | information_schema | region_statistics                     | sst_size                          | 8                |                          |                        | 20                | 0             |                    |                    |                |            |       | select,insert |                       | UInt64               | bigint unsigned | FIELD         |                | Yes         | bigint unsigned |                |        |
 | greptime      | information_schema | region_statistics                     | table_id                          | 2                |                          |                        | 10                | 0             |                    |                    |                |            |       | select,insert |                       | UInt32               | int unsigned    | FIELD         |                | No          | int unsigned    |                |        |
 | greptime      | information_schema | routines                              | character_maximum_length          | 7                |                          |                        | 19                | 0             |                    |                    |                |            |       | select,insert |                       | Int64                | bigint          | FIELD         |                | No          | bigint          |                |        |
 | greptime      | information_schema | routines                              | character_octet_length            | 8                |                          |                        | 19                | 0             |                    |                    |                |            |       | select,insert |                       | Int64                | bigint          | FIELD         |                | No          | bigint          |                |        |
diff --git a/tests/conf/datanode-test.toml.template b/tests/conf/datanode-test.toml.template
index 5ed5352124..3c999635d9 100644
--- a/tests/conf/datanode-test.toml.template
+++ b/tests/conf/datanode-test.toml.template
@@ -32,3 +32,6 @@ tcp_nodelay = false
 [procedure]
 max_retry_times = 3
 retry_delay = "500ms"
+
+[heartbeat]
+interval = '1s'

From 0da112b33513375dd79ec427689b7dc68edabaea Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Wed, 30 Oct 2024 19:56:10 +0800
Subject: [PATCH 125/128] chore: provide more info in check batch message
 (#4906)

* chore: provide more info in check message

* chore: set timeout to 240s

---------

Co-authored-by: WenyXu <wenymedia@gmail.com>
---
 src/mito2/src/read.rs                         | 115 ++++++++++++------
 .../migration/fuzz_migrate_mito_regions.rs    |   4 +-
 2 files changed, 83 insertions(+), 36 deletions(-)

diff --git a/src/mito2/src/read.rs b/src/mito2/src/read.rs
index e1ce9572c4..d8ac5ce46b 100644
--- a/src/mito2/src/read.rs
+++ b/src/mito2/src/read.rs
@@ -494,36 +494,80 @@ impl Batch {
 
     /// Checks the batch is monotonic by timestamps.
     #[cfg(debug_assertions)]
-    pub(crate) fn check_monotonic(&self) -> bool {
+    pub(crate) fn check_monotonic(&self) -> Result<(), String> {
+        use std::cmp::Ordering;
         if self.timestamps_native().is_none() {
-            return true;
+            return Ok(());
         }
 
         let timestamps = self.timestamps_native().unwrap();
         let sequences = self.sequences.as_arrow().values();
-        timestamps.windows(2).enumerate().all(|(i, window)| {
+        for (i, window) in timestamps.windows(2).enumerate() {
             let current = window[0];
             let next = window[1];
             let current_sequence = sequences[i];
             let next_sequence = sequences[i + 1];
-            if current == next {
-                current_sequence >= next_sequence
-            } else {
-                current < next
+            match current.cmp(&next) {
+                Ordering::Less => {
+                    // The current timestamp is less than the next timestamp.
+                    continue;
+                }
+                Ordering::Equal => {
+                    // The current timestamp is equal to the next timestamp.
+                    if current_sequence < next_sequence {
+                        return Err(format!(
+                            "sequence are not monotonic: ts {} == {} but current sequence {} < {}, index: {}",
+                            current, next, current_sequence, next_sequence, i
+                        ));
+                    }
+                }
+                Ordering::Greater => {
+                    // The current timestamp is greater than the next timestamp.
+                    return Err(format!(
+                        "timestamps are not monotonic: {} > {}, index: {}",
+                        current, next, i
+                    ));
+                }
             }
-        })
+        }
+
+        Ok(())
     }
 
-    /// Returns true if the given batch is behind the current batch.
+    /// Returns Ok if the given batch is behind the current batch.
     #[cfg(debug_assertions)]
-    pub(crate) fn check_next_batch(&self, other: &Batch) -> bool {
-        // Checks the primary key and then the timestamp.
-        use std::cmp::Ordering;
-        self.primary_key()
-            .cmp(other.primary_key())
-            .then_with(|| self.last_timestamp().cmp(&other.first_timestamp()))
-            .then_with(|| other.first_sequence().cmp(&self.last_sequence()))
-            <= Ordering::Equal
+    pub(crate) fn check_next_batch(&self, other: &Batch) -> Result<(), String> {
+        // Checks the primary key
+        if self.primary_key() < other.primary_key() {
+            return Ok(());
+        }
+        if self.primary_key() > other.primary_key() {
+            return Err(format!(
+                "primary key is not monotonic: {:?} > {:?}",
+                self.primary_key(),
+                other.primary_key()
+            ));
+        }
+        // Checks the timestamp.
+        if self.last_timestamp() < other.first_timestamp() {
+            return Ok(());
+        }
+        if self.last_timestamp() > other.first_timestamp() {
+            return Err(format!(
+                "timestamps are not monotonic: {:?} > {:?}",
+                self.last_timestamp(),
+                other.first_timestamp()
+            ));
+        }
+        // Checks the sequence.
+        if self.last_sequence() >= other.first_sequence() {
+            return Ok(());
+        }
+        Err(format!(
+            "sequences are not monotonic: {:?} < {:?}",
+            self.last_sequence(),
+            other.first_sequence()
+        ))
     }
 }
 
@@ -552,31 +596,35 @@ impl BatchChecker {
 
     /// Returns true if the given batch is monotonic and behind
     /// the last batch.
-    pub(crate) fn check_monotonic(&mut self, batch: &Batch) -> bool {
-        if !batch.check_monotonic() {
-            return false;
-        }
+    pub(crate) fn check_monotonic(&mut self, batch: &Batch) -> Result<(), String> {
+        batch.check_monotonic()?;
 
         if let (Some(start), Some(first)) = (self.start, batch.first_timestamp()) {
             if start > first {
-                return false;
+                return Err(format!(
+                    "batch's first timestamp is before the start timestamp: {:?} > {:?}",
+                    start, first
+                ));
             }
         }
         if let (Some(end), Some(last)) = (self.end, batch.last_timestamp()) {
             if end <= last {
-                return false;
+                return Err(format!(
+                    "batch's last timestamp is after the end timestamp: {:?} <= {:?}",
+                    end, last
+                ));
             }
         }
 
         // Checks the batch is behind the last batch.
         // Then Updates the last batch.
-        let is_behind = self
+        let res = self
             .last_batch
             .as_ref()
             .map(|last| last.check_next_batch(batch))
-            .unwrap_or(true);
+            .unwrap_or(Ok(()));
         self.last_batch = Some(batch.clone());
-        is_behind
+        res
     }
 
     /// Formats current batch and last batch for debug.
@@ -615,15 +663,14 @@ impl BatchChecker {
         part_range: store_api::region_engine::PartitionRange,
         batch: &Batch,
     ) {
-        if !self.check_monotonic(batch) {
-            panic!(
-                "{}: batch is not sorted, region_id: {}, partition: {}, part_range: {:?}, {}",
-                scanner,
-                region_id,
-                partition,
-                part_range,
-                self.format_batch(batch),
+        if let Err(e) = self.check_monotonic(batch) {
+            let err_msg = format!(
+                "{}: batch is not sorted, {}, region_id: {}, partition: {}, part_range: {:?}",
+                scanner, e, region_id, partition, part_range,
             );
+            common_telemetry::error!("{err_msg}, {}", self.format_batch(batch));
+            // Only print the number of row in the panic message.
+            panic!("{err_msg}, batch rows: {}", batch.num_rows());
         }
     }
 }
diff --git a/tests-fuzz/targets/migration/fuzz_migrate_mito_regions.rs b/tests-fuzz/targets/migration/fuzz_migrate_mito_regions.rs
index 271f04c143..d8fc102da4 100644
--- a/tests-fuzz/targets/migration/fuzz_migrate_mito_regions.rs
+++ b/tests-fuzz/targets/migration/fuzz_migrate_mito_regions.rs
@@ -218,7 +218,7 @@ async fn execute_region_migration(ctx: FuzzContext, input: FuzzInput) -> Result<
     } in &migrations
     {
         let procedure_id =
-            migrate_region(&ctx.greptime, region_id.as_u64(), *from_peer, *to_peer, 120).await;
+            migrate_region(&ctx.greptime, region_id.as_u64(), *from_peer, *to_peer, 240).await;
         info!("Migrating region: {region_id} from {from_peer} to {to_peer}, procedure: {procedure_id}");
         procedure_ids.push(procedure_id);
     }
@@ -228,7 +228,7 @@ async fn execute_region_migration(ctx: FuzzContext, input: FuzzInput) -> Result<
         info!("Waits for migration: {migration:?}");
         let region_id = migration.region_id.as_u64();
         wait_condition_fn(
-            Duration::from_secs(120),
+            Duration::from_secs(240),
             || {
                 let greptime = ctx.greptime.clone();
                 let procedure_id = procedure_id.to_string();

From 69420793e2c74c91a1395b33400392f27f0e2780 Mon Sep 17 00:00:00 2001
From: Ning Sun <sunng@protonmail.com>
Date: Wed, 30 Oct 2024 20:16:22 +0800
Subject: [PATCH 126/128] feat: implement parse_query api (#4860)

* feat: implement parse_query api

* chore: switch to upstream

* fix: add post method for parse_query

* chore: bump promql-parser

* test: use latest promql ast serialization
---
 Cargo.lock                         |  6 ++++--
 Cargo.toml                         |  2 +-
 src/servers/src/http.rs            |  5 +++--
 src/servers/src/http/prometheus.rs | 33 ++++++++++++++++++++++++++++++
 tests-integration/tests/http.rs    | 23 +++++++++++++++++++++
 5 files changed, 64 insertions(+), 5 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 26554d7f81..53ab681cd6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8627,9 +8627,9 @@ dependencies = [
 
 [[package]]
 name = "promql-parser"
-version = "0.4.1"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c1ad4a4cfa84ec4aa5831c82e57af0a3faf3f0af83bee13fa1390b2d0a32dc9"
+checksum = "7fe99e6f80a79abccf1e8fb48dd63473a36057e600cc6ea36147c8318698ae6f"
 dependencies = [
  "cfgrammar",
  "chrono",
@@ -8637,6 +8637,8 @@ dependencies = [
  "lrlex",
  "lrpar",
  "regex",
+ "serde",
+ "serde_json",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 2e7f70d2ab..e45236758f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -145,7 +145,7 @@ parquet = { version = "51.0.0", default-features = false, features = ["arrow", "
 paste = "1.0"
 pin-project = "1.0"
 prometheus = { version = "0.13.3", features = ["process"] }
-promql-parser = { version = "0.4.1" }
+promql-parser = { version = "0.4.3", features = ["ser"] }
 prost = "0.12"
 raft-engine = { version = "0.4.1", default-features = false }
 rand = "0.8"
diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs
index de4f2cf5ff..12ac06db90 100644
--- a/src/servers/src/http.rs
+++ b/src/servers/src/http.rs
@@ -62,8 +62,8 @@ use crate::http::influxdb::{influxdb_health, influxdb_ping, influxdb_write_v1, i
 use crate::http::influxdb_result_v1::InfluxdbV1Response;
 use crate::http::json_result::JsonResponse;
 use crate::http::prometheus::{
-    build_info_query, format_query, instant_query, label_values_query, labels_query, range_query,
-    series_query,
+    build_info_query, format_query, instant_query, label_values_query, labels_query, parse_query,
+    range_query, series_query,
 };
 use crate::interceptor::LogIngestInterceptorRef;
 use crate::metrics::http_metrics_layer;
@@ -819,6 +819,7 @@ impl HttpServer {
             .route("/query_range", routing::post(range_query).get(range_query))
             .route("/labels", routing::post(labels_query).get(labels_query))
             .route("/series", routing::post(series_query).get(series_query))
+            .route("/parse_query", routing::post(parse_query).get(parse_query))
             .route(
                 "/label/:label_name/values",
                 routing::get(label_values_query),
diff --git a/src/servers/src/http/prometheus.rs b/src/servers/src/http/prometheus.rs
index 941cac2539..6f749f2595 100644
--- a/src/servers/src/http/prometheus.rs
+++ b/src/servers/src/http/prometheus.rs
@@ -101,6 +101,9 @@ pub enum PrometheusResponse {
     LabelValues(Vec<String>),
     FormatQuery(String),
     BuildInfo(OwnedBuildInfo),
+    #[schemars(skip)]
+    #[serde(skip_deserializing)]
+    ParseResult(promql_parser::parser::Expr),
 }
 
 impl Default for PrometheusResponse {
@@ -1014,3 +1017,33 @@ pub async fn series_query(
     resp.resp_metrics = merge_map;
     resp
 }
+
+#[derive(Debug, Default, Serialize, Deserialize, JsonSchema)]
+pub struct ParseQuery {
+    query: Option<String>,
+    db: Option<String>,
+}
+
+#[axum_macros::debug_handler]
+#[tracing::instrument(
+    skip_all,
+    fields(protocol = "prometheus", request_type = "parse_query")
+)]
+pub async fn parse_query(
+    State(_handler): State<PrometheusHandlerRef>,
+    Query(params): Query<ParseQuery>,
+    Extension(_query_ctx): Extension<QueryContext>,
+    Form(form_params): Form<ParseQuery>,
+) -> PrometheusJsonResponse {
+    if let Some(query) = params.query.or(form_params.query) {
+        match promql_parser::parser::parse(&query) {
+            Ok(ast) => PrometheusJsonResponse::success(PrometheusResponse::ParseResult(ast)),
+            Err(err) => {
+                let msg = err.to_string();
+                PrometheusJsonResponse::error(StatusCode::InvalidArguments, msg)
+            }
+        }
+    } else {
+        PrometheusJsonResponse::error(StatusCode::InvalidArguments, "query is required")
+    }
+}
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index 823de40d11..638734faba 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -660,6 +660,29 @@ pub async fn test_prom_http_api(store_type: StorageType) {
     let body = serde_json::from_str::<PrometheusJsonResponse>(&res.text().await).unwrap();
     assert_eq!(body.status, "success");
 
+    // parse_query
+    let res = client
+        .get("/v1/prometheus/api/v1/parse_query?query=http_requests")
+        .send()
+        .await;
+    assert_eq!(res.status(), StatusCode::OK);
+    let data = res.text().await;
+    // we don't have deserialization for ast so we keep test simple and compare
+    //  the json output directly.
+    // the correctness should be covered by parser. In this test we only check
+    //  response format.
+    let expected = "{\"status\":\"success\",\"data\":{\"type\":\"vectorSelector\",\"name\":\"http_requests\",\"matchers\":[],\"offset\":0,\"startOrEnd\":null,\"timestamp\":null}}";
+    assert_eq!(expected, data);
+
+    let res = client
+        .get("/v1/prometheus/api/v1/parse_query?query=not http_requests")
+        .send()
+        .await;
+    assert_eq!(res.status(), StatusCode::BAD_REQUEST);
+    let data = res.text().await;
+    let expected = "{\"status\":\"error\",\"data\":{\"resultType\":\"\",\"result\":[]},\"error\":\"invalid promql query\",\"errorType\":\"InvalidArguments\"}";
+    assert_eq!(expected, data);
+
     guard.remove_all().await;
 }
 

From ea6df9ba49104ff1a31a52573df52fbe65bfcfee Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Thu, 31 Oct 2024 13:13:35 +0800
Subject: [PATCH 127/128] fix: prune batches from memtable by time range
 (#4913)

* feat: add an iter to prune by time range

* feat: filter rows from mem range
---
 src/mito2/src/engine/prune_test.rs       |  53 ++++++
 src/mito2/src/memtable.rs                |   8 +-
 src/mito2/src/read/prune.rs              | 217 +++++++++++++++++++++++
 src/mito2/src/read/scan_util.rs          |   4 +-
 src/mito2/src/read/seq_scan.rs           |   7 +-
 src/mito2/src/read/unordered_scan.rs     |   2 +-
 src/mito2/src/test_util/memtable_util.rs |  10 --
 7 files changed, 286 insertions(+), 15 deletions(-)

diff --git a/src/mito2/src/engine/prune_test.rs b/src/mito2/src/engine/prune_test.rs
index 687172417d..b7f9056e58 100644
--- a/src/mito2/src/engine/prune_test.rs
+++ b/src/mito2/src/engine/prune_test.rs
@@ -259,3 +259,56 @@ async fn test_prune_memtable_complex_expr() {
 +-------+---------+---------------------+";
     assert_eq!(expected, batches.pretty_print().unwrap());
 }
+
+#[tokio::test]
+async fn test_mem_range_prune() {
+    let mut env = TestEnv::new();
+    let engine = env.create_engine(MitoConfig::default()).await;
+
+    let region_id = RegionId::new(1, 1);
+    let request = CreateRequestBuilder::new().build();
+
+    let column_schemas = rows_schema(&request);
+
+    engine
+        .handle_request(region_id, RegionRequest::Create(request))
+        .await
+        .unwrap();
+
+    put_rows(
+        &engine,
+        region_id,
+        Rows {
+            schema: column_schemas.clone(),
+            rows: build_rows(5, 8),
+        },
+    )
+    .await;
+
+    // Starts scan and gets the memtable time range.
+    let stream = engine
+        .scan_to_stream(region_id, ScanRequest::default())
+        .await
+        .unwrap();
+
+    put_rows(
+        &engine,
+        region_id,
+        Rows {
+            schema: column_schemas.clone(),
+            rows: build_rows(10, 12),
+        },
+    )
+    .await;
+
+    let batches = RecordBatches::try_collect(stream).await.unwrap();
+    let expected = "\
++-------+---------+---------------------+
+| tag_0 | field_0 | ts                  |
++-------+---------+---------------------+
+| 5     | 5.0     | 1970-01-01T00:00:05 |
+| 6     | 6.0     | 1970-01-01T00:00:06 |
+| 7     | 7.0     | 1970-01-01T00:00:07 |
++-------+---------+---------------------+";
+    assert_eq!(expected, batches.pretty_print().unwrap());
+}
diff --git a/src/mito2/src/memtable.rs b/src/mito2/src/memtable.rs
index 668eb51023..a6963ece42 100644
--- a/src/mito2/src/memtable.rs
+++ b/src/mito2/src/memtable.rs
@@ -34,8 +34,10 @@ pub use crate::memtable::key_values::KeyValues;
 use crate::memtable::partition_tree::{PartitionTreeConfig, PartitionTreeMemtableBuilder};
 use crate::memtable::time_series::TimeSeriesMemtableBuilder;
 use crate::metrics::WRITE_BUFFER_BYTES;
+use crate::read::prune::PruneTimeIterator;
 use crate::read::Batch;
 use crate::region::options::{MemtableOptions, MergeMode};
+use crate::sst::file::FileTimeRange;
 
 pub mod bulk;
 pub mod key_values;
@@ -355,8 +357,10 @@ impl MemtableRange {
     }
 
     /// Builds an iterator to read the range.
-    pub fn build_iter(&self) -> Result<BoxedBatchIterator> {
-        self.context.builder.build()
+    /// Filters the result by the specific time range.
+    pub fn build_iter(&self, time_range: FileTimeRange) -> Result<BoxedBatchIterator> {
+        let iter = self.context.builder.build()?;
+        Ok(Box::new(PruneTimeIterator::new(iter, time_range)))
     }
 }
 
diff --git a/src/mito2/src/read/prune.rs b/src/mito2/src/read/prune.rs
index 0ab2f9e2b5..cb0066e734 100644
--- a/src/mito2/src/read/prune.rs
+++ b/src/mito2/src/read/prune.rs
@@ -12,9 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use common_time::Timestamp;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::vectors::BooleanVectorBuilder;
+
 use crate::error::Result;
+use crate::memtable::BoxedBatchIterator;
 use crate::read::last_row::RowGroupLastRowCachedReader;
 use crate::read::{Batch, BatchReader};
+use crate::sst::file::FileTimeRange;
 use crate::sst::parquet::file_range::FileRangeContextRef;
 use crate::sst::parquet::reader::{ReaderMetrics, RowGroupReader};
 
@@ -112,3 +118,214 @@ impl PruneReader {
         }
     }
 }
+
+/// An iterator that prunes batches by time range.
+pub(crate) struct PruneTimeIterator {
+    iter: BoxedBatchIterator,
+    time_range: FileTimeRange,
+}
+
+impl PruneTimeIterator {
+    /// Creates a new `PruneTimeIterator` with the given iterator and time range.
+    pub(crate) fn new(iter: BoxedBatchIterator, time_range: FileTimeRange) -> Self {
+        Self { iter, time_range }
+    }
+
+    /// Prune batch by time range.
+    fn prune(&self, mut batch: Batch) -> Result<Batch> {
+        if batch.is_empty() {
+            return Ok(batch);
+        }
+
+        // fast path, the batch is within the time range.
+        // Note that the time range is inclusive.
+        if self.time_range.0 <= batch.first_timestamp().unwrap()
+            && batch.last_timestamp().unwrap() <= self.time_range.1
+        {
+            return Ok(batch);
+        }
+
+        // slow path, prune the batch by time range.
+        // Note that the timestamp precision may be different from the time range.
+        // Safety: We know this is the timestamp type.
+        let unit = batch
+            .timestamps()
+            .data_type()
+            .as_timestamp()
+            .unwrap()
+            .unit();
+        let mut filter_builder = BooleanVectorBuilder::with_capacity(batch.timestamps().len());
+        let timestamps = batch.timestamps_native().unwrap();
+        for ts in timestamps {
+            let ts = Timestamp::new(*ts, unit);
+            if self.time_range.0 <= ts && ts <= self.time_range.1 {
+                filter_builder.push(Some(true));
+            } else {
+                filter_builder.push(Some(false));
+            }
+        }
+        let filter = filter_builder.finish();
+
+        batch.filter(&filter)?;
+        Ok(batch)
+    }
+
+    // Prune and return the next non-empty batch.
+    fn next_non_empty_batch(&mut self) -> Result<Option<Batch>> {
+        while let Some(batch) = self.iter.next() {
+            let batch = batch?;
+            let pruned_batch = self.prune(batch)?;
+            if !pruned_batch.is_empty() {
+                return Ok(Some(pruned_batch));
+            }
+        }
+        Ok(None)
+    }
+}
+
+impl Iterator for PruneTimeIterator {
+    type Item = Result<Batch>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.next_non_empty_batch().transpose()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use api::v1::OpType;
+
+    use super::*;
+    use crate::test_util::new_batch;
+
+    #[test]
+    fn test_prune_time_iter_empty() {
+        let input = [];
+        let iter = input.into_iter().map(Ok);
+        let iter = PruneTimeIterator::new(
+            Box::new(iter),
+            (
+                Timestamp::new_millisecond(0),
+                Timestamp::new_millisecond(1000),
+            ),
+        );
+        let actual: Vec<_> = iter.map(|batch| batch.unwrap()).collect();
+        assert!(actual.is_empty());
+    }
+
+    #[test]
+    fn test_prune_time_iter_filter() {
+        let input = [
+            new_batch(
+                b"k1",
+                &[10, 11],
+                &[20, 20],
+                &[OpType::Put, OpType::Put],
+                &[110, 111],
+            ),
+            new_batch(
+                b"k1",
+                &[15, 16],
+                &[20, 20],
+                &[OpType::Put, OpType::Put],
+                &[115, 116],
+            ),
+            new_batch(
+                b"k1",
+                &[17, 18],
+                &[20, 20],
+                &[OpType::Put, OpType::Put],
+                &[117, 118],
+            ),
+        ];
+
+        let iter = input.clone().into_iter().map(Ok);
+        let iter = PruneTimeIterator::new(
+            Box::new(iter),
+            (
+                Timestamp::new_millisecond(10),
+                Timestamp::new_millisecond(15),
+            ),
+        );
+        let actual: Vec<_> = iter.map(|batch| batch.unwrap()).collect();
+        assert_eq!(
+            actual,
+            [
+                new_batch(
+                    b"k1",
+                    &[10, 11],
+                    &[20, 20],
+                    &[OpType::Put, OpType::Put],
+                    &[110, 111],
+                ),
+                new_batch(b"k1", &[15], &[20], &[OpType::Put], &[115],),
+            ]
+        );
+
+        let iter = input.clone().into_iter().map(Ok);
+        let iter = PruneTimeIterator::new(
+            Box::new(iter),
+            (
+                Timestamp::new_millisecond(11),
+                Timestamp::new_millisecond(20),
+            ),
+        );
+        let actual: Vec<_> = iter.map(|batch| batch.unwrap()).collect();
+        assert_eq!(
+            actual,
+            [
+                new_batch(b"k1", &[11], &[20], &[OpType::Put], &[111],),
+                new_batch(
+                    b"k1",
+                    &[15, 16],
+                    &[20, 20],
+                    &[OpType::Put, OpType::Put],
+                    &[115, 116],
+                ),
+                new_batch(
+                    b"k1",
+                    &[17, 18],
+                    &[20, 20],
+                    &[OpType::Put, OpType::Put],
+                    &[117, 118],
+                ),
+            ]
+        );
+
+        let iter = input.into_iter().map(Ok);
+        let iter = PruneTimeIterator::new(
+            Box::new(iter),
+            (
+                Timestamp::new_millisecond(10),
+                Timestamp::new_millisecond(18),
+            ),
+        );
+        let actual: Vec<_> = iter.map(|batch| batch.unwrap()).collect();
+        assert_eq!(
+            actual,
+            [
+                new_batch(
+                    b"k1",
+                    &[10, 11],
+                    &[20, 20],
+                    &[OpType::Put, OpType::Put],
+                    &[110, 111],
+                ),
+                new_batch(
+                    b"k1",
+                    &[15, 16],
+                    &[20, 20],
+                    &[OpType::Put, OpType::Put],
+                    &[115, 116],
+                ),
+                new_batch(
+                    b"k1",
+                    &[17, 18],
+                    &[20, 20],
+                    &[OpType::Put, OpType::Put],
+                    &[117, 118],
+                ),
+            ]
+        );
+    }
+}
diff --git a/src/mito2/src/read/scan_util.rs b/src/mito2/src/read/scan_util.rs
index 74e7af42d6..d27468521a 100644
--- a/src/mito2/src/read/scan_util.rs
+++ b/src/mito2/src/read/scan_util.rs
@@ -26,6 +26,7 @@ use crate::error::Result;
 use crate::read::range::RowGroupIndex;
 use crate::read::scan_region::StreamContext;
 use crate::read::{Batch, ScannerMetrics, Source};
+use crate::sst::file::FileTimeRange;
 use crate::sst::parquet::reader::ReaderMetrics;
 
 struct PartitionMetricsInner {
@@ -128,13 +129,14 @@ pub(crate) fn scan_mem_ranges(
     stream_ctx: Arc<StreamContext>,
     part_metrics: PartitionMetrics,
     index: RowGroupIndex,
+    time_range: FileTimeRange,
 ) -> impl Stream<Item = Result<Batch>> {
     try_stream! {
         let ranges = stream_ctx.build_mem_ranges(index);
         part_metrics.inc_num_mem_ranges(ranges.len());
         for range in ranges {
             let build_reader_start = Instant::now();
-            let iter = range.build_iter()?;
+            let iter = range.build_iter(time_range)?;
             part_metrics.inc_build_reader_cost(build_reader_start.elapsed());
 
             let mut source = Source::Iter(iter);
diff --git a/src/mito2/src/read/seq_scan.rs b/src/mito2/src/read/seq_scan.rs
index 5049ea8a47..1a789c8d5f 100644
--- a/src/mito2/src/read/seq_scan.rs
+++ b/src/mito2/src/read/seq_scan.rs
@@ -355,7 +355,12 @@ fn build_sources(
     sources.reserve(range_meta.row_group_indices.len());
     for index in &range_meta.row_group_indices {
         let stream = if stream_ctx.is_mem_range_index(*index) {
-            let stream = scan_mem_ranges(stream_ctx.clone(), part_metrics.clone(), *index);
+            let stream = scan_mem_ranges(
+                stream_ctx.clone(),
+                part_metrics.clone(),
+                *index,
+                range_meta.time_range,
+            );
             Box::pin(stream) as _
         } else {
             let read_type = if compaction {
diff --git a/src/mito2/src/read/unordered_scan.rs b/src/mito2/src/read/unordered_scan.rs
index 72bf424020..2401504f79 100644
--- a/src/mito2/src/read/unordered_scan.rs
+++ b/src/mito2/src/read/unordered_scan.rs
@@ -89,7 +89,7 @@ impl UnorderedScan {
             let range_meta = &stream_ctx.ranges[part_range_id];
             for index in &range_meta.row_group_indices {
                 if stream_ctx.is_mem_range_index(*index) {
-                    let stream = scan_mem_ranges(stream_ctx.clone(), part_metrics.clone(), *index);
+                    let stream = scan_mem_ranges(stream_ctx.clone(), part_metrics.clone(), *index, range_meta.time_range);
                     for await batch in stream {
                         yield batch;
                     }
diff --git a/src/mito2/src/test_util/memtable_util.rs b/src/mito2/src/test_util/memtable_util.rs
index ed32a3661e..55b42d61dd 100644
--- a/src/mito2/src/test_util/memtable_util.rs
+++ b/src/mito2/src/test_util/memtable_util.rs
@@ -124,16 +124,6 @@ impl MemtableBuilder for EmptyMemtableBuilder {
     }
 }
 
-/// Empty iterator builder.
-#[derive(Default)]
-pub(crate) struct EmptyIterBuilder {}
-
-impl IterBuilder for EmptyIterBuilder {
-    fn build(&self) -> Result<BoxedBatchIterator> {
-        Ok(Box::new(std::iter::empty()))
-    }
-}
-
 /// Creates a region metadata to test memtable with default pk.
 ///
 /// The schema is `k0, k1, ts, v0, v1` and pk is `k0, k1`.

From 8b60c27c2e96008e78c30234d764cd7fcb609aaf Mon Sep 17 00:00:00 2001
From: Ruihang Xia <waynestxia@gmail.com>
Date: Thu, 31 Oct 2024 14:15:45 +0800
Subject: [PATCH 128/128] feat: enhance windowed-sort optimizer rule (#4910)

* add RegionScanner::metadata

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* skip PartSort when there is no tag column

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* add more sqlness test

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* handle desc

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix: should keep part sort on DESC

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix clippy

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
---
 src/file-engine/src/engine.rs                 |   3 +-
 src/mito2/src/read/seq_scan.rs                |   5 +
 src/mito2/src/read/unordered_scan.rs          |   5 +
 src/query/src/optimizer/parallelize_scan.rs   |  29 +++-
 src/query/src/optimizer/windowed_sort.rs      |  29 +++-
 src/query/src/part_sort.rs                    |   1 +
 src/query/src/window_sort.rs                  |   1 +
 src/store-api/src/region_engine.rs            |  17 +-
 src/table/src/table/scan.rs                   |  59 +++++--
 .../common/order/windowed_sort.result         | 150 +++++++++++++++++-
 .../standalone/common/order/windowed_sort.sql |  47 ++++++
 11 files changed, 320 insertions(+), 26 deletions(-)

diff --git a/src/file-engine/src/engine.rs b/src/file-engine/src/engine.rs
index e6313f4322..a29a3add23 100644
--- a/src/file-engine/src/engine.rs
+++ b/src/file-engine/src/engine.rs
@@ -91,8 +91,9 @@ impl RegionEngine for FileRegionEngine {
         request: ScanRequest,
     ) -> Result<RegionScannerRef, BoxedError> {
         let stream = self.handle_query(region_id, request).await?;
+        let metadata = self.get_metadata(region_id).await?;
         // We don't support enabling append mode for file engine.
-        let scanner = Box::new(SinglePartitionScanner::new(stream, false));
+        let scanner = Box::new(SinglePartitionScanner::new(stream, false, metadata));
         Ok(scanner)
     }
 
diff --git a/src/mito2/src/read/seq_scan.rs b/src/mito2/src/read/seq_scan.rs
index 1a789c8d5f..9b7a71a36c 100644
--- a/src/mito2/src/read/seq_scan.rs
+++ b/src/mito2/src/read/seq_scan.rs
@@ -27,6 +27,7 @@ use common_telemetry::tracing;
 use datafusion::physical_plan::{DisplayAs, DisplayFormatType};
 use datatypes::schema::SchemaRef;
 use snafu::ResultExt;
+use store_api::metadata::RegionMetadataRef;
 use store_api::region_engine::{PartitionRange, RegionScanner, ScannerProperties};
 use store_api::storage::TimeSeriesRowSelector;
 use tokio::sync::Semaphore;
@@ -321,6 +322,10 @@ impl RegionScanner for SeqScan {
         let predicate = self.stream_ctx.input.predicate();
         predicate.map(|p| !p.exprs().is_empty()).unwrap_or(false)
     }
+
+    fn metadata(&self) -> RegionMetadataRef {
+        self.stream_ctx.input.mapper.metadata().clone()
+    }
 }
 
 impl DisplayAs for SeqScan {
diff --git a/src/mito2/src/read/unordered_scan.rs b/src/mito2/src/read/unordered_scan.rs
index 2401504f79..707b7d4ba6 100644
--- a/src/mito2/src/read/unordered_scan.rs
+++ b/src/mito2/src/read/unordered_scan.rs
@@ -26,6 +26,7 @@ use datafusion::physical_plan::{DisplayAs, DisplayFormatType};
 use datatypes::schema::SchemaRef;
 use futures::{Stream, StreamExt};
 use snafu::ResultExt;
+use store_api::metadata::RegionMetadataRef;
 use store_api::region_engine::{PartitionRange, RegionScanner, ScannerProperties};
 
 use crate::error::{PartitionOutOfRangeSnafu, Result};
@@ -229,6 +230,10 @@ impl RegionScanner for UnorderedScan {
         let predicate = self.stream_ctx.input.predicate();
         predicate.map(|p| !p.exprs().is_empty()).unwrap_or(false)
     }
+
+    fn metadata(&self) -> RegionMetadataRef {
+        self.stream_ctx.input.mapper.metadata().clone()
+    }
 }
 
 impl DisplayAs for UnorderedScan {
diff --git a/src/query/src/optimizer/parallelize_scan.rs b/src/query/src/optimizer/parallelize_scan.rs
index 19f5db39d3..02cd04df87 100644
--- a/src/query/src/optimizer/parallelize_scan.rs
+++ b/src/query/src/optimizer/parallelize_scan.rs
@@ -17,6 +17,7 @@ use std::sync::Arc;
 use common_telemetry::debug;
 use datafusion::config::ConfigOptions;
 use datafusion::physical_optimizer::PhysicalOptimizerRule;
+use datafusion::physical_plan::sorts::sort::SortExec;
 use datafusion::physical_plan::ExecutionPlan;
 use datafusion_common::tree_node::{Transformed, TreeNode};
 use datafusion_common::{DataFusionError, Result};
@@ -48,9 +49,16 @@ impl ParallelizeScan {
         plan: Arc<dyn ExecutionPlan>,
         config: &ConfigOptions,
     ) -> Result<Arc<dyn ExecutionPlan>> {
+        let mut first_order_expr = None;
+
         let result = plan
             .transform_down(|plan| {
-                if let Some(region_scan_exec) = plan.as_any().downcast_ref::<RegionScanExec>() {
+                if let Some(sort_exec) = plan.as_any().downcast_ref::<SortExec>() {
+                    // save the first order expr
+                    first_order_expr = sort_exec.expr().first().cloned();
+                } else if let Some(region_scan_exec) =
+                    plan.as_any().downcast_ref::<RegionScanExec>()
+                {
                     if region_scan_exec.is_partition_set() {
                         return Ok(Transformed::no(plan));
                     }
@@ -66,10 +74,21 @@ impl ParallelizeScan {
                         "Assign {total_range_num} ranges to {expected_partition_num} partitions"
                     );
 
-                    // sort the ranges in each partition
-                    // TODO(ruihang): smart sort!
-                    for ranges in partition_ranges.iter_mut() {
-                        ranges.sort_by(|a, b| a.start.cmp(&b.start));
+                    // Sort the ranges in each partition based on the order expr
+                    //
+                    // This optimistically assumes that the first order expr is on the time index column
+                    // to skip the validation of the order expr. As it's not harmful if this condition
+                    // is not met.
+                    if let Some(order_expr) = &first_order_expr
+                        && order_expr.options.descending
+                    {
+                        for ranges in partition_ranges.iter_mut() {
+                            ranges.sort_by(|a, b| b.end.cmp(&a.end));
+                        }
+                    } else {
+                        for ranges in partition_ranges.iter_mut() {
+                            ranges.sort_by(|a, b| a.start.cmp(&b.start));
+                        }
                     }
 
                     // update the partition ranges
diff --git a/src/query/src/optimizer/windowed_sort.rs b/src/query/src/optimizer/windowed_sort.rs
index 62d4495cf3..63150fc1f8 100644
--- a/src/query/src/optimizer/windowed_sort.rs
+++ b/src/query/src/optimizer/windowed_sort.rs
@@ -77,7 +77,6 @@ impl WindowedSortPhysicalRule {
                     };
 
                     if let Some(first_sort_expr) = sort_exec.expr().first()
-                        && !first_sort_expr.options.descending
                         && let Some(column_expr) = first_sort_expr
                             .expr
                             .as_any()
@@ -87,18 +86,28 @@ impl WindowedSortPhysicalRule {
                     } else {
                         return Ok(Transformed::no(plan));
                     }
-
                     let first_sort_expr = sort_exec.expr().first().unwrap().clone();
-                    let part_sort_exec = Arc::new(PartSortExec::new(
-                        first_sort_expr.clone(),
-                        scanner_info.partition_ranges.clone(),
-                        sort_exec.input().clone(),
-                    ));
+
+                    // PartSortExec is unnecessary if:
+                    // - there is no tag column, and
+                    // - the sort is ascending on the time index column
+                    let new_input = if scanner_info.tag_columns.is_empty()
+                        && !first_sort_expr.options.descending
+                    {
+                        sort_exec.input().clone()
+                    } else {
+                        Arc::new(PartSortExec::new(
+                            first_sort_expr.clone(),
+                            scanner_info.partition_ranges.clone(),
+                            sort_exec.input().clone(),
+                        ))
+                    };
+
                     let windowed_sort_exec = WindowedSortExec::try_new(
                         first_sort_expr,
                         sort_exec.fetch(),
                         scanner_info.partition_ranges,
-                        part_sort_exec,
+                        new_input,
                     )?;
 
                     return Ok(Transformed {
@@ -119,11 +128,13 @@ impl WindowedSortPhysicalRule {
 struct ScannerInfo {
     partition_ranges: Vec<Vec<PartitionRange>>,
     time_index: String,
+    tag_columns: Vec<String>,
 }
 
 fn fetch_partition_range(input: Arc<dyn ExecutionPlan>) -> DataFusionResult<Option<ScannerInfo>> {
     let mut partition_ranges = None;
     let mut time_index = None;
+    let mut tag_columns = None;
 
     input.transform_up(|plan| {
         // Unappliable case, reset the state.
@@ -139,6 +150,7 @@ fn fetch_partition_range(input: Arc<dyn ExecutionPlan>) -> DataFusionResult<Opti
         if let Some(region_scan_exec) = plan.as_any().downcast_ref::<RegionScanExec>() {
             partition_ranges = Some(region_scan_exec.get_uncollapsed_partition_ranges());
             time_index = region_scan_exec.time_index();
+            tag_columns = Some(region_scan_exec.tag_columns());
 
             // set distinguish_partition_ranges to true, this is an incorrect workaround
             region_scan_exec.with_distinguish_partition_range(true);
@@ -151,6 +163,7 @@ fn fetch_partition_range(input: Arc<dyn ExecutionPlan>) -> DataFusionResult<Opti
         ScannerInfo {
             partition_ranges: partition_ranges?,
             time_index: time_index?,
+            tag_columns: tag_columns?,
         }
     };
 
diff --git a/src/query/src/part_sort.rs b/src/query/src/part_sort.rs
index c4b9d35b20..2b258187b5 100644
--- a/src/query/src/part_sort.rs
+++ b/src/query/src/part_sort.rs
@@ -175,6 +175,7 @@ struct PartSortStream {
     input_complete: bool,
     schema: SchemaRef,
     partition_ranges: Vec<PartitionRange>,
+    #[allow(dead_code)] // this is used under #[debug_assertions]
     partition: usize,
     cur_part_idx: usize,
     metrics: BaselineMetrics,
diff --git a/src/query/src/window_sort.rs b/src/query/src/window_sort.rs
index 305585b267..38b64e29aa 100644
--- a/src/query/src/window_sort.rs
+++ b/src/query/src/window_sort.rs
@@ -270,6 +270,7 @@ pub struct WindowedSortStream {
     /// working ranges promise once input stream get a value out of current range, future values will never be in this range
     all_avail_working_range: Vec<(TimeRange, BTreeSet<usize>)>,
     /// The input partition ranges
+    #[allow(dead_code)] // this is used under #[debug_assertions]
     ranges: Vec<PartitionRange>,
     /// Execution metrics
     metrics: BaselineMetrics,
diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs
index 0832385c93..8dd706395d 100644
--- a/src/store-api/src/region_engine.rs
+++ b/src/store-api/src/region_engine.rs
@@ -265,6 +265,9 @@ pub trait RegionScanner: Debug + DisplayAs + Send {
     /// Returns the schema of the record batches.
     fn schema(&self) -> SchemaRef;
 
+    /// Returns the metadata of the region.
+    fn metadata(&self) -> RegionMetadataRef;
+
     /// Prepares the scanner with the given partition ranges.
     ///
     /// This method is for the planner to adjust the scanner's behavior based on the partition ranges.
@@ -414,11 +417,16 @@ pub struct SinglePartitionScanner {
     stream: Mutex<Option<SendableRecordBatchStream>>,
     schema: SchemaRef,
     properties: ScannerProperties,
+    metadata: RegionMetadataRef,
 }
 
 impl SinglePartitionScanner {
-    /// Creates a new [SinglePartitionScanner] with the given stream.
-    pub fn new(stream: SendableRecordBatchStream, append_mode: bool) -> Self {
+    /// Creates a new [SinglePartitionScanner] with the given stream and metadata.
+    pub fn new(
+        stream: SendableRecordBatchStream,
+        append_mode: bool,
+        metadata: RegionMetadataRef,
+    ) -> Self {
         let schema = stream.schema();
         Self {
             stream: Mutex::new(Some(stream)),
@@ -426,6 +434,7 @@ impl SinglePartitionScanner {
             properties: ScannerProperties::default()
                 .with_parallelism(1)
                 .with_append_mode(append_mode),
+            metadata,
         }
     }
 }
@@ -468,6 +477,10 @@ impl RegionScanner for SinglePartitionScanner {
     fn has_predicate(&self) -> bool {
         false
     }
+
+    fn metadata(&self) -> RegionMetadataRef {
+        self.metadata.clone()
+    }
 }
 
 impl DisplayAs for SinglePartitionScanner {
diff --git a/src/table/src/table/scan.rs b/src/table/src/table/scan.rs
index 9207c1e86d..cc94a054de 100644
--- a/src/table/src/table/scan.rs
+++ b/src/table/src/table/scan.rs
@@ -154,6 +154,16 @@ impl RegionScanExec {
             .timestamp_column()
             .map(|x| x.name.clone())
     }
+
+    pub fn tag_columns(&self) -> Vec<String> {
+        self.scanner
+            .lock()
+            .unwrap()
+            .metadata()
+            .primary_key_columns()
+            .map(|col| col.column_schema.name.clone())
+            .collect()
+    }
 }
 
 impl ExecutionPlan for RegionScanExec {
@@ -301,33 +311,45 @@ impl DfRecordBatchStream for StreamWithMetricWrapper {
 mod test {
     use std::sync::Arc;
 
+    use api::v1::SemanticType;
     use common_recordbatch::{RecordBatch, RecordBatches};
     use datafusion::prelude::SessionContext;
     use datatypes::data_type::ConcreteDataType;
     use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
-    use datatypes::vectors::Int32Vector;
+    use datatypes::vectors::{Int32Vector, TimestampMillisecondVector};
     use futures::TryStreamExt;
+    use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder};
     use store_api::region_engine::SinglePartitionScanner;
+    use store_api::storage::RegionId;
 
     use super::*;
 
     #[tokio::test]
     async fn test_simple_table_scan() {
         let ctx = SessionContext::new();
-        let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
-            "a",
-            ConcreteDataType::int32_datatype(),
-            false,
-        )]));
+        let schema = Arc::new(Schema::new(vec![
+            ColumnSchema::new("a", ConcreteDataType::int32_datatype(), false),
+            ColumnSchema::new(
+                "b",
+                ConcreteDataType::timestamp_millisecond_datatype(),
+                false,
+            ),
+        ]));
 
         let batch1 = RecordBatch::new(
             schema.clone(),
-            vec![Arc::new(Int32Vector::from_slice([1, 2])) as _],
+            vec![
+                Arc::new(Int32Vector::from_slice([1, 2])) as _,
+                Arc::new(TimestampMillisecondVector::from_slice([1000, 2000])) as _,
+            ],
         )
         .unwrap();
         let batch2 = RecordBatch::new(
             schema.clone(),
-            vec![Arc::new(Int32Vector::from_slice([3, 4, 5])) as _],
+            vec![
+                Arc::new(Int32Vector::from_slice([3, 4, 5])) as _,
+                Arc::new(TimestampMillisecondVector::from_slice([3000, 4000, 5000])) as _,
+            ],
         )
         .unwrap();
 
@@ -335,7 +357,26 @@ mod test {
             RecordBatches::try_new(schema.clone(), vec![batch1.clone(), batch2.clone()]).unwrap();
         let stream = recordbatches.as_stream();
 
-        let scanner = Box::new(SinglePartitionScanner::new(stream, false));
+        let mut builder = RegionMetadataBuilder::new(RegionId::new(1234, 5678));
+        builder
+            .push_column_metadata(ColumnMetadata {
+                column_schema: ColumnSchema::new("a", ConcreteDataType::int32_datatype(), false),
+                semantic_type: SemanticType::Tag,
+                column_id: 1,
+            })
+            .push_column_metadata(ColumnMetadata {
+                column_schema: ColumnSchema::new(
+                    "b",
+                    ConcreteDataType::timestamp_millisecond_datatype(),
+                    false,
+                ),
+                semantic_type: SemanticType::Timestamp,
+                column_id: 2,
+            })
+            .primary_key(vec![1]);
+        let region_metadata = Arc::new(builder.build().unwrap());
+
+        let scanner = Box::new(SinglePartitionScanner::new(stream, false, region_metadata));
         let plan = RegionScanExec::new(scanner);
         let actual: SchemaRef = Arc::new(
             plan.properties
diff --git a/tests/cases/standalone/common/order/windowed_sort.result b/tests/cases/standalone/common/order/windowed_sort.result
index 1cc0ab7720..9ecec83d20 100644
--- a/tests/cases/standalone/common/order/windowed_sort.result
+++ b/tests/cases/standalone/common/order/windowed_sort.result
@@ -1,3 +1,4 @@
+-- Test without PK, with a windowed sort query.
 CREATE TABLE test(i INTEGER, t TIMESTAMP TIME INDEX);
 
 Affected Rows: 0
@@ -69,7 +70,39 @@ EXPLAIN ANALYZE SELECT * FROM test ORDER BY t LIMIT 5;
 | 1_| 0_|_GlobalLimitExec: skip=0, fetch=5 REDACTED
 |_|_|_SortPreservingMergeExec: [t@1 ASC NULLS LAST] REDACTED
 |_|_|_WindowedSortExec REDACTED
-|_|_|_PartSortExec t@1 ASC NULLS LAST REDACTED
+|_|_|_SeqScan: region=REDACTED, partition_count=2 (1 memtable ranges, 1 file 1 ranges) REDACTED
+|_|_|_|
+|_|_| Total rows: 5_|
++-+-+-+
+
+SELECT * FROM test ORDER BY t DESC LIMIT 5;
+
++---+-------------------------+
+| i | t                       |
++---+-------------------------+
+| 4 | 1970-01-01T00:00:00.012 |
+| 4 | 1970-01-01T00:00:00.011 |
+| 4 | 1970-01-01T00:00:00.010 |
+| 3 | 1970-01-01T00:00:00.009 |
+| 3 | 1970-01-01T00:00:00.008 |
++---+-------------------------+
+
+-- SQLNESS REPLACE (-+) -
+-- SQLNESS REPLACE (\s\s+) _
+-- SQLNESS REPLACE (peers.*) REDACTED
+-- SQLNESS REPLACE (metrics.*) REDACTED
+-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
+EXPLAIN ANALYZE SELECT * FROM test ORDER BY t DESC LIMIT 5;
+
++-+-+-+
+| stage | node | plan_|
++-+-+-+
+| 0_| 0_|_MergeScanExec: REDACTED
+|_|_|_|
+| 1_| 0_|_GlobalLimitExec: skip=0, fetch=5 REDACTED
+|_|_|_SortPreservingMergeExec: [t@1 DESC] REDACTED
+|_|_|_WindowedSortExec REDACTED
+|_|_|_PartSortExec t@1 DESC REDACTED
 |_|_|_SeqScan: region=REDACTED, partition_count=2 (1 memtable ranges, 1 file 1 ranges) REDACTED
 |_|_|_|
 |_|_| Total rows: 5_|
@@ -79,3 +112,118 @@ DROP TABLE test;
 
 Affected Rows: 0
 
+-- Test with PK, with a windowed sort query.
+CREATE TABLE test_pk(pk INTEGER PRIMARY KEY, i INTEGER, t TIMESTAMP TIME INDEX);
+
+Affected Rows: 0
+
+INSERT INTO test_pk VALUES (1, 1, 1), (2, NULL, 2), (3, 1, 3);
+
+Affected Rows: 3
+
+ADMIN FLUSH_TABLE('test_pk');
+
++------------------------------+
+| ADMIN FLUSH_TABLE('test_pk') |
++------------------------------+
+| 0                            |
++------------------------------+
+
+INSERT INTO test_pk VALUES (4, 2, 4), (5, 2, 5), (6, NULL, 6);
+
+Affected Rows: 3
+
+ADMIN FLUSH_TABLE('test_pk');
+
++------------------------------+
+| ADMIN FLUSH_TABLE('test_pk') |
++------------------------------+
+| 0                            |
++------------------------------+
+
+INSERT INTO test_pk VALUES (7, 3, 7), (8, 3, 8), (9, 3, 9);
+
+Affected Rows: 3
+
+ADMIN FLUSH_TABLE('test_pk');
+
++------------------------------+
+| ADMIN FLUSH_TABLE('test_pk') |
++------------------------------+
+| 0                            |
++------------------------------+
+
+INSERT INTO test_pk VALUES (10, 4, 10), (11, 4, 11), (12, 4, 12);
+
+Affected Rows: 3
+
+SELECT * FROM test_pk ORDER BY t LIMIT 5;
+
++----+---+-------------------------+
+| pk | i | t                       |
++----+---+-------------------------+
+| 1  | 1 | 1970-01-01T00:00:00.001 |
+| 2  |   | 1970-01-01T00:00:00.002 |
+| 3  | 1 | 1970-01-01T00:00:00.003 |
+| 4  | 2 | 1970-01-01T00:00:00.004 |
+| 5  | 2 | 1970-01-01T00:00:00.005 |
++----+---+-------------------------+
+
+-- SQLNESS REPLACE (-+) -
+-- SQLNESS REPLACE (\s\s+) _
+-- SQLNESS REPLACE (peers.*) REDACTED
+-- SQLNESS REPLACE (metrics.*) REDACTED
+-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
+EXPLAIN ANALYZE SELECT * FROM test_pk ORDER BY t LIMIT 5;
+
++-+-+-+
+| stage | node | plan_|
++-+-+-+
+| 0_| 0_|_MergeScanExec: REDACTED
+|_|_|_|
+| 1_| 0_|_GlobalLimitExec: skip=0, fetch=5 REDACTED
+|_|_|_SortPreservingMergeExec: [t@2 ASC NULLS LAST] REDACTED
+|_|_|_WindowedSortExec REDACTED
+|_|_|_PartSortExec t@2 ASC NULLS LAST REDACTED
+|_|_|_SeqScan: region=REDACTED, partition_count=2 (1 memtable ranges, 1 file 1 ranges) REDACTED
+|_|_|_|
+|_|_| Total rows: 5_|
++-+-+-+
+
+SELECT * FROM test_pk ORDER BY t DESC LIMIT 5;
+
++----+---+-------------------------+
+| pk | i | t                       |
++----+---+-------------------------+
+| 12 | 4 | 1970-01-01T00:00:00.012 |
+| 11 | 4 | 1970-01-01T00:00:00.011 |
+| 10 | 4 | 1970-01-01T00:00:00.010 |
+| 9  | 3 | 1970-01-01T00:00:00.009 |
+| 8  | 3 | 1970-01-01T00:00:00.008 |
++----+---+-------------------------+
+
+-- SQLNESS REPLACE (-+) -
+-- SQLNESS REPLACE (\s\s+) _
+-- SQLNESS REPLACE (peers.*) REDACTED
+-- SQLNESS REPLACE (metrics.*) REDACTED
+-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
+EXPLAIN ANALYZE SELECT * FROM test_pk ORDER BY t DESC LIMIT 5;
+
++-+-+-+
+| stage | node | plan_|
++-+-+-+
+| 0_| 0_|_MergeScanExec: REDACTED
+|_|_|_|
+| 1_| 0_|_GlobalLimitExec: skip=0, fetch=5 REDACTED
+|_|_|_SortPreservingMergeExec: [t@2 DESC] REDACTED
+|_|_|_WindowedSortExec REDACTED
+|_|_|_PartSortExec t@2 DESC REDACTED
+|_|_|_SeqScan: region=REDACTED, partition_count=2 (1 memtable ranges, 1 file 1 ranges) REDACTED
+|_|_|_|
+|_|_| Total rows: 5_|
++-+-+-+
+
+DROP TABLE test_pk;
+
+Affected Rows: 0
+
diff --git a/tests/cases/standalone/common/order/windowed_sort.sql b/tests/cases/standalone/common/order/windowed_sort.sql
index 7767825e3d..e8006f74ce 100644
--- a/tests/cases/standalone/common/order/windowed_sort.sql
+++ b/tests/cases/standalone/common/order/windowed_sort.sql
@@ -1,3 +1,4 @@
+-- Test without PK, with a windowed sort query.
 CREATE TABLE test(i INTEGER, t TIMESTAMP TIME INDEX);
 
 INSERT INTO test VALUES (1, 1), (NULL, 2), (1, 3);
@@ -23,4 +24,50 @@ SELECT * FROM test ORDER BY t LIMIT 5;
 -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
 EXPLAIN ANALYZE SELECT * FROM test ORDER BY t LIMIT 5;
 
+SELECT * FROM test ORDER BY t DESC LIMIT 5;
+
+-- SQLNESS REPLACE (-+) -
+-- SQLNESS REPLACE (\s\s+) _
+-- SQLNESS REPLACE (peers.*) REDACTED
+-- SQLNESS REPLACE (metrics.*) REDACTED
+-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
+EXPLAIN ANALYZE SELECT * FROM test ORDER BY t DESC LIMIT 5;
+
 DROP TABLE test;
+
+-- Test with PK, with a windowed sort query.
+CREATE TABLE test_pk(pk INTEGER PRIMARY KEY, i INTEGER, t TIMESTAMP TIME INDEX);
+
+INSERT INTO test_pk VALUES (1, 1, 1), (2, NULL, 2), (3, 1, 3);
+
+ADMIN FLUSH_TABLE('test_pk');
+
+INSERT INTO test_pk VALUES (4, 2, 4), (5, 2, 5), (6, NULL, 6);
+
+ADMIN FLUSH_TABLE('test_pk');
+
+INSERT INTO test_pk VALUES (7, 3, 7), (8, 3, 8), (9, 3, 9);
+
+ADMIN FLUSH_TABLE('test_pk');
+
+INSERT INTO test_pk VALUES (10, 4, 10), (11, 4, 11), (12, 4, 12);
+
+SELECT * FROM test_pk ORDER BY t LIMIT 5;
+
+-- SQLNESS REPLACE (-+) -
+-- SQLNESS REPLACE (\s\s+) _
+-- SQLNESS REPLACE (peers.*) REDACTED
+-- SQLNESS REPLACE (metrics.*) REDACTED
+-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
+EXPLAIN ANALYZE SELECT * FROM test_pk ORDER BY t LIMIT 5;
+
+SELECT * FROM test_pk ORDER BY t DESC LIMIT 5;
+
+-- SQLNESS REPLACE (-+) -
+-- SQLNESS REPLACE (\s\s+) _
+-- SQLNESS REPLACE (peers.*) REDACTED
+-- SQLNESS REPLACE (metrics.*) REDACTED
+-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED
+EXPLAIN ANALYZE SELECT * FROM test_pk ORDER BY t DESC LIMIT 5;
+
+DROP TABLE test_pk;