feat: support request-level insert WAL skipping (#9088)

* refactor: add skip_wal fields to internal write requests Signed-off-by: WenyXu <wenymedia@gmail.com> * chore(deps): update greptime-proto for insert skip_wal Signed-off-by: WenyXu <wenymedia@gmail.com> * feat(mito): support request-level WAL skipping Signed-off-by: WenyXu <wenymedia@gmail.com> * feat(metric-engine): handle request-level WAL policies Signed-off-by: WenyXu <wenymedia@gmail.com> * feat: propagate insert WAL policy through query context Signed-off-by: WenyXu <wenymedia@gmail.com> * feat: support session-level insert WAL policy via SET Signed-off-by: WenyXu <wenymedia@gmail.com> * fix(metric-engine): require uniform WAL policy in batch puts Signed-off-by: WenyXu <wenymedia@gmail.com> * test(metric-engine): simplify WAL policy coverage Signed-off-by: WenyXu <wenymedia@gmail.com> * refactor(servers): simplify gRPC hint extraction Signed-off-by: WenyXu <wenymedia@gmail.com> * test: flatten Mito and Metric WAL scenario orchestration Signed-off-by: WenyXu <wenymedia@gmail.com> * refactor: separate WAL and memtable-only mutations Signed-off-by: WenyXu <wenymedia@gmail.com> * refactor: carry skip-WAL policy in table insert requests Signed-off-by: WenyXu <wenymedia@gmail.com> * test: flatten skip-WAL policy cases Signed-off-by: WenyXu <wenymedia@gmail.com> * refactor: clarify WAL notifier naming Signed-off-by: WenyXu <wenymedia@gmail.com> * chore: pin merged skip-WAL proto revision Signed-off-by: WenyXu <wenymedia@gmail.com> --------- Signed-off-by: WenyXu <wenymedia@gmail.com>
2026-10-15 16:32:34 +00:00 · 2026-09-10 11:40:51 +00:00
parent 536f42e4a2
commit fa128adb8e
46 changed files with 1750 additions and 98 deletions
@@ -265,6 +265,8 @@ impl Inserter {
        accommodate_existing_schema: bool,
        is_single_value: bool,
    ) -> Result<Output> {
+        let skip_wal = ctx.skip_wal();
+
        // remove empty requests
        requests.inserts.retain(|req| {
            req.rows
@@ -297,7 +299,7 @@ impl Inserter {
            instant_table_ids,
            self.partition_manager.as_ref(),
        )
-        .convert(requests)
+        .convert(requests, skip_wal)
        .await?;

        self.do_request(inserts, &table_infos, &ctx).await
@@ -311,6 +313,8 @@ impl Inserter {
        statement_executor: &StatementExecutor,
        physical_table: String,
    ) -> Result<Output> {
+        let skip_wal = ctx.skip_wal();
+
        // remove empty requests
        requests.inserts.retain(|req| {
            req.rows
@@ -343,7 +347,7 @@ impl Inserter {
            .map(|info| (info.name.clone(), info.clone()))
            .collect::<HashMap<_, _>>();
        let inserts = RowToRegion::new(name_to_info, instant_table_ids, &self.partition_manager)
-            .convert(requests)
+            .convert(requests, skip_wal)
            .await?;

        self.do_request(inserts, &table_infos, &ctx).await
@@ -1743,6 +1747,24 @@ mod tests {
        assert!(table_is_native_histogram(&table));
    }

+    #[test]
+    fn test_skip_wal_does_not_change_table_options() {
+        check_skip_wal_does_not_change_table_options(false);
+        check_skip_wal_does_not_change_table_options(true);
+    }
+
+    fn check_skip_wal_does_not_change_table_options(skip_wal: bool) {
+        let ctx = Arc::new(QueryContext::with(
+            DEFAULT_CATALOG_NAME,
+            DEFAULT_SCHEMA_NAME,
+        ));
+        ctx.set_skip_wal(skip_wal);
+        let mut options = Default::default();
+        fill_table_options_for_create(&mut options, &AutoCreateTableType::Physical, &ctx);
+        assert!(!options.contains_key(session::hints::INSERT_SKIP_WAL_HINT));
+        assert!(!options.contains_key("skip_wal"));
+    }
+
    #[test]
    fn test_last_non_null_create_options_preserve_default_without_append_mode() {
        let ctx = Arc::new(QueryContext::with(
@@ -34,6 +34,7 @@ impl<'a> Partitioner<'a> {
        &self,
        table_info: &TableInfo,
        rows: Rows,
+        skip_wal: bool,
    ) -> Result<Vec<InsertRequest>> {
        let table_id = table_info.table_id();
        let requests = self
@@ -44,6 +45,7 @@ impl<'a> Partitioner<'a> {
            .into_iter()
            .map(
                |(region_number, (rows, partition_expr_version))| InsertRequest {
+                    skip_wal,
                    region_id: RegionId::new(table_id, region_number).into(),
                    rows: Some(rows),
                    partition_expr_version: partition_expr_version
@@ -45,6 +45,7 @@ impl<'a> RowToRegion<'a> {
    pub async fn convert(
        &self,
        requests: RowInsertRequests,
+        skip_wal: bool,
    ) -> Result<InstantAndNormalInsertRequests> {
        let mut region_request = Vec::with_capacity(requests.inserts.len());
        let mut instant_request = Vec::with_capacity(requests.inserts.len());
@@ -55,7 +56,7 @@ impl<'a> RowToRegion<'a> {
            let table_id = table_info.table_id();

            let requests = Partitioner::new(self.partition_manager)
-                .partition_insert_requests(table_info, rows)
+                .partition_insert_requests(table_info, rows, skip_wal)
                .await?;

            if self.instant_table_ids.contains(&table_id) {
@@ -81,3 +82,79 @@ impl<'a> RowToRegion<'a> {
            .context(TableNotFoundSnafu { table_name })
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use api::v1::helper::tag_column_schema;
+    use api::v1::value::ValueData;
+    use api::v1::{ColumnDataType, Row, RowInsertRequest, Rows, Value};
+
+    use super::*;
+    use crate::tests::{
+        create_partition_rule_manager, new_test_table_info, prepare_mocked_backend,
+    };
+
+    #[tokio::test]
+    async fn test_partitioned_insert_skip_wal_normal_and_instant() {
+        check_partitioned_insert_skip_wal(false, false).await;
+        check_partitioned_insert_skip_wal(false, true).await;
+        check_partitioned_insert_skip_wal(true, false).await;
+        check_partitioned_insert_skip_wal(true, true).await;
+    }
+
+    async fn check_partitioned_insert_skip_wal(instant: bool, skip_wal: bool) {
+        let backend = prepare_mocked_backend().await;
+        let partition_manager = create_partition_rule_manager(backend).await;
+        let table_info = Arc::new(new_test_table_info(1, "table_1", [1, 2, 3].into_iter()));
+        let instant_table_ids = if instant {
+            HashSet::from_iter([1])
+        } else {
+            HashSet::default()
+        };
+        let converter = RowToRegion::new(
+            HashMap::from_iter([("table_1".to_string(), table_info)]),
+            instant_table_ids,
+            &partition_manager,
+        );
+        let requests = RowInsertRequests {
+            inserts: vec![RowInsertRequest {
+                table_name: "table_1".to_string(),
+                rows: Some(Rows {
+                    schema: vec![tag_column_schema("a", ColumnDataType::Int32)],
+                    rows: [1, 11, 101]
+                        .into_iter()
+                        .map(|value| Row {
+                            values: vec![Value {
+                                value_data: Some(ValueData::I32Value(value)),
+                            }],
+                        })
+                        .collect(),
+                }),
+            }],
+        };
+        let result = converter.convert(requests, skip_wal).await.unwrap();
+        let (selected, other) = if instant {
+            (result.instant_requests, result.normal_requests)
+        } else {
+            (result.normal_requests, result.instant_requests)
+        };
+        assert!(other.requests.is_empty());
+        assert_eq!(selected.requests.len(), 3);
+        assert!(
+            selected
+                .requests
+                .iter()
+                .all(|request| request.skip_wal == skip_wal)
+        );
+        assert_eq!(
+            selected
+                .requests
+                .iter()
+                .map(|request| request.rows.as_ref().unwrap().rows.len())
+                .sum::<usize>(),
+            3
+        );
+    }
+}
@@ -154,7 +154,7 @@ impl<'a> StatementToRegion<'a> {
        }

        let requests = Partitioner::new(self.partition_manager)
-            .partition_insert_requests(&table_info, Rows { schema, rows })
+            .partition_insert_requests(&table_info, Rows { schema, rows }, query_ctx.skip_wal())
            .await?;
        let requests = RegionInsertRequests { requests };
        if table_info.is_ttl_instant_table() {
@@ -46,7 +46,7 @@ impl<'a> TableToRegion<'a> {

        let rows = Rows { schema, rows };
        let requests = Partitioner::new(self.partition_manager)
-            .partition_insert_requests(self.table_info, rows)
+            .partition_insert_requests(self.table_info, rows, request.skip_wal)
            .await?;

        let requests = RegionInsertRequests { requests };
@@ -84,6 +84,11 @@ mod tests {

    #[tokio::test]
    async fn test_insert_request_table_to_region() {
+        check_insert_request_table_to_region(false).await;
+        check_insert_request_table_to_region(true).await;
+    }
+
+    async fn check_insert_request_table_to_region(skip_wal: bool) {
        // region to datanode placement:
        // 1 -> 1
        // 2 -> 2
@@ -100,12 +105,13 @@ mod tests {

        let converter = TableToRegion::new(&table_info, &partition_manager);

-        let table_request = build_table_request(Arc::new(Int32Vector::from(vec![
+        let mut table_request = build_table_request(Arc::new(Int32Vector::from(vec![
            Some(1),
            None,
            Some(11),
            Some(101),
        ])));
+        table_request.skip_wal = skip_wal;
        let versions = partition_manager
            .find_physical_partition_info(1)
            .await
@@ -127,21 +133,26 @@ mod tests {
        let region_request = region_id_to_region_requests.remove(&region_id).unwrap();
        assert_eq!(
            region_request,
-            build_region_request(vec![Some(101)], region_id, versions[&region_id])
+            build_region_request(vec![Some(101)], region_id, versions[&region_id], skip_wal)
        );

        let region_id = RegionId::new(1, 2).as_u64();
        let region_request = region_id_to_region_requests.remove(&region_id).unwrap();
        assert_eq!(
            region_request,
-            build_region_request(vec![Some(11)], region_id, versions[&region_id])
+            build_region_request(vec![Some(11)], region_id, versions[&region_id], skip_wal)
        );

        let region_id = RegionId::new(1, 3).as_u64();
        let region_request = region_id_to_region_requests.remove(&region_id).unwrap();
        assert_eq!(
            region_request,
-            build_region_request(vec![Some(1), None], region_id, versions[&region_id])
+            build_region_request(
+                vec![Some(1), None],
+                region_id,
+                versions[&region_id],
+                skip_wal
+            )
        );
    }

@@ -151,6 +162,7 @@ mod tests {
            schema_name: DEFAULT_SCHEMA_NAME.to_string(),
            table_name: "table_1".to_string(),
            columns_values: HashMap::from([("a".to_string(), vector)]),
+            skip_wal: false,
        }
    }

@@ -158,8 +170,10 @@ mod tests {
        rows: Vec<Option<i32>>,
        region_id: u64,
        version: Option<u64>,
+        skip_wal: bool,
    ) -> RegionInsertRequest {
        RegionInsertRequest {
+            skip_wal,
            region_id,
            rows: Some(Rows {
                schema: vec![tag_column_schema("a", ColumnDataType::Int32)],
@@ -63,7 +63,7 @@ use query::QueryEngineRef;
 use query::parser::QueryStatement;
 use session::context::{Channel, QueryContextBuilder, QueryContextRef};
 use session::table_name::table_idents_to_full_name;
-use set::{set_query_timeout, set_read_preference};
+use set::{set_query_timeout, set_read_preference, set_skip_wal};
 use snafu::{OptionExt, ResultExt, ensure};
 use sql::ast::ObjectNamePartExt;
 use sql::statements::OptionMap;
@@ -534,6 +534,7 @@ impl StatementExecutor {

        match var_name.as_str() {
            "READ_PREFERENCE" => set_read_preference(set_var.value, query_ctx)?,
+            "SKIP_WAL" => set_skip_wal(set_var.value, query_ctx)?,

            "@@TIME_ZONE" | "@@SESSION.TIME_ZONE" | "TIMEZONE" | "TIME_ZONE" => {
                set_timezone(set_var.value, query_ctx)?
@@ -487,6 +487,7 @@ impl StatementExecutor {
                        schema_name: req.schema_name.clone(),
                        table_name: req.table_name.clone(),
                        columns_values,
+                        skip_wal: query_ctx.skip_wal(),
                    },
                    query_ctx.clone(),
                ));
@@ -38,6 +38,35 @@ lazy_static! {
    static ref PG_TIME_INPUT_REGEX: Regex = Regex::new(r"^(\d+)(ms|s|min|h|d)$").unwrap();
 }

+/// Sets the session WAL policy for ordinary inserts without changing table options.
+pub fn set_skip_wal(exprs: Vec<Expr>, ctx: QueryContextRef) -> Result<()> {
+    let [Expr::Value(value)] = exprs.as_slice() else {
+        return NotSupportedSnafu {
+            feat: "SET skip_wal requires exactly one boolean value",
+        }
+        .fail();
+    };
+    let skip_wal = match &value.value {
+        Value::Boolean(value) => *value,
+        Value::SingleQuotedString(value) | Value::DoubleQuotedString(value) => {
+            value.parse::<bool>().map_err(|_| {
+                NotSupportedSnafu {
+                    feat: format!("Invalid skip_wal value {value:?}: expected true or false"),
+                }
+                .build()
+            })?
+        }
+        _ => {
+            return NotSupportedSnafu {
+                feat: "SET skip_wal requires true or false",
+            }
+            .fail();
+        }
+    };
+    ctx.set_skip_wal(skip_wal);
+    Ok(())
+}
+
 pub fn set_read_preference(exprs: Vec<Expr>, ctx: QueryContextRef) -> Result<()> {
    let read_preference_expr = exprs.first().context(NotSupportedSnafu {
        feat: "No read preference find in set variable statement",
@@ -381,8 +410,58 @@ fn parse_pg_query_timeout_input(input: &str) -> Result<u64> {

 #[cfg(test)]
 mod test {
+    use session::Session;
+    use session::context::{Channel, QueryContext};
+    use sql::ast::{Expr, Value};
+
+    use super::set_skip_wal;
    use crate::statement::set::parse_pg_query_timeout_input;

+    #[test]
+    fn test_set_skip_wal() {
+        let ctx = QueryContext::arc();
+        for value in [true, false] {
+            set_skip_wal(vec![Expr::Value(Value::Boolean(value).into())], ctx.clone()).unwrap();
+            assert_eq!(ctx.skip_wal(), value);
+            set_skip_wal(
+                vec![Expr::Value(
+                    Value::SingleQuotedString(value.to_string()).into(),
+                )],
+                ctx.clone(),
+            )
+            .unwrap();
+            assert_eq!(ctx.skip_wal(), value);
+        }
+        ctx.set_skip_wal(true);
+        for values in [
+            vec![],
+            vec![Expr::Value(Value::Number("1".to_string(), false).into())],
+            vec![Expr::Value(
+                Value::SingleQuotedString("invalid".to_string()).into(),
+            )],
+            vec![Expr::Value(Value::Boolean(false).into()); 2],
+        ] {
+            assert!(set_skip_wal(values, ctx.clone()).is_err());
+            assert!(ctx.skip_wal());
+        }
+    }
+
+    #[test]
+    fn test_set_skip_wal_session_isolation() {
+        for channel in [Channel::Mysql, Channel::Postgres] {
+            let session = Session::new(None, channel, Default::default(), 0);
+            let other = Session::new(None, channel, Default::default(), 1);
+            assert!(!session.new_query_context().skip_wal());
+            set_skip_wal(
+                vec![Expr::Value(Value::Boolean(true).into())],
+                session.new_query_context(),
+            )
+            .unwrap();
+            assert!(session.new_query_context().skip_wal());
+            assert!(!other.new_query_context().skip_wal());
+        }
+    }
+
    #[test]
    fn test_parse_pg_query_timeout_input() {
        assert!(parse_pg_query_timeout_input("").is_err());