diff --git a/src/mito2/src/worker/handle_alter.rs b/src/mito2/src/worker/handle_alter.rs index 6fa560e90c..e022d16631 100644 --- a/src/mito2/src/worker/handle_alter.rs +++ b/src/mito2/src/worker/handle_alter.rs @@ -234,6 +234,18 @@ impl RegionWorkerLoop { all_options_altered = false; } } + SetRegionOption::AutoFlushInterval(new_interval) => { + // The flush logic reads the effective interval from the region's + // current version each cycle, so the change takes effect on the + // next flush without a memtable flush. + if new_interval != current_options.auto_flush_interval { + info!( + "Update region auto_flush_interval: {}, previous: {:?} new: {:?}", + region.region_id, current_options.auto_flush_interval, new_interval + ); + current_options.auto_flush_interval = new_interval; + } + } } } region.version_control.alter_options(current_options); @@ -261,7 +273,9 @@ fn new_region_options_on_empty_memtable( let mut current_options = current_options.clone(); for option in options { match option { - SetRegionOption::Ttl(_) | SetRegionOption::Twsc(_, _) => (), + SetRegionOption::Ttl(_) + | SetRegionOption::Twsc(_, _) + | SetRegionOption::AutoFlushInterval(_) => (), SetRegionOption::Format(format_str) => { // Safety: handle_alter_region_options_fast() has validated this. let new_format = format_str.parse::().unwrap(); diff --git a/src/store-api/src/region_request.rs b/src/store-api/src/region_request.rs index 31cf6e1b61..e046e9df56 100644 --- a/src/store-api/src/region_request.rs +++ b/src/store-api/src/region_request.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; use std::fmt::{self, Display}; +use std::time::Duration; use api::helper::{ColumnDataTypeWrapper, from_pb_time_ranges}; use api::v1::add_column_location::LocationType; @@ -52,8 +53,8 @@ use crate::metadata::{ use crate::metric_engine_consts::PHYSICAL_TABLE_METADATA_KEY; use crate::metrics; use crate::mito_engine_options::{ - APPEND_MODE_KEY, SST_FORMAT_KEY, TTL_KEY, TWCS_MAX_OUTPUT_FILE_SIZE, TWCS_TIME_WINDOW, - TWCS_TRIGGER_FILE_NUM, + APPEND_MODE_KEY, AUTO_FLUSH_INTERVAL_KEY, SST_FORMAT_KEY, TTL_KEY, TWCS_MAX_OUTPUT_FILE_SIZE, + TWCS_TIME_WINDOW, TWCS_TRIGGER_FILE_NUM, }; use crate::path_utils::table_dir; use crate::storage::{ColumnId, RegionId, ScanRequest}; @@ -1370,6 +1371,8 @@ pub enum SetRegionOption { Format(String), // Modifying the append mode. AppendMode(bool), + // Modifying the per-region auto flush interval override. + AutoFlushInterval(Option), } impl TryFrom<&PbOption> for SetRegionOption { @@ -1394,6 +1397,20 @@ impl TryFrom<&PbOption> for SetRegionOption { .map_err(|_| InvalidSetRegionOptionRequestSnafu { key, value }.build())?; Ok(Self::AppendMode(append_mode)) } + AUTO_FLUSH_INTERVAL_KEY => { + if value.is_empty() { + // SET 'auto_flush_interval' = NULL comes through as an empty + // string; treat it as clearing the override (fall back to + // the global default), same as Ttl. + return Ok(Self::AutoFlushInterval(None)); + } + let interval = humantime::parse_duration(value) + .map_err(|_| InvalidSetRegionOptionRequestSnafu { key, value }.build())?; + if interval <= Duration::ZERO { + return InvalidSetRegionOptionRequestSnafu { key, value }.fail(); + } + Ok(Self::AutoFlushInterval(Some(interval))) + } _ => InvalidSetRegionOptionRequestSnafu { key, value }.fail(), } } @@ -1678,6 +1695,44 @@ mod tests { .unwrap_err(); } + #[test] + fn test_set_region_option_auto_flush_interval_try_from() { + use std::time::Duration; + + // Valid duration + let pb = PbOption { + key: "auto_flush_interval".to_string(), + value: "5m".to_string(), + }; + let opt = SetRegionOption::try_from(&pb).unwrap(); + assert_eq!( + opt, + SetRegionOption::AutoFlushInterval(Some(Duration::from_secs(300))) + ); + + // Empty value clears the override (mirrors Ttl's behaviour for `SET key = NULL`). + let pb = PbOption { + key: "auto_flush_interval".to_string(), + value: String::new(), + }; + let opt = SetRegionOption::try_from(&pb).unwrap(); + assert_eq!(opt, SetRegionOption::AutoFlushInterval(None)); + + // Zero is rejected up front (engine invariant). + let pb = PbOption { + key: "auto_flush_interval".to_string(), + value: "0s".to_string(), + }; + assert!(SetRegionOption::try_from(&pb).is_err()); + + // Garbage value is rejected. + let pb = PbOption { + key: "auto_flush_interval".to_string(), + value: "not_a_duration".to_string(), + }; + assert!(SetRegionOption::try_from(&pb).is_err()); + } + #[test] fn test_from_proto_alter_request() { RegionAlterRequest::try_from(AlterRequest { diff --git a/src/table/src/metadata.rs b/src/table/src/metadata.rs index e5d0023c49..006cfb3496 100644 --- a/src/table/src/metadata.rs +++ b/src/table/src/metadata.rs @@ -29,7 +29,8 @@ use serde::{Deserialize, Deserializer, Serialize}; use snafu::{OptionExt, ResultExt, ensure}; use store_api::metric_engine_consts::PHYSICAL_TABLE_METADATA_KEY; use store_api::mito_engine_options::{ - APPEND_MODE_KEY, COMPACTION_TYPE, COMPACTION_TYPE_TWCS, MERGE_MODE_KEY, SST_FORMAT_KEY, + APPEND_MODE_KEY, AUTO_FLUSH_INTERVAL_KEY, COMPACTION_TYPE, COMPACTION_TYPE_TWCS, + MERGE_MODE_KEY, SST_FORMAT_KEY, }; use store_api::region_request::{SetRegionOption, UnsetRegionOption}; use store_api::storage::{ColumnDescriptor, ColumnDescriptorBuilder, ColumnId}; @@ -378,6 +379,16 @@ impl TableMeta { new_options.extra_options.remove(MERGE_MODE_KEY); } } + SetRegionOption::AutoFlushInterval(new_interval) => { + if let Some(interval) = new_interval { + new_options.extra_options.insert( + AUTO_FLUSH_INTERVAL_KEY.to_string(), + humantime::format_duration(*interval).to_string(), + ); + } else { + new_options.extra_options.remove(AUTO_FLUSH_INTERVAL_KEY); + } + } } } let mut builder = self.new_meta_builder(); @@ -1872,6 +1883,73 @@ mod tests { ); } + #[test] + fn test_set_auto_flush_interval() { + let schema = Arc::new(new_test_schema()); + let table_options = TableOptions::default(); + let meta = TableMetaBuilder::empty() + .schema(schema) + .primary_key_indices(vec![0]) + .engine("engine") + .next_column_id(3) + .options(table_options) + .build() + .unwrap(); + + let alter_kind = AlterKind::SetTableOptions { + options: vec![SetRegionOption::AutoFlushInterval(Some( + std::time::Duration::from_secs(300), + ))], + }; + let new_meta = meta + .builder_with_alter_kind("my_table", &alter_kind) + .unwrap() + .build() + .unwrap(); + + assert_eq!( + Some("5m"), + new_meta + .options + .extra_options + .get(AUTO_FLUSH_INTERVAL_KEY) + .map(String::as_str) + ); + } + + #[test] + fn test_set_auto_flush_interval_none_removes_existing() { + let schema = Arc::new(new_test_schema()); + let mut table_options = TableOptions::default(); + table_options + .extra_options + .insert(AUTO_FLUSH_INTERVAL_KEY.to_string(), "5m".to_string()); + let meta = TableMetaBuilder::empty() + .schema(schema) + .primary_key_indices(vec![0]) + .engine("engine") + .next_column_id(3) + .options(table_options) + .build() + .unwrap(); + + let alter_kind = AlterKind::SetTableOptions { + options: vec![SetRegionOption::AutoFlushInterval(None)], + }; + let new_meta = meta + .builder_with_alter_kind("my_table", &alter_kind) + .unwrap() + .build() + .unwrap(); + + assert!( + !new_meta + .options + .extra_options + .contains_key(AUTO_FLUSH_INTERVAL_KEY) + ); + } + #[test] fn test_add_columns_multiple_times() { let schema = Arc::new(new_test_schema()); diff --git a/tests/cases/standalone/common/alter/alter_auto_flush_interval.result b/tests/cases/standalone/common/alter/alter_auto_flush_interval.result new file mode 100644 index 0000000000..d35c0f77f9 --- /dev/null +++ b/tests/cases/standalone/common/alter/alter_auto_flush_interval.result @@ -0,0 +1,233 @@ +-- Test altering auto_flush_interval on a mito table +-- Create a table without auto_flush_interval (uses global default) +CREATE TABLE test_alter_auto_flush_interval( + host STRING, + ts TIMESTAMP TIME INDEX, + cpu DOUBLE, + PRIMARY KEY(host) +) ENGINE=mito; + +Affected Rows: 0 + +-- Verify the table was created with the default auto_flush_interval +SHOW CREATE TABLE test_alter_auto_flush_interval; + ++--------------------------------+---------------------------------------------------------------+ +| Table | Create Table | ++--------------------------------+---------------------------------------------------------------+ +| test_alter_auto_flush_interval | CREATE TABLE IF NOT EXISTS "test_alter_auto_flush_interval" ( | +| | "host" STRING NULL, | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "cpu" DOUBLE NULL, | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++--------------------------------+---------------------------------------------------------------+ + +-- Alter auto_flush_interval to 5 minutes +ALTER TABLE test_alter_auto_flush_interval SET 'auto_flush_interval' = '5m'; + +Affected Rows: 0 + +-- Verify the change took effect +SHOW CREATE TABLE test_alter_auto_flush_interval; + ++--------------------------------+---------------------------------------------------------------+ +| Table | Create Table | ++--------------------------------+---------------------------------------------------------------+ +| test_alter_auto_flush_interval | CREATE TABLE IF NOT EXISTS "test_alter_auto_flush_interval" ( | +| | "host" STRING NULL, | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "cpu" DOUBLE NULL, | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | auto_flush_interval = '5m' | +| | ) | ++--------------------------------+---------------------------------------------------------------+ + +-- Insert some data after the alter +INSERT INTO test_alter_auto_flush_interval VALUES ('host1', 0, 1.0), ('host2', 1, 2.0); + +Affected Rows: 2 + +SELECT * FROM test_alter_auto_flush_interval ORDER BY host, ts; + ++-------+-------------------------+-----+ +| host | ts | cpu | ++-------+-------------------------+-----+ +| host1 | 1970-01-01T00:00:00 | 1.0 | +| host2 | 1970-01-01T00:00:00.001 | 2.0 | ++-------+-------------------------+-----+ + +-- Re-altering to the same value should succeed +ALTER TABLE test_alter_auto_flush_interval SET 'auto_flush_interval' = '5m'; + +Affected Rows: 0 + +-- Re-altering to a different value should succeed +ALTER TABLE test_alter_auto_flush_interval SET 'auto_flush_interval' = '10m'; + +Affected Rows: 0 + +-- Verify the new value took effect +SHOW CREATE TABLE test_alter_auto_flush_interval; + ++--------------------------------+---------------------------------------------------------------+ +| Table | Create Table | ++--------------------------------+---------------------------------------------------------------+ +| test_alter_auto_flush_interval | CREATE TABLE IF NOT EXISTS "test_alter_auto_flush_interval" ( | +| | "host" STRING NULL, | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "cpu" DOUBLE NULL, | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | auto_flush_interval = '10m' | +| | ) | ++--------------------------------+---------------------------------------------------------------+ + +-- Trying to set an invalid duration should fail +-- SQLNESS REPLACE \d+\(\d+,\s+\d+\) REDACTED +ALTER TABLE test_alter_auto_flush_interval SET 'auto_flush_interval' = 'not_a_duration'; + +Error: 1004(InvalidArguments), Invalid set table option request: Invalid set region option request, key: auto_flush_interval, value: not_a_duration + +-- Trying to set a zero duration should fail (engine requires > 0) +-- SQLNESS REPLACE \d+\(\d+,\s+\d+\) REDACTED +ALTER TABLE test_alter_auto_flush_interval SET 'auto_flush_interval' = '0s'; + +Error: 1004(InvalidArguments), Invalid set table option request: Invalid set region option request, key: auto_flush_interval, value: 0s + +-- Clean up +DROP TABLE test_alter_auto_flush_interval; + +Affected Rows: 0 + +-- Test clearing the auto_flush_interval override via SET = NULL +CREATE TABLE test_alter_auto_flush_interval_clear( + host STRING, + ts TIMESTAMP TIME INDEX, + cpu DOUBLE, + PRIMARY KEY(host) +) ENGINE=mito WITH ('auto_flush_interval' = '15m'); + +Affected Rows: 0 + +-- Verify initial value +SHOW CREATE TABLE test_alter_auto_flush_interval_clear; + ++--------------------------------------+---------------------------------------------------------------------+ +| Table | Create Table | ++--------------------------------------+---------------------------------------------------------------------+ +| test_alter_auto_flush_interval_clear | CREATE TABLE IF NOT EXISTS "test_alter_auto_flush_interval_clear" ( | +| | "host" STRING NULL, | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "cpu" DOUBLE NULL, | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | auto_flush_interval = '15m' | +| | ) | ++--------------------------------------+---------------------------------------------------------------------+ + +-- Clear the override by setting to NULL +ALTER TABLE test_alter_auto_flush_interval_clear SET 'auto_flush_interval' = NULL; + +Affected Rows: 0 + +-- Verify the override is cleared (the option should no longer appear in SHOW CREATE) +SHOW CREATE TABLE test_alter_auto_flush_interval_clear; + ++--------------------------------------+---------------------------------------------------------------------+ +| Table | Create Table | ++--------------------------------------+---------------------------------------------------------------------+ +| test_alter_auto_flush_interval_clear | CREATE TABLE IF NOT EXISTS "test_alter_auto_flush_interval_clear" ( | +| | "host" STRING NULL, | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "cpu" DOUBLE NULL, | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host") | +| | ) | +| | | +| | ENGINE=mito | +| | | ++--------------------------------------+---------------------------------------------------------------------+ + +-- Clean up +DROP TABLE test_alter_auto_flush_interval_clear; + +Affected Rows: 0 + +-- Test altering auto_flush_interval on a table that already had it set at create time +CREATE TABLE test_alter_auto_flush_interval_with_default( + host STRING, + ts TIMESTAMP TIME INDEX, + cpu DOUBLE, + PRIMARY KEY(host) +) ENGINE=mito WITH ('auto_flush_interval' = '1h'); + +Affected Rows: 0 + +-- Verify initial value +SHOW CREATE TABLE test_alter_auto_flush_interval_with_default; + ++---------------------------------------------+----------------------------------------------------------------------------+ +| Table | Create Table | ++---------------------------------------------+----------------------------------------------------------------------------+ +| test_alter_auto_flush_interval_with_default | CREATE TABLE IF NOT EXISTS "test_alter_auto_flush_interval_with_default" ( | +| | "host" STRING NULL, | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "cpu" DOUBLE NULL, | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | auto_flush_interval = '1h' | +| | ) | ++---------------------------------------------+----------------------------------------------------------------------------+ + +-- Alter it +ALTER TABLE test_alter_auto_flush_interval_with_default SET 'auto_flush_interval' = '30m'; + +Affected Rows: 0 + +-- Verify the new value +SHOW CREATE TABLE test_alter_auto_flush_interval_with_default; + ++---------------------------------------------+----------------------------------------------------------------------------+ +| Table | Create Table | ++---------------------------------------------+----------------------------------------------------------------------------+ +| test_alter_auto_flush_interval_with_default | CREATE TABLE IF NOT EXISTS "test_alter_auto_flush_interval_with_default" ( | +| | "host" STRING NULL, | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "cpu" DOUBLE NULL, | +| | TIME INDEX ("ts"), | +| | PRIMARY KEY ("host") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | auto_flush_interval = '30m' | +| | ) | ++---------------------------------------------+----------------------------------------------------------------------------+ + +-- Clean up +DROP TABLE test_alter_auto_flush_interval_with_default; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/alter/alter_auto_flush_interval.sql b/tests/cases/standalone/common/alter/alter_auto_flush_interval.sql new file mode 100644 index 0000000000..0d57d571a3 --- /dev/null +++ b/tests/cases/standalone/common/alter/alter_auto_flush_interval.sql @@ -0,0 +1,83 @@ +-- Test altering auto_flush_interval on a mito table + +-- Create a table without auto_flush_interval (uses global default) +CREATE TABLE test_alter_auto_flush_interval( + host STRING, + ts TIMESTAMP TIME INDEX, + cpu DOUBLE, + PRIMARY KEY(host) +) ENGINE=mito; + +-- Verify the table was created with the default auto_flush_interval +SHOW CREATE TABLE test_alter_auto_flush_interval; + +-- Alter auto_flush_interval to 5 minutes +ALTER TABLE test_alter_auto_flush_interval SET 'auto_flush_interval' = '5m'; + +-- Verify the change took effect +SHOW CREATE TABLE test_alter_auto_flush_interval; + +-- Insert some data after the alter +INSERT INTO test_alter_auto_flush_interval VALUES ('host1', 0, 1.0), ('host2', 1, 2.0); + +SELECT * FROM test_alter_auto_flush_interval ORDER BY host, ts; + +-- Re-altering to the same value should succeed +ALTER TABLE test_alter_auto_flush_interval SET 'auto_flush_interval' = '5m'; + +-- Re-altering to a different value should succeed +ALTER TABLE test_alter_auto_flush_interval SET 'auto_flush_interval' = '10m'; + +-- Verify the new value took effect +SHOW CREATE TABLE test_alter_auto_flush_interval; + +-- Trying to set an invalid duration should fail +-- SQLNESS REPLACE \d+\(\d+,\s+\d+\) REDACTED +ALTER TABLE test_alter_auto_flush_interval SET 'auto_flush_interval' = 'not_a_duration'; + +-- Trying to set a zero duration should fail (engine requires > 0) +-- SQLNESS REPLACE \d+\(\d+,\s+\d+\) REDACTED +ALTER TABLE test_alter_auto_flush_interval SET 'auto_flush_interval' = '0s'; + +-- Clean up +DROP TABLE test_alter_auto_flush_interval; + +-- Test clearing the auto_flush_interval override via SET = NULL +CREATE TABLE test_alter_auto_flush_interval_clear( + host STRING, + ts TIMESTAMP TIME INDEX, + cpu DOUBLE, + PRIMARY KEY(host) +) ENGINE=mito WITH ('auto_flush_interval' = '15m'); + +-- Verify initial value +SHOW CREATE TABLE test_alter_auto_flush_interval_clear; + +-- Clear the override by setting to NULL +ALTER TABLE test_alter_auto_flush_interval_clear SET 'auto_flush_interval' = NULL; + +-- Verify the override is cleared (the option should no longer appear in SHOW CREATE) +SHOW CREATE TABLE test_alter_auto_flush_interval_clear; + +-- Clean up +DROP TABLE test_alter_auto_flush_interval_clear; + +-- Test altering auto_flush_interval on a table that already had it set at create time +CREATE TABLE test_alter_auto_flush_interval_with_default( + host STRING, + ts TIMESTAMP TIME INDEX, + cpu DOUBLE, + PRIMARY KEY(host) +) ENGINE=mito WITH ('auto_flush_interval' = '1h'); + +-- Verify initial value +SHOW CREATE TABLE test_alter_auto_flush_interval_with_default; + +-- Alter it +ALTER TABLE test_alter_auto_flush_interval_with_default SET 'auto_flush_interval' = '30m'; + +-- Verify the new value +SHOW CREATE TABLE test_alter_auto_flush_interval_with_default; + +-- Clean up +DROP TABLE test_alter_auto_flush_interval_with_default;